diff --git a/outposts/unicore/kustomize-maint/gunicorn_config.yaml b/outposts/unicore/kustomize-maint/gunicorn_config.yaml
deleted file mode 100644
index 2ac599cfdc35efb72b8099440dbf75038c6e4dcc..0000000000000000000000000000000000000000
--- a/outposts/unicore/kustomize-maint/gunicorn_config.yaml
+++ /dev/null
@@ -1,212 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: gunicorn-config
- namespace: unicore
-data:
- gunicorn_http.py: |
- import os
-
- # https://github.com/benoitc/gunicorn/blob/master/examples/example_config.py
- # https://docs.gunicorn.org/en/latest/settings.html#worker-class#
- # https://docs.gunicorn.org/en/latest/design.html#choosing-a-worker-type
-
-
- # Sample Gunicorn configuration file.
-
- #
- # Server socket
- #
- # bind - The socket to bind.
- #
- # A string of the form: 'HOST', 'HOST:PORT', 'unix:PATH'.
- # An IP is a valid HOST.
- #
- # backlog - The number of pending connections. This refers
- # to the number of clients that can be waiting to be
- # served. Exceeding this number results in the client
- # getting an error when attempting to connect. It should
- # only affect servers under significant load.
- #
- # Must be a positive integer. Generally set in the 64-2048
- # range.
- #
-
- bind = "0.0.0.0:8080"
- backlog = 2048
- preload_app = False
-
- #
- # Worker processes
- #
- # workers - The number of worker processes that this server
- # should keep alive for handling requests.
- #
- # A positive integer generally in the 2-4 x $(NUM_CORES)
- # range. You'll want to vary this a bit to find the best
- # for your particular application's work load.
- #
- # worker_class - The type of workers to use. The default
- # sync class should handle most 'normal' types of work
- # loads. You'll want to read
- # http://docs.gunicorn.org/en/latest/design.html#choosing-a-worker-type
- # for information on when you might want to choose one
- # of the other worker classes.
- #
- # A string referring to a Python path to a subclass of
- # gunicorn.workers.base.Worker. The default provided values
- # can be seen at
- # http://docs.gunicorn.org/en/latest/settings.html#worker-class
- #
- # worker_connections - For the eventlet and gevent worker classes
- # this limits the maximum number of simultaneous clients that
- # a single process can handle.
- #
- # A positive integer generally set to around 1000.
- #
- # timeout - If a worker does not notify the master process in this
- # number of seconds it is killed and a new worker is spawned
- # to replace it.
- #
- # Generally set to thirty seconds. Only set this noticeably
- # higher if you're sure of the repercussions for sync workers.
- # For the non sync workers it just means that the worker
- # process is still communicating and is not tied to the length
- # of time required to handle a single request.
- #
- # keepalive - The number of seconds to wait for the next request
- # on a Keep-Alive HTTP connection.
- #
- # A positive integer. Generally set in the 1-5 seconds range.
- #
-
- SQL_TYPE = os.environ.get("SQL_TYPE", "sqlite")
- if SQL_TYPE in ["sqlite", "sqlite+pysqlite"]:
- workers_default = 1
- threads_default = 1
- else:
- workers_default = 4
- threads_default = 25
-
- workers = int(os.environ.get("GUNICORN_PROCESSES", workers_default))
- threads = int(os.environ.get("GUNICORN_THREADS", threads_default))
- worker_class = "uvicorn.workers.UvicornWorker"
- # worker_connections = 1000
- timeout = int(os.environ.get("GUNICORN_TIMEOUT", 30))
- keepalive = 2
-
- #
- # spew - Install a trace function that spews every line of Python
- # that is executed when running the server. This is the
- # nuclear option.
- #
- # True or False
- #
-
- # spew = False
-
- #
- # Server mechanics
- #
- # daemon - Detach the main Gunicorn process from the controlling
- # terminal with a standard fork/fork sequence.
- #
- # True or False
- #
- # raw_env - Pass environment variables to the execution environment.
- #
- # pidfile - The path to a pid file to write
- #
- # A path string or None to not write a pid file.
- #
- # user - Switch worker processes to run as this user.
- #
- # A valid user id (as an integer) or the name of a user that
- # can be retrieved with a call to pwd.getpwnam(value) or None
- # to not change the worker process user.
- #
- # group - Switch worker process to run as this group.
- #
- # A valid group id (as an integer) or the name of a user that
- # can be retrieved with a call to pwd.getgrnam(value) or None
- # to change the worker processes group.
- #
- # umask - A mask for file permissions written by Gunicorn. Note that
- # this affects unix socket permissions.
- #
- # A valid value for the os.umask(mode) call or a string
- # compatible with int(value, 0) (0 means Python guesses
- # the base, so values like "0", "0xFF", "0022" are valid
- # for decimal, hex, and octal representations)
- #
- # tmp_upload_dir - A directory to store temporary request data when
- # requests are read. This will most likely be disappearing soon.
- #
- # A path to a directory where the process owner can write. Or
- # None to signal that Python should choose one on its own.
- #
-
- daemon = False
- raw_env = [
- "GUNICORN_START=true",
- ]
- pidfile = "/home/jhuboutpost/gunicorn.pid"
- umask = 0
- user = 1000
- group = 1000
- tmp_upload_dir = None
- chdir = "/home/jhuboutpost/app"
-
- #
- # Logging
- #
- # logfile - The path to a log file to write to.
- #
- # A path string. "-" means log to stdout.
- #
- # loglevel - The granularity of log output
- #
- # A string of "debug", "info", "warning", "error", "critical"
- #
-
- errorlog = "-"
- loglevel = "info"
- accesslog = "-"
- access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'
-
- #
- # Process naming
- #
- # proc_name - A base to use with setproctitle to change the way
- # that Gunicorn processes are reported in the system process
- # table. This affects things like 'ps' and 'top'. If you're
- # going to be running more than one instance of Gunicorn you'll
- # probably want to set a name to tell them apart. This requires
- # that you install the setproctitle module.
- #
- # A string or None to choose a default of something like 'gunicorn'.
- #
-
- # proc_name = None
-
- #
- # Server hooks
- #
- # post_fork - Called just after a worker has been forked.
- #
- # A callable that takes a server and worker instance
- # as arguments.
- #
- # pre_fork - Called just prior to forking the worker subprocess.
- #
- # A callable that accepts the same arguments as after_fork
- #
- # pre_exec - Called just prior to forking off a secondary
- # master process during things like config reloading.
- #
- # A callable that takes a server instance as the sole argument.
- #
-
- # Max Requests used to reduce memory consumption
- max_requests = int(os.environ.get("GUNICORN_MAX_REQUESTS", 0))
- max_requests_jitter = int(os.environ.get("GUNICORN_MAX_REQUESTS_JITTER", 0))
diff --git a/outposts/unicore/kustomize-maint/jobs-2.6.yaml b/outposts/unicore/kustomize-maint/jobs-2.6.yaml
deleted file mode 100644
index 2263cb6f907d3ece4eb4a5f2050674473aec0a86..0000000000000000000000000000000000000000
--- a/outposts/unicore/kustomize-maint/jobs-2.6.yaml
+++ /dev/null
@@ -1,392 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: unicore-jobs-2-6
- namespace: unicore
-data:
- input_start.sh: |
- #!/bin/bash
-
- # Catch SIGTERM signal and stop $child process
- _term() {
- echo "$(date) - Stop JupyterLab ..."
- if [[ -z $child ]]; then
- echo "$(date) - Stop JupyterLab - unknown PID. Start stop.sh to wait for PID"
- bash ${DIR}/stop.sh &
- else
- pkill --parent $child
- kill $child
- echo "$(date) - Stop JupyterLab ( $child ) ... done"
- if [[ -z $token_pid ]]; then
- echo "$(date) - Validate Token PID not found..."
- else
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ..."
- kill $token_pid
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ... done"
- fi
- fi
- }
-
- trap _term SIGTERM
-
- requirements () {
- echo "$(date) - Setup system specific requirements ( ${HOSTNAME} ) ..."
- if [[ ! -d ${HOME}/.cache/black/19.3b0 ]]; then
- mkdir -p ${HOME}/.cache/black/19.3b0
- fi
- # export a memory warn threshold on login nodes
- hostname | egrep '<hostname_base>' > /dev/null && export JUPYTER_MEMWARNTHRES=0.5
- # set cpu limit/warning
- export FPATH_CPUQUOTA="/sys/fs/cgroup/cpu,cpuacct/user.slice/user-$(id -u $USER).slice/cpu.cfs_quota_us"
- [ -f "${FPATH_CPUQUOTA}" ] && [ $(hostname | egrep '<hostname_cpulimit>') ] && export JUPYTER_CPULIMIT=$(($(cat ${FPATH_CPUQUOTA})/100000))
-
- echo "$(date) - Setup system specific requirements done"
- }
-
- # set env_variables correctly
- set_env () {
- echo "$(date) - Set environment variables ..."
- export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
- export PID_PATH=${DIR}/service.pid
- export VALIDATE_TOKEN_PID_PATH=${DIR}/validate_token.pid
- export LC_ALL=en_US.UTF-8
- export JUPYTER_LOG_DIR=${DIR}
- export JUPYTER_STDOUT=${JUPYTER_LOG_DIR}/stderr
- export PYTHONPATH=""
- export PYTHONNOUSERSITE=1
- export HOSTNAME_S=$(hostname -s)
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export HOSTNAME_I=${HOSTNAME_S}
- else
- export HOSTNAME_I=$(hostname -s)i
- fi
- export CURL_ARGS="--silent --write-out %{http_code} --output /dev/null"
- export CURL_HEADERS="-H \"Authorization: token ${JUPYTERHUB_API_TOKEN}\" -H \"Content-Type: application/json\" -H \"Accept: application/json\""
-
- if [[ $JUPYTERHUB_SERVICE_URL == "https"* ]]; then
- export HUB_PROTO="https"
- else
- export HUB_PROTO="http"
- fi
-
- API_URL_WITHOUT_PROTO=${JUPYTERHUB_API_URL##https\:\/\/}
- export JUPYTERHUB_DOMAIN=${API_URL_WITHOUT_PROTO%%\/*}
- export JUPYTER_SERVER_PUBLIC_URL="https://${JUPYTERHUB_DOMAIN}${JUPYTERHUB_SERVICE_PREFIX}"
- export DWAVE_INSPECTOR_JUPYTER_SERVER_PROXY_EXTERNAL_URL=${JUPYTER_SERVER_PUBLIC_URL}
-
- echo "$(date) - JupyterLab is running on ${HOSTNAME_S}"
- # If this JupyterLab is running on a LoginNode, we'll use the external api url.
- # Otherwise we have to use a LoginNode as Proxy, because there's no internet
- # connection for batch nodes
- if [[ ${HOSTNAME_S} == "<hostname_base>"* ]]; then
- # LoginNode
- # This is the internal jupyterhub certificate, only used when connecting through proxy on login node
- unset JUPYTERHUB_SSL_CLIENT_CA
- export SSH_NODE=${HOSTNAME_I}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub. Jump node for tunnel: ${SSH_NODE}"
- else
- # batch node, no internet connection to the outside world
- export PREVIOUS_API_URL=${JUPYTERHUB_API_URL}
- export REMOTE_PORT="<remote_port>"
- export JUPYTERHUB_CERTIFICATE="${DIR}/service_ca.crt"
-
- if [[ $HUB_PROTO == "https" ]]; then
- export CURL_ARGS="${CURL_ARGS} --cacert ${JUPYTERHUB_CERTIFICATE}"
- export JUPYTERHUB_SSL_CLIENT_CA="${JUPYTERHUB_CERTIFICATE}"
- fi
-
- # Look for random LoginNode to use it as proxy
- # We start at a random index, then we will run through all nodes and test them
- export ALL_REMOTE_NODES=(<hostname_all>)
- size=${#ALL_REMOTE_NODES[@]}
- index=$(($RANDOM % $size))
- start_index=$index
- for _ in ${ALL_REMOTE_NODES[@]}
- do
- HTTPCODE=$(curl -X "GET" ${CURL_ARGS} ${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api)
- if [[ ${HTTPCODE} -ge 200 ]] && [[ ${HTTPCODE} -le 299 ]]; then
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- fi
- # Add additional maintenance check
- STATUSOUT=$(curl -s -w '%{http_code}' http://${ALL_REMOTE_NODES[$index]}:9100/metrics)
- STATUSOUTCODE=$(echo "${STATUSOUT}" | tail -n 1)
- STATUSOUTMAINT=$(echo "${STATUSOUT}" | grep -E '^maint_status' | cut -d' ' -f2)
- if [[ $STATUSOUTCODE -eq 200 ]] && [[ ${STATUSOUTMAINT} -eq 0 ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- else
- echo "$(date) - Hub reachable for ${ALL_REMOTE_NODES[$index]}. But node metrics answered with: ${STATUSOUTCODE} , ${STATUSOUTMAINT}. Continue search for node. Full output: ${STATUSOUT}"
- fi
- fi
- index=$(expr $index + 1)
- if [[ $index == $size ]]; then
- # if loop reached array end continue at 0
- index=0
- fi
- if [[ $index == $start_index ]]; then
- # Tried all login nodes
- echo "$(date) - Could not find any LoginNode to connect to JupyterHub. Please try again in a few minutes."
- exit 1
- fi
- done
- export JUPYTERHUB_ACTIVITY_URL=${JUPYTERHUB_ACTIVITY_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_EVENTS_URL=${JUPYTERHUB_EVENTS_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_SETUPTUNNEL_URL=${JUPYTERHUB_SETUPTUNNEL_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- fi
-
- export JUPYTERHUB_API_TOKEN=$(cat ${DIR}/.jupyter.token)
- export JUPYTERHUB_OAUTH_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_OAUTH_ACCESS_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_SINGLEUSER_APP="jupyter_server.serverapp.ServerApp"
-
- export PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')
- # Replace previously defined port as ip and port passed via
- # cmd line arguments are ignored as in the new implementation of jupyterhub-singleuser
- export JUPYTERHUB_SERVICE_URL=${HUB_PROTO}://0.0.0.0:${PORT}${JUPYTERHUB_SERVICE_PREFIX}
-
- if [[ $HUB_PROTO == "https" ]]; then
- export JUPYTERHUB_SSL_KEYFILE="${DIR}/service_cert.key"
- export JUPYTERHUB_SSL_CERTFILE="${DIR}/service_cert.crt"
- fi
-
- export JUPYTERJSC_CMD_ARGS=""
- echo "$(date) - Set environment variables done"
- }
-
- grace_runtime_kill() {
- # If JupyterLab is running on a batch node, we will kill it 10 seconds
- # before the slrum runtime ends. This way JupyterHub stops the JupyterLab
- # itself, instead of waiting to be notified that JupyterLab stopped.
- if [[ -z $SLURM_JOBID ]]; then
- # On Login Nodes this is not required
- echo "$(date) - SLURM_JOBID not set. Do not set activate grace_runtime_kill"
- return
- fi
- time=$(squeue -o "%L" -h -j $SLURM_JOBID)
- # Check if the time is "NOT_SET" or "UNLIMITED"
- if [[ "$time" == "NOT_SET" || "$time" == "UNLIMITED" || "$time" == "" ]]; then
- echo "$(date) - Time limit not set or unlimited"
- return
- fi
- # Split the time string into its components
- IFS="-:" read -r -a time_array <<< "$time"
- # Set default values for missing components
- for ((i=${#time_array[@]}; i<4; i++)); do
- time_array=(0 "${time_array[@]}")
- done
- # Fill the variables from back to front
- secs=${time_array[3]}
- mins=${time_array[2]}
- hours=${time_array[1]}
- days=${time_array[0]}
- total_seconds=$((days * 86400 + hours * 3600 + mins * 60 + secs - 10))
- echo "$(date) - Kill JupyterLab in $total_seconds seconds."
- sleep $total_seconds && send_spawn_update_fail "Stop JupyterLab" "The runtime of the slurm job $SLURM_JOBID will end within the next 10 seconds. Stop JupyterLab gracefully." &
- }
-
-
- # Check Quota
-
- check_quota () {
- echo "$(date) - Check quota ..."
- if [[ ! -f ${HOME}/.${JUPYTERHUB_SERVER_NAME} ]]; then
- touch ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC1=$?
- echo "Quota Check ${JUPYTERHUB_SERVER_NAME}" >> ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC2=$?
- if [[ $EC1 -ne 0 || $EC1 -ne 0 ]]; then
- send_spawn_update_fail "Disk quota exceeded in $HOME. You have to clean up your home directory before you can start a JupyterLab." "Jupyter-JSC tried to create a testfile in ${HOME} and failed. Job directory may contain further information: '"${DIR}"'"
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- exit 0
- fi
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- send_spawn_update 50 "Disk quota checked." "If Jupyter-JSC could not create files in ${HOME}, JupyterLab would not be able to start."
- else
- echo "$(date) - Could not check quota"
- fi
- echo "$(date) - Check quota done"
- }
-
-
- # Hook to load customized environments before loading modules
-
- pre_start () {
- echo "$(date) - Pre start ..."
- if [[ -f ${HOME}/.jupyter/pre_jupyter-jsc.sh ]]; then
- echo "$(date) - Pre start if ..."
- send_spawn_update_warning 60 "Use customized start script." "You are using a customized environment, defined in ${HOME}/.jupyter/pre_jupyter-jsc.sh."
- echo "------ pre_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/pre_jupyter-jsc.sh
- echo "------ pre_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/pre_jupyter-jsc.sh
- fi
- echo "$(date) - Pre start done"
- }
-
-
- # Load modules
-
- load_modules () {
- echo "$(date) - Load modules ..."
- if [[ -f ${HOME}/.jupyter/start_jupyter-jsc.sh ]]; then
- send_spawn_update_warning 70 "Load customized modules ..." "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- echo "------ start_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/start_jupyter-jsc.sh
- echo "------ start_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/start_jupyter-jsc.sh
- send_spawn_update_warning 80 "Load customized modules done" "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- else
- send_spawn_update 70 "Load default modules ..." "Loading JupyterCollection/2020.2.6 in Stages/Devel-2020."
-
- module purge
- module use ${OTHERSTAGES}
- module load Stages/Devel-2020
- module load GCCcore/.9.3.0 JupyterCollection/2020.2.6
-
- echo "$(date) - checking for GPU"
- lspci -k | grep -A 2 -i "NVIDIA" | grep "Kernel driver in use:" | grep "nvidia"
- if [ $? -eq 0 ]; then
- send_spawn_update 75 "Load Nvidia module ..." "Loading JupyterExtension-nvdashboard/0.6.0-2021.3.2 in Stages/Devel-2020."
- module load JupyterExtension-nvdashboard/0.6.0-2021.3.2
- fi
- send_spawn_update 80 "Load default modules done" "Loading JupyterCollection/2020.2.6 in Stages/Devel-2020."
- fi
- echo "$(date) - Load modules done"
- }
-
- update_config () {
- sed -i -e "s|_port_|${PORT}|g" -e "s|_home_|${JUPYTERHUB_HOME:-${HOME}}|g" -e "s|_servername_|${JUPYTERHUB_SERVER_NAME}|g" ${DIR}/config.py
- if [[ -f ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py ]]; then
- echo "$(date) - Add system specific config ..."
- send_spawn_update 85 "Add system specific configuration." "Use system specific config file ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py"
- echo "" >> ${DIR}/config.py
- cat ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py >> ${DIR}/config.py
- for path in ${JUPYTER_EXTRA_LABEXTENSIONS_PATH//:/$'\n'}; do
- echo "c.LabServerApp.extra_labextensions_path.append('$path')" >> ${JUPYTER_LOG_DIR}/config.py
- done
- echo "$(date) - Add system specific config done"
- fi
- if [[ -f ${EBROOTJUPYTERLAB}/bin/update_favorites_json ]]; then
- # update favorite-dirs with $HOME,$PROJECT,$SCRATCH,
- echo "$(date) - Update favorites"
- ${EBROOTJUPYTERLAB}/bin/update_favorites_json
- fi
- }
-
- load_project_specific_kernel () {
- echo "$(date) - Activate hook for project specific kernels ..."
- if [[ -d ${PROJECT}/.local/share/jupyter ]] && [[ -r ${PROJECT}/.local/share/jupyter ]] && [[ -x ${PROJECT}/.local/share/jupyter ]]; then
- echo "$(date) - Add project kernel for $PROJECT"
- send_spawn_update_warning 90 "Activate project specific kernel." "Add ${PROJECT}/.local/share/jupyter to JUPYTER_PATH env variable. This may impede the start of JupyterLab."
- export JUPYTER_PATH=${JUPYTER_PATH}:${PROJECT}/.local/share/jupyter
- else
- echo "$(date) - Do not add project kernel for $PROJECT"
- # send_spawn_update_warning 90 "Could not activate project specific kernel." "Ensure that ${PROJECT}/.local/share/jupyter exists and is readable and accessible for ${USER}"
- fi
- echo "$(date) - Activate hook for project specific kernels done"
- }
-
- send_event () {
- BODY=${1//\'/}
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_EVENTS_URL}"
- eval " $CURL_CMD"
- }
-
- # show user a message in the UI
- send_spawn_update () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a darkorange message in the UI
- send_spawn_update_warning () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary><span style=\\\"color:darkorange;\\\">${SUMMARY}</span></summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a fail message in the UI
- send_spawn_update_fail () {
- SUMMARY=$1
- DETAILS=$2
- BODY="{\"progress\": 100, \"failed\": true, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send fail status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Send fail update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # setup tunnel to the node JupyterLab is running at
- setup_tunnel () {
- echo "$(date) - Setup tunnel ..."
- BODY="{\"ssh_node\": \"${SSH_NODE}\", \"service\": \"${HOSTNAME_I}:${PORT}\"}"
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_SETUPTUNNEL_URL}"
- HTTPCODE=$(eval " $CURL_CMD")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "Could not setup tunnel. Cancel start."
- exit 1
- fi
- send_spawn_update 40 "Setup ssh port-forwarding." "Create ssh tunnel with system user ljupyter. JupyterHub will then be able to connect to JupyterLab at ${HOSTNAME_I}:${PORT}"
- echo "$(date) - Setup tunnel done"
- }
-
- start () {
- echo "$(date) - Start jupyterhub-singleuser ..."
- export JUPYTERHUB_HOME=${JUPYTERHUB_HOME:-${HOME}}
- cd ${JUPYTERHUB_HOME}
-
- if [[ -n $JUPYTERJSC_USER_CMD ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom command" "Use custom command \$JUPYTERJSC_USER_CMD . You will be redirected, when your JupyterLab is ready."
- timeout 30d ${JUPYTERJSC_USER_CMD} ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- elif [[ -n $JUPYTERJSC_USER_CMD_ARGS ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom arguments" "Use custom arguments \$JUPYTERJSC_USER_CMD_ARGS . You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} ${JUPYTERJSC_USER_CMD_ARGS} &
- child=$!
- else
- send_spawn_update 95 "Start JupyterLab" "You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- fi
- echo "$child" > ${PID_PATH}
- echo "$(date) - Start jupyterhub-singleuser done (PID: $child )"
-
- echo "$(date) - Start validate_token.sh ..."
- /bin/bash ${DIR}/validate_token.sh ${child} ${JUPYTERHUB_API_URL} &
- token_pid=$!
- echo "$token_pid" > ${VALIDATE_TOKEN_PID_PATH}
- echo "$(date) - Start validate_token.sh done (PID: $token_pid )"
-
- wait $child
- }
-
- requirements
- set_env
- send_spawn_update_fail "JupyterLab 2.6 no longer supported on ${SYSTEMNAME}. Please update to newest version" "Update this JupyterLab configuration or delete this one and create a new one with the newest version."
- exit 1
\ No newline at end of file
diff --git a/outposts/unicore/kustomize-maint/jobs-3.2.yaml b/outposts/unicore/kustomize-maint/jobs-3.2.yaml
deleted file mode 100644
index 93b72a45dd6b0db6c5d121cd07898dcd2a3cb187..0000000000000000000000000000000000000000
--- a/outposts/unicore/kustomize-maint/jobs-3.2.yaml
+++ /dev/null
@@ -1,794 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: unicore-jobs-3-2
- namespace: unicore
-data:
- input_start.sh: |
- #!/bin/bash
-
- # Catch SIGTERM signal and stop $child process
- _term() {
- echo "$(date) - Stop JupyterLab ..."
- if [[ -z $child ]]; then
- echo "$(date) - Stop JupyterLab - unknown PID. Start stop.sh to wait for PID"
- bash ${DIR}/stop.sh &
- else
- pkill --parent $child
- kill $child
- echo "$(date) - Stop JupyterLab ( $child ) ... done"
- if [[ -z $token_pid ]]; then
- echo "$(date) - Validate Token PID not found..."
- else
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ..."
- kill $token_pid
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ... done"
- fi
- fi
- }
-
- trap _term SIGTERM
-
- requirements () {
- echo "$(date) - Setup system specific requirements ( ${HOSTNAME} ) ..."
- if [[ ! -d ${HOME}/.cache/black/19.3b0 ]]; then
- mkdir -p ${HOME}/.cache/black/19.3b0
- fi
- # export a memory warn threshold on login nodes
- hostname | egrep '<hostname_base>' > /dev/null && export JUPYTER_MEMWARNTHRES=0.5
- # set cpu limit/warning
- export FPATH_CPUQUOTA="/sys/fs/cgroup/cpu,cpuacct/user.slice/user-$(id -u $USER).slice/cpu.cfs_quota_us"
- [ -f "${FPATH_CPUQUOTA}" ] && [ $(hostname | egrep '<hostname_cpulimit>') ] && export JUPYTER_CPULIMIT=$(($(cat ${FPATH_CPUQUOTA})/100000))
-
- echo "$(date) - Setup system specific requirements done"
- }
-
- # set env_variables correctly
- set_env () {
- echo "$(date) - Set environment variables ..."
- export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
- export PID_PATH=${DIR}/service.pid
- export VALIDATE_TOKEN_PID_PATH=${DIR}/validate_token.pid
- export LC_ALL=en_US.UTF-8
- export JUPYTER_LOG_DIR=${DIR}
- export JUPYTER_STDOUT=${JUPYTER_LOG_DIR}/stderr
- export PYTHONPATH=""
- export PYTHONNOUSERSITE=1
- export HOSTNAME_S=$(hostname -s)
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export HOSTNAME_I=${HOSTNAME_S}
- else
- export HOSTNAME_I=$(hostname -s)i
- fi
- export CURL_ARGS="--silent --write-out %{http_code} --output /dev/null"
- export CURL_HEADERS="-H \"Authorization: token ${JUPYTERHUB_API_TOKEN}\" -H \"Content-Type: application/json\" -H \"Accept: application/json\""
-
- if [[ $JUPYTERHUB_SERVICE_URL == "https"* ]]; then
- export HUB_PROTO="https"
- else
- export HUB_PROTO="http"
- fi
-
- API_URL_WITHOUT_PROTO=${JUPYTERHUB_API_URL##https\:\/\/}
- export JUPYTERHUB_DOMAIN=${API_URL_WITHOUT_PROTO%%\/*}
- export JUPYTER_SERVER_PUBLIC_URL="https://${JUPYTERHUB_DOMAIN}${JUPYTERHUB_SERVICE_PREFIX}"
- export DWAVE_INSPECTOR_JUPYTER_SERVER_PROXY_EXTERNAL_URL=${JUPYTER_SERVER_PUBLIC_URL}
-
- echo "$(date) - JupyterLab is running on ${HOSTNAME_S}"
- # If this JupyterLab is running on a LoginNode, we'll use the external api url.
- # Otherwise we have to use a LoginNode as Proxy, because there's no internet
- # connection for batch nodes
- if [[ ${HOSTNAME_S} == "<hostname_base>"* ]]; then
- # LoginNode
- # This is the internal jupyterhub certificate, only used when connecting through proxy on login node
- unset JUPYTERHUB_SSL_CLIENT_CA
- export SSH_NODE=${HOSTNAME_I}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub. Jump node for tunnel: ${SSH_NODE}"
- else
- # batch node, no internet connection to the outside world
- export PREVIOUS_API_URL=${JUPYTERHUB_API_URL}
- export REMOTE_PORT="<remote_port>"
- export JUPYTERHUB_CERTIFICATE="${DIR}/service_ca.crt"
-
- if [[ $HUB_PROTO == "https" ]]; then
- export CURL_ARGS="${CURL_ARGS} --cacert ${JUPYTERHUB_CERTIFICATE}"
- export JUPYTERHUB_SSL_CLIENT_CA="${JUPYTERHUB_CERTIFICATE}"
- fi
-
- # Look for random LoginNode to use it as proxy
- # We start at a random index, then we will run through all nodes and test them
- export ALL_REMOTE_NODES=(<hostname_all>)
- size=${#ALL_REMOTE_NODES[@]}
- index=$(($RANDOM % $size))
- start_index=$index
- for _ in ${ALL_REMOTE_NODES[@]}
- do
- HTTPCODE=$(curl -X "GET" ${CURL_ARGS} ${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api)
- if [[ ${HTTPCODE} -ge 200 ]] && [[ ${HTTPCODE} -le 299 ]]; then
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- fi
- # Add additional maintenance check
- STATUSOUT=$(curl -s -w '%{http_code}' http://${ALL_REMOTE_NODES[$index]}:9100/metrics)
- STATUSOUTCODE=$(echo "${STATUSOUT}" | tail -n 1)
- STATUSOUTMAINT=$(echo "${STATUSOUT}" | grep -E '^maint_status' | cut -d' ' -f2)
- if [[ $STATUSOUTCODE -eq 200 ]] && [[ ${STATUSOUTMAINT} -eq 0 ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- else
- echo "$(date) - Hub reachable for ${ALL_REMOTE_NODES[$index]}. But node metrics answered with: ${STATUSOUTCODE} , ${STATUSOUTMAINT}. Continue search for node. Full output: ${STATUSOUT}"
- fi
- fi
- index=$(expr $index + 1)
- if [[ $index == $size ]]; then
- # if loop reached array end continue at 0
- index=0
- fi
- if [[ $index == $start_index ]]; then
- # Tried all login nodes
- echo "$(date) - Could not find any LoginNode to connect to JupyterHub. Please try again in a few minutes."
- exit 1
- fi
- done
- export JUPYTERHUB_ACTIVITY_URL=${JUPYTERHUB_ACTIVITY_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_EVENTS_URL=${JUPYTERHUB_EVENTS_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_SETUPTUNNEL_URL=${JUPYTERHUB_SETUPTUNNEL_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- fi
-
- export JUPYTERHUB_API_TOKEN=$(cat ${DIR}/.jupyter.token)
- export JUPYTERHUB_OAUTH_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_OAUTH_ACCESS_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_SINGLEUSER_APP="jupyter_server.serverapp.ServerApp"
-
- export PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')
- # Replace previously defined port as ip and port passed via
- # cmd line arguments are ignored as in the new implementation of jupyterhub-singleuser
- export JUPYTERHUB_SERVICE_URL=${HUB_PROTO}://0.0.0.0:${PORT}${JUPYTERHUB_SERVICE_PREFIX}
-
- if [[ $HUB_PROTO == "https" ]]; then
- export JUPYTERHUB_SSL_KEYFILE="${DIR}/service_cert.key"
- export JUPYTERHUB_SSL_CERTFILE="${DIR}/service_cert.crt"
- fi
-
- export JUPYTERJSC_CMD_ARGS=""
- echo "$(date) - Set environment variables done"
- }
-
- grace_runtime_kill() {
- # If JupyterLab is running on a batch node, we will kill it 10 seconds
- # before the slrum runtime ends. This way JupyterHub stops the JupyterLab
- # itself, instead of waiting to be notified that JupyterLab stopped.
- if [[ -z $SLURM_JOBID ]]; then
- # On Login Nodes this is not required
- echo "$(date) - SLURM_JOBID not set. Do not set activate grace_runtime_kill"
- return
- fi
- time=$(squeue -o "%L" -h -j $SLURM_JOBID)
- # Check if the time is "NOT_SET" or "UNLIMITED"
- if [[ "$time" == "NOT_SET" || "$time" == "UNLIMITED" || "$time" == "" ]]; then
- echo "$(date) - Time limit not set or unlimited"
- return
- fi
- # Split the time string into its components
- IFS="-:" read -r -a time_array <<< "$time"
- # Set default values for missing components
- for ((i=${#time_array[@]}; i<4; i++)); do
- time_array=(0 "${time_array[@]}")
- done
- # Fill the variables from back to front
- secs=${time_array[3]}
- mins=${time_array[2]}
- hours=${time_array[1]}
- days=${time_array[0]}
- total_seconds=$((days * 86400 + hours * 3600 + mins * 60 + secs - 10))
- echo "$(date) - Kill JupyterLab in $total_seconds seconds."
- sleep $total_seconds && send_spawn_update_fail "Stop JupyterLab" "The runtime of the slurm job $SLURM_JOBID will end within the next 10 seconds. Stop JupyterLab gracefully." &
- }
-
-
- # Check Quota
-
- check_quota () {
- echo "$(date) - Check quota ..."
- if [[ ! -f ${HOME}/.${JUPYTERHUB_SERVER_NAME} ]]; then
- touch ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC1=$?
- echo "Quota Check ${JUPYTERHUB_SERVER_NAME}" >> ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC2=$?
- if [[ $EC1 -ne 0 || $EC1 -ne 0 ]]; then
- send_spawn_update_fail "Disk quota exceeded in $HOME. You have to clean up your home directory before you can start a JupyterLab." "Jupyter-JSC tried to create a testfile in ${HOME} and failed. Job directory may contain further information: '"${DIR}"'"
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- exit 0
- fi
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- send_spawn_update 50 "Disk quota checked." "If Jupyter-JSC could not create files in ${HOME}, JupyterLab would not be able to start."
- else
- echo "$(date) - Could not check quota"
- fi
- echo "$(date) - Check quota done"
- }
-
-
- # Hook to load customized environments before loading modules
-
- pre_start () {
- echo "$(date) - Pre start ..."
- if [[ -f ${HOME}/.jupyter/pre_jupyter-jsc.sh ]]; then
- echo "$(date) - Pre start if ..."
- send_spawn_update_warning 60 "Use customized start script." "You are using a customized environment, defined in ${HOME}/.jupyter/pre_jupyter-jsc.sh."
- echo "------ pre_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/pre_jupyter-jsc.sh
- echo "------ pre_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/pre_jupyter-jsc.sh
- fi
- echo "$(date) - Pre start done"
- }
-
-
- # Load modules
-
- load_modules () {
- echo "$(date) - Load modules ..."
- if [[ -f ${HOME}/.jupyter/start_jupyter-jsc.sh ]]; then
- send_spawn_update_warning 70 "Load customized modules ..." "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- echo "------ start_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/start_jupyter-jsc.sh
- echo "------ start_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/start_jupyter-jsc.sh
- send_spawn_update_warning 80 "Load customized modules done" "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- else
- send_spawn_update 70 "Load default modules ..." "Loading JupyterCollection/2021.3.2 in Stages/2020."
- module purge
- module use ${OTHERSTAGES}
- module load Stages/2020
- module load GCCcore/.10.3.0 JupyterCollection/2021.3.2
-
- echo "$(date) - checking for GPU"
- lspci -k | grep -A 2 -i "NVIDIA" | grep "Kernel driver in use:" | grep "nvidia"
- if [ $? -eq 0 ]; then
- send_spawn_update 75 "Load Nvidia module ..." "Loading JupyterExtension-nvdashboard/0.6.0-2021.3.2 in Stages/2020."
- module load JupyterExtension-nvdashboard/0.6.0-2021.3.2
- fi
- send_spawn_update 80 "Load default modules done" "Loading JupyterCollection/2021.3.2 in Stages/2020."
- fi
- echo "$(date) - Load modules done"
- }
-
- update_config () {
- sed -i -e "s|_port_|${PORT}|g" -e "s|_home_|${JUPYTERHUB_HOME:-${HOME}}|g" -e "s|_servername_|${JUPYTERHUB_SERVER_NAME}|g" ${DIR}/config.py
- if [[ -f ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py ]]; then
- echo "$(date) - Add system specific config ..."
- send_spawn_update 85 "Add system specific configuration." "Use system specific config file ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py"
- echo "" >> ${DIR}/config.py
- cat ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py >> ${DIR}/config.py
- for path in ${JUPYTER_EXTRA_LABEXTENSIONS_PATH//:/$'\n'}; do
- echo "c.LabServerApp.extra_labextensions_path.append('$path')" >> ${JUPYTER_LOG_DIR}/config.py
- done
- echo "$(date) - Add system specific config done"
- fi
- if [[ -f ${EBROOTJUPYTERLAB}/bin/update_favorites_json ]]; then
- # update favorite-dirs with $HOME,$PROJECT,$SCRATCH,
- echo "$(date) - Update favorites"
- ${EBROOTJUPYTERLAB}/bin/update_favorites_json
- fi
- }
-
- load_project_specific_kernel () {
- echo "$(date) - Activate hook for project specific kernels ..."
- if [[ -d ${PROJECT}/.local/share/jupyter ]] && [[ -r ${PROJECT}/.local/share/jupyter ]] && [[ -x ${PROJECT}/.local/share/jupyter ]]; then
- echo "$(date) - Add project kernel for $PROJECT"
- send_spawn_update_warning 90 "Activate project specific kernel." "Add ${PROJECT}/.local/share/jupyter to JUPYTER_PATH env variable. This may impede the start of JupyterLab."
- export JUPYTER_PATH=${JUPYTER_PATH}:${PROJECT}/.local/share/jupyter
- else
- echo "$(date) - Do not add project kernel for $PROJECT"
- # send_spawn_update_warning 90 "Could not activate project specific kernel." "Ensure that ${PROJECT}/.local/share/jupyter exists and is readable and accessible for ${USER}"
- fi
- echo "$(date) - Activate hook for project specific kernels done"
- }
-
- send_event () {
- BODY=${1//\'/}
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_EVENTS_URL}"
- eval " $CURL_CMD"
- }
-
- # show user a message in the UI
- send_spawn_update () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a darkorange message in the UI
- send_spawn_update_warning () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary><span style=\\\"color:darkorange;\\\">${SUMMARY}</span></summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a fail message in the UI
- send_spawn_update_fail () {
- SUMMARY=$1
- DETAILS=$2
- BODY="{\"progress\": 100, \"failed\": true, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send fail status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Send fail update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # setup tunnel to the node JupyterLab is running at
- setup_tunnel () {
- echo "$(date) - Setup tunnel ..."
- BODY="{\"ssh_node\": \"${SSH_NODE}\", \"service\": \"${HOSTNAME_I}:${PORT}\"}"
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_SETUPTUNNEL_URL}"
- HTTPCODE=$(eval " $CURL_CMD")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "Could not setup tunnel. Cancel start."
- exit 1
- fi
- send_spawn_update 40 "Setup ssh port-forwarding." "Create ssh tunnel with system user ljupyter. JupyterHub will then be able to connect to JupyterLab at ${HOSTNAME_I}:${PORT}"
- echo "$(date) - Setup tunnel done"
- }
-
- start () {
- echo "$(date) - Start jupyterhub-singleuser ..."
- export JUPYTERHUB_HOME=${JUPYTERHUB_HOME:-${HOME}}
- cd ${JUPYTERHUB_HOME}
-
- if [[ -n $JUPYTERJSC_USER_CMD ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom command" "Use custom command \$JUPYTERJSC_USER_CMD . You will be redirected, when your JupyterLab is ready."
- timeout 30d ${JUPYTERJSC_USER_CMD} ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- elif [[ -n $JUPYTERJSC_USER_CMD_ARGS ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom arguments" "Use custom arguments \$JUPYTERJSC_USER_CMD_ARGS . You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} ${JUPYTERJSC_USER_CMD_ARGS} &
- child=$!
- else
- send_spawn_update 95 "Start JupyterLab" "You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- fi
- echo "$child" > ${PID_PATH}
- echo "$(date) - Start jupyterhub-singleuser done (PID: $child )"
-
- echo "$(date) - Start validate_token.sh ..."
- /bin/bash ${DIR}/validate_token.sh ${child} ${JUPYTERHUB_API_URL} &
- token_pid=$!
- echo "$token_pid" > ${VALIDATE_TOKEN_PID_PATH}
- echo "$(date) - Start validate_token.sh done (PID: $token_pid )"
-
- wait $child
- }
-
-
- requirements
- set_env
- grace_runtime_kill
- setup_tunnel
- check_quota
- pre_start
- load_modules
- update_config
- load_project_specific_kernel
- start
-
- juniq_input_start.sh: |
- #!/bin/bash
-
- # Catch SIGTERM signal and stop $child process
- _term() {
- echo "$(date) - Stop JupyterLab ..."
- if [[ -z $child ]]; then
- echo "$(date) - Stop JupyterLab - unknown PID. Start stop.sh to wait for PID"
- bash ${DIR}/stop.sh &
- else
- pkill --parent $child
- kill $child
- echo "$(date) - Stop JupyterLab ( $child ) ... done"
- if [[ -z $token_pid ]]; then
- echo "$(date) - Validate Token PID not found..."
- else
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ..."
- kill $token_pid
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ... done"
- fi
- fi
- }
-
- trap _term SIGTERM
-
- requirements () {
- echo "$(date) - Setup system specific requirements ( ${HOSTNAME} ) ..."
- if [[ ! -d ${HOME}/.cache/black/19.3b0 ]]; then
- mkdir -p ${HOME}/.cache/black/19.3b0
- fi
- # export a memory warn threshold on login nodes
- hostname | egrep '<hostname_base>' > /dev/null && export JUPYTER_MEMWARNTHRES=0.5
- # set cpu limit/warning
- export FPATH_CPUQUOTA="/sys/fs/cgroup/cpu,cpuacct/user.slice/user-$(id -u $USER).slice/cpu.cfs_quota_us"
- [ -f "${FPATH_CPUQUOTA}" ] && [ $(hostname | egrep '<hostname_cpulimit>') ] && export JUPYTER_CPULIMIT=$(($(cat ${FPATH_CPUQUOTA})/100000))
-
- echo "$(date) - Setup system specific requirements done"
- }
-
- # set env_variables correctly
- set_env () {
- echo "$(date) - Set environment variables ..."
- export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
- export PID_PATH=${DIR}/service.pid
- export VALIDATE_TOKEN_PID_PATH=${DIR}/validate_token.pid
- export LC_ALL=en_US.UTF-8
- export JUPYTER_LOG_DIR=${DIR}
- export JUPYTER_STDOUT=${JUPYTER_LOG_DIR}/stderr
- export PYTHONPATH=""
- export PYTHONNOUSERSITE=1
- export HOSTNAME_S=$(hostname -s)
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export HOSTNAME_I=${HOSTNAME_S}
- else
- export HOSTNAME_I=$(hostname -s)i
- fi
- export CURL_ARGS="--silent --write-out %{http_code} --output /dev/null"
- export CURL_HEADERS="-H \"Authorization: token ${JUPYTERHUB_API_TOKEN}\" -H \"Content-Type: application/json\" -H \"Accept: application/json\""
-
- if [[ $JUPYTERHUB_SERVICE_URL == "https"* ]]; then
- export HUB_PROTO="https"
- else
- export HUB_PROTO="http"
- fi
-
- API_URL_WITHOUT_PROTO=${JUPYTERHUB_API_URL##https\:\/\/}
- export JUPYTERHUB_DOMAIN=${API_URL_WITHOUT_PROTO%%\/*}
- export JUPYTER_SERVER_PUBLIC_URL="https://${JUPYTERHUB_DOMAIN}${JUPYTERHUB_SERVICE_PREFIX}"
- export DWAVE_INSPECTOR_JUPYTER_SERVER_PROXY_EXTERNAL_URL=${JUPYTER_SERVER_PUBLIC_URL}
-
- echo "$(date) - JupyterLab is running on ${HOSTNAME_S}"
- # If this JupyterLab is running on a LoginNode, we'll use the external api url.
- # Otherwise we have to use a LoginNode as Proxy, because there's no internet
- # connection for batch nodes
- if [[ ${HOSTNAME_S} == "<hostname_base>"* ]]; then
- # LoginNode
- # This is the internal jupyterhub certificate, only used when connecting through proxy on login node
- unset JUPYTERHUB_SSL_CLIENT_CA
- export SSH_NODE=${HOSTNAME_I}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub. Jump node for tunnel: ${SSH_NODE}"
- else
- # batch node, no internet connection to the outside world
- export PREVIOUS_API_URL=${JUPYTERHUB_API_URL}
- export REMOTE_PORT="<remote_port>"
- export JUPYTERHUB_CERTIFICATE="${DIR}/service_ca.crt"
-
- if [[ $HUB_PROTO == "https" ]]; then
- export CURL_ARGS="${CURL_ARGS} --cacert ${JUPYTERHUB_CERTIFICATE}"
- export JUPYTERHUB_SSL_CLIENT_CA="${JUPYTERHUB_CERTIFICATE}"
- fi
-
- # Look for random LoginNode to use it as proxy
- # We start at a random index, then we will run through all nodes and test them
- export ALL_REMOTE_NODES=(<hostname_all>)
- size=${#ALL_REMOTE_NODES[@]}
- index=$(($RANDOM % $size))
- start_index=$index
- for _ in ${ALL_REMOTE_NODES[@]}
- do
- HTTPCODE=$(curl -X "GET" ${CURL_ARGS} ${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api)
- if [[ ${HTTPCODE} -ge 200 ]] && [[ ${HTTPCODE} -le 299 ]]; then
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- fi
- # Add additional maintenance check
- STATUSOUT=$(curl -s -w '%{http_code}' http://${ALL_REMOTE_NODES[$index]}:9100/metrics)
- STATUSOUTCODE=$(echo "${STATUSOUT}" | tail -n 1)
- STATUSOUTMAINT=$(echo "${STATUSOUT}" | grep -E '^maint_status' | cut -d' ' -f2)
- if [[ $STATUSOUTCODE -eq 200 ]] && [[ ${STATUSOUTMAINT} -eq 0 ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- else
- echo "$(date) - Hub reachable for ${ALL_REMOTE_NODES[$index]}. But node metrics answered with: ${STATUSOUTCODE} , ${STATUSOUTMAINT}. Continue search for node. Full output: ${STATUSOUT}"
- fi
- fi
- index=$(expr $index + 1)
- if [[ $index == $size ]]; then
- # if loop reached array end continue at 0
- index=0
- fi
- if [[ $index == $start_index ]]; then
- # Tried all login nodes
- echo "$(date) - Could not find any LoginNode to connect to JupyterHub. Please try again in a few minutes."
- exit 1
- fi
- done
- export JUPYTERHUB_ACTIVITY_URL=${JUPYTERHUB_ACTIVITY_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_EVENTS_URL=${JUPYTERHUB_EVENTS_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_SETUPTUNNEL_URL=${JUPYTERHUB_SETUPTUNNEL_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- fi
-
- export JUPYTERHUB_API_TOKEN=$(cat ${DIR}/.jupyter.token)
- export JUPYTERHUB_OAUTH_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_OAUTH_ACCESS_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_SINGLEUSER_APP="jupyter_server.serverapp.ServerApp"
-
- export PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')
- # Replace previously defined port as ip and port passed via
- # cmd line arguments are ignored as in the new implementation of jupyterhub-singleuser
- export JUPYTERHUB_SERVICE_URL=${HUB_PROTO}://0.0.0.0:${PORT}${JUPYTERHUB_SERVICE_PREFIX}
-
- if [[ $HUB_PROTO == "https" ]]; then
- export JUPYTERHUB_SSL_KEYFILE="${DIR}/service_cert.key"
- export JUPYTERHUB_SSL_CERTFILE="${DIR}/service_cert.crt"
- fi
-
- export JUPYTERJSC_CMD_ARGS=""
- echo "$(date) - Set environment variables done"
- }
-
- grace_runtime_kill() {
- # If JupyterLab is running on a batch node, we will kill it 10 seconds
- # before the slrum runtime ends. This way JupyterHub stops the JupyterLab
- # itself, instead of waiting to be notified that JupyterLab stopped.
- if [[ -z $SLURM_JOBID ]]; then
- # On Login Nodes this is not required
- echo "$(date) - SLURM_JOBID not set. Do not set activate grace_runtime_kill"
- return
- fi
- time=$(squeue -o "%L" -h -j $SLURM_JOBID)
- # Check if the time is "NOT_SET" or "UNLIMITED"
- if [[ "$time" == "NOT_SET" || "$time" == "UNLIMITED" || "$time" == "" ]]; then
- echo "$(date) - Time limit not set or unlimited"
- return
- fi
- # Split the time string into its components
- IFS="-:" read -r -a time_array <<< "$time"
- # Set default values for missing components
- for ((i=${#time_array[@]}; i<4; i++)); do
- time_array=(0 "${time_array[@]}")
- done
- # Fill the variables from back to front
- secs=${time_array[3]}
- mins=${time_array[2]}
- hours=${time_array[1]}
- days=${time_array[0]}
- total_seconds=$((days * 86400 + hours * 3600 + mins * 60 + secs - 10))
- echo "$(date) - Kill JupyterLab in $total_seconds seconds."
- sleep $total_seconds && send_spawn_update_fail "Stop JupyterLab" "The runtime of the slurm job $SLURM_JOBID will end within the next 10 seconds. Stop JupyterLab gracefully." &
- }
-
-
- # Check Quota
-
- check_quota () {
- echo "$(date) - Check quota ..."
- if [[ ! -f ${HOME}/.${JUPYTERHUB_SERVER_NAME} ]]; then
- touch ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC1=$?
- echo "Quota Check ${JUPYTERHUB_SERVER_NAME}" >> ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC2=$?
- if [[ $EC1 -ne 0 || $EC1 -ne 0 ]]; then
- send_spawn_update_fail "Disk quota exceeded in $HOME. You have to clean up your home directory before you can start a JupyterLab." "Jupyter-JSC tried to create a testfile in ${HOME} and failed. Job directory may contain further information: '"${DIR}"'"
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- exit 0
- fi
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- send_spawn_update 50 "Disk quota checked." "If Jupyter-JSC could not create files in ${HOME}, JupyterLab would not be able to start."
- else
- echo "$(date) - Could not check quota"
- fi
- echo "$(date) - Check quota done"
- }
-
-
- # Hook to load customized environments before loading modules
-
- pre_start () {
- echo "$(date) - Pre start ..."
- if [[ -f ${HOME}/.jupyter/pre_jupyter-jsc.sh ]]; then
- echo "$(date) - Pre start if ..."
- send_spawn_update_warning 60 "Use customized start script." "You are using a customized environment, defined in ${HOME}/.jupyter/pre_jupyter-jsc.sh."
- echo "------ pre_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/pre_jupyter-jsc.sh
- echo "------ pre_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/pre_jupyter-jsc.sh
- fi
- echo "$(date) - Pre start done"
- }
-
-
- # Load modules
-
- load_modules () {
- echo "$(date) - Load modules ..."
- if [[ -f ${HOME}/.jupyter/start_jupyter-jsc.sh ]]; then
- send_spawn_update_warning 70 "Load customized modules ..." "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- echo "------ start_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/start_jupyter-jsc.sh
- echo "------ start_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/start_jupyter-jsc.sh
- send_spawn_update_warning 80 "Load customized modules done" "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- else
- send_spawn_update 70 "Load default modules ..." "Loading JupyterCollection/2021.3.2 in Stages/2020."
-
- module purge
- module use ${OTHERSTAGES}
- module load Stages/2020
- module load GCCcore/.10.3.0 JupyterCollection/2021.3.2-JUNIQ
- module load GCC/10.3.0 ParaStationMPI
- module load Cirq/0.9.1-Python-3.8.5 PyQuil/2.27.0-Python-3.8.5 DWave/4.2.0-Python-3.8.5 Qiskit/0.32.1-Python-3.8.5
-
- echo "$(date) - checking for GPU"
- lspci -k | grep -A 2 -i "NVIDIA" | grep "Kernel driver in use:" | grep "nvidia"
- if [ $? -eq 0 ]; then
- send_spawn_update 75 "Load Nvidia module ..." "Loading JupyterExtension-nvdashboard/0.6.0-2021.3.2 in Stages/2020."
- module load JupyterExtension-nvdashboard/0.6.0-2021.3.2
- fi
- send_spawn_update 80 "Load default modules done" "Loading JupyterCollection/2021.3.2 in Stages/2020."
- fi
- echo "$(date) - Load modules done"
- }
-
- update_config () {
- sed -i -e "s|_port_|${PORT}|g" -e "s|_home_|${JUPYTERHUB_HOME:-${HOME}}|g" -e "s|_servername_|${JUPYTERHUB_SERVER_NAME}|g" ${DIR}/config.py
- if [[ -f ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py ]]; then
- echo "$(date) - Add system specific config ..."
- send_spawn_update 85 "Add system specific configuration." "Use system specific config file ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py"
- echo "" >> ${DIR}/config.py
- cat ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py >> ${DIR}/config.py
- for path in ${JUPYTER_EXTRA_LABEXTENSIONS_PATH//:/$'\n'}; do
- echo "c.LabServerApp.extra_labextensions_path.append('$path')" >> ${JUPYTER_LOG_DIR}/config.py
- done
- echo "$(date) - Add system specific config done"
- fi
- if [[ -f ${EBROOTJUPYTERLAB}/bin/update_favorites_json ]]; then
- # update favorite-dirs with $HOME,$PROJECT,$SCRATCH,
- echo "$(date) - Update favorites"
- ${EBROOTJUPYTERLAB}/bin/update_favorites_json
- fi
- }
-
- load_project_specific_kernel () {
- echo "$(date) - Activate hook for project specific kernels ..."
- if [[ -d ${PROJECT}/.local/share/jupyter ]] && [[ -r ${PROJECT}/.local/share/jupyter ]] && [[ -x ${PROJECT}/.local/share/jupyter ]]; then
- echo "$(date) - Add project kernel for $PROJECT"
- send_spawn_update_warning 90 "Activate project specific kernel." "Add ${PROJECT}/.local/share/jupyter to JUPYTER_PATH env variable. This may impede the start of JupyterLab."
- export JUPYTER_PATH=${JUPYTER_PATH}:${PROJECT}/.local/share/jupyter
- else
- echo "$(date) - Do not add project kernel for $PROJECT"
- # send_spawn_update_warning 90 "Could not activate project specific kernel." "Ensure that ${PROJECT}/.local/share/jupyter exists and is readable and accessible for ${USER}"
- fi
- echo "$(date) - Activate hook for project specific kernels done"
- }
-
- send_event () {
- BODY=${1//\'/}
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_EVENTS_URL}"
- eval " $CURL_CMD"
- }
-
- # show user a message in the UI
- send_spawn_update () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a darkorange message in the UI
- send_spawn_update_warning () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary><span style=\\\"color:darkorange;\\\">${SUMMARY}</span></summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a fail message in the UI
- send_spawn_update_fail () {
- SUMMARY=$1
- DETAILS=$2
- BODY="{\"progress\": 100, \"failed\": true, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send fail status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Send fail update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # setup tunnel to the node JupyterLab is running at
- setup_tunnel () {
- echo "$(date) - Setup tunnel ..."
- BODY="{\"ssh_node\": \"${SSH_NODE}\", \"service\": \"${HOSTNAME_I}:${PORT}\"}"
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_SETUPTUNNEL_URL}"
- HTTPCODE=$(eval " $CURL_CMD")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "Could not setup tunnel. Cancel start."
- exit 1
- fi
- send_spawn_update 40 "Setup ssh port-forwarding." "Create ssh tunnel with system user ljupyter. JupyterHub will then be able to connect to JupyterLab at ${HOSTNAME_I}:${PORT}"
- echo "$(date) - Setup tunnel done"
- }
-
- start () {
- echo "$(date) - Start jupyterhub-singleuser ..."
- export JUPYTERHUB_HOME=${JUPYTERHUB_HOME:-${HOME}}
- cd ${JUPYTERHUB_HOME}
-
- if [[ -n $JUPYTERJSC_USER_CMD ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom command" "Use custom command \$JUPYTERJSC_USER_CMD . You will be redirected, when your JupyterLab is ready."
- timeout 30d ${JUPYTERJSC_USER_CMD} ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- elif [[ -n $JUPYTERJSC_USER_CMD_ARGS ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom arguments" "Use custom arguments \$JUPYTERJSC_USER_CMD_ARGS . You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} ${JUPYTERJSC_USER_CMD_ARGS} &
- child=$!
- else
- send_spawn_update 95 "Start JupyterLab" "You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- fi
- echo "$child" > ${PID_PATH}
- echo "$(date) - Start jupyterhub-singleuser done (PID: $child )"
-
- echo "$(date) - Start validate_token.sh ..."
- /bin/bash ${DIR}/validate_token.sh ${child} ${JUPYTERHUB_API_URL} &
- token_pid=$!
- echo "$token_pid" > ${VALIDATE_TOKEN_PID_PATH}
- echo "$(date) - Start validate_token.sh done (PID: $token_pid )"
-
- wait $child
- }
-
-
- requirements
- set_env
- grace_runtime_kill
- setup_tunnel
- check_quota
- pre_start
- load_modules
- update_config
- load_project_specific_kernel
- start
\ No newline at end of file
diff --git a/outposts/unicore/kustomize-maint/jobs-3.3.yaml b/outposts/unicore/kustomize-maint/jobs-3.3.yaml
deleted file mode 100644
index 3027d564153a33ef85d79155f4e87633587bad79..0000000000000000000000000000000000000000
--- a/outposts/unicore/kustomize-maint/jobs-3.3.yaml
+++ /dev/null
@@ -1,501 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: unicore-jobs-3-3
- namespace: unicore
-data:
- input_start.sh: |
- #!/bin/bash
-
- # Catch SIGTERM signal and stop $child process
- _term() {
- echo "$(date) - Stop JupyterLab ..."
- if [[ -z $child ]]; then
- echo "$(date) - Stop JupyterLab - unknown PID. Start stop.sh to wait for PID"
- bash ${DIR}/stop.sh &
- else
- pkill --parent $child
- kill $child
- echo "$(date) - Stop JupyterLab ( $child ) ... done"
- if [[ -z $token_pid ]]; then
- echo "$(date) - Validate Token PID not found..."
- else
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ..."
- kill $token_pid
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ... done"
- fi
- fi
- }
-
- trap _term SIGTERM
-
- requirements () {
- echo "$(date) - Setup system specific requirements ( ${HOSTNAME} ) ..."
- if [[ ! -d ${HOME}/.cache/black/19.3b0 ]]; then
- mkdir -p ${HOME}/.cache/black/19.3b0
- fi
- # export a memory warn threshold on login nodes
- hostname | egrep '<hostname_base>' > /dev/null && export JUPYTER_MEMWARNTHRES=0.5
- # set cpu limit/warning
- export FPATH_CPUQUOTA="/sys/fs/cgroup/cpu,cpuacct/user.slice/user-$(id -u $USER).slice/cpu.cfs_quota_us"
- [ -f "${FPATH_CPUQUOTA}" ] && [ $(hostname | egrep '<hostname_cpulimit>') ] && export JUPYTER_CPULIMIT=$(($(cat ${FPATH_CPUQUOTA})/100000))
-
- echo "$(date) - Setup system specific requirements done"
- }
-
- # set env_variables correctly
- set_env () {
- echo "$(date) - Set environment variables ..."
- export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
- export PID_PATH=${DIR}/service.pid
- export VALIDATE_TOKEN_PID_PATH=${DIR}/validate_token.pid
- export LC_ALL=en_US.UTF-8
- export JUPYTER_LOG_DIR=${DIR}
- export JUPYTER_STDOUT=${JUPYTER_LOG_DIR}/stderr
- export PYTHONPATH=""
- export PYTHONNOUSERSITE=1
- export HOSTNAME_S=$(hostname -s)
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export HOSTNAME_I=${HOSTNAME_S}
- else
- export HOSTNAME_I=$(hostname -s)i
- fi
- export CURL_ARGS="--silent --write-out %{http_code} --output /dev/null"
- export CURL_HEADERS="-H \"Authorization: token ${JUPYTERHUB_API_TOKEN}\" -H \"Content-Type: application/json\" -H \"Accept: application/json\""
-
- if [[ $JUPYTERHUB_SERVICE_URL == "https"* ]]; then
- export HUB_PROTO="https"
- else
- export HUB_PROTO="http"
- fi
-
- API_URL_WITHOUT_PROTO=${JUPYTERHUB_API_URL##https\:\/\/}
- export JUPYTERHUB_DOMAIN=${API_URL_WITHOUT_PROTO%%\/*}
- export JUPYTER_SERVER_PUBLIC_URL="https://${JUPYTERHUB_DOMAIN}${JUPYTERHUB_SERVICE_PREFIX}"
- export DWAVE_INSPECTOR_JUPYTER_SERVER_PROXY_EXTERNAL_URL=${JUPYTER_SERVER_PUBLIC_URL}
-
- echo "$(date) - JupyterLab is running on ${HOSTNAME_S}"
- # If this JupyterLab is running on a LoginNode, we'll use the external api url.
- # Otherwise we have to use a LoginNode as Proxy, because there's no internet
- # connection for batch nodes
- if [[ ${HOSTNAME_S} == "<hostname_base>"* ]]; then
- # LoginNode
- # This is the internal jupyterhub certificate, only used when connecting through proxy on login node
- unset JUPYTERHUB_SSL_CLIENT_CA
- export SSH_NODE=${HOSTNAME_I}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub. Jump node for tunnel: ${SSH_NODE}"
- else
- # batch node, no internet connection to the outside world
- export PREVIOUS_API_URL=${JUPYTERHUB_API_URL}
- export REMOTE_PORT="<remote_port>"
- export JUPYTERHUB_CERTIFICATE="${DIR}/service_ca.crt"
-
- if [[ $HUB_PROTO == "https" ]]; then
- export CURL_ARGS="${CURL_ARGS} --cacert ${JUPYTERHUB_CERTIFICATE}"
- export JUPYTERHUB_SSL_CLIENT_CA="${JUPYTERHUB_CERTIFICATE}"
- fi
-
- # Look for random LoginNode to use it as proxy
- # We start at a random index, then we will run through all nodes and test them
- export ALL_REMOTE_NODES=(<hostname_all>)
- size=${#ALL_REMOTE_NODES[@]}
- index=$(($RANDOM % $size))
- start_index=$index
- for _ in ${ALL_REMOTE_NODES[@]}
- do
- HTTPCODE=$(curl -X "GET" ${CURL_ARGS} ${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api)
- if [[ ${HTTPCODE} -ge 200 ]] && [[ ${HTTPCODE} -le 299 ]]; then
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- fi
- # Add additional maintenance check
- STATUSOUT=$(curl -s -w '%{http_code}' http://${ALL_REMOTE_NODES[$index]}:9100/metrics)
- STATUSOUTCODE=$(echo "${STATUSOUT}" | tail -n 1)
- STATUSOUTMAINT=$(echo "${STATUSOUT}" | grep -E '^maint_status' | cut -d' ' -f2)
- if [[ $STATUSOUTCODE -eq 200 ]] && [[ ${STATUSOUTMAINT} -eq 0 ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- else
- echo "$(date) - Hub reachable for ${ALL_REMOTE_NODES[$index]}. But node metrics answered with: ${STATUSOUTCODE} , ${STATUSOUTMAINT}. Continue search for node. Full output: ${STATUSOUT}"
- fi
- fi
- index=$(expr $index + 1)
- if [[ $index == $size ]]; then
- # if loop reached array end continue at 0
- index=0
- fi
- if [[ $index == $start_index ]]; then
- # Tried all login nodes
- echo "$(date) - Could not find any LoginNode to connect to JupyterHub. Please try again in a few minutes."
- exit 1
- fi
- done
- export JUPYTERHUB_ACTIVITY_URL=${JUPYTERHUB_ACTIVITY_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_EVENTS_URL=${JUPYTERHUB_EVENTS_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_SETUPTUNNEL_URL=${JUPYTERHUB_SETUPTUNNEL_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- fi
-
- export JUPYTERHUB_API_TOKEN=$(cat ${DIR}/.jupyter.token)
- export JUPYTERHUB_OAUTH_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_OAUTH_ACCESS_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_SINGLEUSER_APP="jupyter_server.serverapp.ServerApp"
-
- export PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')
- # Replace previously defined port as ip and port passed via
- # cmd line arguments are ignored as in the new implementation of jupyterhub-singleuser
- export JUPYTERHUB_SERVICE_URL=${HUB_PROTO}://0.0.0.0:${PORT}${JUPYTERHUB_SERVICE_PREFIX}
-
- if [[ $HUB_PROTO == "https" ]]; then
- export JUPYTERHUB_SSL_KEYFILE="${DIR}/service_cert.key"
- export JUPYTERHUB_SSL_CERTFILE="${DIR}/service_cert.crt"
- fi
-
- export JUPYTERJSC_CMD_ARGS=""
- echo "$(date) - Set environment variables done"
- }
-
- grace_runtime_kill() {
- # If JupyterLab is running on a batch node, we will kill it 10 seconds
- # before the slrum runtime ends. This way JupyterHub stops the JupyterLab
- # itself, instead of waiting to be notified that JupyterLab stopped.
- if [[ -z $SLURM_JOBID ]]; then
- # On Login Nodes this is not required
- echo "$(date) - SLURM_JOBID not set. Do not set activate grace_runtime_kill"
- return
- fi
- time=$(squeue -o "%L" -h -j $SLURM_JOBID)
- # Check if the time is "NOT_SET" or "UNLIMITED"
- if [[ "$time" == "NOT_SET" || "$time" == "UNLIMITED" || "$time" == "" ]]; then
- echo "$(date) - Time limit not set or unlimited"
- return
- fi
- # Split the time string into its components
- IFS="-:" read -r -a time_array <<< "$time"
- # Set default values for missing components
- for ((i=${#time_array[@]}; i<4; i++)); do
- time_array=(0 "${time_array[@]}")
- done
- # Fill the variables from back to front
- secs=${time_array[3]}
- mins=${time_array[2]}
- hours=${time_array[1]}
- days=${time_array[0]}
- total_seconds=$((days * 86400 + hours * 3600 + mins * 60 + secs - 10))
- echo "$(date) - Kill JupyterLab in $total_seconds seconds."
- sleep $total_seconds && send_spawn_update_fail "Stop JupyterLab" "The runtime of the slurm job $SLURM_JOBID will end within the next 10 seconds. Stop JupyterLab gracefully." &
- }
-
-
- # Check Quota
-
- check_quota () {
- echo "$(date) - Check quota ..."
- if [[ ! -f ${HOME}/.${JUPYTERHUB_SERVER_NAME} ]]; then
- touch ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC1=$?
- echo "Quota Check ${JUPYTERHUB_SERVER_NAME}" >> ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC2=$?
- if [[ $EC1 -ne 0 || $EC1 -ne 0 ]]; then
- send_spawn_update_fail "Disk quota exceeded in $HOME. You have to clean up your home directory before you can start a JupyterLab." "Jupyter-JSC tried to create a testfile in ${HOME} and failed. Job directory may contain further information: '"${DIR}"'"
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- exit 0
- fi
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- send_spawn_update 50 "Disk quota checked." "If Jupyter-JSC could not create files in ${HOME}, JupyterLab would not be able to start."
- else
- echo "$(date) - Could not check quota"
- fi
- echo "$(date) - Check quota done"
- }
-
-
- # Hook to load customized environments before loading modules
-
- pre_start () {
- echo "$(date) - Pre start ..."
- if [[ -f ${HOME}/.jupyter/pre_jupyter-jsc.sh ]]; then
- echo "$(date) - Pre start if ..."
- send_spawn_update_warning 60 "Use customized start script." "You are using a customized environment, defined in ${HOME}/.jupyter/pre_jupyter-jsc.sh."
- echo "------ pre_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/pre_jupyter-jsc.sh
- echo "------ pre_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/pre_jupyter-jsc.sh
- fi
- echo "$(date) - Pre start done"
- }
-
-
- # Load modules
-
- load_modules () {
- echo "$(date) - Load modules ..."
- if [[ -f ${HOME}/.jupyter/start_jupyter-jsc.sh ]]; then
- send_spawn_update_warning 70 "Load customized modules ..." "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- echo "------ start_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/start_jupyter-jsc.sh
- echo "------ start_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/start_jupyter-jsc.sh
- send_spawn_update_warning 80 "Load customized modules done" "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- else
- send_spawn_update 70 "Load default modules ..." "Loading JupyterCollection/2022.3.3 in Stages/2022."
- module purge
- module use ${OTHERSTAGES}
- module load Stages/2022
- module load GCCcore/.11.2.0 Python JupyterCollection/2022.3.3
-
- echo "$(date) - checking for GPU"
- lspci -k | grep -A 2 -i "NVIDIA" | grep "Kernel driver in use:" | grep "nvidia"
- if [ $? -eq 0 ]; then
- send_spawn_update 75 "Load Nvidia module ..." "Loading JupyterExtension-nvdashboard/0.6.0-2022.3.3 in Stages/2022."
- module load JupyterExtension-nvdashboard/0.6.0-2022.3.3
- fi
- send_spawn_update 80 "Load default modules done" "Loading JupyterCollection/2022.3.3 in Stages/2022."
- fi
- echo "$(date) - Load modules done"
- }
-
- update_config () {
- sed -i -e "s|_port_|${PORT}|g" -e "s|_home_|${JUPYTERHUB_HOME:-${HOME}}|g" -e "s|_servername_|${JUPYTERHUB_SERVER_NAME}|g" ${DIR}/config.py
- if [[ -f ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py ]]; then
- echo "$(date) - Add system specific config ..."
- send_spawn_update 85 "Add system specific configuration." "Use system specific config file ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py"
- echo "" >> ${DIR}/config.py
- cat ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py >> ${DIR}/config.py
- for path in ${JUPYTER_EXTRA_LABEXTENSIONS_PATH//:/$'\n'}; do
- echo "c.LabServerApp.extra_labextensions_path.append('$path')" >> ${JUPYTER_LOG_DIR}/config.py
- done
- echo "$(date) - Add system specific config done"
- fi
- if [[ -f ${EBROOTJUPYTERLAB}/bin/update_favorites_json ]]; then
- # update favorite-dirs with $HOME,$PROJECT,$SCRATCH,
- echo "$(date) - Update favorites"
- ${EBROOTJUPYTERLAB}/bin/update_favorites_json
- fi
- }
-
- load_project_specific_kernel () {
- echo "$(date) - Activate hook for project specific kernels ..."
- if [[ -d ${PROJECT}/.local/share/jupyter ]] && [[ -r ${PROJECT}/.local/share/jupyter ]] && [[ -x ${PROJECT}/.local/share/jupyter ]]; then
- echo "$(date) - Add project kernel for $PROJECT"
- send_spawn_update_warning 90 "Activate project specific kernel." "Add ${PROJECT}/.local/share/jupyter to JUPYTER_PATH env variable. This may impede the start of JupyterLab."
- export JUPYTER_PATH=${JUPYTER_PATH}:${PROJECT}/.local/share/jupyter
- else
- echo "$(date) - Do not add project kernel for $PROJECT"
- # send_spawn_update_warning 90 "Could not activate project specific kernel." "Ensure that ${PROJECT}/.local/share/jupyter exists and is readable and accessible for ${USER}"
- fi
- echo "$(date) - Activate hook for project specific kernels done"
- }
-
- send_event () {
- BODY=${1//\'/}
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_EVENTS_URL}"
- eval " $CURL_CMD"
- }
-
- # show user a message in the UI
- send_spawn_update () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a darkorange message in the UI
- send_spawn_update_warning () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary><span style=\\\"color:darkorange;\\\">${SUMMARY}</span></summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a fail message in the UI
- send_spawn_update_fail () {
- SUMMARY=$1
- DETAILS=$2
- BODY="{\"progress\": 100, \"failed\": true, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send fail status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Send fail update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # setup tunnel to the node JupyterLab is running at
- setup_tunnel () {
- echo "$(date) - Setup tunnel ..."
- BODY="{\"ssh_node\": \"${SSH_NODE}\", \"service\": \"${HOSTNAME_I}:${PORT}\"}"
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_SETUPTUNNEL_URL}"
- HTTPCODE=$(eval " $CURL_CMD")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "Could not setup tunnel. Cancel start."
- exit 1
- fi
- send_spawn_update 40 "Setup ssh port-forwarding." "Create ssh tunnel with system user ljupyter. JupyterHub will then be able to connect to JupyterLab at ${HOSTNAME_I}:${PORT}"
- echo "$(date) - Setup tunnel done"
- }
-
- start () {
- echo "$(date) - Start jupyterhub-singleuser ..."
- export JUPYTERHUB_HOME=${JUPYTERHUB_HOME:-${HOME}}
- cd ${JUPYTERHUB_HOME}
-
- if [[ -n $JUPYTERJSC_USER_CMD ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom command" "Use custom command \$JUPYTERJSC_USER_CMD . You will be redirected, when your JupyterLab is ready."
- timeout 30d ${JUPYTERJSC_USER_CMD} ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- elif [[ -n $JUPYTERJSC_USER_CMD_ARGS ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom arguments" "Use custom arguments \$JUPYTERJSC_USER_CMD_ARGS . You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} ${JUPYTERJSC_USER_CMD_ARGS} &
- child=$!
- else
- send_spawn_update 95 "Start JupyterLab" "You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- fi
- echo "$child" > ${PID_PATH}
- echo "$(date) - Start jupyterhub-singleuser done (PID: $child )"
-
- echo "$(date) - Start validate_token.sh ..."
- /bin/bash ${DIR}/validate_token.sh ${child} ${JUPYTERHUB_API_URL} &
- token_pid=$!
- echo "$token_pid" > ${VALIDATE_TOKEN_PID_PATH}
- echo "$(date) - Start validate_token.sh done (PID: $token_pid )"
-
- wait $child
- }
-
-
- requirements
- set_env
- grace_runtime_kill
- setup_tunnel
- check_quota
- pre_start
- load_modules
- update_config
- load_project_specific_kernel
- start
-
- #!/bin/bash
-
- # Catch SIGTERM signal and stop $child process
- _term() {
- echo "$(date) - Stop JupyterLab ..."
- if [[ -z $child ]]; then
- echo "$(date) - Stop JupyterLab - unknown PID. Start stop.sh to wait for PID"
- bash ${DIR}/stop.sh &
- else
- pkill --parent $child
- kill $child
- echo "$(date) - Stop JupyterLab ( $child ) ... done"
- if [[ -z $token_pid ]]; then
- echo "$(date) - Validate Token PID not found..."
- else
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ..."
- kill $token_pid
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ... done"
- fi
- fi
- }
- trap _term SIGTERM
-
- # System specific requirements
- requirements () {
- echo "$(date) - Setup system specific requirements ( ${HOSTNAME} ) ..."
- if [[ ! -d ${HOME}/.cache/black/19.3b0 ]]; then
- mkdir -p ${HOME}/.cache/black/19.3b0
- fi
- # export a memory warn threshold on login nodes
- hostname | egrep '<hostname_base>' > /dev/null && export JUPYTER_MEMWARNTHRES=0.5
- # set cpu limit/warning
- export FPATH_CPUQUOTA="/sys/fs/cgroup/cpu,cpuacct/user.slice/user-$(id -u $USER).slice/cpu.cfs_quota_us"
- [ -f "${FPATH_CPUQUOTA}" ] && [ $(hostname | egrep '<hostname_cpulimit>') ] && export JUPYTER_CPULIMIT=$(($(cat ${FPATH_CPUQUOTA})/100000))
-
- echo "$(date) - Setup system specific requirements done"
- }
-
- # set env_variables correctly
- set_env () {
- echo "$(date) - Set environment variables ..."
- export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
- export PID_PATH=${DIR}/service.pid
- export VALIDATE_TOKEN_PID_PATH=${DIR}/validate_token.pid
- export LC_ALL=en_US.UTF-8
- export JUPYTER_LOG_DIR=${DIR}
- export JUPYTER_STDOUT=${JUPYTER_LOG_DIR}/stderr
- export PYTHONPATH=""
- export PYTHONNOUSERSITE=1
- HOSTNAME_S=$(hostname -s)
- # If this JupyterLab is running on a LoginNode, we'll use the external api url. Otherwise we have to use a LoginNode as Proxy.
- if [[ ${HOSTNAME_S} == "<hostname_base>"* ]]; then
- unset JUPYTERHUB_SSL_CLIENT_CA
- export JUPYTERHUB_API_URL=<jupyterhub_api_url>
- export JUMP_NODE=${HOSTNAME_S}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub. Jump node for tunnel: ${JUMP_NODE}"
- export CURL_ARGS="--silent --write-out %{http_code} --output /dev/null"
- else
- export REMOTE_PORT="<remote_port>"
- export JUPYTERHUB_CERTIFICATE="${DIR}/notebooks-ca_trust.crt"
- export CURL_ARGS="--silent --write-out %{http_code} --cacert ${JUPYTERHUB_CERTIFICATE} --output /dev/null"
- # Look for random LoginNode
- # We start at a random index, then we will run through all nodes and test them
- export ALL_REMOTE_NODES=(<hostname_all>)
- size=${#ALL_REMOTE_NODES[@]}
- index=$(($RANDOM % $size))
- start_index=$index
- for _ in ${ALL_REMOTE_NODES[@]}
- do
- HTTPCODE=$(curl -X "GET" ${CURL_ARGS} https://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api)
- if [[ ${HTTPCODE} -ge 200 || ${HTTPCODE} -lt 299 ]] ; then
- export JUPYTERHUB_API_URL="https://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export JUMP_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${JUMP_NODE} as jump node."
- break
- fi
- index=$(expr $index + 1)
- if [[ $index == $size ]]; then
- index=0
- fi
- if [[ $index == $start_index ]]; then
- echo "$(date) - Could not find any LoginNode to connect to JupyterHub. Please try again in a few minutes."
- exit 1
- fi
- done
- export JUPYTERHUB_SSL_CLIENT_CA="${DIR}/service_ca.crt"
- fi
- export JUPYTERHUB_ACTIVITY_URL=${JUPYTERHUB_API_URL}/users/${JUPYTERHUB_USER}/activity
- export JUPYTERHUB_API_TOKEN=$(cat ${DIR}/.jupyter.token)
- export JUPYTERHUB_OAUTH_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_SINGLEUSER_APP="jupyter_server.serverapp.ServerApp"
- export JUPYTERHUB_SSL_KEYFILE="${DIR}/service_cert.key"
- export JUPYTERHUB_SSL_CERTFILE="${DIR}/service_cert.crt"
- export JUPYTERJSC_CMD_ARGS=""
- export PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')
- echo "$(date) - Set environment variables done"
- }
-
- requirements
- set_env
- send_spawn_update_fail "JupyterLab 3.3 no longer supported on ${SYSTEMNAME}. Please update to version 3.6" "Update this JupyterLab configuration or delete this one and create a new one with 3.6."
- exit 1
\ No newline at end of file
diff --git a/outposts/unicore/kustomize-maint/jobs-3.4.yaml b/outposts/unicore/kustomize-maint/jobs-3.4.yaml
deleted file mode 100644
index b7c0f1d37728c108daa6e862239e51c3307decf1..0000000000000000000000000000000000000000
--- a/outposts/unicore/kustomize-maint/jobs-3.4.yaml
+++ /dev/null
@@ -1,876 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: unicore-jobs-3-4
- namespace: unicore
-data:
- input_start.sh: |
- #!/bin/bash
-
- # Catch SIGTERM signal and stop $child process
- _term() {
- echo "$(date) - Stop JupyterLab ..."
- if [[ -z $child ]]; then
- echo "$(date) - Stop JupyterLab - unknown PID. Start stop.sh to wait for PID"
- bash ${DIR}/stop.sh &
- else
- pkill --parent $child
- kill $child
- echo "$(date) - Stop JupyterLab ( $child ) ... done"
- if [[ -z $token_pid ]]; then
- echo "$(date) - Validate Token PID not found..."
- else
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ..."
- kill $token_pid
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ... done"
- fi
- fi
- }
-
- trap _term SIGTERM
-
- requirements () {
- echo "$(date) - Setup system specific requirements ( ${HOSTNAME} ) ..."
- if [[ ! -d ${HOME}/.cache/black/19.3b0 ]]; then
- mkdir -p ${HOME}/.cache/black/19.3b0
- fi
- # export a memory warn threshold on login nodes
- hostname | egrep '<hostname_base>' > /dev/null && export JUPYTER_MEMWARNTHRES=0.5
- # set cpu limit/warning
- export FPATH_CPUQUOTA="/sys/fs/cgroup/cpu,cpuacct/user.slice/user-$(id -u $USER).slice/cpu.cfs_quota_us"
- [ -f "${FPATH_CPUQUOTA}" ] && [ $(hostname | egrep '<hostname_cpulimit>') ] && export JUPYTER_CPULIMIT=$(($(cat ${FPATH_CPUQUOTA})/100000))
-
- echo "$(date) - Setup system specific requirements done"
- }
-
- # set env_variables correctly
- set_env () {
- echo "$(date) - Set environment variables ..."
- export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
- export PID_PATH=${DIR}/service.pid
- export VALIDATE_TOKEN_PID_PATH=${DIR}/validate_token.pid
- export LC_ALL=en_US.UTF-8
- export JUPYTER_LOG_DIR=${DIR}
- export JUPYTER_STDOUT=${JUPYTER_LOG_DIR}/stderr
- export PYTHONPATH=""
- export PYTHONNOUSERSITE=1
- export HOSTNAME_S=$(hostname -s)
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export HOSTNAME_I=${HOSTNAME_S}
- else
- export HOSTNAME_I=$(hostname -s)i
- fi
- export CURL_ARGS="--silent --write-out %{http_code} --output /dev/null"
- export CURL_HEADERS="-H \"Authorization: token ${JUPYTERHUB_API_TOKEN}\" -H \"Content-Type: application/json\" -H \"Accept: application/json\""
-
- if [[ $JUPYTERHUB_SERVICE_URL == "https"* ]]; then
- export HUB_PROTO="https"
- else
- export HUB_PROTO="http"
- fi
-
- API_URL_WITHOUT_PROTO=${JUPYTERHUB_API_URL##https\:\/\/}
- export JUPYTERHUB_DOMAIN=${API_URL_WITHOUT_PROTO%%\/*}
- export JUPYTER_SERVER_PUBLIC_URL="https://${JUPYTERHUB_DOMAIN}${JUPYTERHUB_SERVICE_PREFIX}"
- export DWAVE_INSPECTOR_JUPYTER_SERVER_PROXY_EXTERNAL_URL=${JUPYTER_SERVER_PUBLIC_URL}
-
- echo "$(date) - JupyterLab is running on ${HOSTNAME_S}"
- # If this JupyterLab is running on a LoginNode, we'll use the external api url.
- # Otherwise we have to use a LoginNode as Proxy, because there's no internet
- # connection for batch nodes
- if [[ ${HOSTNAME_S} == "<hostname_base>"* ]]; then
- # LoginNode
- # This is the internal jupyterhub certificate, only used when connecting through proxy on login node
- unset JUPYTERHUB_SSL_CLIENT_CA
- export SSH_NODE=${HOSTNAME_I}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub. Jump node for tunnel: ${SSH_NODE}"
- else
- # batch node, no internet connection to the outside world
- export PREVIOUS_API_URL=${JUPYTERHUB_API_URL}
- export REMOTE_PORT="<remote_port>"
- export JUPYTERHUB_CERTIFICATE="${DIR}/service_ca.crt"
-
- if [[ $HUB_PROTO == "https" ]]; then
- export CURL_ARGS="${CURL_ARGS} --cacert ${JUPYTERHUB_CERTIFICATE}"
- export JUPYTERHUB_SSL_CLIENT_CA="${JUPYTERHUB_CERTIFICATE}"
- fi
-
- # Look for random LoginNode to use it as proxy
- # We start at a random index, then we will run through all nodes and test them
- export ALL_REMOTE_NODES=(<hostname_all>)
- size=${#ALL_REMOTE_NODES[@]}
- index=$(($RANDOM % $size))
- start_index=$index
- for _ in ${ALL_REMOTE_NODES[@]}
- do
- HTTPCODE=$(curl -X "GET" ${CURL_ARGS} ${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api)
- if [[ ${HTTPCODE} -ge 200 ]] && [[ ${HTTPCODE} -le 299 ]]; then
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- fi
- # Add additional maintenance check
- STATUSOUT=$(curl -s -w '%{http_code}' http://${ALL_REMOTE_NODES[$index]}:9100/metrics)
- STATUSOUTCODE=$(echo "${STATUSOUT}" | tail -n 1)
- STATUSOUTMAINT=$(echo "${STATUSOUT}" | grep -E '^maint_status' | cut -d' ' -f2)
- if [[ $STATUSOUTCODE -eq 200 ]] && [[ ${STATUSOUTMAINT} -eq 0 ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- else
- echo "$(date) - Hub reachable for ${ALL_REMOTE_NODES[$index]}. But node metrics answered with: ${STATUSOUTCODE} , ${STATUSOUTMAINT}. Continue search for node. Full output: ${STATUSOUT}"
- fi
- fi
- index=$(expr $index + 1)
- if [[ $index == $size ]]; then
- # if loop reached array end continue at 0
- index=0
- fi
- if [[ $index == $start_index ]]; then
- # Tried all login nodes
- echo "$(date) - Could not find any LoginNode to connect to JupyterHub. Please try again in a few minutes."
- exit 1
- fi
- done
- export JUPYTERHUB_ACTIVITY_URL=${JUPYTERHUB_ACTIVITY_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_EVENTS_URL=${JUPYTERHUB_EVENTS_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_SETUPTUNNEL_URL=${JUPYTERHUB_SETUPTUNNEL_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- fi
-
- export JUPYTERHUB_API_TOKEN=$(cat ${DIR}/.jupyter.token)
- export JUPYTERHUB_OAUTH_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_OAUTH_ACCESS_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_SINGLEUSER_APP="jupyter_server.serverapp.ServerApp"
-
- export PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')
- # Replace previously defined port as ip and port passed via
- # cmd line arguments are ignored as in the new implementation of jupyterhub-singleuser
- export JUPYTERHUB_SERVICE_URL=${HUB_PROTO}://0.0.0.0:${PORT}${JUPYTERHUB_SERVICE_PREFIX}
-
- if [[ $HUB_PROTO == "https" ]]; then
- export JUPYTERHUB_SSL_KEYFILE="${DIR}/service_cert.key"
- export JUPYTERHUB_SSL_CERTFILE="${DIR}/service_cert.crt"
- fi
-
- export JUPYTERJSC_CMD_ARGS=""
- echo "$(date) - Set environment variables done"
- }
-
- grace_runtime_kill() {
- # If JupyterLab is running on a batch node, we will kill it 10 seconds
- # before the slrum runtime ends. This way JupyterHub stops the JupyterLab
- # itself, instead of waiting to be notified that JupyterLab stopped.
- if [[ -z $SLURM_JOBID ]]; then
- # On Login Nodes this is not required
- echo "$(date) - SLURM_JOBID not set. Do not set activate grace_runtime_kill"
- return
- fi
- time=$(squeue -o "%L" -h -j $SLURM_JOBID)
- # Check if the time is "NOT_SET" or "UNLIMITED"
- if [[ "$time" == "NOT_SET" || "$time" == "UNLIMITED" || "$time" == "" ]]; then
- echo "$(date) - Time limit not set or unlimited"
- return
- fi
- # Split the time string into its components
- IFS="-:" read -r -a time_array <<< "$time"
- # Set default values for missing components
- for ((i=${#time_array[@]}; i<4; i++)); do
- time_array=(0 "${time_array[@]}")
- done
- # Fill the variables from back to front
- secs=${time_array[3]}
- mins=${time_array[2]}
- hours=${time_array[1]}
- days=${time_array[0]}
- total_seconds=$((days * 86400 + hours * 3600 + mins * 60 + secs - 10))
- echo "$(date) - Kill JupyterLab in $total_seconds seconds."
- sleep $total_seconds && send_spawn_update_fail "Stop JupyterLab" "The runtime of the slurm job $SLURM_JOBID will end within the next 10 seconds. Stop JupyterLab gracefully." &
- }
-
-
- # Check Quota
-
- check_quota () {
- echo "$(date) - Check quota ..."
- if [[ ! -f ${HOME}/.${JUPYTERHUB_SERVER_NAME} ]]; then
- touch ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC1=$?
- echo "Quota Check ${JUPYTERHUB_SERVER_NAME}" >> ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC2=$?
- if [[ $EC1 -ne 0 || $EC1 -ne 0 ]]; then
- send_spawn_update_fail "Disk quota exceeded in $HOME. You have to clean up your home directory before you can start a JupyterLab." "Jupyter-JSC tried to create a testfile in ${HOME} and failed. Job directory may contain further information: '"${DIR}"'"
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- exit 0
- fi
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- send_spawn_update 50 "Disk quota checked." "If Jupyter-JSC could not create files in ${HOME}, JupyterLab would not be able to start."
- else
- echo "$(date) - Could not check quota"
- fi
- echo "$(date) - Check quota done"
- }
-
-
- # Hook to load customized environments before loading modules
-
- pre_start () {
- echo "$(date) - Pre start ..."
- if [[ -f ${HOME}/.jupyter/pre_jupyter-jsc.sh ]]; then
- echo "$(date) - Pre start if ..."
- send_spawn_update_warning 60 "Use customized start script." "You are using a customized environment, defined in ${HOME}/.jupyter/pre_jupyter-jsc.sh."
- echo "------ pre_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/pre_jupyter-jsc.sh
- echo "------ pre_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/pre_jupyter-jsc.sh
- fi
- echo "$(date) - Pre start done"
- }
-
-
- # Load modules
-
- load_modules () {
- echo "$(date) - Load modules ..."
- if [[ -f ${HOME}/.jupyter/start_jupyter-jsc.sh ]]; then
- send_spawn_update_warning 70 "Load customized modules ..." "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- echo "------ start_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/start_jupyter-jsc.sh
- echo "------ start_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/start_jupyter-jsc.sh
- send_spawn_update_warning 80 "Load customized modules done" "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- else
- send_spawn_update 70 "Load default modules ..." "Loading Jupyter/2022.3.4 in Stages/2022."
-
- module purge
- module use ${OTHERSTAGES}
- module load Stages/2022
- module load GCCcore/.11.2.0 Python Jupyter/2022.3.4
-
- if [[ $JUPYTER_MODULE_BASH_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Bash/.0.7.2-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_CLING_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Cling/.0.9-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_JAVASCRIPT_ENABLED -eq 1 ]]; then
- module load JupyterKernel-JavaScript/.5.2.1-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_JULIA_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Julia/.1.7.1-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_OCTAVE_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Octave/.6.4.0-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_PYDEEPLEARNING_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyDeepLearning/.1.1-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_PYQUANTUM_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyQuantum/.3.0-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_PYVISUALIZATION_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyVisualization/.1.0-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_R_ENABLED -eq 1 ]]; then
- module load JupyterKernel-R/.4.1.2-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_RUBY_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Ruby/.3.0.1-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_XPRAHTML5_ENABLED -eq 1 ]]; then
- module load JupyterProxy-XpraHTML5/.0.3.5-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_SLURMWRAPPER_ENABLED -eq 1 ]]; then
- module load JupyterExtension-slurmprovisioner/0.6.0-2022.3.4
- export SLURMEL_DOCUMENTATION_HREF="https://docs.jupyter.jsc.fz-juelich.de/github/FZJ-JSC/jupyter-jsc-notebooks/blob/documentation/05-News&Updates/Announcement-2022-12_Slurm_Wrapped_Kernels.ipynb"
- export JUPYTERJSC_CMD_ARGS="${JUPYTERJSC_CMD_ARGS} --ServerApp.kernel_manager_class=jupyter_slurm_provisioner.SlurmAsyncMappingKernelManager"
- export SLURM_PROVISIONER_NODE_SUFFIX="i"
- export SLURM_PROVISIONER_JHUB_METRICS="${JUPYTERHUB_API_URL}/slurmwrapper/${JUPYTERHUB_USER}/${JUPYTERHUB_SERVER_NAME}"
- test -d ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel && ! test -d ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel && mv ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel
- else
- test -d ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel && mkdir -p ${HOME}/.local/share/jupyter/kernels.deactivated && mv ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel
- fi
- lspci -k | grep -A 2 -i "NVIDIA" | grep "Kernel driver in use:" | grep "nvidia"
- if [ $? -eq 0 ]; then
- module load JupyterExtension-nvdashboard/0.8.0-2023.3.6 || true
- fi
- send_spawn_update 80 "Load default modules done" "Loaded Jupyter/2022.3.4 in Stages/2022."
-
- fi
- echo "$(date) - Load modules done"
- }
-
- update_config () {
- sed -i -e "s|_port_|${PORT}|g" -e "s|_home_|${JUPYTERHUB_HOME:-${HOME}}|g" -e "s|_servername_|${JUPYTERHUB_SERVER_NAME}|g" ${DIR}/config.py
- if [[ -f ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py ]]; then
- echo "$(date) - Add system specific config ..."
- send_spawn_update 85 "Add system specific configuration." "Use system specific config file ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py"
- echo "" >> ${DIR}/config.py
- cat ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py >> ${DIR}/config.py
- for path in ${JUPYTER_EXTRA_LABEXTENSIONS_PATH//:/$'\n'}; do
- echo "c.LabServerApp.extra_labextensions_path.append('$path')" >> ${JUPYTER_LOG_DIR}/config.py
- done
- echo "$(date) - Add system specific config done"
- fi
- if [[ -f ${EBROOTJUPYTERLAB}/bin/update_favorites_json ]]; then
- # update favorite-dirs with $HOME,$PROJECT,$SCRATCH,
- echo "$(date) - Update favorites"
- ${EBROOTJUPYTERLAB}/bin/update_favorites_json
- fi
- }
-
- load_project_specific_kernel () {
- echo "$(date) - Activate hook for project specific kernels ..."
- if [[ -d ${PROJECT}/.local/share/jupyter ]] && [[ -r ${PROJECT}/.local/share/jupyter ]] && [[ -x ${PROJECT}/.local/share/jupyter ]]; then
- echo "$(date) - Add project kernel for $PROJECT"
- send_spawn_update_warning 90 "Activate project specific kernel." "Add ${PROJECT}/.local/share/jupyter to JUPYTER_PATH env variable. This may impede the start of JupyterLab."
- export JUPYTER_PATH=${JUPYTER_PATH}:${PROJECT}/.local/share/jupyter
- else
- echo "$(date) - Do not add project kernel for $PROJECT"
- # send_spawn_update_warning 90 "Could not activate project specific kernel." "Ensure that ${PROJECT}/.local/share/jupyter exists and is readable and accessible for ${USER}"
- fi
- echo "$(date) - Activate hook for project specific kernels done"
- }
-
- send_event () {
- BODY=${1//\'/}
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_EVENTS_URL}"
- eval " $CURL_CMD"
- }
-
- # show user a message in the UI
- send_spawn_update () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a darkorange message in the UI
- send_spawn_update_warning () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary><span style=\\\"color:darkorange;\\\">${SUMMARY}</span></summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a fail message in the UI
- send_spawn_update_fail () {
- SUMMARY=$1
- DETAILS=$2
- BODY="{\"progress\": 100, \"failed\": true, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send fail status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Send fail update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # setup tunnel to the node JupyterLab is running at
- setup_tunnel () {
- echo "$(date) - Setup tunnel ..."
- BODY="{\"ssh_node\": \"${SSH_NODE}\", \"service\": \"${HOSTNAME_I}:${PORT}\"}"
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_SETUPTUNNEL_URL}"
- HTTPCODE=$(eval " $CURL_CMD")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "Could not setup tunnel. Cancel start."
- exit 1
- fi
- send_spawn_update 40 "Setup ssh port-forwarding." "Create ssh tunnel with system user ljupyter. JupyterHub will then be able to connect to JupyterLab at ${HOSTNAME_I}:${PORT}"
- echo "$(date) - Setup tunnel done"
- }
-
- start () {
- echo "$(date) - Start jupyterhub-singleuser ..."
- export JUPYTERHUB_HOME=${JUPYTERHUB_HOME:-${HOME}}
- cd ${JUPYTERHUB_HOME}
-
- if [[ -n $JUPYTERJSC_USER_CMD ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom command" "Use custom command \$JUPYTERJSC_USER_CMD . You will be redirected, when your JupyterLab is ready."
- timeout 30d ${JUPYTERJSC_USER_CMD} ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- elif [[ -n $JUPYTERJSC_USER_CMD_ARGS ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom arguments" "Use custom arguments \$JUPYTERJSC_USER_CMD_ARGS . You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} ${JUPYTERJSC_USER_CMD_ARGS} &
- child=$!
- else
- send_spawn_update 95 "Start JupyterLab" "You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- fi
- echo "$child" > ${PID_PATH}
- echo "$(date) - Start jupyterhub-singleuser done (PID: $child )"
-
- echo "$(date) - Start validate_token.sh ..."
- /bin/bash ${DIR}/validate_token.sh ${child} ${JUPYTERHUB_API_URL} &
- token_pid=$!
- echo "$token_pid" > ${VALIDATE_TOKEN_PID_PATH}
- echo "$(date) - Start validate_token.sh done (PID: $token_pid )"
-
- wait $child
- }
-
-
- requirements
- set_env
- grace_runtime_kill
- setup_tunnel
- check_quota
- pre_start
- load_modules
- update_config
- load_project_specific_kernel
- start
-
- juniq_input_start.sh: |
- #!/bin/bash
-
- # Catch SIGTERM signal and stop $child process
- _term() {
- echo "$(date) - Stop JupyterLab ..."
- if [[ -z $child ]]; then
- echo "$(date) - Stop JupyterLab - unknown PID. Start stop.sh to wait for PID"
- bash ${DIR}/stop.sh &
- else
- pkill --parent $child
- kill $child
- echo "$(date) - Stop JupyterLab ( $child ) ... done"
- if [[ -z $token_pid ]]; then
- echo "$(date) - Validate Token PID not found..."
- else
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ..."
- kill $token_pid
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ... done"
- fi
- fi
- }
-
- trap _term SIGTERM
-
- requirements () {
- echo "$(date) - Setup system specific requirements ( ${HOSTNAME} ) ..."
- if [[ ! -d ${HOME}/.cache/black/19.3b0 ]]; then
- mkdir -p ${HOME}/.cache/black/19.3b0
- fi
- # export a memory warn threshold on login nodes
- hostname | egrep '<hostname_base>' > /dev/null && export JUPYTER_MEMWARNTHRES=0.5
- # set cpu limit/warning
- export FPATH_CPUQUOTA="/sys/fs/cgroup/cpu,cpuacct/user.slice/user-$(id -u $USER).slice/cpu.cfs_quota_us"
- [ -f "${FPATH_CPUQUOTA}" ] && [ $(hostname | egrep '<hostname_cpulimit>') ] && export JUPYTER_CPULIMIT=$(($(cat ${FPATH_CPUQUOTA})/100000))
-
- echo "$(date) - Setup system specific requirements done"
- }
-
- # set env_variables correctly
- set_env () {
- echo "$(date) - Set environment variables ..."
- export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
- export PID_PATH=${DIR}/service.pid
- export VALIDATE_TOKEN_PID_PATH=${DIR}/validate_token.pid
- export LC_ALL=en_US.UTF-8
- export JUPYTER_LOG_DIR=${DIR}
- export JUPYTER_STDOUT=${JUPYTER_LOG_DIR}/stderr
- export PYTHONPATH=""
- export PYTHONNOUSERSITE=1
- export HOSTNAME_S=$(hostname -s)
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export HOSTNAME_I=${HOSTNAME_S}
- else
- export HOSTNAME_I=$(hostname -s)i
- fi
- export CURL_ARGS="--silent --write-out %{http_code} --output /dev/null"
- export CURL_HEADERS="-H \"Authorization: token ${JUPYTERHUB_API_TOKEN}\" -H \"Content-Type: application/json\" -H \"Accept: application/json\""
-
- if [[ $JUPYTERHUB_SERVICE_URL == "https"* ]]; then
- export HUB_PROTO="https"
- else
- export HUB_PROTO="http"
- fi
-
- API_URL_WITHOUT_PROTO=${JUPYTERHUB_API_URL##https\:\/\/}
- export JUPYTERHUB_DOMAIN=${API_URL_WITHOUT_PROTO%%\/*}
- export JUPYTER_SERVER_PUBLIC_URL="https://${JUPYTERHUB_DOMAIN}${JUPYTERHUB_SERVICE_PREFIX}"
- export DWAVE_INSPECTOR_JUPYTER_SERVER_PROXY_EXTERNAL_URL=${JUPYTER_SERVER_PUBLIC_URL}
-
- echo "$(date) - JupyterLab is running on ${HOSTNAME_S}"
- # If this JupyterLab is running on a LoginNode, we'll use the external api url.
- # Otherwise we have to use a LoginNode as Proxy, because there's no internet
- # connection for batch nodes
- if [[ ${HOSTNAME_S} == "<hostname_base>"* ]]; then
- # LoginNode
- # This is the internal jupyterhub certificate, only used when connecting through proxy on login node
- unset JUPYTERHUB_SSL_CLIENT_CA
- export SSH_NODE=${HOSTNAME_I}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub. Jump node for tunnel: ${SSH_NODE}"
- else
- # batch node, no internet connection to the outside world
- export PREVIOUS_API_URL=${JUPYTERHUB_API_URL}
- export REMOTE_PORT="<remote_port>"
- export JUPYTERHUB_CERTIFICATE="${DIR}/service_ca.crt"
-
- if [[ $HUB_PROTO == "https" ]]; then
- export CURL_ARGS="${CURL_ARGS} --cacert ${JUPYTERHUB_CERTIFICATE}"
- export JUPYTERHUB_SSL_CLIENT_CA="${JUPYTERHUB_CERTIFICATE}"
- fi
-
- # Look for random LoginNode to use it as proxy
- # We start at a random index, then we will run through all nodes and test them
- export ALL_REMOTE_NODES=(<hostname_all>)
- size=${#ALL_REMOTE_NODES[@]}
- index=$(($RANDOM % $size))
- start_index=$index
- for _ in ${ALL_REMOTE_NODES[@]}
- do
- HTTPCODE=$(curl -X "GET" ${CURL_ARGS} ${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api)
- if [[ ${HTTPCODE} -ge 200 ]] && [[ ${HTTPCODE} -le 299 ]]; then
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- fi
- # Add additional maintenance check
- STATUSOUT=$(curl -s -w '%{http_code}' http://${ALL_REMOTE_NODES[$index]}:9100/metrics)
- STATUSOUTCODE=$(echo "${STATUSOUT}" | tail -n 1)
- STATUSOUTMAINT=$(echo "${STATUSOUT}" | grep -E '^maint_status' | cut -d' ' -f2)
- if [[ $STATUSOUTCODE -eq 200 ]] && [[ ${STATUSOUTMAINT} -eq 0 ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- else
- echo "$(date) - Hub reachable for ${ALL_REMOTE_NODES[$index]}. But node metrics answered with: ${STATUSOUTCODE} , ${STATUSOUTMAINT}. Continue search for node. Full output: ${STATUSOUT}"
- fi
- fi
- index=$(expr $index + 1)
- if [[ $index == $size ]]; then
- # if loop reached array end continue at 0
- index=0
- fi
- if [[ $index == $start_index ]]; then
- # Tried all login nodes
- echo "$(date) - Could not find any LoginNode to connect to JupyterHub. Please try again in a few minutes."
- exit 1
- fi
- done
- export JUPYTERHUB_ACTIVITY_URL=${JUPYTERHUB_ACTIVITY_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_EVENTS_URL=${JUPYTERHUB_EVENTS_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_SETUPTUNNEL_URL=${JUPYTERHUB_SETUPTUNNEL_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- fi
-
- export JUPYTERHUB_API_TOKEN=$(cat ${DIR}/.jupyter.token)
- export JUPYTERHUB_OAUTH_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_OAUTH_ACCESS_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_SINGLEUSER_APP="jupyter_server.serverapp.ServerApp"
-
- export PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')
- # Replace previously defined port as ip and port passed via
- # cmd line arguments are ignored as in the new implementation of jupyterhub-singleuser
- export JUPYTERHUB_SERVICE_URL=${HUB_PROTO}://0.0.0.0:${PORT}${JUPYTERHUB_SERVICE_PREFIX}
-
- if [[ $HUB_PROTO == "https" ]]; then
- export JUPYTERHUB_SSL_KEYFILE="${DIR}/service_cert.key"
- export JUPYTERHUB_SSL_CERTFILE="${DIR}/service_cert.crt"
- fi
-
- export JUPYTERJSC_CMD_ARGS=""
- echo "$(date) - Set environment variables done"
- }
-
- grace_runtime_kill() {
- # If JupyterLab is running on a batch node, we will kill it 10 seconds
- # before the slrum runtime ends. This way JupyterHub stops the JupyterLab
- # itself, instead of waiting to be notified that JupyterLab stopped.
- if [[ -z $SLURM_JOBID ]]; then
- # On Login Nodes this is not required
- echo "$(date) - SLURM_JOBID not set. Do not set activate grace_runtime_kill"
- return
- fi
- time=$(squeue -o "%L" -h -j $SLURM_JOBID)
- # Check if the time is "NOT_SET" or "UNLIMITED"
- if [[ "$time" == "NOT_SET" || "$time" == "UNLIMITED" || "$time" == "" ]]; then
- echo "$(date) - Time limit not set or unlimited"
- return
- fi
- # Split the time string into its components
- IFS="-:" read -r -a time_array <<< "$time"
- # Set default values for missing components
- for ((i=${#time_array[@]}; i<4; i++)); do
- time_array=(0 "${time_array[@]}")
- done
- # Fill the variables from back to front
- secs=${time_array[3]}
- mins=${time_array[2]}
- hours=${time_array[1]}
- days=${time_array[0]}
- total_seconds=$((days * 86400 + hours * 3600 + mins * 60 + secs - 10))
- echo "$(date) - Kill JupyterLab in $total_seconds seconds."
- sleep $total_seconds && send_spawn_update_fail "Stop JupyterLab" "The runtime of the slurm job $SLURM_JOBID will end within the next 10 seconds. Stop JupyterLab gracefully." &
- }
-
-
- # Check Quota
-
- check_quota () {
- echo "$(date) - Check quota ..."
- if [[ ! -f ${HOME}/.${JUPYTERHUB_SERVER_NAME} ]]; then
- touch ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC1=$?
- echo "Quota Check ${JUPYTERHUB_SERVER_NAME}" >> ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC2=$?
- if [[ $EC1 -ne 0 || $EC1 -ne 0 ]]; then
- send_spawn_update_fail "Disk quota exceeded in $HOME. You have to clean up your home directory before you can start a JupyterLab." "Jupyter-JSC tried to create a testfile in ${HOME} and failed. Job directory may contain further information: '"${DIR}"'"
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- exit 0
- fi
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- send_spawn_update 50 "Disk quota checked." "If Jupyter-JSC could not create files in ${HOME}, JupyterLab would not be able to start."
- else
- echo "$(date) - Could not check quota"
- fi
- echo "$(date) - Check quota done"
- }
-
-
- # Hook to load customized environments before loading modules
-
- pre_start () {
- echo "$(date) - Pre start ..."
- if [[ -f ${HOME}/.jupyter/pre_jupyter-jsc.sh ]]; then
- echo "$(date) - Pre start if ..."
- send_spawn_update_warning 60 "Use customized start script." "You are using a customized environment, defined in ${HOME}/.jupyter/pre_jupyter-jsc.sh."
- echo "------ pre_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/pre_jupyter-jsc.sh
- echo "------ pre_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/pre_jupyter-jsc.sh
- fi
- echo "$(date) - Pre start done"
- }
-
-
- # Load modules
-
- load_modules () {
- echo "$(date) - Load modules ..."
- if [[ -f ${HOME}/.jupyter/start_jupyter-jsc.sh ]]; then
- send_spawn_update_warning 70 "Load customized modules ..." "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- echo "------ start_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/start_jupyter-jsc.sh
- echo "------ start_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/start_jupyter-jsc.sh
- send_spawn_update_warning 80 "Load customized modules done" "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- else
- send_spawn_update 70 "Load default modules ..." "Loading Jupyter/2022.3.4 in Stages/2022."
- module purge
- module use ${OTHERSTAGES}
- module load Stages/2022
- module load GCCcore/.11.2.0 ParaStationMPI Jupyter/2022.3.4
- module load Cartopy/0.20.0 Cirq/0.14.1 DWave/4.2.0 PyQuil/3.0.1 Qiskit/0.36.2 Qiskit-juqcs/0.5.0
-
- if [[ $JUPYTER_MODULE_BASH_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Bash/.0.7.2-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_CLING_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Cling/.0.9-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_JAVASCRIPT_ENABLED -eq 1 ]]; then
- module load JupyterKernel-JavaScript/.5.2.1-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_JULIA_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Julia/.1.7.1-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_OCTAVE_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Octave/.6.4.0-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_PYDEEPLEARNING_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyDeepLearning/.1.1-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_PYQUANTUM_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyQuantum/.3.0-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_PYVISUALIZATION_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyVisualization/.1.0-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_R_ENABLED -eq 1 ]]; then
- module load JupyterKernel-R/.4.1.2-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_RUBY_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Ruby/.3.0.1-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_XPRAHTML5_ENABLED -eq 1 ]]; then
- module load JupyterProxy-XpraHTML5/.0.3.5-2022.3.4
- fi
- if [[ $JUPYTER_MODULE_SLURMWRAPPER_ENABLED -eq 1 ]]; then
- module load JupyterExtension-slurmprovisioner/0.6.0-2022.3.4
- export SLURMEL_DOCUMENTATION_HREF="https://docs.jupyter.jsc.fz-juelich.de/github/FZJ-JSC/jupyter-jsc-notebooks/blob/documentation/05-News&Updates/Announcement-2022-12_Slurm_Wrapped_Kernels.ipynb"
- export JUPYTERJSC_CMD_ARGS="${JUPYTERJSC_CMD_ARGS} --ServerApp.kernel_manager_class=jupyter_slurm_provisioner.SlurmAsyncMappingKernelManager"
- export SLURM_PROVISIONER_NODE_SUFFIX="i"
- export SLURM_PROVISIONER_JHUB_METRICS="${JUPYTERHUB_API_URL}/slurmwrapper/${JUPYTERHUB_USER}/${JUPYTERHUB_SERVER_NAME}"
- test -d ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel && ! test -d ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel && mv ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel
- else
- test -d ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel && mkdir -p ${HOME}/.local/share/jupyter/kernels.deactivated && mv ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel
- fi
- lspci -k | grep -A 2 -i "NVIDIA" | grep "Kernel driver in use:" | grep "nvidia"
- if [ $? -eq 0 ]; then
- module load JupyterExtension-nvdashboard/0.8.0-2023.3.6 || true
- fi
- send_spawn_update 80 "Load default modules done" "Loaded Jupyter/2022.3.4 in Stages/2022."
- fi
- echo "$(date) - Load modules done"
- }
-
- update_config () {
- sed -i -e "s|_port_|${PORT}|g" -e "s|_home_|${JUPYTERHUB_HOME:-${HOME}}|g" -e "s|_servername_|${JUPYTERHUB_SERVER_NAME}|g" ${DIR}/config.py
- if [[ -f ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py ]]; then
- echo "$(date) - Add system specific config ..."
- send_spawn_update 85 "Add system specific configuration." "Use system specific config file ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py"
- echo "" >> ${DIR}/config.py
- cat ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py >> ${DIR}/config.py
- for path in ${JUPYTER_EXTRA_LABEXTENSIONS_PATH//:/$'\n'}; do
- echo "c.LabServerApp.extra_labextensions_path.append('$path')" >> ${JUPYTER_LOG_DIR}/config.py
- done
- echo "$(date) - Add system specific config done"
- fi
- if [[ -f ${EBROOTJUPYTERLAB}/bin/update_favorites_json ]]; then
- # update favorite-dirs with $HOME,$PROJECT,$SCRATCH,
- echo "$(date) - Update favorites"
- ${EBROOTJUPYTERLAB}/bin/update_favorites_json
- fi
- }
-
- load_project_specific_kernel () {
- echo "$(date) - Activate hook for project specific kernels ..."
- if [[ -d ${PROJECT}/.local/share/jupyter ]] && [[ -r ${PROJECT}/.local/share/jupyter ]] && [[ -x ${PROJECT}/.local/share/jupyter ]]; then
- echo "$(date) - Add project kernel for $PROJECT"
- send_spawn_update_warning 90 "Activate project specific kernel." "Add ${PROJECT}/.local/share/jupyter to JUPYTER_PATH env variable. This may impede the start of JupyterLab."
- export JUPYTER_PATH=${JUPYTER_PATH}:${PROJECT}/.local/share/jupyter
- else
- echo "$(date) - Do not add project kernel for $PROJECT"
- # send_spawn_update_warning 90 "Could not activate project specific kernel." "Ensure that ${PROJECT}/.local/share/jupyter exists and is readable and accessible for ${USER}"
- fi
- echo "$(date) - Activate hook for project specific kernels done"
- }
-
- send_event () {
- BODY=${1//\'/}
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_EVENTS_URL}"
- eval " $CURL_CMD"
- }
-
- # show user a message in the UI
- send_spawn_update () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a darkorange message in the UI
- send_spawn_update_warning () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary><span style=\\\"color:darkorange;\\\">${SUMMARY}</span></summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a fail message in the UI
- send_spawn_update_fail () {
- SUMMARY=$1
- DETAILS=$2
- BODY="{\"progress\": 100, \"failed\": true, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send fail status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Send fail update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # setup tunnel to the node JupyterLab is running at
- setup_tunnel () {
- echo "$(date) - Setup tunnel ..."
- BODY="{\"ssh_node\": \"${SSH_NODE}\", \"service\": \"${HOSTNAME_I}:${PORT}\"}"
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_SETUPTUNNEL_URL}"
- HTTPCODE=$(eval " $CURL_CMD")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "Could not setup tunnel. Cancel start."
- exit 1
- fi
- send_spawn_update 40 "Setup ssh port-forwarding." "Create ssh tunnel with system user ljupyter. JupyterHub will then be able to connect to JupyterLab at ${HOSTNAME_I}:${PORT}"
- echo "$(date) - Setup tunnel done"
- }
-
- start () {
- echo "$(date) - Start jupyterhub-singleuser ..."
- export JUPYTERHUB_HOME=${JUPYTERHUB_HOME:-${HOME}}
- cd ${JUPYTERHUB_HOME}
-
- if [[ -n $JUPYTERJSC_USER_CMD ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom command" "Use custom command \$JUPYTERJSC_USER_CMD . You will be redirected, when your JupyterLab is ready."
- timeout 30d ${JUPYTERJSC_USER_CMD} ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- elif [[ -n $JUPYTERJSC_USER_CMD_ARGS ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom arguments" "Use custom arguments \$JUPYTERJSC_USER_CMD_ARGS . You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} ${JUPYTERJSC_USER_CMD_ARGS} &
- child=$!
- else
- send_spawn_update 95 "Start JupyterLab" "You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- fi
- echo "$child" > ${PID_PATH}
- echo "$(date) - Start jupyterhub-singleuser done (PID: $child )"
-
- echo "$(date) - Start validate_token.sh ..."
- /bin/bash ${DIR}/validate_token.sh ${child} ${JUPYTERHUB_API_URL} &
- token_pid=$!
- echo "$token_pid" > ${VALIDATE_TOKEN_PID_PATH}
- echo "$(date) - Start validate_token.sh done (PID: $token_pid )"
-
- wait $child
- }
-
-
- requirements
- set_env
- grace_runtime_kill
- setup_tunnel
- check_quota
- pre_start
- load_modules
- update_config
- load_project_specific_kernel
- start
diff --git a/outposts/unicore/kustomize-maint/jobs-3.6-juniq.yaml b/outposts/unicore/kustomize-maint/jobs-3.6-juniq.yaml
deleted file mode 100644
index 5615515acc4fc4d6bdcc0dad6bcf97be68e8a751..0000000000000000000000000000000000000000
--- a/outposts/unicore/kustomize-maint/jobs-3.6-juniq.yaml
+++ /dev/null
@@ -1,471 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: unicore-jobs-3-6-juniq
- namespace: unicore
-data:
- input_start.sh: |
- #!/bin/bash
-
- # Catch SIGTERM signal and stop $child process
- _term() {
- echo "$(date) - Stop JupyterLab ..."
- if [[ -z $child ]]; then
- echo "$(date) - Stop JupyterLab - unknown PID. Start stop.sh to wait for PID"
- bash ${DIR}/stop.sh &
- else
- pkill --parent $child
- kill $child
- echo "$(date) - Stop JupyterLab ( $child ) ... done"
- if [[ -z $token_pid ]]; then
- echo "$(date) - Validate Token PID not found..."
- else
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ..."
- kill $token_pid
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ... done"
- fi
- fi
- }
-
- trap _term SIGTERM
-
- requirements () {
- echo "$(date) - Setup system specific requirements ( ${HOSTNAME} ) ..."
- if [[ ! -d ${HOME}/.cache/black/19.3b0 ]]; then
- mkdir -p ${HOME}/.cache/black/19.3b0
- fi
- # export a memory warn threshold on login nodes
- hostname | egrep '<hostname_base>' > /dev/null && export JUPYTER_MEMWARNTHRES=0.5
- # set cpu limit/warning
- export FPATH_CPUQUOTA="/sys/fs/cgroup/cpu,cpuacct/user.slice/user-$(id -u $USER).slice/cpu.cfs_quota_us"
- [ -f "${FPATH_CPUQUOTA}" ] && [ $(hostname | egrep '<hostname_cpulimit>') ] && export JUPYTER_CPULIMIT=$(($(cat ${FPATH_CPUQUOTA})/100000))
-
- echo "$(date) - Setup system specific requirements done"
- }
-
- # set env_variables correctly
- set_env () {
- echo "$(date) - Set environment variables ..."
- export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
- if [[ -f ${DIR}/.env ]]; then
- source ${DIR}/.env
- fi
- export PID_PATH=${DIR}/service.pid
- export VALIDATE_TOKEN_PID_PATH=${DIR}/validate_token.pid
- export LC_ALL=en_US.UTF-8
- export JUPYTER_LOG_DIR=${DIR}
- export JUPYTER_STDOUT=${JUPYTER_LOG_DIR}/stderr
- export PYTHONPATH=""
- export PYTHONNOUSERSITE=1
- export HOSTNAME_S=$(hostname -s)
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export HOSTNAME_I=${HOSTNAME_S}
- else
- export HOSTNAME_I=$(hostname -s)i
- fi
- export CURL_ARGS="--silent --write-out %{http_code} --output /dev/null"
- export CURL_HEADERS="-H \"Authorization: token ${JUPYTERHUB_API_TOKEN}\" -H \"Content-Type: application/json\" -H \"Accept: application/json\""
-
- if [[ $JUPYTERHUB_SERVICE_URL == "https"* ]]; then
- export HUB_PROTO="https"
- else
- export HUB_PROTO="http"
- fi
-
- API_URL_WITHOUT_PROTO=${JUPYTERHUB_API_URL##https\:\/\/}
- export JUPYTERHUB_DOMAIN=${API_URL_WITHOUT_PROTO%%\/*}
- export JUPYTER_SERVER_PUBLIC_URL="https://${JUPYTERHUB_DOMAIN}${JUPYTERHUB_SERVICE_PREFIX}"
- export DWAVE_INSPECTOR_JUPYTER_SERVER_PROXY_EXTERNAL_URL=${JUPYTER_SERVER_PUBLIC_URL}
-
- echo "$(date) - JupyterLab is running on ${HOSTNAME_S}"
- # If this JupyterLab is running on a LoginNode, we'll use the external api url.
- # Otherwise we have to use a LoginNode as Proxy, because there's no internet
- # connection for batch nodes
- if [[ ${HOSTNAME_S} == "<hostname_base>"* ]]; then
- # LoginNode
- # This is the internal jupyterhub certificate, only used when connecting through proxy on login node
- unset JUPYTERHUB_SSL_CLIENT_CA
- export SSH_NODE=${HOSTNAME_I}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub. Jump node for tunnel: ${SSH_NODE}"
- else
- # batch node, no internet connection to the outside world
- export PREVIOUS_API_URL=${JUPYTERHUB_API_URL}
- export REMOTE_PORT="<remote_port>"
- export JUPYTERHUB_CERTIFICATE="${DIR}/service_ca.crt"
-
- if [[ $HUB_PROTO == "https" ]]; then
- export CURL_ARGS="${CURL_ARGS} --cacert ${JUPYTERHUB_CERTIFICATE}"
- export JUPYTERHUB_SSL_CLIENT_CA="${JUPYTERHUB_CERTIFICATE}"
- fi
-
- # Look for random LoginNode to use it as proxy
- # We start at a random index, then we will run through all nodes and test them
- export ALL_REMOTE_NODES=(<hostname_all>)
- size=${#ALL_REMOTE_NODES[@]}
- index=$(($RANDOM % $size))
- start_index=$index
- for _ in ${ALL_REMOTE_NODES[@]}
- do
- HTTPCODE=$(curl -X "GET" ${CURL_ARGS} ${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api)
- if [[ ${HTTPCODE} -ge 200 ]] && [[ ${HTTPCODE} -le 299 ]]; then
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- fi
- # Add additional maintenance check
- STATUSOUT=$(curl -s -w '%{http_code}' http://${ALL_REMOTE_NODES[$index]}:9100/metrics)
- STATUSOUTCODE=$(echo "${STATUSOUT}" | tail -n 1)
- STATUSOUTMAINT=$(echo "${STATUSOUT}" | grep -E '^maint_status' | cut -d' ' -f2)
- if [[ $STATUSOUTCODE -eq 200 ]] && [[ ${STATUSOUTMAINT} -eq 0 ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- else
- echo "$(date) - Hub reachable for ${ALL_REMOTE_NODES[$index]}. But node metrics answered with: ${STATUSOUTCODE} , ${STATUSOUTMAINT}. Continue search for node. Full output: ${STATUSOUT}"
- fi
- fi
- index=$(expr $index + 1)
- if [[ $index == $size ]]; then
- # if loop reached array end continue at 0
- index=0
- fi
- if [[ $index == $start_index ]]; then
- # Tried all login nodes
- echo "$(date) - Could not find any LoginNode to connect to JupyterHub. Please try again in a few minutes."
- exit 1
- fi
- done
- export JUPYTERHUB_ACTIVITY_URL=${JUPYTERHUB_ACTIVITY_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_EVENTS_URL=${JUPYTERHUB_EVENTS_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_SETUPTUNNEL_URL=${JUPYTERHUB_SETUPTUNNEL_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- fi
-
- export JUPYTERHUB_API_TOKEN=$(cat ${DIR}/.jupyter.token)
- export JUPYTERHUB_OAUTH_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_OAUTH_ACCESS_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_SINGLEUSER_APP="jupyter_server.serverapp.ServerApp"
-
- export PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')
- # Replace previously defined port as ip and port passed via
- # cmd line arguments are ignored as in the new implementation of jupyterhub-singleuser
- export JUPYTERHUB_SERVICE_URL=${HUB_PROTO}://0.0.0.0:${PORT}${JUPYTERHUB_SERVICE_PREFIX}
-
- if [[ $HUB_PROTO == "https" ]]; then
- export JUPYTERHUB_SSL_KEYFILE="${DIR}/service_cert.key"
- export JUPYTERHUB_SSL_CERTFILE="${DIR}/service_cert.crt"
- fi
-
- export JUPYTERJSC_CMD_ARGS=""
- echo "$(date) - Set environment variables done"
- }
-
- grace_runtime_kill() {
- # If JupyterLab is running on a batch node, we will kill it 10 seconds
- # before the slrum runtime ends. This way JupyterHub stops the JupyterLab
- # itself, instead of waiting to be notified that JupyterLab stopped.
- if [[ -z $SLURM_JOBID ]]; then
- # On Login Nodes this is not required
- echo "$(date) - SLURM_JOBID not set. Do not set activate grace_runtime_kill"
- return
- fi
- time=$(squeue -o "%L" -h -j $SLURM_JOBID)
- # Check if the time is "NOT_SET" or "UNLIMITED"
- if [[ "$time" == "NOT_SET" || "$time" == "UNLIMITED" || "$time" == "" ]]; then
- echo "$(date) - Time limit not set or unlimited"
- return
- fi
- # Split the time string into its components
- IFS="-:" read -r -a time_array <<< "$time"
- # Set default values for missing components
- for ((i=${#time_array[@]}; i<4; i++)); do
- time_array=(0 "${time_array[@]}")
- done
- # Fill the variables from back to front
- secs=${time_array[3]}
- mins=${time_array[2]}
- hours=${time_array[1]}
- days=${time_array[0]}
- total_seconds=$((days * 86400 + hours * 3600 + mins * 60 + secs - 10))
- echo "$(date) - Kill JupyterLab in $total_seconds seconds."
- sleep $total_seconds && send_spawn_update_fail "Stop JupyterLab" "The runtime of the slurm job $SLURM_JOBID will end within the next 10 seconds. Stop JupyterLab gracefully." &
- }
-
-
- # Check Quota
-
- check_quota () {
- echo "$(date) - Check quota ..."
- if [[ ! -f ${HOME}/.${JUPYTERHUB_SERVER_NAME} ]]; then
- touch ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC1=$?
- echo "Quota Check ${JUPYTERHUB_SERVER_NAME}" >> ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC2=$?
- if [[ $EC1 -ne 0 || $EC1 -ne 0 ]]; then
- send_spawn_update_fail "Disk quota exceeded in $HOME. You have to clean up your home directory before you can start a JupyterLab." "Jupyter-JSC tried to create a testfile in ${HOME} and failed. Job directory may contain further information: '"${DIR}"'"
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- exit 0
- fi
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- send_spawn_update 50 "Disk quota checked." "If Jupyter-JSC could not create files in ${HOME}, JupyterLab would not be able to start."
- else
- echo "$(date) - Could not check quota"
- fi
- echo "$(date) - Check quota done"
- }
-
-
- # Hook to load customized environments before loading modules
-
- pre_start () {
- echo "$(date) - Pre start ..."
- if [[ -f ${HOME}/.jupyter/pre_jupyter-jsc.sh ]]; then
- echo "$(date) - Pre start if ..."
- send_spawn_update_warning 60 "Use customized start script." "You are using a customized environment, defined in ${HOME}/.jupyter/pre_jupyter-jsc.sh."
- echo "------ pre_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/pre_jupyter-jsc.sh
- echo "------ pre_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/pre_jupyter-jsc.sh
- fi
- echo "$(date) - Pre start done"
- }
-
-
- # Load modules
-
- load_modules () {
- echo "$(date) - Load modules ..."
- if [[ -f ${HOME}/.jupyter/start_jupyter-jsc.sh ]]; then
- send_spawn_update_warning 70 "Load customized modules ..." "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- echo "------ start_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/start_jupyter-jsc.sh
- echo "------ start_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/start_jupyter-jsc.sh
- send_spawn_update_warning 80 "Load customized modules done" "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- else
- send_spawn_update 70 "Load default modules ..." "Loading Jupyter/2023.3.6 in Stages/2023."
-
- module purge
- module use ${OTHERSTAGES}
- module load Stages/2023
- module load GCC/11.3.0 ParaStationMPI JupyterLab/2023.3.6
- module load Cartopy/0.21.0 Cirq/1.0.0 DWave/6.3.0 PyQuil/3.3.3 Qiskit/0.41.0 pulser/0.12.0 myqlm/1.7.3 Qiskit-juqcs/0.8.0
-
- if [[ $JUPYTER_MODULE_BASH_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Bash/.0.9.0-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_CLING_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Cling/.20230205-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_JULIA_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Julia/.1.8.5-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_LFORTRAN_ENABLED -eq 1 ]]; then
- module load JupyterKernel-LFortran/.0.19.0-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_OCTAVE_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Octave/.8.2.0-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_PYDEEPLEARNING_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyDeepLearning/.2023.5-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_PYQUANTUM_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyQuantum/.2023.5-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_PYVISUALIZATION_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyVisualization/.2023.5-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_PYEARTHSYSTEM_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyEarthSystem/.2023.5-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_R_ENABLED -eq 1 ]]; then
- module load JupyterKernel-R/.4.2.1-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_RUBY_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Ruby/.3.0.5-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_XPRAHTML5_ENABLED -eq 1 ]]; then
- module load JupyterProxy-XpraHTML5/.0.3.5-2023.3.6
- fi
- # Load MatLab Proxy module, if user is in matlab group
- IFS=' ' read -ra groups_array <<< "$(id -Gn)"
- for group in "${groups_array[@]}"; do
- if [[ "$group" == "matlab" ]]; then
- if [[ $SYSTEMNAME == "hdfml" ]]; then
- module load JupyterProxy-Matlab/.0.8.0-2023.3.6
- else
- module load JupyterProxy-Matlab/.0.9.0-2023.3.6
- fi
- break
- fi
- done
- if [[ $JUPYTER_MODULE_NGLVIEW_ENABLED -eq 1 ]]; then
- module load JupyterExtension-nglview/3.0.6-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_JUPYTERAI_ENABLED -eq 1 ]]; then
- module load JupyterExtension-jupyterai/1.0.1-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_NEST_ENABLED -eq 1 ]]; then
- if [[ $SYSTEMNAME != "hdfml" ]]; then
- module load JupyterProxy-NESTDesktop
- fi
- fi
- if [[ $JUPYTER_MODULE_SLURMWRAPPER_ENABLED -eq 1 ]]; then
- module load JupyterExtension-slurmprovisioner/0.6.0-2023.3.6
- export SLURMEL_DOCUMENTATION_HREF="https://docs.jupyter.jsc.fz-juelich.de/github/FZJ-JSC/jupyter-jsc-notebooks/blob/documentation/05-News&Updates/Announcement-2022-12_Slurm_Wrapped_Kernels.ipynb"
- export JUPYTERJSC_CMD_ARGS="${JUPYTERJSC_CMD_ARGS} --ServerApp.kernel_manager_class=jupyter_slurm_provisioner.SlurmAsyncMappingKernelManager"
- export SLURM_PROVISIONER_NODE_SUFFIX="i"
- export SLURM_PROVISIONER_JHUB_METRICS="${JUPYTERHUB_API_URL}/slurmwrapper/${JUPYTERHUB_USER}/${JUPYTERHUB_SERVER_NAME}"
- test -d ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel && ! test -d ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel && mv ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel
- else
- test -d ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel && mkdir -p ${HOME}/.local/share/jupyter/kernels.deactivated && mv ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel
- fi
- lspci -k | grep -A 2 -i "NVIDIA" | grep "Kernel driver in use:" | grep "nvidia"
- if [ $? -eq 0 ]; then
- module load JupyterExtension-nvdashboard/0.8.0-2023.3.6 || true
- fi
- send_spawn_update 80 "Load default modules done" "Loaded Jupyter/2023.3.6 in Stages/2023."
- fi
- echo "$(date) - Load modules done"
- }
-
- update_config () {
- sed -i -e "s|_port_|${PORT}|g" -e "s|_home_|${JUPYTERHUB_HOME:-${HOME}}|g" -e "s|_servername_|${JUPYTERHUB_SERVER_NAME}|g" ${DIR}/config.py
- if [[ -f ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py ]]; then
- echo "$(date) - Add system specific config ..."
- send_spawn_update 85 "Add system specific configuration." "Use system specific config file ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py"
- echo "" >> ${DIR}/config.py
- cat ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py >> ${DIR}/config.py
- for path in ${JUPYTER_EXTRA_LABEXTENSIONS_PATH//:/$'\n'}; do
- echo "c.LabServerApp.extra_labextensions_path.append('$path')" >> ${JUPYTER_LOG_DIR}/config.py
- done
- echo "$(date) - Add system specific config done"
- fi
- if [[ -f ${EBROOTJUPYTERLAB}/bin/update_favorites_json ]]; then
- # update favorite-dirs with $HOME,$PROJECT,$SCRATCH,
- echo "$(date) - Update favorites"
- ${EBROOTJUPYTERLAB}/bin/update_favorites_json
- fi
- }
-
- load_project_specific_kernel () {
- echo "$(date) - Activate hook for project specific kernels ..."
- if [[ -d ${PROJECT}/.local/share/jupyter ]] && [[ -r ${PROJECT}/.local/share/jupyter ]] && [[ -x ${PROJECT}/.local/share/jupyter ]]; then
- echo "$(date) - Add project kernel for $PROJECT"
- send_spawn_update_warning 90 "Activate project specific kernel." "Add ${PROJECT}/.local/share/jupyter to JUPYTER_PATH env variable. This may impede the start of JupyterLab."
- export JUPYTER_PATH=${JUPYTER_PATH}:${PROJECT}/.local/share/jupyter
- else
- echo "$(date) - Do not add project kernel for $PROJECT"
- # send_spawn_update_warning 90 "Could not activate project specific kernel." "Ensure that ${PROJECT}/.local/share/jupyter exists and is readable and accessible for ${USER}"
- fi
- echo "$(date) - Activate hook for project specific kernels done"
- }
-
- send_event () {
- BODY=${1//\'/}
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_EVENTS_URL}"
- eval " $CURL_CMD"
- }
-
- # show user a message in the UI
- send_spawn_update () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a darkorange message in the UI
- send_spawn_update_warning () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary><span style=\\\"color:darkorange;\\\">${SUMMARY}</span></summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a fail message in the UI
- send_spawn_update_fail () {
- SUMMARY=$1
- DETAILS=$2
- BODY="{\"progress\": 100, \"failed\": true, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send fail status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Send fail update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # setup tunnel to the node JupyterLab is running at
- setup_tunnel () {
- echo "$(date) - Setup tunnel ..."
- BODY="{\"ssh_node\": \"${SSH_NODE}\", \"service\": \"${HOSTNAME_I}:${PORT}\"}"
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_SETUPTUNNEL_URL}"
- HTTPCODE=$(eval " $CURL_CMD")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "Could not setup tunnel. Cancel start."
- exit 1
- fi
- send_spawn_update 40 "Setup ssh port-forwarding." "Create ssh tunnel with system user ljupyter. JupyterHub will then be able to connect to JupyterLab at ${HOSTNAME_I}:${PORT}"
- echo "$(date) - Setup tunnel done"
- }
-
- start () {
- echo "$(date) - Start jupyterhub-singleuser ..."
- export JUPYTERHUB_HOME=${JUPYTERHUB_HOME:-${HOME}}
- cd ${JUPYTERHUB_HOME}
-
- if [[ -n $JUPYTERJSC_USER_CMD ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom command" "Use custom command \$JUPYTERJSC_USER_CMD . You will be redirected, when your JupyterLab is ready."
- timeout 30d ${JUPYTERJSC_USER_CMD} ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- elif [[ -n $JUPYTERJSC_USER_CMD_ARGS ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom arguments" "Use custom arguments \$JUPYTERJSC_USER_CMD_ARGS . You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} ${JUPYTERJSC_USER_CMD_ARGS} &
- child=$!
- else
- send_spawn_update 95 "Start JupyterLab" "You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- fi
- echo "$child" > ${PID_PATH}
- echo "$(date) - Start jupyterhub-singleuser done (PID: $child )"
-
- echo "$(date) - Start validate_token.sh ..."
- /bin/bash ${DIR}/validate_token.sh ${child} ${JUPYTERHUB_API_URL} &
- token_pid=$!
- echo "$token_pid" > ${VALIDATE_TOKEN_PID_PATH}
- echo "$(date) - Start validate_token.sh done (PID: $token_pid )"
-
- wait $child
- }
-
-
- requirements
- set_env
- grace_runtime_kill
- setup_tunnel
- check_quota
- pre_start
- load_modules
- update_config
- load_project_specific_kernel
- start
-
diff --git a/outposts/unicore/kustomize-maint/jobs-3.6.yaml b/outposts/unicore/kustomize-maint/jobs-3.6.yaml
deleted file mode 100644
index 1af5b356f1853cb5b5b8704313548f4e8d8ae9d3..0000000000000000000000000000000000000000
--- a/outposts/unicore/kustomize-maint/jobs-3.6.yaml
+++ /dev/null
@@ -1,465 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: unicore-jobs-3-6
- namespace: unicore
-data:
- input_start.sh: |
- #!/bin/bash
-
- # Catch SIGTERM signal and stop $child process
- _term() {
- echo "$(date) - Stop JupyterLab ..."
- if [[ -z $child ]]; then
- echo "$(date) - Stop JupyterLab - unknown PID. Start stop.sh to wait for PID"
- bash ${DIR}/stop.sh &
- else
- pkill --parent $child
- kill $child
- echo "$(date) - Stop JupyterLab ( $child ) ... done"
- if [[ -z $token_pid ]]; then
- echo "$(date) - Validate Token PID not found..."
- else
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ..."
- kill $token_pid
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ... done"
- fi
- fi
- }
-
- trap _term SIGTERM
-
- requirements () {
- echo "$(date) - Setup system specific requirements ( ${HOSTNAME} ) ..."
- if [[ ! -d ${HOME}/.cache/black/19.3b0 ]]; then
- mkdir -p ${HOME}/.cache/black/19.3b0
- fi
- # export a memory warn threshold on login nodes
- hostname | egrep '<hostname_base>' > /dev/null && export JUPYTER_MEMWARNTHRES=0.5
- # set cpu limit/warning
- export FPATH_CPUQUOTA="/sys/fs/cgroup/cpu,cpuacct/user.slice/user-$(id -u $USER).slice/cpu.cfs_quota_us"
- [ -f "${FPATH_CPUQUOTA}" ] && [ $(hostname | egrep '<hostname_cpulimit>') ] && export JUPYTER_CPULIMIT=$(($(cat ${FPATH_CPUQUOTA})/100000))
-
- echo "$(date) - Setup system specific requirements done"
- }
-
- # set env_variables correctly
- set_env () {
- echo "$(date) - Set environment variables ..."
- export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
- if [[ -f ${DIR}/.env ]]; then
- source ${DIR}/.env
- fi
- export PID_PATH=${DIR}/service.pid
- export VALIDATE_TOKEN_PID_PATH=${DIR}/validate_token.pid
- export LC_ALL=en_US.UTF-8
- export JUPYTER_LOG_DIR=${DIR}
- export JUPYTER_STDOUT=${JUPYTER_LOG_DIR}/stderr
- export PYTHONPATH=""
- export PYTHONNOUSERSITE=1
- export HOSTNAME_S=$(hostname -s)
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export HOSTNAME_I=${HOSTNAME_S}
- else
- export HOSTNAME_I=$(hostname -s)i
- fi
- export CURL_ARGS="--silent --write-out %{http_code} --output /dev/null"
- export CURL_HEADERS="-H \"Authorization: token ${JUPYTERHUB_API_TOKEN}\" -H \"Content-Type: application/json\" -H \"Accept: application/json\""
-
- if [[ $JUPYTERHUB_SERVICE_URL == "https"* ]]; then
- export HUB_PROTO="https"
- else
- export HUB_PROTO="http"
- fi
-
- API_URL_WITHOUT_PROTO=${JUPYTERHUB_API_URL##https\:\/\/}
- export JUPYTERHUB_DOMAIN=${API_URL_WITHOUT_PROTO%%\/*}
- export JUPYTER_SERVER_PUBLIC_URL="https://${JUPYTERHUB_DOMAIN}${JUPYTERHUB_SERVICE_PREFIX}"
- export DWAVE_INSPECTOR_JUPYTER_SERVER_PROXY_EXTERNAL_URL=${JUPYTER_SERVER_PUBLIC_URL}
-
- echo "$(date) - JupyterLab is running on ${HOSTNAME_S}"
- # If this JupyterLab is running on a LoginNode, we'll use the external api url.
- # Otherwise we have to use a LoginNode as Proxy, because there's no internet
- # connection for batch nodes
- if [[ ${HOSTNAME_S} == "<hostname_base>"* ]]; then
- # LoginNode
- # This is the internal jupyterhub certificate, only used when connecting through proxy on login node
- unset JUPYTERHUB_SSL_CLIENT_CA
- export SSH_NODE=${HOSTNAME_I}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub. Jump node for tunnel: ${SSH_NODE}"
- else
- # batch node, no internet connection to the outside world
- export PREVIOUS_API_URL=${JUPYTERHUB_API_URL}
- export REMOTE_PORT="<remote_port>"
- export JUPYTERHUB_CERTIFICATE="${DIR}/service_ca.crt"
-
- if [[ $HUB_PROTO == "https" ]]; then
- export CURL_ARGS="${CURL_ARGS} --cacert ${JUPYTERHUB_CERTIFICATE}"
- export JUPYTERHUB_SSL_CLIENT_CA="${JUPYTERHUB_CERTIFICATE}"
- fi
-
- # Look for random LoginNode to use it as proxy
- # We start at a random index, then we will run through all nodes and test them
- export ALL_REMOTE_NODES=(<hostname_all>)
- size=${#ALL_REMOTE_NODES[@]}
- index=$(($RANDOM % $size))
- start_index=$index
- for _ in ${ALL_REMOTE_NODES[@]}
- do
- HTTPCODE=$(curl -X "GET" ${CURL_ARGS} ${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api)
- if [[ ${HTTPCODE} -ge 200 ]] && [[ ${HTTPCODE} -le 299 ]]; then
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- fi
- # Add additional maintenance check
- STATUSOUT=$(curl -s -w '%{http_code}' http://${ALL_REMOTE_NODES[$index]}:9100/metrics)
- STATUSOUTCODE=$(echo "${STATUSOUT}" | tail -n 1)
- STATUSOUTMAINT=$(echo "${STATUSOUT}" | grep -E '^maint_status' | cut -d' ' -f2)
- if [[ $STATUSOUTCODE -eq 200 ]] && [[ ${STATUSOUTMAINT} -eq 0 ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- else
- echo "$(date) - Hub reachable for ${ALL_REMOTE_NODES[$index]}. But node metrics answered with: ${STATUSOUTCODE} , ${STATUSOUTMAINT}. Continue search for node. Full output: ${STATUSOUT}"
- fi
- fi
- index=$(expr $index + 1)
- if [[ $index == $size ]]; then
- # if loop reached array end continue at 0
- index=0
- fi
- if [[ $index == $start_index ]]; then
- # Tried all login nodes
- echo "$(date) - Could not find any LoginNode to connect to JupyterHub. Please try again in a few minutes."
- exit 1
- fi
- done
- export JUPYTERHUB_ACTIVITY_URL=${JUPYTERHUB_ACTIVITY_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_EVENTS_URL=${JUPYTERHUB_EVENTS_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_SETUPTUNNEL_URL=${JUPYTERHUB_SETUPTUNNEL_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- fi
-
- export JUPYTERHUB_API_TOKEN=$(cat ${DIR}/.jupyter.token)
- export JUPYTERHUB_OAUTH_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_OAUTH_ACCESS_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_SINGLEUSER_APP="jupyter_server.serverapp.ServerApp"
-
- export PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')
- # Replace previously defined port as ip and port passed via
- # cmd line arguments are ignored as in the new implementation of jupyterhub-singleuser
- export JUPYTERHUB_SERVICE_URL=${HUB_PROTO}://0.0.0.0:${PORT}${JUPYTERHUB_SERVICE_PREFIX}
-
- if [[ $HUB_PROTO == "https" ]]; then
- export JUPYTERHUB_SSL_KEYFILE="${DIR}/service_cert.key"
- export JUPYTERHUB_SSL_CERTFILE="${DIR}/service_cert.crt"
- fi
-
- export JUPYTERJSC_CMD_ARGS=""
- echo "$(date) - Set environment variables done"
- }
-
- grace_runtime_kill() {
- # If JupyterLab is running on a batch node, we will kill it 10 seconds
- # before the slrum runtime ends. This way JupyterHub stops the JupyterLab
- # itself, instead of waiting to be notified that JupyterLab stopped.
- if [[ -z $SLURM_JOBID ]]; then
- # On Login Nodes this is not required
- echo "$(date) - SLURM_JOBID not set. Do not set activate grace_runtime_kill"
- return
- fi
- time=$(squeue -o "%L" -h -j $SLURM_JOBID)
- # Check if the time is "NOT_SET" or "UNLIMITED"
- if [[ "$time" == "NOT_SET" || "$time" == "UNLIMITED" || "$time" == "" ]]; then
- echo "$(date) - Time limit not set or unlimited"
- return
- fi
- # Split the time string into its components
- IFS="-:" read -r -a time_array <<< "$time"
- # Set default values for missing components
- for ((i=${#time_array[@]}; i<4; i++)); do
- time_array=(0 "${time_array[@]}")
- done
- # Fill the variables from back to front
- secs=${time_array[3]}
- mins=${time_array[2]}
- hours=${time_array[1]}
- days=${time_array[0]}
- total_seconds=$((days * 86400 + hours * 3600 + mins * 60 + secs - 10))
- echo "$(date) - Kill JupyterLab in $total_seconds seconds."
- sleep $total_seconds && send_spawn_update_fail "Stop JupyterLab" "The runtime of the slurm job $SLURM_JOBID will end within the next 10 seconds. Stop JupyterLab gracefully." &
- }
-
-
- # Check Quota
-
- check_quota () {
- echo "$(date) - Check quota ..."
- if [[ ! -f ${HOME}/.${JUPYTERHUB_SERVER_NAME} ]]; then
- touch ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC1=$?
- echo "Quota Check ${JUPYTERHUB_SERVER_NAME}" >> ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC2=$?
- if [[ $EC1 -ne 0 || $EC1 -ne 0 ]]; then
- send_spawn_update_fail "Disk quota exceeded in $HOME. You have to clean up your home directory before you can start a JupyterLab." "Jupyter-JSC tried to create a testfile in ${HOME} and failed. Job directory may contain further information: '"${DIR}"'"
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- exit 0
- fi
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- send_spawn_update 50 "Disk quota checked." "If Jupyter-JSC could not create files in ${HOME}, JupyterLab would not be able to start."
- else
- echo "$(date) - Could not check quota"
- fi
- echo "$(date) - Check quota done"
- }
-
-
- # Hook to load customized environments before loading modules
-
- pre_start () {
- echo "$(date) - Pre start ..."
- if [[ -f ${HOME}/.jupyter/pre_jupyter-jsc.sh ]]; then
- echo "$(date) - Pre start if ..."
- send_spawn_update_warning 60 "Use customized start script." "You are using a customized environment, defined in ${HOME}/.jupyter/pre_jupyter-jsc.sh."
- echo "------ pre_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/pre_jupyter-jsc.sh
- echo "------ pre_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/pre_jupyter-jsc.sh
- fi
- echo "$(date) - Pre start done"
- }
-
-
- # Load modules
-
- load_modules () {
- echo "$(date) - Load modules ..."
- if [[ -f ${HOME}/.jupyter/start_jupyter-jsc.sh ]]; then
- send_spawn_update_warning 70 "Load customized modules ..." "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- echo "------ start_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/start_jupyter-jsc.sh
- echo "------ start_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/start_jupyter-jsc.sh
- send_spawn_update_warning 80 "Load customized modules done" "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- else
- send_spawn_update 70 "Load default modules ..." "Loading Jupyter/2023.3.6 in Stages/2023."
-
- module purge
- module use ${OTHERSTAGES}
- module load Stages/2023
- module load GCCcore/.11.3.0 Python JupyterLab/2023.3.6
-
- if [[ $JUPYTER_MODULE_BASH_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Bash/.0.9.0-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_CLING_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Cling/.20230205-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_JULIA_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Julia/.1.8.5-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_LFORTRAN_ENABLED -eq 1 ]]; then
- module load JupyterKernel-LFortran/.0.19.0-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_OCTAVE_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Octave/.8.2.0-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_PYDEEPLEARNING_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyDeepLearning/.2023.5-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_PYQUANTUM_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyQuantum/.2023.5-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_PYVISUALIZATION_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyVisualization/.2023.5-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_PYEARTHSYSTEM_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyEarthSystem/.2023.5-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_R_ENABLED -eq 1 ]]; then
- module load JupyterKernel-R/.4.2.1-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_RUBY_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Ruby/.3.0.5-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_XPRAHTML5_ENABLED -eq 1 ]]; then
- module load JupyterProxy-XpraHTML5/.0.3.5-2023.3.6
- fi
- # Load MatLab Proxy module, if user is in matlab group
- IFS=' ' read -ra groups_array <<< "$(id -Gn)"
- for group in "${groups_array[@]}"; do
- if [[ "$group" == "matlab" ]]; then
- module load JupyterProxy-Matlab/.0.8.0-2023.3.6
- break
- fi
- done
- if [[ $JUPYTER_MODULE_NGLVIEW_ENABLED -eq 1 ]]; then
- module load JupyterExtension-nglview/3.0.6-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_JUPYTERAI_ENABLED -eq 1 ]]; then
- module load JupyterExtension-jupyterai/1.0.1-2023.3.6
- fi
- if [[ $JUPYTER_MODULE_NEST_ENABLED -eq 1 ]]; then
- module use /p/usersoftware/swmanage/goebbert1/stage2023/nest-desktop/easybuild/${SYSTEMNAME}/modules/all/Compiler/GCCcore/11.3.0/
- module use /p/usersoftware/swmanage/goebbert1/stage2023/nest-desktop/easybuild/${SYSTEMNAME}/modules/all/MPI/GCC/11.3.0/psmpi/5/
- module load JupyterProxy-NESTDesktop
- fi
- if [[ $JUPYTER_MODULE_SLURMWRAPPER_ENABLED -eq 1 ]]; then
- module load JupyterExtension-slurmprovisioner/0.6.0-2023.3.6
- export SLURMEL_DOCUMENTATION_HREF="https://docs.jupyter.jsc.fz-juelich.de/github/FZJ-JSC/jupyter-jsc-notebooks/blob/documentation/05-News&Updates/Announcement-2022-12_Slurm_Wrapped_Kernels.ipynb"
- export JUPYTERJSC_CMD_ARGS="${JUPYTERJSC_CMD_ARGS} --ServerApp.kernel_manager_class=jupyter_slurm_provisioner.SlurmAsyncMappingKernelManager"
- export SLURM_PROVISIONER_NODE_SUFFIX="i"
- export SLURM_PROVISIONER_JHUB_METRICS="${JUPYTERHUB_API_URL}/slurmwrapper/${JUPYTERHUB_USER}/${JUPYTERHUB_SERVER_NAME}"
- test -d ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel && ! test -d ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel && mv ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel
- else
- test -d ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel && mkdir -p ${HOME}/.local/share/jupyter/kernels.deactivated && mv ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel
- fi
- lspci -k | grep -A 2 -i "NVIDIA" | grep "Kernel driver in use:" | grep "nvidia"
- if [ $? -eq 0 ]; then
- module load JupyterExtension-nvdashboard/0.8.0-2023.3.6 || true
- fi
- send_spawn_update 80 "Load default modules done" "Loaded Jupyter/2023.3.6 in Stages/2023."
- fi
- echo "$(date) - Load modules done"
- }
-
- update_config () {
- sed -i -e "s|_port_|${PORT}|g" -e "s|_home_|${JUPYTERHUB_HOME:-${HOME}}|g" -e "s|_servername_|${JUPYTERHUB_SERVER_NAME}|g" ${DIR}/config.py
- if [[ -f ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py ]]; then
- echo "$(date) - Add system specific config ..."
- send_spawn_update 85 "Add system specific configuration." "Use system specific config file ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py"
- echo "" >> ${DIR}/config.py
- cat ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py >> ${DIR}/config.py
- for path in ${JUPYTER_EXTRA_LABEXTENSIONS_PATH//:/$'\n'}; do
- echo "c.LabServerApp.extra_labextensions_path.append('$path')" >> ${JUPYTER_LOG_DIR}/config.py
- done
- echo "$(date) - Add system specific config done"
- fi
- if [[ -f ${EBROOTJUPYTERLAB}/bin/update_favorites_json ]]; then
- # update favorite-dirs with $HOME,$PROJECT,$SCRATCH,
- echo "$(date) - Update favorites"
- ${EBROOTJUPYTERLAB}/bin/update_favorites_json
- fi
- }
-
- load_project_specific_kernel () {
- echo "$(date) - Activate hook for project specific kernels ..."
- if [[ -d ${PROJECT}/.local/share/jupyter ]] && [[ -r ${PROJECT}/.local/share/jupyter ]] && [[ -x ${PROJECT}/.local/share/jupyter ]]; then
- echo "$(date) - Add project kernel for $PROJECT"
- send_spawn_update_warning 90 "Activate project specific kernel." "Add ${PROJECT}/.local/share/jupyter to JUPYTER_PATH env variable. This may impede the start of JupyterLab."
- export JUPYTER_PATH=${JUPYTER_PATH}:${PROJECT}/.local/share/jupyter
- else
- echo "$(date) - Do not add project kernel for $PROJECT"
- # send_spawn_update_warning 90 "Could not activate project specific kernel." "Ensure that ${PROJECT}/.local/share/jupyter exists and is readable and accessible for ${USER}"
- fi
- echo "$(date) - Activate hook for project specific kernels done"
- }
-
- send_event () {
- BODY=${1//\'/}
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_EVENTS_URL}"
- eval " $CURL_CMD"
- }
-
- # show user a message in the UI
- send_spawn_update () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a darkorange message in the UI
- send_spawn_update_warning () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary><span style=\\\"color:darkorange;\\\">${SUMMARY}</span></summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a fail message in the UI
- send_spawn_update_fail () {
- SUMMARY=$1
- DETAILS=$2
- BODY="{\"progress\": 100, \"failed\": true, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send fail status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Send fail update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # setup tunnel to the node JupyterLab is running at
- setup_tunnel () {
- echo "$(date) - Setup tunnel ..."
- BODY="{\"ssh_node\": \"${SSH_NODE}\", \"service\": \"${HOSTNAME_I}:${PORT}\"}"
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_SETUPTUNNEL_URL}"
- HTTPCODE=$(eval " $CURL_CMD")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "Could not setup tunnel. Cancel start."
- exit 1
- fi
- send_spawn_update 40 "Setup ssh port-forwarding." "Create ssh tunnel with system user ljupyter. JupyterHub will then be able to connect to JupyterLab at ${HOSTNAME_I}:${PORT}"
- echo "$(date) - Setup tunnel done"
- }
-
- start () {
- echo "$(date) - Start jupyterhub-singleuser ..."
- export JUPYTERHUB_HOME=${JUPYTERHUB_HOME:-${HOME}}
- cd ${JUPYTERHUB_HOME}
-
- if [[ -n $JUPYTERJSC_USER_CMD ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom command" "Use custom command \$JUPYTERJSC_USER_CMD . You will be redirected, when your JupyterLab is ready."
- timeout 30d ${JUPYTERJSC_USER_CMD} ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- elif [[ -n $JUPYTERJSC_USER_CMD_ARGS ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom arguments" "Use custom arguments \$JUPYTERJSC_USER_CMD_ARGS . You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} ${JUPYTERJSC_USER_CMD_ARGS} &
- child=$!
- else
- send_spawn_update 95 "Start JupyterLab" "You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- fi
- echo "$child" > ${PID_PATH}
- echo "$(date) - Start jupyterhub-singleuser done (PID: $child )"
-
- echo "$(date) - Start validate_token.sh ..."
- /bin/bash ${DIR}/validate_token.sh ${child} ${JUPYTERHUB_API_URL} &
- token_pid=$!
- echo "$token_pid" > ${VALIDATE_TOKEN_PID_PATH}
- echo "$(date) - Start validate_token.sh done (PID: $token_pid )"
-
- wait $child
- }
-
-
- requirements
- set_env
- grace_runtime_kill
- setup_tunnel
- check_quota
- pre_start
- load_modules
- update_config
- load_project_specific_kernel
- start
\ No newline at end of file
diff --git a/outposts/unicore/kustomize-maint/jobs-4.2-juniq.yaml b/outposts/unicore/kustomize-maint/jobs-4.2-juniq.yaml
deleted file mode 100644
index 6dfc0cc800f95513d23f0fda3d9860e89c819fe1..0000000000000000000000000000000000000000
--- a/outposts/unicore/kustomize-maint/jobs-4.2-juniq.yaml
+++ /dev/null
@@ -1,498 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: unicore-jobs-4-2-juniq
- namespace: unicore
-data:
- input_start.sh: |
- #!/bin/bash
-
- # Catch SIGTERM signal and stop $child process
- _term() {
- echo "$(date) - Stop JupyterLab ..."
- if [[ -z $child ]]; then
- echo "$(date) - Stop JupyterLab - unknown PID. Start stop.sh to wait for PID"
- bash ${DIR}/stop.sh &
- else
- pkill --parent $child
- kill $child
- echo "$(date) - Stop JupyterLab ( $child ) ... done"
- if [[ -z $token_pid ]]; then
- echo "$(date) - Validate Token PID not found..."
- else
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ..."
- kill $token_pid
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ... done"
- fi
- fi
- }
-
- trap _term SIGTERM
-
- requirements () {
- echo "$(date) - Setup system specific requirements ( ${HOSTNAME} ) ..."
- if [[ ! -d ${HOME}/.cache/black/19.3b0 ]]; then
- mkdir -p ${HOME}/.cache/black/19.3b0
- fi
- # export a memory warn threshold on login nodes
- hostname | egrep '<hostname_base>' > /dev/null && export JUPYTER_MEMWARNTHRES=0.5
- # set cpu limit/warning
- export FPATH_CPUQUOTA="/sys/fs/cgroup/cpu,cpuacct/user.slice/user-$(id -u $USER).slice/cpu.cfs_quota_us"
- [ -f "${FPATH_CPUQUOTA}" ] && [ $(hostname | egrep '<hostname_cpulimit>') ] && export JUPYTER_CPULIMIT=$(($(cat ${FPATH_CPUQUOTA})/100000))
-
- echo "$(date) - Setup system specific requirements done"
- }
-
- # set env_variables correctly
- set_env () {
- echo "$(date) - Set environment variables ..."
- export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
- if [[ -f ${DIR}/.env ]]; then
- source ${DIR}/.env
- fi
- export PID_PATH=${DIR}/service.pid
- export VALIDATE_TOKEN_PID_PATH=${DIR}/validate_token.pid
- export LC_ALL=en_US.UTF-8
- export JUPYTER_LOG_DIR=${DIR}
- export JUPYTER_STDOUT=${JUPYTER_LOG_DIR}/stderr
- export PYTHONPATH=""
- export PYTHONNOUSERSITE=1
- export HOSTNAME_S=$(hostname -s)
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export HOSTNAME_I=${HOSTNAME_S}
- elif [[ ${SYSTEMNAME} == "jedi" ]]; then
- export HOSTNAME_I=$(hostname -s)-interconnect-1
- else
- export HOSTNAME_I=$(hostname -s)i
- fi
- export CURL_ARGS="--silent --write-out %{http_code} --output /dev/null"
- export CURL_HEADERS="-H \"Authorization: token ${JUPYTERHUB_API_TOKEN}\" -H \"Content-Type: application/json\" -H \"Accept: application/json\""
-
- if [[ $JUPYTERHUB_SERVICE_URL == "https"* ]]; then
- export HUB_PROTO="https"
- else
- export HUB_PROTO="http"
- fi
-
- API_URL_WITHOUT_PROTO=${JUPYTERHUB_API_URL##https\:\/\/}
- export JUPYTERHUB_DOMAIN=${API_URL_WITHOUT_PROTO%%\/*}
- export JUPYTER_SERVER_PUBLIC_URL="https://${JUPYTERHUB_DOMAIN}${JUPYTERHUB_SERVICE_PREFIX}"
- export DWAVE_INSPECTOR_JUPYTER_SERVER_PROXY_EXTERNAL_URL=${JUPYTER_SERVER_PUBLIC_URL}
-
- echo "$(date) - JupyterLab is running on ${HOSTNAME_S}"
- # If this JupyterLab is running on a LoginNode, we'll use the external api url.
- # Otherwise we have to use a LoginNode as Proxy, because there's no internet
- # connection for batch nodes
- if [[ ${HOSTNAME_S} == "<hostname_base>"* ]]; then
- # LoginNode
- # This is the internal jupyterhub certificate, only used when connecting through proxy on login node
- unset JUPYTERHUB_SSL_CLIENT_CA
- export SSH_NODE=${HOSTNAME_I}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub. Jump node for tunnel: ${SSH_NODE}"
- else
- # batch node, no internet connection to the outside world
- export PREVIOUS_API_URL=${JUPYTERHUB_API_URL}
- export REMOTE_PORT="<remote_port>"
- export JUPYTERHUB_CERTIFICATE="${DIR}/service_ca.crt"
-
- if [[ $HUB_PROTO == "https" ]]; then
- export CURL_ARGS="${CURL_ARGS} --cacert ${JUPYTERHUB_CERTIFICATE}"
- export JUPYTERHUB_SSL_CLIENT_CA="${JUPYTERHUB_CERTIFICATE}"
- fi
-
- # Look for random LoginNode to use it as proxy
- # We start at a random index, then we will run through all nodes and test them
- export ALL_REMOTE_NODES=(<hostname_all>)
- size=${#ALL_REMOTE_NODES[@]}
- index=$(($RANDOM % $size))
- start_index=$index
- for _ in ${ALL_REMOTE_NODES[@]}
- do
- HTTPCODE=$(curl -X "GET" ${CURL_ARGS} ${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api)
- if [[ ${HTTPCODE} -ge 200 ]] && [[ ${HTTPCODE} -le 299 ]]; then
- if [[ ${SYSTEMNAME} == "deep" || ${SYSTEMNAME} == "jedi" ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- fi
- # Add additional maintenance check
- STATUSOUT=$(curl -s -w '%{http_code}' http://${ALL_REMOTE_NODES[$index]}:9100/metrics)
- STATUSOUTCODE=$(echo "${STATUSOUT}" | tail -n 1)
- STATUSOUTMAINT=$(echo "${STATUSOUT}" | grep -E '^maint_status' | cut -d' ' -f2)
- if [[ $STATUSOUTCODE -eq 200 ]] && [[ ${STATUSOUTMAINT} -eq 0 ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- else
- echo "$(date) - Hub reachable for ${ALL_REMOTE_NODES[$index]}. But node metrics answered with: ${STATUSOUTCODE} , ${STATUSOUTMAINT}. Continue search for node. Full output: ${STATUSOUT}"
- fi
- fi
- index=$(expr $index + 1)
- if [[ $index == $size ]]; then
- # if loop reached array end continue at 0
- index=0
- fi
- if [[ $index == $start_index ]]; then
- # Tried all login nodes
- echo "$(date) - Could not find any LoginNode to connect to JupyterHub. Please try again in a few minutes."
- exit 1
- fi
- done
- export JUPYTERHUB_ACTIVITY_URL=${JUPYTERHUB_ACTIVITY_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_EVENTS_URL=${JUPYTERHUB_EVENTS_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_SETUPTUNNEL_URL=${JUPYTERHUB_SETUPTUNNEL_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- fi
-
- export JUPYTERHUB_API_TOKEN=$(cat ${DIR}/.jupyter.token)
- export JUPYTERHUB_OAUTH_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_OAUTH_ACCESS_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_SINGLEUSER_APP="jupyter_server.serverapp.ServerApp"
-
- export PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')
- # Replace previously defined port as ip and port passed via
- # cmd line arguments are ignored as in the new implementation of jupyterhub-singleuser
- export JUPYTERHUB_SERVICE_URL=${HUB_PROTO}://0.0.0.0:${PORT}${JUPYTERHUB_SERVICE_PREFIX}
-
- if [[ $HUB_PROTO == "https" ]]; then
- export JUPYTERHUB_SSL_KEYFILE="${DIR}/service_cert.key"
- export JUPYTERHUB_SSL_CERTFILE="${DIR}/service_cert.crt"
- fi
-
- export JUPYTERJSC_CMD_ARGS=""
- echo "$(date) - Set environment variables done"
- }
-
- grace_runtime_kill() {
- # If JupyterLab is running on a batch node, we will kill it 10 seconds
- # before the slrum runtime ends. This way JupyterHub stops the JupyterLab
- # itself, instead of waiting to be notified that JupyterLab stopped.
- if [[ -z $SLURM_JOBID ]]; then
- # On Login Nodes this is not required
- echo "$(date) - SLURM_JOBID not set. Do not set activate grace_runtime_kill"
- return
- fi
- time=$(squeue -o "%L" -h -j $SLURM_JOBID)
- # Check if the time is "NOT_SET" or "UNLIMITED"
- if [[ "$time" == "NOT_SET" || "$time" == "UNLIMITED" || "$time" == "" ]]; then
- echo "$(date) - Time limit not set or unlimited"
- return
- fi
- # Split the time string into its components
- IFS="-:" read -r -a time_array <<< "$time"
- # Set default values for missing components
- for ((i=${#time_array[@]}; i<4; i++)); do
- time_array=(0 "${time_array[@]}")
- done
- # Fill the variables from back to front
- secs=${time_array[3]}
- mins=${time_array[2]}
- hours=${time_array[1]}
- days=${time_array[0]}
- total_seconds=$((days * 86400 + hours * 3600 + mins * 60 + secs - 10))
- echo "$(date) - Kill JupyterLab in $total_seconds seconds."
- sleep $total_seconds && send_spawn_update_fail "Stop JupyterLab" "The runtime of the slurm job $SLURM_JOBID will end within the next 10 seconds. Stop JupyterLab gracefully." &
- }
-
-
- # Check Quota
-
- check_quota () {
- echo "$(date) - Check quota ..."
- if [[ ! -f ${HOME}/.${JUPYTERHUB_SERVER_NAME} ]]; then
- touch ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC1=$?
- echo "Quota Check ${JUPYTERHUB_SERVER_NAME}" >> ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC2=$?
- if [[ $EC1 -ne 0 || $EC1 -ne 0 ]]; then
- send_spawn_update_fail "Disk quota exceeded in $HOME. You have to clean up your home directory before you can start a JupyterLab." "Jupyter-JSC tried to create a testfile in ${HOME} and failed. Job directory may contain further information: '"${DIR}"'"
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- exit 0
- fi
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- send_spawn_update 50 "Disk quota checked." "If Jupyter-JSC could not create files in ${HOME}, JupyterLab would not be able to start."
- else
- echo "$(date) - Could not check quota"
- fi
- echo "$(date) - Check quota done"
- }
-
-
- # Hook to load customized environments before loading modules
-
- pre_start () {
- echo "$(date) - Pre start ..."
- if [[ -f ${HOME}/.jupyter/pre_jupyter-jsc.sh ]]; then
- echo "$(date) - Pre start if ..."
- send_spawn_update_warning 60 "Use customized start script." "You are using a customized environment, defined in ${HOME}/.jupyter/pre_jupyter-jsc.sh."
- echo "------ pre_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/pre_jupyter-jsc.sh
- echo "------ pre_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/pre_jupyter-jsc.sh
- fi
- echo "$(date) - Pre start done"
- }
-
-
- # Load modules
-
- load_modules () {
- echo "$(date) - Load modules ..."
- if [[ -f ${HOME}/.jupyter/start_jupyter-jsc.sh ]]; then
- send_spawn_update_warning 70 "Load customized modules ..." "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- echo "------ start_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/start_jupyter-jsc.sh
- echo "------ start_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/start_jupyter-jsc.sh
- send_spawn_update_warning 80 "Load customized modules done" "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- else
- send_spawn_update 70 "Load default modules ..." "Loading Jupyter-bundle/20240520 in Stages/2024."
-
- module purge
- module use ${OTHERSTAGES}
- module load Stages/2024 GCC/12.3.0 ParaStationMPI
- module load Jupyter-bundle/20240520
- module load Cartopy/0.22.0 Cirq/1.4.0 DWave/6.8.0 PyQuil/4.8.0 CUDA/12 cuQuantum-Python/23.10.0-CUDA-12 Qiskit/0.45.1 pulser/0.18.0 qoqo-quest/0.14.4a3-CUDA-12-Rust-1.70.0 myqlm/1.10.4 ParityOS/2.3.0
-
- # Kernels
- if [[ $JUPYTER_MODULE_BASH_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Bash/.0.9.3-4.2.1
- fi
- if [[ $JUPYTER_MODULE_CLING_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Cling/.20231018-4.2.1
- fi
- if [[ $JUPYTER_MODULE_JAVA_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Java/.1.3.0
- fi
- if [[ $JUPYTER_MODULE_JULIA_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Julia/.1.9.3
- fi
- if [[ $JUPYTER_MODULE_LFORTRAN_ENABLED -eq 1 ]]; then
- module load JupyterKernel-LFortran/.0.30.0
- fi
- if [[ $JUPYTER_MODULE_OCTAVE_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Octave/.8.4.0-4.2.1
- fi
- if [[ $JUPYTER_MODULE_R_ENABLED -eq 1 ]]; then
- module load JupyterKernel-R/.4.3.2-4.2.1
- fi
- if [[ $JUPYTER_MODULE_RUBY_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Ruby/.3.2.2
- fi
- if [[ $JUPYTER_MODULE_PYDEEPLEARNING_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyDeepLearning/.2024.3-4.2.1
- fi
- if [[ $JUPYTER_MODULE_PYEARTHSYSTEM_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyEarthSystem/.2024.3-4.2.1
- fi
- if [[ $JUPYTER_MODULE_PYQUANTUM_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyQuantum/.2024.10-4.2.1
- fi
- if [[ $JUPYTER_MODULE_PYVISUALIZATION_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyVisualization/.2024.3-4.2.1
- fi
-
- # Proxies
- if [[ $JUPYTER_MODULE_XPRAHTML5_ENABLED -eq 1 ]]; then
- module load JupyterProxy-XpraHTML5/.0.4.0-4.2.1
- fi
- # Load MatLab Proxy module, if user is in matlab group
- IFS=' ' read -ra groups_array <<< "$(id -Gn)"
- for group in "${groups_array[@]}"; do
- if [[ "$group" == "matlab" ]]; then
- module load JupyterProxy-Matlab/.0.12.2-4.2.1
- break
- fi
- done
- # NEST does not support nodejs v 18 ( https://github.com/nest-desktop/nest-desktop/issues/509 )
- if [[ $JUPYTER_MODULE_NEST_ENABLED -eq 1 ]]; then
- if [[ $SYSTEMNAME != "hdfml" ]]; then
- module load JupyterProxy-NESTDesktop/.0.4.0-4.2.1
- fi
- fi
-
- if [[ $JUPYTER_MODULE_RSTUDIO_ENABLED -eq 1 ]]; then
- if [[ $SYSTEMNAME != "deep" ]]; then
- module load JupyterProxy-RStudio/.2023.09.1-4.2.1
- fi
- fi
- # Extensions
- if [[ $JUPYTER_MODULE_NGLVIEW_ENABLED -eq 1 ]]; then
- module load nglview/.3.1.2
- fi
- if [[ $JUPYTER_MODULE_JUPYTERAI_ENABLED -eq 1 ]]; then
- module load jupyter-ai/.2.15.0
- fi
- if [[ $JUPYTER_MODULE_VARIABLEINSPECTOR_ENABLED -eq 1 ]]; then
- module load jupyterlab-variableinspector/.3.2.1
- fi
- if [[ $JUPYTER_MODULE_SPELLCHECKER_ENABLED -eq 1 ]]; then
- module load jupyterlab-spellchecker/.0.8.4
- fi
- if [[ $JUPYTER_MODULE_TRAMEMANAGER_ENABLED -eq 1 ]]; then
- if [[ $SYSTEMNAME != "deep" ]]; then
- module load jupyterlab-trame-manager/.0.6.1-4.2.1
- fi
- fi
- if [[ $JUPYTER_MODULE_SLURMWRAPPER_ENABLED -eq 1 ]]; then
- module load jupyter-slurm-provisioner/.0.6.0-4.2.1
- export SLURMEL_DOCUMENTATION_HREF="https://docs.jupyter.jsc.fz-juelich.de/github/FZJ-JSC/jupyter-jsc-notebooks/blob/documentation/05-News&Updates/Announcement-2022-12_Slurm_Wrapped_Kernels.ipynb"
- export JUPYTERJSC_CMD_ARGS="${JUPYTERJSC_CMD_ARGS} --ServerApp.kernel_manager_class=jupyter_slurm_provisioner.SlurmAsyncMappingKernelManager"
- export SLURM_PROVISIONER_NODE_SUFFIX="i"
- export SLURM_PROVISIONER_JHUB_METRICS="${JUPYTERHUB_API_URL}/slurmwrapper/${JUPYTERHUB_USER}/${JUPYTERHUB_SERVER_NAME}"
- test -d ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel && ! test -d ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel && mv ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel
- else
- test -d ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel && mkdir -p ${HOME}/.local/share/jupyter/kernels.deactivated && mv ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel
- fi
- lspci -k | grep -A 2 -i "NVIDIA" | grep "Kernel driver in use:" | grep "nvidia"
- if [ $? -eq 0 ]; then
- module load jupyterlab-nvdashboard/.0.10.0
- fi
- send_spawn_update 80 "Load default modules done" "Loaded Jupyter-bundle/20240300 in Stages/2024."
- fi
- echo "$(date) - Load modules done"
- }
-
- update_config () {
- sed -i -e "s|_port_|${PORT}|g" -e "s|_home_|${JUPYTERHUB_HOME:-${HOME}}|g" -e "s|_servername_|${JUPYTERHUB_SERVER_NAME}|g" ${DIR}/config.py
- send_spawn_update 85 "Add system specific configuration." "Use system specific jupyter_notebook_config.py files"
- if [[ -f ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py ]]; then
- echo "" >> ${DIR}/config.py
- cat ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py >> ${DIR}/config.py
- echo "$(date) - Added system specific config ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py"
- fi
- if [[ -f ${EBROOTJUPYTERMINRESOURCEMINUSAGE}/etc/jupyter/jupyter_notebook_config.py ]]; then
- echo "" >> ${DIR}/config.py
- cat ${EBROOTJUPYTERMINRESOURCEMINUSAGE}/etc/jupyter/jupyter_notebook_config.py >> ${DIR}/config.py
- echo "$(date) - Added system specific config ${EBROOTJUPYTERMINRESOURCEMINUSAGE}/etc/jupyter/jupyter_notebook_config.py"
- fi
- for path in ${JUPYTER_EXTRA_LABEXTENSIONS_PATH//:/$'\n'}; do
- echo "c.LabServerApp.extra_labextensions_path.append('$path')" >> ${JUPYTER_LOG_DIR}/config.py
- done
- if command -v update_favorites_json &> /dev/null; then
- # update favorite-dirs with $HOME,$PROJECT,$SCRATCH,
- echo "$(date) - Update favorites"
- update_favorites_json
- fi
- }
-
- load_project_specific_kernel () {
- echo "$(date) - Activate hook for project specific kernels ..."
- if [[ -d ${PROJECT}/.local/share/jupyter ]] && [[ -r ${PROJECT}/.local/share/jupyter ]] && [[ -x ${PROJECT}/.local/share/jupyter ]]; then
- echo "$(date) - Add project kernel for $PROJECT"
- send_spawn_update_warning 90 "Activate project specific kernel." "Add ${PROJECT}/.local/share/jupyter to JUPYTER_PATH env variable. This may impede the start of JupyterLab."
- export JUPYTER_PATH=${JUPYTER_PATH}:${PROJECT}/.local/share/jupyter
- else
- echo "$(date) - Do not add project kernel for $PROJECT"
- # send_spawn_update_warning 90 "Could not activate project specific kernel." "Ensure that ${PROJECT}/.local/share/jupyter exists and is readable and accessible for ${USER}"
- fi
- echo "$(date) - Activate hook for project specific kernels done"
- }
-
- send_event () {
- BODY=${1//\'/}
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_EVENTS_URL}"
- eval " $CURL_CMD"
- }
-
- # show user a message in the UI
- send_spawn_update () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a darkorange message in the UI
- send_spawn_update_warning () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary><span style=\\\"color:darkorange;\\\">${SUMMARY}</span></summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a fail message in the UI
- send_spawn_update_fail () {
- SUMMARY=$1
- DETAILS=$2
- BODY="{\"progress\": 100, \"failed\": true, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send fail status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Send fail update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # setup tunnel to the node JupyterLab is running at
- setup_tunnel () {
- echo "$(date) - Setup tunnel ..."
- BODY="{\"ssh_node\": \"${SSH_NODE}\", \"service\": \"${HOSTNAME_I}:${PORT}\"}"
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_SETUPTUNNEL_URL}"
- HTTPCODE=$(eval " $CURL_CMD")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "Could not setup tunnel. Cancel start."
- exit 1
- fi
- send_spawn_update 40 "Setup ssh port-forwarding." "Create ssh tunnel with system user ljupyter. JupyterHub will then be able to connect to JupyterLab at ${HOSTNAME_I}:${PORT}"
- echo "$(date) - Setup tunnel done"
- }
-
- start () {
- echo "$(date) - Start jupyterhub-singleuser ..."
- export JUPYTERHUB_HOME=${JUPYTERHUB_HOME:-${HOME}}
- cd ${JUPYTERHUB_HOME}
-
- if [[ -n $JUPYTERJSC_USER_CMD ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom command" "Use custom command \$JUPYTERJSC_USER_CMD . You will be redirected, when your JupyterLab is ready."
- timeout 30d ${JUPYTERJSC_USER_CMD} ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- elif [[ -n $JUPYTERJSC_USER_CMD_ARGS ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom arguments" "Use custom arguments \$JUPYTERJSC_USER_CMD_ARGS . You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} ${JUPYTERJSC_USER_CMD_ARGS} &
- child=$!
- else
- send_spawn_update 95 "Start JupyterLab" "You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- fi
- echo "$child" > ${PID_PATH}
- echo "$(date) - Start jupyterhub-singleuser done (PID: $child )"
-
- echo "$(date) - Start validate_token.sh ..."
- /bin/bash ${DIR}/validate_token.sh ${child} ${JUPYTERHUB_API_URL} &
- token_pid=$!
- echo "$token_pid" > ${VALIDATE_TOKEN_PID_PATH}
- echo "$(date) - Start validate_token.sh done (PID: $token_pid )"
-
- wait $child
- }
-
-
- requirements
- set_env
- grace_runtime_kill
- setup_tunnel
- check_quota
- pre_start
- load_modules
- update_config
- load_project_specific_kernel
- start
-
diff --git a/outposts/unicore/kustomize-maint/jobs-4.2.yaml b/outposts/unicore/kustomize-maint/jobs-4.2.yaml
deleted file mode 100644
index f0fe756457f5db38c6850c9646cf9025325bceb7..0000000000000000000000000000000000000000
--- a/outposts/unicore/kustomize-maint/jobs-4.2.yaml
+++ /dev/null
@@ -1,503 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: unicore-jobs-4-2
- namespace: unicore
-data:
- input_start.sh: |
- #!/bin/bash
-
- # Catch SIGTERM signal and stop $child process
- _term() {
- echo "$(date) - Stop JupyterLab ..."
- if [[ -z $child ]]; then
- echo "$(date) - Stop JupyterLab - unknown PID. Start stop.sh to wait for PID"
- bash ${DIR}/stop.sh &
- else
- pkill --parent $child
- kill $child
- echo "$(date) - Stop JupyterLab ( $child ) ... done"
- if [[ -z $token_pid ]]; then
- echo "$(date) - Validate Token PID not found..."
- else
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ..."
- kill $token_pid
- echo "$(date) - Stop Validate Token Script ( $token_pid ) ... done"
- fi
- fi
- }
-
- trap _term SIGTERM
-
- requirements () {
- echo "$(date) - Setup system specific requirements ( ${HOSTNAME} ) ..."
- if [[ ! -d ${HOME}/.cache/black/19.3b0 ]]; then
- mkdir -p ${HOME}/.cache/black/19.3b0
- fi
- # export a memory warn threshold on login nodes
- hostname | egrep '<hostname_base>' > /dev/null && export JUPYTER_MEMWARNTHRES=0.5
- # set cpu limit/warning
- export FPATH_CPUQUOTA="/sys/fs/cgroup/cpu,cpuacct/user.slice/user-$(id -u $USER).slice/cpu.cfs_quota_us"
- [ -f "${FPATH_CPUQUOTA}" ] && [ $(hostname | egrep '<hostname_cpulimit>') ] && export JUPYTER_CPULIMIT=$(($(cat ${FPATH_CPUQUOTA})/100000))
-
- echo "$(date) - Setup system specific requirements done"
- }
-
- # set env_variables correctly
- set_env () {
- echo "$(date) - Set environment variables ..."
- export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
- if [[ -f ${DIR}/.env ]]; then
- source ${DIR}/.env
- fi
- export PID_PATH=${DIR}/service.pid
- export VALIDATE_TOKEN_PID_PATH=${DIR}/validate_token.pid
- export LC_ALL=en_US.UTF-8
- export JUPYTER_LOG_DIR=${DIR}
- export JUPYTER_STDOUT=${JUPYTER_LOG_DIR}/stderr
- export PYTHONPATH=""
- export PYTHONNOUSERSITE=1
- export HOSTNAME_S=$(hostname -s)
- if [[ ${SYSTEMNAME} == "deep" ]]; then
- export HOSTNAME_I=${HOSTNAME_S}
- elif [[ ${SYSTEMNAME} == "jedi" ]]; then
- export HOSTNAME_I=$(hostname -s)-interconnect-1
- else
- export HOSTNAME_I=$(hostname -s)i
- fi
- export CURL_ARGS="--silent --write-out %{http_code} --output /dev/null"
- export CURL_HEADERS="-H \"Authorization: token ${JUPYTERHUB_API_TOKEN}\" -H \"Content-Type: application/json\" -H \"Accept: application/json\""
-
- if [[ $JUPYTERHUB_SERVICE_URL == "https"* ]]; then
- export HUB_PROTO="https"
- else
- export HUB_PROTO="http"
- fi
-
- API_URL_WITHOUT_PROTO=${JUPYTERHUB_API_URL##https\:\/\/}
- export JUPYTERHUB_DOMAIN=${API_URL_WITHOUT_PROTO%%\/*}
- export JUPYTER_SERVER_PUBLIC_URL="https://${JUPYTERHUB_DOMAIN}${JUPYTERHUB_SERVICE_PREFIX}"
- export DWAVE_INSPECTOR_JUPYTER_SERVER_PROXY_EXTERNAL_URL=${JUPYTER_SERVER_PUBLIC_URL}
-
- echo "$(date) - JupyterLab is running on ${HOSTNAME_S}"
- # If this JupyterLab is running on a LoginNode, we'll use the external api url.
- # Otherwise we have to use a LoginNode as Proxy, because there's no internet
- # connection for batch nodes
- if [[ ${HOSTNAME_S} == "<hostname_base>"* ]]; then
- # LoginNode
- # This is the internal jupyterhub certificate, only used when connecting through proxy on login node
- unset JUPYTERHUB_SSL_CLIENT_CA
- export SSH_NODE=${HOSTNAME_I}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub. Jump node for tunnel: ${SSH_NODE}"
- else
- # batch node, no internet connection to the outside world
- export PREVIOUS_API_URL=${JUPYTERHUB_API_URL}
- export REMOTE_PORT="<remote_port>"
- export JUPYTERHUB_CERTIFICATE="${DIR}/service_ca.crt"
-
- if [[ $HUB_PROTO == "https" ]]; then
- export CURL_ARGS="${CURL_ARGS} --cacert ${JUPYTERHUB_CERTIFICATE}"
- export JUPYTERHUB_SSL_CLIENT_CA="${JUPYTERHUB_CERTIFICATE}"
- fi
-
- # Look for random LoginNode to use it as proxy
- # We start at a random index, then we will run through all nodes and test them
- export ALL_REMOTE_NODES=(<hostname_all>)
- size=${#ALL_REMOTE_NODES[@]}
- index=$(($RANDOM % $size))
- start_index=$index
- for _ in ${ALL_REMOTE_NODES[@]}
- do
- HTTPCODE=$(curl -X "GET" ${CURL_ARGS} ${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api)
- if [[ ${HTTPCODE} -ge 200 ]] && [[ ${HTTPCODE} -le 299 ]]; then
- if [[ ${SYSTEMNAME} == "deep" || ${SYSTEMNAME} == "jedi" ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- fi
- # Add additional maintenance check
- STATUSOUT=$(curl -s -w '%{http_code}' http://${ALL_REMOTE_NODES[$index]}:9100/metrics)
- STATUSOUTCODE=$(echo "${STATUSOUT}" | tail -n 1)
- STATUSOUTMAINT=$(echo "${STATUSOUT}" | grep -E '^maint_status' | cut -d' ' -f2)
- if [[ $STATUSOUTCODE -eq 200 ]] && [[ ${STATUSOUTMAINT} -eq 0 ]]; then
- export JUPYTERHUB_API_URL="${HUB_PROTO}://${ALL_REMOTE_NODES[$index]}:${REMOTE_PORT}/hub/api"
- export SSH_NODE=${ALL_REMOTE_NODES[$index]}
- echo "$(date) - Use ${JUPYTERHUB_API_URL} to communicate with JupyterHub and ${SSH_NODE} as jump node."
- break
- else
- echo "$(date) - Hub reachable for ${ALL_REMOTE_NODES[$index]}. But node metrics answered with: ${STATUSOUTCODE} , ${STATUSOUTMAINT}. Continue search for node. Full output: ${STATUSOUT}"
- fi
- fi
- index=$(expr $index + 1)
- if [[ $index == $size ]]; then
- # if loop reached array end continue at 0
- index=0
- fi
- if [[ $index == $start_index ]]; then
- # Tried all login nodes
- echo "$(date) - Could not find any LoginNode to connect to JupyterHub. Please try again in a few minutes."
- exit 1
- fi
- done
- export JUPYTERHUB_ACTIVITY_URL=${JUPYTERHUB_ACTIVITY_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_EVENTS_URL=${JUPYTERHUB_EVENTS_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- export JUPYTERHUB_SETUPTUNNEL_URL=${JUPYTERHUB_SETUPTUNNEL_URL/${PREVIOUS_API_URL}/${JUPYTERHUB_API_URL}}
- fi
-
- export JUPYTERHUB_API_TOKEN=$(cat ${DIR}/.jupyter.token)
- export JUPYTERHUB_OAUTH_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_OAUTH_ACCESS_SCOPES=$(cat ${DIR}/.oauth.scopes)
- export JUPYTERHUB_SINGLEUSER_APP="jupyter_server.serverapp.ServerApp"
-
- export PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')
- # Replace previously defined port as ip and port passed via
- # cmd line arguments are ignored as in the new implementation of jupyterhub-singleuser
- export JUPYTERHUB_SERVICE_URL=${HUB_PROTO}://0.0.0.0:${PORT}${JUPYTERHUB_SERVICE_PREFIX}
-
- if [[ $HUB_PROTO == "https" ]]; then
- export JUPYTERHUB_SSL_KEYFILE="${DIR}/service_cert.key"
- export JUPYTERHUB_SSL_CERTFILE="${DIR}/service_cert.crt"
- fi
-
- export JUPYTERJSC_CMD_ARGS=""
- echo "$(date) - Set environment variables done"
- }
-
- grace_runtime_kill() {
- # If JupyterLab is running on a batch node, we will kill it 10 seconds
- # before the slrum runtime ends. This way JupyterHub stops the JupyterLab
- # itself, instead of waiting to be notified that JupyterLab stopped.
- if [[ -z $SLURM_JOBID ]]; then
- # On Login Nodes this is not required
- echo "$(date) - SLURM_JOBID not set. Do not set activate grace_runtime_kill"
- return
- fi
- time=$(squeue -o "%L" -h -j $SLURM_JOBID)
- # Check if the time is "NOT_SET" or "UNLIMITED"
- if [[ "$time" == "NOT_SET" || "$time" == "UNLIMITED" || "$time" == "" ]]; then
- echo "$(date) - Time limit not set or unlimited"
- return
- fi
- # Split the time string into its components
- IFS="-:" read -r -a time_array <<< "$time"
- # Set default values for missing components
- for ((i=${#time_array[@]}; i<4; i++)); do
- time_array=(0 "${time_array[@]}")
- done
- # Fill the variables from back to front
- secs=${time_array[3]}
- mins=${time_array[2]}
- hours=${time_array[1]}
- days=${time_array[0]}
- total_seconds=$((days * 86400 + hours * 3600 + mins * 60 + secs - 10))
- echo "$(date) - Kill JupyterLab in $total_seconds seconds."
- sleep $total_seconds && send_spawn_update_fail "Stop JupyterLab" "The runtime of the slurm job $SLURM_JOBID will end within the next 10 seconds. Stop JupyterLab gracefully." &
- }
-
-
- # Check Quota
-
- check_quota () {
- echo "$(date) - Check quota ..."
- if [[ ! -f ${HOME}/.${JUPYTERHUB_SERVER_NAME} ]]; then
- touch ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC1=$?
- echo "Quota Check ${JUPYTERHUB_SERVER_NAME}" >> ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- EC2=$?
- if [[ $EC1 -ne 0 || $EC1 -ne 0 ]]; then
- send_spawn_update_fail "Disk quota exceeded in $HOME. You have to clean up your home directory before you can start a JupyterLab." "Jupyter-JSC tried to create a testfile in ${HOME} and failed. Job directory may contain further information: '"${DIR}"'"
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- exit 0
- fi
- rm ${HOME}/.${JUPYTERHUB_SERVER_NAME}
- send_spawn_update 50 "Disk quota checked." "If Jupyter-JSC could not create files in ${HOME}, JupyterLab would not be able to start."
- else
- echo "$(date) - Could not check quota"
- fi
- echo "$(date) - Check quota done"
- }
-
-
- # Hook to load customized environments before loading modules
-
- pre_start () {
- echo "$(date) - Pre start ..."
- if [[ -f ${HOME}/.jupyter/pre_jupyter-jsc.sh ]]; then
- echo "$(date) - Pre start if ..."
- send_spawn_update_warning 60 "Use customized start script." "You are using a customized environment, defined in ${HOME}/.jupyter/pre_jupyter-jsc.sh."
- echo "------ pre_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/pre_jupyter-jsc.sh
- echo "------ pre_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/pre_jupyter-jsc.sh
- fi
- echo "$(date) - Pre start done"
- }
-
-
- # Load modules
-
- load_modules () {
- echo "$(date) - Load modules ..."
- if [[ -f ${HOME}/.jupyter/start_jupyter-jsc.sh ]]; then
- send_spawn_update_warning 70 "Load customized modules ..." "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- echo "------ start_jupyter-jsc.sh file ------"
- cat ${HOME}/.jupyter/start_jupyter-jsc.sh
- echo "------ start_jupyter-jsc.sh file ------"
- source ${HOME}/.jupyter/start_jupyter-jsc.sh
- send_spawn_update_warning 80 "Load customized modules done" "You are using a customized modules script, defined in ${HOME}/.jupyter/start_jupyter-jsc.sh."
- else
- send_spawn_update 70 "Load default modules ..." "Loading Jupyter-bundle/20240520 in Stages/2024."
-
- module purge
- module use ${OTHERSTAGES}
- module load Stages/2024
- module load GCCcore/.12.3.0
- module load Jupyter-bundle/20240520
-
- # Kernels
- if [[ $JUPYTER_MODULE_BASH_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Bash/.0.9.3-4.2.1
- fi
- if [[ $JUPYTER_MODULE_CLING_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Cling/.20231018-4.2.1
- fi
- if [[ $JUPYTER_MODULE_JAVA_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Java/.1.3.0
- fi
- if [[ $JUPYTER_MODULE_JULIA_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Julia/.1.9.3
- fi
- if [[ $JUPYTER_MODULE_LFORTRAN_ENABLED -eq 1 ]]; then
- module load JupyterKernel-LFortran/.0.30.0
- fi
- if [[ $JUPYTER_MODULE_OCTAVE_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Octave/.8.4.0-4.2.1
- fi
- if [[ $JUPYTER_MODULE_R_ENABLED -eq 1 ]]; then
- module load JupyterKernel-R/.4.3.2-4.2.1
- fi
- if [[ $JUPYTER_MODULE_RUBY_ENABLED -eq 1 ]]; then
- module load JupyterKernel-Ruby/.3.2.2
- fi
- if [[ $JUPYTER_MODULE_PYDEEPLEARNING_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyDeepLearning/.2024.3-4.2.1
- fi
- if [[ $JUPYTER_MODULE_PYEARTHSYSTEM_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyEarthSystem/.2024.3-4.2.1
- fi
- if [[ $JUPYTER_MODULE_PYQUANTUM_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyQuantum/.2024.10-4.2.1
- fi
- if [[ $JUPYTER_MODULE_PYVISUALIZATION_ENABLED -eq 1 ]]; then
- module load JupyterKernel-PyVisualization/.2024.3-4.2.1
- fi
-
- # Proxies
- if [[ $JUPYTER_MODULE_XPRAHTML5_ENABLED -eq 1 ]]; then
- module load JupyterProxy-XpraHTML5/.0.4.0-4.2.1
- fi
- # Load MatLab Proxy module, if user is in matlab group
- IFS=' ' read -ra groups_array <<< "$(id -Gn)"
- for group in "${groups_array[@]}"; do
- if [[ "$group" == "matlab" ]]; then
- module load JupyterProxy-Matlab/.0.12.2-4.2.1
- break
- fi
- done
- # NEST does not support nodejs v 18 ( https://github.com/nest-desktop/nest-desktop/issues/509 )
- if [[ $JUPYTER_MODULE_NEST_ENABLED -eq 1 ]]; then
- if [[ $SYSTEMNAME != "hdfml" ]]; then
- module load JupyterProxy-NESTDesktop/.0.4.0-4.2.1
- fi
- fi
-
- if [[ $JUPYTER_MODULE_RSTUDIO_ENABLED -eq 1 ]]; then
- if [[ $SYSTEMNAME != "deep" ]]; then
- module load JupyterProxy-RStudio/.2023.09.1-4.2.1
- fi
- fi
- # Extensions
- if [[ $JUPYTER_MODULE_NBDEV_ENABLED -eq 1 ]]; then
- module load nbdev/2.3.25
- fi
- if [[ $JUPYTER_MODULE_VOILA_ENABLED -eq 1 ]]; then
- module load voila/0.5.7
- fi
- if [[ $JUPYTER_MODULE_NGLVIEW_ENABLED -eq 1 ]]; then
- module load nglview/.3.1.2
- fi
- if [[ $JUPYTER_MODULE_JUPYTERAI_ENABLED -eq 1 ]]; then
- module load jupyter-ai/.2.15.0
- fi
- if [[ $JUPYTER_MODULE_VARIABLEINSPECTOR_ENABLED -eq 1 ]]; then
- module load jupyterlab-variableinspector/.3.2.1
- fi
- if [[ $JUPYTER_MODULE_SPELLCHECKER_ENABLED -eq 1 ]]; then
- module load jupyterlab-spellchecker/.0.8.4
- fi
- if [[ $JUPYTER_MODULE_TRAMEMANAGER_ENABLED -eq 1 ]]; then
- if [[ $SYSTEMNAME != "deep" ]]; then
- module load jupyterlab-trame-manager/.0.6.1-4.2.1
- fi
- fi
- if [[ $JUPYTER_MODULE_SLURMWRAPPER_ENABLED -eq 1 ]]; then
- module load jupyter-slurm-provisioner/.0.6.0-4.2.1
- export SLURMEL_DOCUMENTATION_HREF="https://docs.jupyter.jsc.fz-juelich.de/github/FZJ-JSC/jupyter-jsc-notebooks/blob/documentation/05-News&Updates/Announcement-2022-12_Slurm_Wrapped_Kernels.ipynb"
- export JUPYTERJSC_CMD_ARGS="${JUPYTERJSC_CMD_ARGS} --ServerApp.kernel_manager_class=jupyter_slurm_provisioner.SlurmAsyncMappingKernelManager"
- export SLURM_PROVISIONER_NODE_SUFFIX="i"
- export SLURM_PROVISIONER_JHUB_METRICS="${JUPYTERHUB_API_URL}/slurmwrapper/${JUPYTERHUB_USER}/${JUPYTERHUB_SERVER_NAME}"
- test -d ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel && ! test -d ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel && mv ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel
- else
- test -d ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel && mkdir -p ${HOME}/.local/share/jupyter/kernels.deactivated && mv ${HOME}/.local/share/jupyter/kernels/slurm-provisioner-kernel ${HOME}/.local/share/jupyter/kernels.deactivated/slurm-provisioner-kernel
- fi
- lspci -k | grep -A 2 -i "NVIDIA" | grep "Kernel driver in use:" | grep "nvidia"
- if [ $? -eq 0 ]; then
- module load jupyterlab-nvdashboard/.0.10.0
- fi
- send_spawn_update 80 "Load default modules done" "Loaded Jupyter-bundle/20240300 in Stages/2024."
- fi
- echo "$(date) - Load modules done"
- }
-
- update_config () {
- sed -i -e "s|_port_|${PORT}|g" -e "s|_home_|${JUPYTERHUB_HOME:-${HOME}}|g" -e "s|_servername_|${JUPYTERHUB_SERVER_NAME}|g" ${DIR}/config.py
- send_spawn_update 85 "Add system specific configuration." "Use system specific jupyter_notebook_config.py files"
- if [[ -f ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py ]]; then
- echo "" >> ${DIR}/config.py
- cat ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py >> ${DIR}/config.py
- echo "$(date) - Added system specific config ${EBROOTJUPYTERLAB}/etc/jupyter/jupyter_notebook_config.py"
- fi
- if [[ -f ${EBROOTJUPYTERMINRESOURCEMINUSAGE}/etc/jupyter/jupyter_notebook_config.py ]]; then
- echo "" >> ${DIR}/config.py
- cat ${EBROOTJUPYTERMINRESOURCEMINUSAGE}/etc/jupyter/jupyter_notebook_config.py >> ${DIR}/config.py
- echo "$(date) - Added system specific config ${EBROOTJUPYTERMINRESOURCEMINUSAGE}/etc/jupyter/jupyter_notebook_config.py"
- fi
- for path in ${JUPYTER_EXTRA_LABEXTENSIONS_PATH//:/$'\n'}; do
- echo "c.LabServerApp.extra_labextensions_path.append('$path')" >> ${JUPYTER_LOG_DIR}/config.py
- done
- if command -v update_favorites_json &> /dev/null; then
- # update favorite-dirs with $HOME,$PROJECT,$SCRATCH,
- echo "$(date) - Update favorites"
- update_favorites_json
- fi
- }
-
- load_project_specific_kernel () {
- echo "$(date) - Activate hook for project specific kernels ..."
- if [[ -d ${PROJECT}/.local/share/jupyter ]] && [[ -r ${PROJECT}/.local/share/jupyter ]] && [[ -x ${PROJECT}/.local/share/jupyter ]]; then
- echo "$(date) - Add project kernel for $PROJECT"
- send_spawn_update_warning 90 "Activate project specific kernel." "Add ${PROJECT}/.local/share/jupyter to JUPYTER_PATH env variable. This may impede the start of JupyterLab."
- export JUPYTER_PATH=${JUPYTER_PATH}:${PROJECT}/.local/share/jupyter
- else
- echo "$(date) - Do not add project kernel for $PROJECT"
- # send_spawn_update_warning 90 "Could not activate project specific kernel." "Ensure that ${PROJECT}/.local/share/jupyter exists and is readable and accessible for ${USER}"
- fi
- echo "$(date) - Activate hook for project specific kernels done"
- }
-
- send_event () {
- BODY=${1//\'/}
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_EVENTS_URL}"
- eval " $CURL_CMD"
- }
-
- # show user a message in the UI
- send_spawn_update () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a darkorange message in the UI
- send_spawn_update_warning () {
- PROGRESS=$1
- SUMMARY=$2
- DETAILS=$3
- BODY="{\"progress\": ${PROGRESS}, \"failed\": false, \"html_message\": \"<details><summary><span style=\\\"color:darkorange;\\\">${SUMMARY}</span></summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Spawn update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # show user a fail message in the UI
- send_spawn_update_fail () {
- SUMMARY=$1
- DETAILS=$2
- BODY="{\"progress\": 100, \"failed\": true, \"html_message\": \"<details><summary>${SUMMARY}</summary>${DETAILS}</details>\"}"
- HTTPCODE=$(send_event "$BODY")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "$(date) - Could not send fail status update (${HTTPCODE} - ${PROGRESS}%: ${SUMMARY} - ${DETAILS}). Cancel start."
- exit 1
- else
- echo "$(date) - Send fail update (${PROGRESS}%) successful: ${HTTPCODE}"
- fi
- }
-
- # setup tunnel to the node JupyterLab is running at
- setup_tunnel () {
- echo "$(date) - Setup tunnel ..."
- BODY="{\"ssh_node\": \"${SSH_NODE}\", \"service\": \"${HOSTNAME_I}:${PORT}\"}"
- CURL_CMD="curl ${CURL_ARGS} ${CURL_HEADERS} -d '${BODY}' -X \"POST\" ${JUPYTERHUB_SETUPTUNNEL_URL}"
- HTTPCODE=$(eval " $CURL_CMD")
- if [[ ${HTTPCODE} -lt 200 || ${HTTPCODE} -gt 299 ]]; then
- echo "Could not setup tunnel. Cancel start."
- exit 1
- fi
- send_spawn_update 40 "Setup ssh port-forwarding." "Create ssh tunnel with system user ljupyter. JupyterHub will then be able to connect to JupyterLab at ${HOSTNAME_I}:${PORT}"
- echo "$(date) - Setup tunnel done"
- }
-
- start () {
- echo "$(date) - Start jupyterhub-singleuser ..."
- export JUPYTERHUB_HOME=${JUPYTERHUB_HOME:-${HOME}}
- cd ${JUPYTERHUB_HOME}
-
- if [[ -n $JUPYTERJSC_USER_CMD ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom command" "Use custom command \$JUPYTERJSC_USER_CMD . You will be redirected, when your JupyterLab is ready."
- timeout 30d ${JUPYTERJSC_USER_CMD} ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- elif [[ -n $JUPYTERJSC_USER_CMD_ARGS ]]; then
- send_spawn_update_warning 95 "Start JupyterLab with custom arguments" "Use custom arguments \$JUPYTERJSC_USER_CMD_ARGS . You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} ${JUPYTERJSC_USER_CMD_ARGS} &
- child=$!
- else
- send_spawn_update 95 "Start JupyterLab" "You will be redirected, when your JupyterLab is ready."
- timeout 30d jupyterhub-singleuser --config ${DIR}/config.py ${JUPYTERJSC_CMD_ARGS} &
- child=$!
- fi
- echo "$child" > ${PID_PATH}
- echo "$(date) - Start jupyterhub-singleuser done (PID: $child )"
-
- echo "$(date) - Start validate_token.sh ..."
- /bin/bash ${DIR}/validate_token.sh ${child} ${JUPYTERHUB_API_URL} &
- token_pid=$!
- echo "$token_pid" > ${VALIDATE_TOKEN_PID_PATH}
- echo "$(date) - Start validate_token.sh done (PID: $token_pid )"
-
- wait $child
- }
-
-
- requirements
- set_env
- grace_runtime_kill
- setup_tunnel
- check_quota
- pre_start
- load_modules
- update_config
- load_project_specific_kernel
- start
\ No newline at end of file
diff --git a/outposts/unicore/kustomize-maint/jobs-common.yaml b/outposts/unicore/kustomize-maint/jobs-common.yaml
deleted file mode 100644
index ccb9d2393e121b3ed20e7f4488a18a8b605f977c..0000000000000000000000000000000000000000
--- a/outposts/unicore/kustomize-maint/jobs-common.yaml
+++ /dev/null
@@ -1,105 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: unicore-jobs-common
- namespace: unicore
-data:
- input_.jupyter.token: "<JUPYTERHUB_API_TOKEN>"
- input_.oauth.scopes: "<JUPYTERHUB_OAUTH_SCOPES>"
- input_config.py: |
- c = get_config()
- c.ServerApp.ip = "0.0.0.0"
- c.ServerApp.port = int("_port_")
- c.ServerApp.root_dir = "/"
- c.ServerApp.default_url = "/lab/workspaces/_servername_/tree_home_"
- c.ContentsManager.allow_hidden = True
- c.ServerApp.terminado_settings = {"shell_command": ["/bin/bash"]}
- c.ServerApp.tornado_settings = {"websocket_max_message_size": 1024 * 1024 * 1024}
- c.ServerApp.max_buffer_size = 1024 * 1024 * 1024
- c.ServerApp.max_body_size = 1024 * 1024 * 1024
- c.ServerApp.quit_button = False
- input_stop.sh: |
- # We'll be waiting for JupyterLab to come up for max 300 seconds
- # and kill it once it's up
- END=$((SECONDS+300))
- DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
- PID_PATH=${DIR}/service.pid
- VALIDATE_TOKEN_PID_PATH=${DIR}/validate_token.pid
- echo "$(date) - stop.sh - Wait for ${PID_PATH}"
- while [[ ${SECONDS} -lt ${END} ]]; do
- if [[ -f ${PID_PATH} ]]; then
- child=$(cat ${PID_PATH})
- echo "$(date) - stop.sh - Found PID: ${child}"
- pkill --parent $child
- kill $child
- echo "$(date) - stop.sh - Wait 5 seconds for validate token pid"
- sleep 5
- if [[ -f ${VALIDATE_TOKEN_PID_PATH} ]]; then
- token_pid=$(cat ${VALIDATE_TOKEN_PID_PATH})
- echo "$(date) - stop.sh - Stop Validate Token Script ( $token_pid ) ..."
- kill $token_pid
- echo "$(date) - stop.sh - Stop Validate Token Script ( $token_pid ) ... done"
- else
- echo "$(date) - stop.sh - No validate token PID found."
- fi
- break
- fi
- echo "$(date) - stop.sh - No PID yet - sleep 10"
- sleep 10
- done
- echo "$(date) - stop.sh - done"
- input_validate_token.sh: |
- #!/bin/bash
- # If JupyterHub could not stop JupyterLab it will be running forever.
- # But the API_TOKEN is no longer valid, so we check if it's valid and stop
- # JupyterLab if that's not the case
- JLAB_PID=${1}
- JUPYTERHUB_API_URL=${2}
- # Get current directory
- DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
- TOKEN=$(cat ${DIR}/.jupyter.token)
- COUNT=0
- HOSTNAME_S=$(hostname -s)
- # If this JupyterLab is running on a LoginNode, we'll use the external api url. Otherwise we have to use a LoginNode as Proxy.
- if [[ ${HOSTNAME_S} == "<hostname_base>"* ]]; then
- export CURL_ARGS=""
- else
- if [[ ${JUPYTERHUB_API_URL} == "https://"* ]]; then
- export CURL_ARGS="--cacert ${DIR}/notebooks-ca_trust.crt"
- else
- export CURL_ARGS=""
- fi
- fi
- while true;
- do
- HTTPCODE=$(curl ${CURL_ARGS} --write-out '%{http_code}' --silent --output /dev/null -X "GET" -H "Authorization: token ${TOKEN}" ${JUPYTERHUB_API_URL}/user)
- echo "$(date) - Validate token - Result: ${HTTPCODE}"
- if [[ ${HTTPCODE} -eq 403 ]]; then
- COUNT=$((COUNT+1))
- echo "$(date) - 403 Counter: ${COUNT}"
- else
- COUNT=0
- fi
- if [[ ${COUNT} -ge 3 ]]; then
- echo "$(date) - Kill JupyterLab PID $JLAB_PID ( ${HTTPCODE} )"
- pkill --parent $JLAB_PID
- kill -9 $JLAB_PID
- exit
- fi
- sleep 300
- done
- job_description.json: |
- {
- "ApplicationName": "Bash shell",
- "Executable": "/bin/bash",
- "Arguments": [
- "start.sh"
- ],
- "umask": "022",
- "NotificationSettings": {
- "URL": "<JUPYTERHUB_UNICORE_NOTIFICATION_URL>",
- "bssStatus": ["PENDING", "CONFIGURING"]
- }
- }
-
-
diff --git a/outposts/unicore/kustomize-maint/kustomization.yaml b/outposts/unicore/kustomize-maint/kustomization.yaml
deleted file mode 100644
index 10f3242369a26ee0ccc7d39e5a62519797d1bc9f..0000000000000000000000000000000000000000
--- a/outposts/unicore/kustomize-maint/kustomization.yaml
+++ /dev/null
@@ -1,20 +0,0 @@
-resources:
- - gunicorn_config.yaml
- - jobs-2.6.yaml
- - jobs-3.2.yaml
- - jobs-3.3.yaml
- - jobs-3.4.yaml
- - jobs-3.6.yaml
- - jobs-3.6-juniq.yaml
- - jobs-4.2.yaml
- - jobs-4.2-juniq.yaml
- - jobs-common.yaml
-
-patches:
-- target:
- kind: Deployment
- labelSelector: app.kubernetes.io/instance=unicore
- patch: |-
- - op: replace
- path: /spec/replicas
- value: 0
\ No newline at end of file
diff --git a/outposts/unicore/values/base/config.yaml b/outposts/unicore/values/base/config.yaml
index 4185d35f6d179c33dfc8fd3199ac041d6fded228..01594813c392feb52f10cde58c97f9d22ed3f015 100644
--- a/outposts/unicore/values/base/config.yaml
+++ b/outposts/unicore/values/base/config.yaml
@@ -306,6 +306,9 @@ outpostConfig: |-
},
"portalgauss": {
"remote_port": "25492",
+ },
+ "ebrains": {
+ "remote_port": "25494",
}
},
"staging": {
@@ -332,6 +335,9 @@ outpostConfig: |-
},
"portalgauss": {
"remote_port": "25493",
+ },
+ "ebrains": {
+ "remote_port": "25495",
}
}
}