diff --git a/HPC_setup/create_runscripts_HPC.sh b/HPC_setup/create_runscripts_HPC.sh
index bcbb5fb07800079736361450d7f0fed8684dc344..7bf08a34c6285e31895d817735f319ddde5bfb04 100755
--- a/HPC_setup/create_runscripts_HPC.sh
+++ b/HPC_setup/create_runscripts_HPC.sh
@@ -117,15 +117,17 @@ EOT
fi
-echo
-echo "You have to run the the following command on a login node to download data:"
-echo " \`python run.py'"
-echo
+echo "###################################################################################"
+echo "# You have to run the the following command on a login node to download data: #"
+echo "# \`python run_HPC.py' #"
+echo "# #"
-echo "Please execute the following command to check if the setup went well:"
+echo "# Please execute the following command to check if the setup went well: #"
if [[ ${hpcsys} = 'juwels' ]]; then
- echo " \`sbatch run_juwels_develgpus.bash'"
+ echo "# \`sbatch run_juwels_develgpus.bash' #"
else
- echo " \`sbatch run_hdfml_batch.bash'"
+ echo "# \`sbatch run_hdfml_batch.bash' #"
fi
+echo "###################################################################################"
+
diff --git a/HPC_setup/mlt_modules_juwels.sh b/HPC_setup/mlt_modules_juwels.sh
index d20b246d4f396363a23e68d64f89b6d3abaee8c4..01eecbab617f7b3042222e24e562901b302d401e 100755
--- a/HPC_setup/mlt_modules_juwels.sh
+++ b/HPC_setup/mlt_modules_juwels.sh
@@ -8,7 +8,7 @@
module --force purge
module use $OTHERSTAGES
-ml Stages/Devel-2019a
+ml Stages/2019a
ml GCCcore/.8.3.0
ml Jupyter/2019a-Python-3.6.8
@@ -18,4 +18,4 @@ ml Keras/2.2.4-GPU-Python-3.6.8
ml SciPy-Stack/2019a-Python-3.6.8
ml dask/1.1.5-Python-3.6.8
ml GEOS/3.7.1-Python-3.6.8
-ml Graphviz/2.40.1
\ No newline at end of file
+ml Graphviz/2.40.1
diff --git a/HPC_setup/requirements_JUWELS_additionals.txt b/HPC_setup/requirements_JUWELS_additionals.txt
index 0a2d6bc0f5cb4ce565b9eb69aad27cd1b3bbaef6..82d78d096ba56157cf046c2e9e2064c3b68e421c 100644
--- a/HPC_setup/requirements_JUWELS_additionals.txt
+++ b/HPC_setup/requirements_JUWELS_additionals.txt
@@ -1,16 +1,60 @@
+absl-py==0.9.0
+astor==0.8.1
+atomicwrites==1.3.0
+attrs==19.3.0
+certifi==2019.11.28
+chardet==3.0.4
+cloudpickle==1.3.0
coverage==5.0.3
+cycler==0.10.0
+Cython==0.29.15
+dask==2.11.0
+fsspec==0.6.2
+gast==0.3.3
+grpcio==1.27.2
+h5py==2.10.0
+idna==2.8
importlib-metadata==1.5.0
-matplotlib==3.2.0 # in SciPy-Stack
-pandas==1.0.1 # in SciPy-Stack / but older version
-py==1.8.1 # ?
-pyproj==2.5.0 # in basemap
-pyshp==2.1.0 # in basemap
-pytest==5.3.5 # in python (but we need higher version)
+
+kiwisolver==1.1.0
+locket==0.2.0
+Markdown==3.2.1
+matplotlib==3.2.0
+mock==4.0.1
+more-itertools==8.2.0
+numpy==1.18.1
+packaging==20.3
+pandas==1.0.1
+partd==1.1.0
+patsy==0.5.1
+Pillow==7.0.0
+pluggy==0.13.1
+protobuf==3.11.3
+py==1.8.1
+pydot==1.4.1
+pyparsing==2.4.6
+pyproj==2.5.0
+pyshp==2.1.0
+pytest==5.3.5
pytest-cov==2.8.1
pytest-html==2.0.1
pytest-lazy-fixture==0.6.3
pytest-metadata==1.8.0
pytest-sugar
-statsmodels==0.11.1 # (in Jupyter, but not found)
-xarray==0.15.0 # in SciPy-Stack only 0.12.1 a
+python-dateutil==2.8.1
+pytz==2019.3
+PyYAML==5.3
+requests==2.23.0
+scipy==1.4.1
+seaborn==0.10.0
+--no-binary shapely Shapely==1.7.0
+six==1.11.0
+statsmodels==0.11.1
tabulate
+toolz==0.10.0
+typing-extensions
+urllib3==1.25.8
+wcwidth==0.1.8
+Werkzeug==1.0.0
+xarray==0.15.0
+zipp==3.1.0
diff --git a/HPC_setup/setup_venv_juwels.sh b/HPC_setup/setup_venv_juwels.sh
index b543db1ee5ac4bea4f64467e360a084a2156c02a..7788c124fdbd997789811d32dccab8b04894b0ae 100755
--- a/HPC_setup/setup_venv_juwels.sh
+++ b/HPC_setup/setup_venv_juwels.sh
@@ -24,18 +24,15 @@ source ${cur}/../venv_juwels/bin/activate
# export path for side-packages
export PYTHONPATH=${cur}/../venv_juwels/lib/python3.6/site-packages:${PYTHONPATH}
+
+echo "##### START INSTALLING requirements_JUWELS_additionals.txt #####"
+pip install -r ${cur}/requirements_JUWELS_additionals.txt
+echo "##### FINISH INSTALLING requirements_JUWELS_additionals.txt #####"
+
pip install -r ${cur}/requirements_JUWELS_additionals.txt
+pip install netcdf4
pip install --ignore-installed matplotlib==3.2.0
pip install --ignore-installed pandas==1.0.1
-
+pip install -U typing_extensions
# Comment: Maybe we have to export PYTHONPATH a second time ater activating the venv (after job allocation)
-# source venv/bin/activate
-# alloc_develgpu
-# source venv/bin/activate
-# export PYTHONPATH=${PWD}/venv/lib/python3.6/site-packages:${PYTHONPATH}
-# srun python run.py
-
-# create batch run scripts
-# source create_runscripts_HPC.sh
-
diff --git a/create_runscripts_HPC.sh b/create_runscripts_HPC.sh
deleted file mode 100755
index af657fd11779f67861785c1573acd80235380b53..0000000000000000000000000000000000000000
--- a/create_runscripts_HPC.sh
+++ /dev/null
@@ -1,91 +0,0 @@
-#!/bin/csh -x
-
-echo "############################################################"
-echo "# #"
-echo "# user interaction required #"
-echo "# #"
-echo "############################################################"
-echo
-
-echo "This script creates the HPC batch scripts to run mlt on compute nodes (gpus and develgpus)."
-echo "You can modify the created run scripts afterwards if needed."
-
-while true; do
- read -p "Do you wish to create run scripts for JUWELS? [yes/no]" yn
- case $yn in
- [Yy]* ) juwels=True; break;;
- [Nn]* ) juwels=False;;
- * ) echo "Please answer yes or no.";;
- esac
-done
-
-while true; do
- read -p "Do you wish to create run script for HDFML? [yes/no]" yn
- case $yn in
- [Yy]* ) hdfml=True; break;;
- [Nn]* ) hdfml=False;;
- * ) echo "Please answer yes or no.";;
- esac
-done
-
-
-budget=''
-while [[ $budget == '' ]]
-do
- echo
- read -p "Enter project budget for --account flag: " budget
-done
-
-email=`jutil user show -o json | grep email | cut -f2 -d':' | cut -f1 -d',' | cut -f2 -d'"'`
-echo
-read -p "Enter e-mail address for --mail-user (default: ${email}): " new_email
-
-if [[ -z "$new_email" ]]; then
- new_email=$email
-fi
-
-# create HPC_logging dir
-hpclogging="../HPC_logging/"
-mkdir -p $hpclogging
-
-
-# ordering for looping:
-# "partition nGPUs timing"
-if [[ $juwels == True ]]; then
- for i in "develgpus 2 02:00:00" "gpus 4 08:00:00"; do
- set -- $i
-
-cat <<EOT > run_$1.bash
-#!/bin/bash -x
-#SBATCH --account=${budget}
-#SBATCH --nodes=1
-#SBATCH --output=${hpclogging}mlt-out.%j
-#SBATCH --error=${hpclogging}/mlt-err.%j
-#SBATCH --time=$3
-#SBATCH --partition=$1
-#SBATCH --gres=gpu:$2
-#SBATCH --mail-type=ALL
-#SBATCH --mail-user=${email}
-
-source mlt_modules_.sh
-source venv/bin/activate
-
-timestamp=\`date +"%Y-%m-%d_%H%M-%S"\`
-
-export PYTHONPATH=\${PWD}/venv/lib/python3.6/site-packages:\${PYTHONPATH}
-
-srun python run.py --experiment_date=\$timestamp
-EOT
-
- echo "Created runscript: run_$1.bash"
-
- done
-fi
-
-echo
-echo "You have to run the the following command on a login node to download data:"
-echo " \`python run.py'"
-echo
-echo "Please execute the following command to check if the setup went well:"
-echo " \`sbatch run_develgpus.bash'"
-
diff --git a/mlair/configuration/defaults.py b/mlair/configuration/defaults.py
index b048712f8abf00bc3bab7ab7bddd3f42492ee9cf..d191af2edd8a6fe2c1093b3f1c3f5d419cc42b76 100644
--- a/mlair/configuration/defaults.py
+++ b/mlair/configuration/defaults.py
@@ -46,15 +46,19 @@ DEFAULT_USE_ALL_STATIONS_ON_ALL_DATA_SETS = True
DEFAULT_EVALUATE_BOOTSTRAPS = True
DEFAULT_CREATE_NEW_BOOTSTRAPS = False
DEFAULT_NUMBER_OF_BOOTSTRAPS = 20
-DEFAULT_PLOT_LIST = ["PlotMonthlySummary", "PlotStationMap", "PlotClimatologicalSkillScore", "PlotTimeSeries",
+#DEFAULT_PLOT_LIST = ["PlotMonthlySummary", "PlotStationMap", "PlotClimatologicalSkillScore", "PlotTimeSeries",
+# "PlotCompetitiveSkillScore", "PlotBootstrapSkillScore", "PlotConditionalQuantiles",
+# "PlotAvailability"]
+DEFAULT_PLOT_LIST = ["PlotMonthlySummary", "PlotStationMap", "PlotClimatologicalSkillScore",
"PlotCompetitiveSkillScore", "PlotBootstrapSkillScore", "PlotConditionalQuantiles",
"PlotAvailability"]
+
def get_defaults():
"""Return all default parameters set in defaults.py"""
return {key: value for key, value in globals().items() if key.startswith('DEFAULT')}
if __name__ == "__main__":
- print(get_defaults())
\ No newline at end of file
+ print(get_defaults())
diff --git a/requirements_gpu.txt b/requirements_gpu.txt
index 5ddb56acc71e0a51abb99b9447f871ddcb715a5d..38598ff6b902da6d96eb870124f60c72737000c2 100644
--- a/requirements_gpu.txt
+++ b/requirements_gpu.txt
@@ -2,7 +2,7 @@ absl-py==0.9.0
astor==0.8.1
atomicwrites==1.3.0
attrs==19.3.0
-Cartopy==0.17.0
+#Cartopy==0.17.0
certifi==2019.11.28
chardet==3.0.4
cloudpickle==1.3.0
@@ -16,9 +16,9 @@ grpcio==1.27.2
h5py==2.10.0
idna==2.8
importlib-metadata==1.5.0
-Keras==2.2.4
-Keras-Applications==1.0.8
-Keras-Preprocessing==1.1.0
+#Keras==2.2.4
+#Keras-Applications==1.0.8
+#Keras-Preprocessing==1.1.0
kiwisolver==1.1.0
locket==0.2.0
Markdown==3.2.1
@@ -54,10 +54,10 @@ seaborn==0.10.0
six==1.11.0
statsmodels==0.11.1
tabulate
-tensorboard==1.13.1
-tensorflow-estimator==1.13.0
-tensorflow-gpu==1.13.1
-termcolor==1.1.0
+#tensorboard==1.13.1
+#tensorflow-estimator==1.13.0
+#tensorflow-gpu==1.13.1
+#termcolor==1.1.0
toolz==0.10.0
typing-extensions
urllib3==1.25.8
diff --git a/setupHPC.sh b/setupHPC.sh
index 0248fdc09e658bac1ba6f9742426ce41996e1ade..a50f2ddea62cc298eaf6ed9a1b443d0f9769c5b6 100644
--- a/setupHPC.sh
+++ b/setupHPC.sh
@@ -1,3 +1,7 @@
+# This is the main installation script for the HPC systems JUWELS and HDFML operated by JSC.
+# It loads all preinstalled HPC-modules, installs additional packages via pip3, and finally creates the sbatch-runscripts.
+# If you use other HPC-systems you can use this script and related scripts in HPC_setup/ as blue print.
+# __author__ = Felix Kleinert
basepath=${PWD}/
settingpath=HPC_setup/