update README and clean up of req_JW.txt

README.md

@@ -14,17 +14,19 @@ and [Network In Network (Lin et al., 2014)](https://arxiv.org/abs/1312.4400).
 * Install __proj__ on your machine using the console. E.g. for opensuse / leap `zypper install proj`
 * c++ compiler required for cartopy installation
 
-## HPC - JUWELS
+## HPC - JUWELS and HDFML
+The following instruction guide you throug the installation on JUWELS and HDFML. 
 * Clone the repo to HPC system (we recommend to place it in `/p/projects/<project name>`.
-* Setup venv by executing `source setup_venv.sh`. This script loads all pre-installed modules and creates a venv for 
-all other packages. Furthermore, it creates two slurm/batch scripts to execute code on compute nodes. <br> 
+* Setup venv by executing `source setupHPC.sh`. This script loads all pre-installed modules and creates a venv for 
+all other packages. Furthermore, it creates slurm/batch scripts to execute code on compute nodes. <br> 
 You have to enter the HPC project's budget name (--account flag).
-* The default external data path on JUWELS is set to `/p/project/deepacf/intelliaq/<user>/DATA/toar_<sampling>`. 
+* The default external data path on JUWELS and HDFML is set to `/p/project/deepacf/intelliaq/<user>/DATA/toar_<sampling>`. 
 <br>To choose a different location open `run.py` and add the following keyword argument to `ExperimentSetup`: 
 `data_path=<your>/<custom>/<path>`. 
 * Execute `python run.py` on a login node to download example data. The program will throw an OSerror after downloading.
-* Execute `sbatch run_develgpus.bash` to verify that the setup went well.
-* Currently cartopy is not working, therefore PlotStations does not create any output
+* Execute either `sbatch run_juwels_develgpus.bash` or `sbatch run_hdfml_batch.bash` to verify that the setup went well.
+* Currently cartopy is not working on our HPC system, therefore PlotStations does not create any output.
+
 # Security
 
 * To use hourly data from ToarDB via JOIN interface, a private token is required. Request your personal access token and

requirements_JUWELS.txt

-absl-py==0.9.0                 # in TensorFlow
-astor==0.8.1                   # in TF
-atomicwrites==1.3.0            # in Python
-attrs==19.3.0                  # in Python
-Cartopy==0.17.0
-certifi==2019.11.28            # in Python
-chardet==3.0.4                 # in Python
-cloudpickle==1.3.0             # in Python 
-coverage==5.0.3
-cycler==0.10.0                 # in SciPy-Stack
-Cython==0.29.15                # in Python
-dask==2.11.0                   # in dask
-fsspec==0.6.2                  # in Jupyter
-gast==0.3.3                    # in TF
-grpcio==1.27.2                 # in TF
-h5py==2.10.0                   # in Jupyter
-idna==2.8                      # in Jupyter
-importlib-metadata==1.5.0
-Keras==2.2.4                   # in Keras
-Keras-Applications==1.0.8      # in TF
-Keras-Preprocessing==1.1.0     # in TF 
-kiwisolver==1.1.0              # in SciPy-Stack
-locket==0.2.0                  # in Jupyter
-Markdown==3.2.1                # in TF 
-matplotlib==3.2.0              # in SciPy-Stack 
-mock==4.0.1                    # in Python 
-more-itertools==8.2.0          # in Python 
-numpy==1.18.1                  # in SciPy-Stack
-packaging==20.3                # in Python
-pandas==1.0.1                  # in SciPy-Stack
-partd==1.1.0                   # in Jupyter
-patsy==0.5.1                   # in Jupyter 
-Pillow==7.0.0                  # in Python 
-pluggy==0.13.1                 # in Python
-protobuf==3.11.3               # in Python 
-py==1.8.1                      # ?
-pydot==1.4.1                   # in Python
-pyparsing==2.4.6               # in Python
-pyproj==2.5.0                  # in basemap
-pyshp==2.1.0                   # in basemap
-pytest==5.3.5                  # in python 
-pytest-cov==2.8.1
-pytest-html==2.0.1
-pytest-lazy-fixture==0.6.3
-pytest-metadata==1.8.0
-pytest-sugar
-python-dateutil==2.8.1         # in Python 
-pytz==2019.3                   # in Python 
-PyYAML==5.3                    # in Python 
-requests==2.23.0               # in Python 
-scipy==1.4.1                   # in SciPy-Stack
-seaborn==0.10.0                # in SciPy-Stack
---no-binary shapely Shapely==1.7.0
-six==1.11.0                    # in Python
-statsmodels==0.11.1            # in Jupyter
-tensorboard==1.13.1            # in TF
-tensorflow-estimator==1.13.0   # in TF
-tensorflow-gpu==1.13.1         # TF
-termcolor==1.1.0               # in TF 
-toolz==0.10.0                  # in Python
-urllib3==1.25.8                # in Python
-wcwidth==0.1.8                 # in Python 
-Werkzeug==1.0.0                # in python
-xarray==0.15.0                 # in SciPy-Stack
-zipp==3.1.0                    # in Jupyter

requirements_JUWELS_outcommented.txt

-# absl-py==0.9.0                 # in TensorFlow
-# astor==0.8.1                   # in TF
-# atomicwrites==1.3.0            # in Python
-# attrs==19.3.0                  # in Python
-Cartopy==0.17.0
-# certifi==2019.11.28            # in Python
-# chardet==3.0.4                 # in Python
-# cloudpickle==1.3.0             # in Python 
-coverage==5.0.3
-# cycler==0.10.0                 # in SciPy-Stack
-# Cython==0.29.15                # in Python
-# dask==2.11.0                   # in dask
-# fsspec==0.6.2                  # in Jupyter
-# gast==0.3.3                    # in TF
-# grpcio==1.27.2                 # in TF
-# h5py==2.10.0                   # in Jupyter
-# idna==2.8                      # in Jupyter
-importlib-metadata==1.5.0
-# Keras==2.2.4                   # in Keras
-# Keras-Applications==1.0.8      # in TF
-# Keras-Preprocessing==1.1.0     # in TF 
-# kiwisolver==1.1.0              # in SciPy-Stack
-# locket==0.2.0                  # in Jupyter
-# Markdown==3.2.1                # in TF 
-matplotlib==3.2.0              # in SciPy-Stack 
-# mock==4.0.1                    # in Python 
-# more-itertools==8.2.0          # in Python 
-# numpy==1.18.1                  # in SciPy-Stack
-# packaging==20.3                # in Python
-pandas==1.0.1                  # in SciPy-Stack / but older version
-# partd==1.1.0                   # in Jupyter
-# patsy==0.5.1                   # in Jupyter 
-# Pillow==7.0.0                  # in Python 
-# pluggy==0.13.1                 # in Python
-# protobuf==3.11.3               # in Python 
-py==1.8.1                      # ?
-# pydot==1.4.1                   # in Python
-# pyparsing==2.4.6               # in Python
-pyproj==2.5.0                  # in basemap
-pyshp==2.1.0                   # in basemap
-pytest==5.3.5                  # in python (but we need higher version) 
-pytest-cov==2.8.1
-pytest-html==2.0.1
-pytest-lazy-fixture==0.6.3
-pytest-metadata==1.8.0
-pytest-sugar
-# python-dateutil==2.8.1         # in Python 
-# pytz==2019.3                   # in Python 
-# PyYAML==5.3                    # in Python 
-# requests==2.23.0               # in Python 
-# scipy==1.4.1                   # in SciPy-Stack
-# seaborn==0.10.0                # in SciPy-Stack
---no-binary shapely Shapely==1.7.0
-# six==1.11.0                    # in Python
-statsmodels==0.11.1              # (in Jupyter, but not found)
-# tensorboard==1.13.1            # in TF
-# tensorflow-estimator==1.13.0   # in TF
-# tensorflow-gpu==1.13.1         # TF
-# termcolor==1.1.0               # in TF 
-# toolz==0.10.0                  # in Python
-# urllib3==1.25.8                # in Python
-# wcwidth==0.1.8                 # in Python 
-# Werkzeug==1.0.0                # in python
-# xarray==0.15.0                 # in SciPy-Stack
-# zipp==3.1.0                    # in Jupyter

setupHPC_test.sh → setupHPC.sh

@@ -14,8 +14,8 @@ else
   exit
 fi
 
-echo "execute: HPC_setup/setup_venv_${hpcsys}.sh $basepath/$settingpath"
-source HPC_setup/setup_venv_${hpcsys}.sh $basepath/$settingpath
+echo "execute: HPC_setup/setup_venv_${hpcsys}.sh $basepath$settingpath"
+source HPC_setup/setup_venv_${hpcsys}.sh $basepath$settingpath
 
 echo "execute: HPC_setup/create_runscripts_HPC.sh $hpcsys $basepath"
 source HPC_setup/create_runscripts_HPC.sh $hpcsys $basepath