Merge branch '200-update-intellio3-ts-hpc-scripts' into 'develop_IntelliO3-ts'

Resolve "Update IntelliO3-ts HPC scripts"

See merge request toar/mlair!175

HPC_setup/create_runscripts_HPC.sh

@@ -112,20 +112,22 @@ timestamp=\`date +"%Y-%m-%d_%H%M-%S"\`
 
 export PYTHONPATH=\${PWD}/venv_${hpcsys}/lib/python3.6/site-packages:\${PYTHONPATH}
 
-srun python run.py --experiment_date=\$timestamp
+srun python run_HPC.py --experiment_date=\$timestamp
 EOT
 
 fi
 
-echo
-echo "You have to run the the following command on a login node to download data:"
-echo "          \`python run.py'"
-echo
+echo "###################################################################################"
+echo "#   You have to run the the following command on a login node to download data:   #"
+echo "#                              \`python run_HPC.py'                                #"
+echo "#                                                                                 #"
 
-echo "Please execute the following command to check if the setup went well:"
+echo "#   Please execute the following command to check if the setup went well:         #"
 if [[ ${hpcsys} = 'juwels' ]]; then
-  echo "          \`sbatch run_juwels_develgpus.bash'"
+  echo "#                   \`sbatch run_juwels_develgpus.bash'                            #"
 else
-  echo "          \`sbatch run_hdfml_batch.bash'"
+  echo "#                   \`sbatch run_hdfml_batch.bash'                                 #"
 fi
 
+echo "###################################################################################"
+

HPC_setup/mlt_modules_juwels.sh

@@ -8,7 +8,7 @@
 module --force purge
 module use $OTHERSTAGES
 
-ml Stages/Devel-2019a
+ml Stages/2019a
 ml GCCcore/.8.3.0
 
 ml Jupyter/2019a-Python-3.6.8

HPC_setup/requirements_HDFML_additionals.txt

+absl-py==0.9.0
+astor==0.8.1
+atomicwrites==1.3.0
+attrs==19.3.0
+certifi==2019.11.28
+chardet==3.0.4
+cloudpickle==1.3.0
 coverage==5.0.3
+cycler==0.10.0
+Cython==0.29.15
+dask==2.11.0
+fsspec==0.6.2
+gast==0.3.3
+grpcio==1.27.2
+h5py==2.10.0
+idna==2.8
 importlib-metadata==1.5.0
-matplotlib==3.2.0              # in SciPy-Stack
-netcdf4
-pandas==1.0.1                  # in SciPy-Stack
-patsy==0.5.1                   #
-py==1.8.1                      # ?
-pyproj==2.5.0                  # in basemap
-pyshp==2.1.0                   # in basemap
-pytest==5.3.5                  # in python (but we need higher version) 
+
+kiwisolver==1.1.0
+locket==0.2.0
+Markdown==3.2.1
+matplotlib==3.2.0
+mock==4.0.1
+more-itertools==8.2.0
+numpy==1.18.1
+packaging==20.3
+pandas==1.0.1
+partd==1.1.0
+patsy==0.5.1
+Pillow==7.0.0
+pluggy==0.13.1
+protobuf==3.11.3
+py==1.8.1
+pydot==1.4.1
+pyparsing==2.4.6
+pyproj==2.5.0
+pyshp==2.1.0
+pytest==5.3.5
 pytest-cov==2.8.1
 pytest-html==2.0.1
 pytest-lazy-fixture==0.6.3
 pytest-metadata==1.8.0
 pytest-sugar
-statsmodels==0.11.1              # (in jupyter, bit not oh hdfml)
-xarray==0.15.0                 # in SciPy-Stacki (v0.12.1, only)
-zipp==3.1.0                    
+python-dateutil==2.8.1
+pytz==2019.3
+PyYAML==5.3
+requests==2.23.0
+scipy==1.4.1
+seaborn==0.10.0
+--no-binary shapely Shapely==1.7.0
+six==1.11.0
+statsmodels==0.11.1
 tabulate
+toolz==0.10.0
+typing-extensions
+urllib3==1.25.8
+wcwidth==0.1.8
+Werkzeug==1.0.0
+xarray==0.15.0
+zipp==3.1.0

HPC_setup/requirements_JUWELS_additionals.txt

+absl-py==0.9.0
+astor==0.8.1
+atomicwrites==1.3.0
+attrs==19.3.0
+certifi==2019.11.28
+chardet==3.0.4
+cloudpickle==1.3.0
 coverage==5.0.3
+cycler==0.10.0
+Cython==0.29.15
+dask==2.11.0
+fsspec==0.6.2
+gast==0.3.3
+grpcio==1.27.2
+h5py==2.10.0
+idna==2.8
 importlib-metadata==1.5.0
-matplotlib==3.2.0              # in SciPy-Stack
-pandas==1.0.1                  # in SciPy-Stack / but older version
-py==1.8.1                      # ?
-pyproj==2.5.0                  # in basemap
-pyshp==2.1.0                   # in basemap
-pytest==5.3.5                  # in python (but we need higher version) 
+
+kiwisolver==1.1.0
+locket==0.2.0
+Markdown==3.2.1
+matplotlib==3.2.0
+mock==4.0.1
+more-itertools==8.2.0
+numpy==1.18.1
+packaging==20.3
+pandas==1.0.1
+partd==1.1.0
+patsy==0.5.1
+Pillow==7.0.0
+pluggy==0.13.1
+protobuf==3.11.3
+py==1.8.1
+pydot==1.4.1
+pyparsing==2.4.6
+pyproj==2.5.0
+pyshp==2.1.0
+pytest==5.3.5
 pytest-cov==2.8.1
 pytest-html==2.0.1
 pytest-lazy-fixture==0.6.3
 pytest-metadata==1.8.0
 pytest-sugar
-statsmodels==0.11.1              # (in Jupyter, but not found)
-xarray==0.15.0                 # in SciPy-Stack only 0.12.1 a
+python-dateutil==2.8.1
+pytz==2019.3
+PyYAML==5.3
+requests==2.23.0
+scipy==1.4.1
+seaborn==0.10.0
+--no-binary shapely Shapely==1.7.0
+six==1.11.0
+statsmodels==0.11.1
 tabulate
+toolz==0.10.0
+typing-extensions
+urllib3==1.25.8
+wcwidth==0.1.8
+Werkzeug==1.0.0
+xarray==0.15.0
+zipp==3.1.0

HPC_setup/setup_venv_hdfml.sh

@@ -29,12 +29,12 @@ pip install --ignore-installed matplotlib==3.2.0
 pip install --ignore-installed pandas==1.0.1
 pip install --ignore-installed statsmodels==0.11.1
 pip install --ignore-installed tabulate
-
+pip install -U typing_extensions
 # see wiki on hdfml for information oh h5py:
 # https://gitlab.version.fz-juelich.de/haf/Wiki/-/wikis/HDF-ML%20System
 
 export CC=mpicc
 export HDF5_MPI="ON"
 pip install --no-binary=h5py h5py
-
+pip install --ignore-installed netcdf4==1.5.4
 

HPC_setup/setup_venv_juwels.sh

@@ -24,18 +24,15 @@ source ${cur}/../venv_juwels/bin/activate
 # export path for side-packages 
 export PYTHONPATH=${cur}/../venv_juwels/lib/python3.6/site-packages:${PYTHONPATH}
 
+
+echo "##### START INSTALLING requirements_JUWELS_additionals.txt #####"
+pip install -r ${cur}/requirements_JUWELS_additionals.txt
+echo "##### FINISH INSTALLING requirements_JUWELS_additionals.txt #####"
+
 pip install -r ${cur}/requirements_JUWELS_additionals.txt
+pip install netcdf4
 pip install --ignore-installed matplotlib==3.2.0
 pip install --ignore-installed pandas==1.0.1
-
+pip install -U typing_extensions
 
 # Comment: Maybe we have to export PYTHONPATH a second time ater activating the venv (after job allocation)
-# source venv/bin/activate
-# alloc_develgpu
-# source venv/bin/activate
-# export PYTHONPATH=${PWD}/venv/lib/python3.6/site-packages:${PYTHONPATH}
-# srun python run.py
-
-# create batch run scripts
-# source create_runscripts_HPC.sh
-