From 0bd496323ddd4a0e3d83a1c40f7c85c7093498d4 Mon Sep 17 00:00:00 2001
From: Felix Kleinert <f.kleinert@fz-juelich.de>
Date: Thu, 29 Oct 2020 08:02:20 +0100
Subject: [PATCH] update HPC scripts for IntelliO3 #200
---
HPC_setup/create_runscripts_HPC.sh | 18 +++---
HPC_setup/mlt_modules_juwels.sh | 4 +-
HPC_setup/requirements_HDFML_additionals.txt | 63 +++++++++++++++----
HPC_setup/requirements_JUWELS_additionals.txt | 60 +++++++++++++++---
HPC_setup/setup_venv_hdfml.sh | 4 +-
HPC_setup/setup_venv_juwels.sh | 17 +++--
6 files changed, 125 insertions(+), 41 deletions(-)
diff --git a/HPC_setup/create_runscripts_HPC.sh b/HPC_setup/create_runscripts_HPC.sh
index 1322f47d..7bf08a34 100755
--- a/HPC_setup/create_runscripts_HPC.sh
+++ b/HPC_setup/create_runscripts_HPC.sh
@@ -112,20 +112,22 @@ timestamp=\`date +"%Y-%m-%d_%H%M-%S"\`
export PYTHONPATH=\${PWD}/venv_${hpcsys}/lib/python3.6/site-packages:\${PYTHONPATH}
-srun python run.py --experiment_date=\$timestamp
+srun python run_HPC.py --experiment_date=\$timestamp
EOT
fi
-echo
-echo "You have to run the the following command on a login node to download data:"
-echo " \`python run.py'"
-echo
+echo "###################################################################################"
+echo "# You have to run the the following command on a login node to download data: #"
+echo "# \`python run_HPC.py' #"
+echo "# #"
-echo "Please execute the following command to check if the setup went well:"
+echo "# Please execute the following command to check if the setup went well: #"
if [[ ${hpcsys} = 'juwels' ]]; then
- echo " \`sbatch run_juwels_develgpus.bash'"
+ echo "# \`sbatch run_juwels_develgpus.bash' #"
else
- echo " \`sbatch run_hdfml_batch.bash'"
+ echo "# \`sbatch run_hdfml_batch.bash' #"
fi
+echo "###################################################################################"
+
diff --git a/HPC_setup/mlt_modules_juwels.sh b/HPC_setup/mlt_modules_juwels.sh
index d20b246d..01eecbab 100755
--- a/HPC_setup/mlt_modules_juwels.sh
+++ b/HPC_setup/mlt_modules_juwels.sh
@@ -8,7 +8,7 @@
module --force purge
module use $OTHERSTAGES
-ml Stages/Devel-2019a
+ml Stages/2019a
ml GCCcore/.8.3.0
ml Jupyter/2019a-Python-3.6.8
@@ -18,4 +18,4 @@ ml Keras/2.2.4-GPU-Python-3.6.8
ml SciPy-Stack/2019a-Python-3.6.8
ml dask/1.1.5-Python-3.6.8
ml GEOS/3.7.1-Python-3.6.8
-ml Graphviz/2.40.1
\ No newline at end of file
+ml Graphviz/2.40.1
diff --git a/HPC_setup/requirements_HDFML_additionals.txt b/HPC_setup/requirements_HDFML_additionals.txt
index 5065149a..82d78d09 100644
--- a/HPC_setup/requirements_HDFML_additionals.txt
+++ b/HPC_setup/requirements_HDFML_additionals.txt
@@ -1,19 +1,60 @@
+absl-py==0.9.0
+astor==0.8.1
+atomicwrites==1.3.0
+attrs==19.3.0
+certifi==2019.11.28
+chardet==3.0.4
+cloudpickle==1.3.0
coverage==5.0.3
+cycler==0.10.0
+Cython==0.29.15
+dask==2.11.0
+fsspec==0.6.2
+gast==0.3.3
+grpcio==1.27.2
+h5py==2.10.0
+idna==2.8
importlib-metadata==1.5.0
-matplotlib==3.2.0 # in SciPy-Stack
-netcdf4
-pandas==1.0.1 # in SciPy-Stack
-patsy==0.5.1 #
-py==1.8.1 # ?
-pyproj==2.5.0 # in basemap
-pyshp==2.1.0 # in basemap
-pytest==5.3.5 # in python (but we need higher version)
+
+kiwisolver==1.1.0
+locket==0.2.0
+Markdown==3.2.1
+matplotlib==3.2.0
+mock==4.0.1
+more-itertools==8.2.0
+numpy==1.18.1
+packaging==20.3
+pandas==1.0.1
+partd==1.1.0
+patsy==0.5.1
+Pillow==7.0.0
+pluggy==0.13.1
+protobuf==3.11.3
+py==1.8.1
+pydot==1.4.1
+pyparsing==2.4.6
+pyproj==2.5.0
+pyshp==2.1.0
+pytest==5.3.5
pytest-cov==2.8.1
pytest-html==2.0.1
pytest-lazy-fixture==0.6.3
pytest-metadata==1.8.0
pytest-sugar
-statsmodels==0.11.1 # (in jupyter, bit not oh hdfml)
-xarray==0.15.0 # in SciPy-Stacki (v0.12.1, only)
-zipp==3.1.0
+python-dateutil==2.8.1
+pytz==2019.3
+PyYAML==5.3
+requests==2.23.0
+scipy==1.4.1
+seaborn==0.10.0
+--no-binary shapely Shapely==1.7.0
+six==1.11.0
+statsmodels==0.11.1
tabulate
+toolz==0.10.0
+typing-extensions
+urllib3==1.25.8
+wcwidth==0.1.8
+Werkzeug==1.0.0
+xarray==0.15.0
+zipp==3.1.0
diff --git a/HPC_setup/requirements_JUWELS_additionals.txt b/HPC_setup/requirements_JUWELS_additionals.txt
index 0a2d6bc0..82d78d09 100644
--- a/HPC_setup/requirements_JUWELS_additionals.txt
+++ b/HPC_setup/requirements_JUWELS_additionals.txt
@@ -1,16 +1,60 @@
+absl-py==0.9.0
+astor==0.8.1
+atomicwrites==1.3.0
+attrs==19.3.0
+certifi==2019.11.28
+chardet==3.0.4
+cloudpickle==1.3.0
coverage==5.0.3
+cycler==0.10.0
+Cython==0.29.15
+dask==2.11.0
+fsspec==0.6.2
+gast==0.3.3
+grpcio==1.27.2
+h5py==2.10.0
+idna==2.8
importlib-metadata==1.5.0
-matplotlib==3.2.0 # in SciPy-Stack
-pandas==1.0.1 # in SciPy-Stack / but older version
-py==1.8.1 # ?
-pyproj==2.5.0 # in basemap
-pyshp==2.1.0 # in basemap
-pytest==5.3.5 # in python (but we need higher version)
+
+kiwisolver==1.1.0
+locket==0.2.0
+Markdown==3.2.1
+matplotlib==3.2.0
+mock==4.0.1
+more-itertools==8.2.0
+numpy==1.18.1
+packaging==20.3
+pandas==1.0.1
+partd==1.1.0
+patsy==0.5.1
+Pillow==7.0.0
+pluggy==0.13.1
+protobuf==3.11.3
+py==1.8.1
+pydot==1.4.1
+pyparsing==2.4.6
+pyproj==2.5.0
+pyshp==2.1.0
+pytest==5.3.5
pytest-cov==2.8.1
pytest-html==2.0.1
pytest-lazy-fixture==0.6.3
pytest-metadata==1.8.0
pytest-sugar
-statsmodels==0.11.1 # (in Jupyter, but not found)
-xarray==0.15.0 # in SciPy-Stack only 0.12.1 a
+python-dateutil==2.8.1
+pytz==2019.3
+PyYAML==5.3
+requests==2.23.0
+scipy==1.4.1
+seaborn==0.10.0
+--no-binary shapely Shapely==1.7.0
+six==1.11.0
+statsmodels==0.11.1
tabulate
+toolz==0.10.0
+typing-extensions
+urllib3==1.25.8
+wcwidth==0.1.8
+Werkzeug==1.0.0
+xarray==0.15.0
+zipp==3.1.0
diff --git a/HPC_setup/setup_venv_hdfml.sh b/HPC_setup/setup_venv_hdfml.sh
index 585e43b1..ad5b1276 100644
--- a/HPC_setup/setup_venv_hdfml.sh
+++ b/HPC_setup/setup_venv_hdfml.sh
@@ -29,12 +29,12 @@ pip install --ignore-installed matplotlib==3.2.0
pip install --ignore-installed pandas==1.0.1
pip install --ignore-installed statsmodels==0.11.1
pip install --ignore-installed tabulate
-
+pip install -U typing_extensions
# see wiki on hdfml for information oh h5py:
# https://gitlab.version.fz-juelich.de/haf/Wiki/-/wikis/HDF-ML%20System
export CC=mpicc
export HDF5_MPI="ON"
pip install --no-binary=h5py h5py
-
+pip install --ignore-installed netcdf4==1.5.4
diff --git a/HPC_setup/setup_venv_juwels.sh b/HPC_setup/setup_venv_juwels.sh
index b543db1e..7788c124 100755
--- a/HPC_setup/setup_venv_juwels.sh
+++ b/HPC_setup/setup_venv_juwels.sh
@@ -24,18 +24,15 @@ source ${cur}/../venv_juwels/bin/activate
# export path for side-packages
export PYTHONPATH=${cur}/../venv_juwels/lib/python3.6/site-packages:${PYTHONPATH}
+
+echo "##### START INSTALLING requirements_JUWELS_additionals.txt #####"
+pip install -r ${cur}/requirements_JUWELS_additionals.txt
+echo "##### FINISH INSTALLING requirements_JUWELS_additionals.txt #####"
+
pip install -r ${cur}/requirements_JUWELS_additionals.txt
+pip install netcdf4
pip install --ignore-installed matplotlib==3.2.0
pip install --ignore-installed pandas==1.0.1
-
+pip install -U typing_extensions
# Comment: Maybe we have to export PYTHONPATH a second time ater activating the venv (after job allocation)
-# source venv/bin/activate
-# alloc_develgpu
-# source venv/bin/activate
-# export PYTHONPATH=${PWD}/venv/lib/python3.6/site-packages:${PYTHONPATH}
-# srun python run.py
-
-# create batch run scripts
-# source create_runscripts_HPC.sh
-
--
GitLab