diff --git a/HPC_setup/mlt_modules_juwels.sh b/HPC_setup/mlt_modules_juwels.sh new file mode 100755 index 0000000000000000000000000000000000000000..d20b246d4f396363a23e68d64f89b6d3abaee8c4 --- /dev/null +++ b/HPC_setup/mlt_modules_juwels.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# __author__ = Felix Kleinert +# __date__ = '2020-04-06' + +# This script loads the required modules for mlt which are available on JUWELS. +# Note that some other packages have to be installed into a venv (see setup_venv.sh). + +module --force purge +module use $OTHERSTAGES + +ml Stages/Devel-2019a +ml GCCcore/.8.3.0 + +ml Jupyter/2019a-Python-3.6.8 +ml Python/3.6.8 +ml TensorFlow/1.13.1-GPU-Python-3.6.8 +ml Keras/2.2.4-GPU-Python-3.6.8 +ml SciPy-Stack/2019a-Python-3.6.8 +ml dask/1.1.5-Python-3.6.8 +ml GEOS/3.7.1-Python-3.6.8 +ml Graphviz/2.40.1 \ No newline at end of file diff --git a/HPC_setup/requirements_JUWELS_outcommented.txt b/HPC_setup/requirements_JUWELS_outcommented.txt new file mode 100644 index 0000000000000000000000000000000000000000..821dd802579b119c19f2bb2d8f097a4bac71adc0 --- /dev/null +++ b/HPC_setup/requirements_JUWELS_outcommented.txt @@ -0,0 +1,65 @@ +# absl-py==0.9.0 # in TensorFlow +# astor==0.8.1 # in TF +# atomicwrites==1.3.0 # in Python +# attrs==19.3.0 # in Python +Cartopy==0.17.0 +# certifi==2019.11.28 # in Python +# chardet==3.0.4 # in Python +# cloudpickle==1.3.0 # in Python +coverage==5.0.3 +# cycler==0.10.0 # in SciPy-Stack +# Cython==0.29.15 # in Python +# dask==2.11.0 # in dask +# fsspec==0.6.2 # in Jupyter +# gast==0.3.3 # in TF +# grpcio==1.27.2 # in TF +# h5py==2.10.0 # in Jupyter +# idna==2.8 # in Jupyter +importlib-metadata==1.5.0 +# Keras==2.2.4 # in Keras +# Keras-Applications==1.0.8 # in TF +# Keras-Preprocessing==1.1.0 # in TF +# kiwisolver==1.1.0 # in SciPy-Stack +# locket==0.2.0 # in Jupyter +# Markdown==3.2.1 # in TF +matplotlib==3.2.0 # in SciPy-Stack +# mock==4.0.1 # in Python +# more-itertools==8.2.0 # in Python +# numpy==1.18.1 # in SciPy-Stack +# packaging==20.3 # in Python +# pandas==1.0.1 # in SciPy-Stack +# partd==1.1.0 # in Jupyter +# patsy==0.5.1 # in Jupyter +# Pillow==7.0.0 # in Python +# pluggy==0.13.1 # in Python +# protobuf==3.11.3 # in Python +py==1.8.1 # ? +# pydot==1.4.1 # in Python +# pyparsing==2.4.6 # in Python +pyproj==2.5.0 # in basemap +pyshp==2.1.0 # in basemap +pytest==5.3.5 # in python (but we need higher version) +pytest-cov==2.8.1 +pytest-html==2.0.1 +pytest-lazy-fixture==0.6.3 +pytest-metadata==1.8.0 +pytest-sugar +# python-dateutil==2.8.1 # in Python +# pytz==2019.3 # in Python +# PyYAML==5.3 # in Python +# requests==2.23.0 # in Python +# scipy==1.4.1 # in SciPy-Stack +# seaborn==0.10.0 # in SciPy-Stack +--no-binary shapely Shapely==1.7.0 +# six==1.11.0 # in Python +statsmodels==0.11.1 # (in Jupyter, but not found) +# tensorboard==1.13.1 # in TF +# tensorflow-estimator==1.13.0 # in TF +# tensorflow-gpu==1.13.1 # TF +# termcolor==1.1.0 # in TF +# toolz==0.10.0 # in Python +# urllib3==1.25.8 # in Python +# wcwidth==0.1.8 # in Python +# Werkzeug==1.0.0 # in python +# xarray==0.15.0 # in SciPy-Stack +# zipp==3.1.0 # in Jupyter diff --git a/HPC_setup/setup_venv_juwels.sh b/HPC_setup/setup_venv_juwels.sh new file mode 100755 index 0000000000000000000000000000000000000000..98edd7540c141e2336c1f638a7e491b6f0714934 --- /dev/null +++ b/HPC_setup/setup_venv_juwels.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# __author__ = Felix Kleinert +# __date__ = '2020-04-06' + +# This script creates a virtual env which contains all modules which are not available via slrum/easybuild (see mlt_modules.sh) + +# enter setting dir if called externally +if [[ $1 != '' ]]; then + old=$PWD + cur=$1 +else + cur=$PWD +fi + +# load existing modules +source ${cur}mlt_modules_juwels.sh + +# create venv +python3 -m venv ${cur}/../venv_juwels + +source ${cur}/../venv_juwels/bin/activate + +# export path for side-packages +export PYTHONPATH=${cur}/../venv_juwels/lib/python3.6/site-packages:${PYTHONPATH} + +pip install -r ${cur}requirements_JUWELS_outcommented.txt +pip install --ignore-installed matplotlib==3.2.0 + +# Comment: Maybe we have to export PYTHONPATH a second time ater activating the venv (after job allocation) +# source venv/bin/activate +# alloc_develgpu +# source venv/bin/activate +# export PYTHONPATH=${PWD}/venv/lib/python3.6/site-packages:${PYTHONPATH} +# srun python run.py + +# create batch run scripts +# source create_runscripts_HPC.sh +