diff --git a/HPC_setup/create_runscripts_HPC.sh b/HPC_setup/create_runscripts_HPC.sh new file mode 100755 index 0000000000000000000000000000000000000000..829379a12452614fa9a6b181a0adf15f3bc5e174 --- /dev/null +++ b/HPC_setup/create_runscripts_HPC.sh @@ -0,0 +1,131 @@ +#!/bin/csh -x + +# __author__ = Felix Kleinert +# __date__ = '2020-04-30' +# This script creates run scripts for JUWELS or HDFML + +# When you call this script directly you can use +# $1 which has to be `juwels' or `hdfml'. +# $2 which is the path where the run scripts should be stored + +if [[ $1 != '' ]]; then + hpcsys=$1 +else + if [[ $HOSTNAME == *"juwels"* ]]; then + hpcsys="juwels" + elif [[ $HOSTNAME == *"hdfml"* ]]; then + hpcsys="hdfml" + else + echo "Unknown hpc host \`$HOSTNAME\`. Pass 'juwels' or 'hdfml' as first argument." + exit + fi +fi + +if [[ $2 != '' ]]; then + cur=$2 +else + cur=$PWD +fi + +echo "############################################################" +echo "# #" +echo "# user interaction required #" +echo "# #" +echo "############################################################" +echo + +echo "This script creates the HPC batch scripts to run mlt on compute nodes on JUWELS or hdfml." +echo "You can modify the created run scripts afterwards if needed." + +echo +echo +echo "Creating run script for $hpcsys:" +echo + +budget='' +while [[ $budget == '' ]] +do + echo + read -p "Enter project budget for --account flag: " budget +done + +email=`jutil user show -o json | grep email | cut -f2 -d':' | cut -f1 -d',' | cut -f2 -d'"'` +echo +read -p "Enter e-mail address for --mail-user (default: ${email}): " new_email + +if [[ -z "$new_email" ]]; then + new_email=$email +fi + +# create HPC_logging dir +hpclogging="/HPC_logging/" +mkdir -p ${cur}${}hpclogging} + + +# ordering for looping: +# "partition nGPUs timing" +if [[ $hpcsys = "juwels" ]]; then + for i in "develgpus 2 02:00:00" "gpus 4 08:00:00"; do + set -- $i + +cat <<EOT > ${hpcsys}/run_${hpcsys}_$1.bash +#!/bin/bash -x +#SBATCH --account=${budget} +#SBATCH --nodes=1 +#SBATCH --output=${hpclogging}mlt-out.%j +#SBATCH --error=${hpclogging}/mlt-err.%j +#SBATCH --time=$3 +#SBATCH --partition=$1 +#SBATCH --gres=gpu:$2 +#SBATCH --mail-type=ALL +#SBATCH --mail-user=${email} + +source HPC_setup/mlt_modules_${hpcsys}.sh +source venv_${hpcsys}/bin/activate + +timestamp=\`date +"%Y-%m-%d_%H%M-%S"\` + +export PYTHONPATH=\${PWD}/venv_${hpcsys}/lib/python3.6/site-packages:\${PYTHONPATH} + +srun python run.py --experiment_date=\$timestamp +EOT + + echo "Created runscript: run_${hpcsys}_$1.bash" + + done + +elif [[ $hpcsys = "hdfml" ]]; then +cat <<EOT > ${hpcsys}/run_${hpcsys}_batch.bash +#!/bin/bash -x +#SBATCH --account=${budget} +#SBATCH --nodes=1 +#SBATCH --output=${hpclogging}mlt-out.%j +#SBATCH --error=${hpclogging}/mlt-err.%j +#SBATCH --time=08:00:00 +#SBATCH --mail-type=ALL +#SBATCH --mail-user=${email} + +source HPC_setup/mlt_modules_${hpcsys}.sh +source venv_${hpcsys}/bin/activate + +timestamp=\`date +"%Y-%m-%d_%H%M-%S"\` + +export PYTHONPATH=\${PWD}/venv_${hpcsys}/lib/python3.6/site-packages:\${PYTHONPATH} + +srun python run.py --experiment_date=\$timestamp +EOT + +fi + +echo +echo "You have to run the the following command on a login node to download data:" +echo " \`python run.py'" +echo + +echo "Please execute the following command to check if the setup went well:" +if [[ ${hpcsys} = 'juwels' ]]; then + echo " \`sbatch run_${hpcsys}_develgpus.bash'" +else + echo " \`sbatch run_${hpcsys}_batch.bash'" +fi + diff --git a/HPC_setup/requirements_JUWELS_outcommented.txt b/HPC_setup/requirements_JUWELS_outcommented.txt index 821dd802579b119c19f2bb2d8f097a4bac71adc0..f2a6b7df9097df6319e16f1986b3d0ff43ac1095 100644 --- a/HPC_setup/requirements_JUWELS_outcommented.txt +++ b/HPC_setup/requirements_JUWELS_outcommented.txt @@ -27,7 +27,7 @@ matplotlib==3.2.0 # in SciPy-Stack # more-itertools==8.2.0 # in Python # numpy==1.18.1 # in SciPy-Stack # packaging==20.3 # in Python -# pandas==1.0.1 # in SciPy-Stack +pandas==1.0.1 # in SciPy-Stack / but older version # partd==1.1.0 # in Jupyter # patsy==0.5.1 # in Jupyter # Pillow==7.0.0 # in Python diff --git a/HPC_setup/setup_venv_hdfml.sh b/HPC_setup/setup_venv_hdfml.sh index f6b0d5d5c9ccc80450088186fa2f6b182b26e084..d18e41d580b5b81a0ff8d44e39f828e713418ffb 100644 --- a/HPC_setup/setup_venv_hdfml.sh +++ b/HPC_setup/setup_venv_hdfml.sh @@ -3,19 +3,28 @@ # __date__ = '2020-04-06' # This script creates a virtual env which contains all modules which are not available via slrum/easybuild (see mlt_modules_hdfml.sh) +# enter setting dir if called externally + +# $1 has to be an abs path to HPC_setup. If not provided, $PWD is used + +if [[ $1 != '' ]]; then + cur=$1 +else + cur=$PWD +fi # load existing modules -source mlt_modules_hdfml.sh +source ${cur}mlt_modules_hdfml.sh # create venv -python3 -m venv ../venv_hdfml +python3 -m venv ${cur}../venv_hdfml -source ../venv_hdfml/bin/activate +source ${cur}/../venv_hdfml/bin/activate # export path for side-packages -export PYTHONPATH=${PWD}/../venv_hdfml/lib/python3.6/site-packages:${PYTHONPATH} +export PYTHONPATH=${cur}/../venv_hdfml/lib/python3.6/site-packages:${PYTHONPATH} -pip install -r requirements_HDFML_outcommented.txt +pip install -r ${cur}/requirements_HDFML_outcommented.txt pip install --ignore-installed matplotlib==3.2.0 pip install --ignore-installed pandas==1.0.1 diff --git a/HPC_setup/setup_venv_juwels.sh b/HPC_setup/setup_venv_juwels.sh index 98edd7540c141e2336c1f638a7e491b6f0714934..ca362a8b160145087fc54d864fe5d2d52b1143e5 100755 --- a/HPC_setup/setup_venv_juwels.sh +++ b/HPC_setup/setup_venv_juwels.sh @@ -4,9 +4,10 @@ # This script creates a virtual env which contains all modules which are not available via slrum/easybuild (see mlt_modules.sh) +# $1 has to be an abs path to HPC_setup. If not provided, $PWD is used + # enter setting dir if called externally if [[ $1 != '' ]]; then - old=$PWD cur=$1 else cur=$PWD @@ -25,6 +26,8 @@ export PYTHONPATH=${cur}/../venv_juwels/lib/python3.6/site-packages:${PYTHONPATH pip install -r ${cur}requirements_JUWELS_outcommented.txt pip install --ignore-installed matplotlib==3.2.0 +pip install --ignore-installed pandas==1.0.1 + # Comment: Maybe we have to export PYTHONPATH a second time ater activating the venv (after job allocation) # source venv/bin/activate diff --git a/create_runscripts_HPC.sh b/create_runscripts_HPC.sh index b2df1a25d5d615f3a020bdf19d625467d72db8c5..af657fd11779f67861785c1573acd80235380b53 100755 --- a/create_runscripts_HPC.sh +++ b/create_runscripts_HPC.sh @@ -10,6 +10,24 @@ echo echo "This script creates the HPC batch scripts to run mlt on compute nodes (gpus and develgpus)." echo "You can modify the created run scripts afterwards if needed." +while true; do + read -p "Do you wish to create run scripts for JUWELS? [yes/no]" yn + case $yn in + [Yy]* ) juwels=True; break;; + [Nn]* ) juwels=False;; + * ) echo "Please answer yes or no.";; + esac +done + +while true; do + read -p "Do you wish to create run script for HDFML? [yes/no]" yn + case $yn in + [Yy]* ) hdfml=True; break;; + [Nn]* ) hdfml=False;; + * ) echo "Please answer yes or no.";; + esac +done + budget='' while [[ $budget == '' ]] @@ -27,15 +45,15 @@ if [[ -z "$new_email" ]]; then fi # create HPC_logging dir -hpclogging="HPC_logging/" +hpclogging="../HPC_logging/" mkdir -p $hpclogging # ordering for looping: # "partition nGPUs timing" - -for i in "develgpus 2 02:00:00" "gpus 4 08:00:00"; do - set -- $i +if [[ $juwels == True ]]; then + for i in "develgpus 2 02:00:00" "gpus 4 08:00:00"; do + set -- $i cat <<EOT > run_$1.bash #!/bin/bash -x @@ -49,7 +67,7 @@ cat <<EOT > run_$1.bash #SBATCH --mail-type=ALL #SBATCH --mail-user=${email} -source mlt_modules.sh +source mlt_modules_.sh source venv/bin/activate timestamp=\`date +"%Y-%m-%d_%H%M-%S"\` @@ -59,9 +77,11 @@ export PYTHONPATH=\${PWD}/venv/lib/python3.6/site-packages:\${PYTHONPATH} srun python run.py --experiment_date=\$timestamp EOT -echo "Created runscript: run_$1.bash" + echo "Created runscript: run_$1.bash" + + done +fi -done echo echo "You have to run the the following command on a login node to download data:" echo " \`python run.py'" diff --git a/setupHPC_test.sh b/setupHPC_test.sh index c67e2430f7a279d250e0d58f74586d5a0d18e892..fad7c6053a5d076ec4dcf0abba7b363e0252e690 100644 --- a/setupHPC_test.sh +++ b/setupHPC_test.sh @@ -16,4 +16,6 @@ else fi source HPC_setup/setup_venv_${hpcsys}.sh $basepath/$settingpath -#source $settingpath/testcall_juwels.sh $basepath/$settingpath + +source HPC_setup/create_runscripts_HPC.sh $hpcsys $basepath +