create_runscripts_HPC.sh

#!/bin/csh -x

# __author__ = Felix Kleinert
# __date__  = '2020-04-30'
# This script creates run scripts for JUWELS or HDFML

# When you call this script directly you can use
#   $1 which has to be `juwels' or `hdfml'.
#   $2 which is the path where the run scripts should be stored

if [[ $1 != '' ]]; then
  hpcsys=$1
else
  if [[ $HOSTNAME == *"juwels"* ]]; then
    hpcsys="juwels"
  elif [[ $HOSTNAME == *"hdfml"* ]]; then
    hpcsys="hdfml"
  else
     echo  "Unknown hpc host \`$HOSTNAME\`. Pass 'juwels' or 'hdfml' as first argument."
     exit
  fi
fi

if [[ $2 != '' ]]; then
  cur=$2
else
  cur=$PWD
fi

echo "############################################################"
echo "#                                                          #"
echo "#            user interaction required                     #"
echo "#                                                          #"
echo "############################################################"
echo 

echo "This script creates the HPC batch scripts to run mlt on compute nodes on JUWELS or hdfml."
echo "You can modify the created run scripts afterwards if needed."

echo
echo
echo "Creating run script for $hpcsys:"
echo

budget=''
while [[ $budget == '' ]]
do
 echo
 read -p "Enter project budget for --account flag: " budget 
done

email=`jutil user show -o json | grep email | cut -f2 -d':' | cut -f1 -d',' | cut -f2 -d'"'`
echo
read -p "Enter e-mail address for --mail-user (default: ${email}): " new_email

if [[ -z "$new_email" ]]; then
    new_email=$email
fi

# create HPC_logging dir
hpclogging="HPC_logging/"
mkdir -p ${cur}/${hpclogging}


# ordering for looping:
# "partition nGPUs timing"
if [[ $hpcsys = "juwels" ]]; then
  for i in "develgpus 2  02:00:00" "gpus 4 08:00:00"; do
      set -- $i

cat <<EOT > ${cur}/run_${hpcsys}_$1.bash
#!/bin/bash -x
#SBATCH --account=${budget}
#SBATCH --nodes=1
#SBATCH --output=${hpclogging}mlt-out.%j
#SBATCH --error=${hpclogging}mlt-err.%j
#SBATCH --time=$3 
#SBATCH --partition=$1 
#SBATCH --gres=gpu:$2
#SBATCH --mail-type=ALL
#SBATCH --mail-user=${email}

source HPC_setup/mlt_modules_${hpcsys}.sh
source venv_${hpcsys}/bin/activate

timestamp=\`date +"%Y-%m-%d_%H%M-%S"\`

export PYTHONPATH=\${PWD}/venv_${hpcsys}/lib/python3.9/site-packages:\${PYTHONPATH}

srun --cpu-bind=none python run.py --experiment_date=\$timestamp
EOT

  echo "Created runscript: run_${hpcsys}_$1.bash"

  done

elif [[ $hpcsys = "hdfml" ]]; then
cat <<EOT > ${cur}/run_${hpcsys}_batch.bash
#!/bin/bash -x
#SBATCH --account=${budget}
#SBATCH --nodes=1
#SBATCH --output=${hpclogging}mlt-out.%j
#SBATCH --error=${hpclogging}mlt-err.%j
#SBATCH --time=08:00:00
#SBATCH --gres=gpu:4
#SBATCH --mail-type=ALL
#SBATCH --mail-user=${email}

source HPC_setup/mlt_modules_${hpcsys}.sh
source venv_${hpcsys}/bin/activate

timestamp=\`date +"%Y-%m-%d_%H%M-%S"\`

export PYTHONPATH=\${PWD}/venv_${hpcsys}/lib/python3.9/site-packages:\${PYTHONPATH}

srun --cpu-bind=none python run_HPC.py --experiment_date=\$timestamp
EOT

fi

echo "###################################################################################"
echo "#   You have to run the the following command on a login node to download data:   #"
echo "#                              \`python run_HPC.py'                                #"
echo "#                                                                                 #"

echo "#   Please execute the following command to check if the setup went well:         #"
if [[ ${hpcsys} = 'juwels' ]]; then
  echo "#                   \`sbatch run_juwels_develgpus.bash'                            #"
else
  echo "#                   \`sbatch run_hdfml_batch.bash'                                 #"
fi

echo "###################################################################################"