Skip to content
Snippets Groups Projects
Commit 9cc1b734 authored by gong1's avatar gong1
Browse files

Merge branch 'bing_clean_GMD1' into develop

parents d8387404 118545b5
Branches
Tags
No related merge requests found
Pipeline #97059 failed
Showing
with 600 additions and 553 deletions
...@@ -127,6 +127,6 @@ virt_env*/ ...@@ -127,6 +127,6 @@ virt_env*/
# Ignore (Batch) runscripts # Ignore (Batch) runscripts
video_prediction_tools/HPC_scripts/** video_prediction_tools/HPC_scripts/**
!video_prediction_tools/HPC_scripts/*_template.sh !video_prediction_tools/HPC_scripts/*_template.sh
video_prediction_tools/nonHPC_scripts/** video_prediction_tools/no_HPC_scripts/**
!video_prediction_tools/nonHPC_scripts/*_template.sh !video_prediction_tools/no_HPC_scripts/*_template.sh
This diff is collapsed.
#!/bin/bash -x #!/bin/bash -x
## Controlling Batch-job : Need input ## Controlling Batch-job
#SBATCH --account=<Project name> #SBATCH --account=<your_project>
#SBATCH --nodes=1 #SBATCH --nodes=1
#SBATCH --ntasks=13 #SBATCH --ntasks=13
##SBATCH --ntasks-per-node=13 ##SBATCH --ntasks-per-node=13
...@@ -13,28 +13,26 @@ ...@@ -13,28 +13,26 @@
#SBATCH --mail-type=ALL #SBATCH --mail-type=ALL
#SBATCH --mail-user=me@somewhere.com #SBATCH --mail-user=me@somewhere.com
##Load basic Python module: Need input ######### Template identifier (don't remove) #########
#module load Python echo "Do not run the template scripts"
exit 99
######### Template identifier (don't remove) #########
##Create and activate a virtual environment: Need input
#VENV_NAME=<my_venv> # Name of virtual environment
#Python -m venv ../virtual_envs/${VENV_NAME} VIRT_ENV_NAME="my_venv"
#source ../virtual_envs/${VENV_NAME}/bin/activate
# Activate virtual environment if needed (and possible)
if [ -z ${VIRTUAL_ENV} ]; then
## Install required packages if [[ -f ../virtual_envs/${VIRT_ENV_NAME}/bin/activate ]]; then
# set PYTHONPATH... echo "Activating virtual environment..."
BASE_DIR="$(pwd)" source ../virtual_envs/${VIRT_ENV_NAME}/bin/activate
WORKING_DIR=="$(BASE_DIR "$dir")" else
export PYTHONPATH=${WORKING_DIR}/virtual_envs/${VENV_NAME}/lib/python3.8/site-packages:$PYTHONPATH echo "ERROR: Requested virtual environment ${VIRT_ENV_NAME} not found..."
export PYTHONPATH=${WORKING_DIR}:$PYTHONPATH exit 1
export PYTHONPATH=${WORKING_DIR}/utils:$PYTHONPATH fi
export PYTHONPATH=${WORKING_DIR}/model_modules:$PYTHONPATH fi
export PYTHONPATH=${WORKING_DIR}/postprocess:$PYTHONPATH # Loading modules
# ... install requirements source ../env_setup/modules_preprocess+extract.sh
pip install --no-cache-dir -r ../env_setup/requirements.txt
# Declare path-variables (dest_dir will be set and configured automatically via generate_runscript.py) # Declare path-variables (dest_dir will be set and configured automatically via generate_runscript.py)
source_dir=/my/path/to/era5 source_dir=/my/path/to/era5
......
#!/bin/bash -x #!/bin/bash -x
## Controlling Batch-job: Need input ## Controlling Batch-job
#SBATCH --account=<Project name> #SBATCH --account=<your_project>
#SBATCH --nodes=1 #SBATCH --nodes=1
#SBATCH --ntasks=13 #SBATCH --ntasks=1
##SBATCH --ntasks-per-node=13 ##SBATCH --ntasks-per-node=13
#SBATCH --cpus-per-task=1 #SBATCH --cpus-per-task=1
#SBATCH --output=Data_Preprocess_step1_era5-out.%j #SBATCH --output=meta_postprocess_era5-out.%j
#SBATCH --error=Data_Preprocess_step1era5-err.%j #SBATCH --error=meta_postprocess_era5-err.%j
#SBATCH --time=04:20:00 #SBATCH --time=00:20:00
#SBATCH --partition=batch #SBATCH --partition=batch
#SBATCH --gres=gpu:0 #SBATCH --gres=gpu:0
#SBATCH --mail-type=ALL #SBATCH --mail-type=ALL
#SBATCH --mail-user=me@somewhere.com #SBATCH --mail-user=me@somewhere.com
##Load basic Python module: Need input ######### Template identifier (don't remove) #########
#module load Python echo "Do not run the template scripts"
exit 99
######### Template identifier (don't remove) #########
##Create and activate a virtual environment: Need input
#VENV_NAME=<my_venv>
#Python -m venv ../virtual_envs/${VENV_NAME}
#source ../virtual_envs/${VENV_NAME}/bin/activate
## Install required packages
# set PYTHONPATH...
WORKING_DIR="$(pwd)"
BASE_DIR=="$(WORKING_DIR "$dir")"
export PYTHONPATH=${BASE_DIR}/virtual_envs/${VENV_NAME}/lib/python3.8/site-packages:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/utils:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/model_modules:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/postprocess:$PYTHONPATH
# ... install requirements
pip install --no-cache-dir -r ../env_setup/requirements.txt
# Name of virtual environment
VENV_NAME=venv_hdfml
# Name of container image (must be available in working directory)
CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif"
WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh"
# sanity checks
if [[ ! -f ${CONTAINER_IMG} ]]; then
echo "ERROR: Cannot find required TF1.15 container image '${CONTAINER_IMG}'."
exit 1
fi
if [[ ! -f ${WRAPPER} ]]; then
echo "ERROR: Cannot find wrapper-script '${WRAPPER}' for TF1.15 container image."
exit 1
fi
# Declare input parameters # Declare input parameters
root_dir=/p/project/deepacf/deeprain/video_prediction_shared_folder/ root_dir=/p/project/deepacf/deeprain/video_prediction_shared_folder/
......
#!/bin/bash -x #!/bin/bash -x
## Controlling Batch-job : Need input ## Controlling Batch-job
#SBATCH --account=<Project name> #SBATCH --account=<your_project>
#SBATCH --nodes=1 #SBATCH --nodes=1
#SBATCH --ntasks=13 #SBATCH --ntasks=13
##SBATCH --ntasks-per-node=13 ##SBATCH --ntasks-per-node=12
#SBATCH --cpus-per-task=1 #SBATCH --cpus-per-task=1
#SBATCH --output=Data_Preprocess_step1_era5-out.%j #SBATCH --output=DataPreprocess_era5_step1-out.%j
#SBATCH --error=Data_Preprocess_step1era5-err.%j #SBATCH --error=DataPreprocess_era5_step1-err.%j
#SBATCH --time=04:20:00 #SBATCH --time=04:20:00
#SBATCH --partition=batch
#SBATCH --gres=gpu:0 #SBATCH --gres=gpu:0
#SBATCH --partition=batch
#SBATCH --mail-type=ALL #SBATCH --mail-type=ALL
#SBATCH --mail-user=me@somewhere.com #SBATCH --mail-user=me@somewhere.com
##Load basic Python module: Need input ######### Template identifier (don't remove) #########
#module load Python echo "Do not run the template scripts"
exit 99
######### Template identifier (don't remove) #########
##Create and activate a virtual environment : Need input
#VENV_NAME=<my_venv> # Name of virtual environment
#Python -m venv ../virtual_envs/${VENV_NAME} VIRT_ENV_NAME="my_venv"
#source ../virtual_envs/${VENV_NAME}/bin/activate
# Activate virtual environment if needed (and possible)
if [ -z ${VIRTUAL_ENV} ]; then
## Install required packages if [[ -f ../virtual_envs/${VIRT_ENV_NAME}/bin/activate ]]; then
# set PYTHONPATH... echo "Activating virtual environment..."
BASE_DIR="$(pwd)" source ../virtual_envs/${VIRT_ENV_NAME}/bin/activate
WORKING_DIR=="$(BASE_DIR "$dir")" else
export PYTHONPATH=${WORKING_DIR}/virtual_envs/${VENV_NAME}/lib/python3.8/site-packages:$PYTHONPATH echo "ERROR: Requested virtual environment ${VIRT_ENV_NAME} not found..."
export PYTHONPATH=${WORKING_DIR}:$PYTHONPATH exit 1
export PYTHONPATH=${WORKING_DIR}/utils:$PYTHONPATH fi
export PYTHONPATH=${WORKING_DIR}/model_modules:$PYTHONPATH fi
export PYTHONPATH=${WORKING_DIR}/postprocess:$PYTHONPATH # Loading modules
# ... install requirements source ../env_setup/modules_preprocess+extract.sh
pip install --no-cache-dir -r ../env_setup/requirements.txt
# select years and variables for dataset and define target domain
years=( "2015" )
variables=( "t2" "t2" "t2" )
sw_corner=( -999.9 -999.9)
nyx=( -999 -999 )
# select years for dataset
declare -a years=(
"2017"
)
max_year=`echo "${years[*]}" | sort -nr | head -n1`
min_year=`echo "${years[*]}" | sort -nr | tail -n1`
# set some paths # set some paths
# note, that destination_dir is used during runtime to set a proper experiment directory # note, that destination_dir is adjusted during runtime based on the data
exp_id=xxx # experiment identifier is set by 'generate_workflow_runscripts.sh' source_dir=/my/path/to/extracted/data/
source_dir=${SAVE_DIR}/extractedData destination_dir=/my/path/to/pickle/files
destination_dir=${SAVE_DIR}/preprocessedData/era5-Y${min_year}to${max_year}M01to12
script_dir=`pwd` # execute Python-scripts
for year in "${years[@]}"; do
for year in "${years[@]}"; echo "start preprocessing data for year ${year}"
do srun python ../main_scripts/main_preprocess_data_step1.py \
echo "Year $year" --source_dir ${source_dir} --destination_dir ${destination_dir} --years "${year}" \
echo "source_dir ${source_dir}/${year}" --vars "${variables[0]}" "${variables[1]}" "${variables[2]}" \
mpirun -np 2 python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \ --sw_corner "${sw_corner[0]}" "${sw_corner[1]}" --nyx "${nyx[0]}" "${nyx[1]}"
--source_dir ${source_dir} -scr_dir ${script_dir} -exp_dir ${exp_id} \
--destination_dir ${destination_dir} --years ${years} --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
done done
#srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_split_data_multi_years.py --destination_dir ${destination_dir} --varnames T2 MSL gph500
#!/bin/bash -x #!/bin/bash -x
## Controlling Batch-job: Need input #SBATCH --account=<your_project>
#SBATCH --account=<Project name>
#SBATCH --nodes=1 #SBATCH --nodes=1
#SBATCH --ntasks=13 #SBATCH --ntasks=13
##SBATCH --ntasks-per-node=13 ##SBATCH --ntasks-per-node=13
#SBATCH --cpus-per-task=1 #SBATCH --cpus-per-task=1
#SBATCH --output=Data_Preprocess_step1_era5-out.%j #SBATCH --output=DataPreprocess_era5_step2-out.%j
#SBATCH --error=Data_Preprocess_step1era5-err.%j #SBATCH --error=DataPreprocess_era5_step2-err.%j
#SBATCH --time=04:20:00 #SBATCH --time=04:00:00
#SBATCH --partition=batch
#SBATCH --gres=gpu:0 #SBATCH --gres=gpu:0
#SBATCH --partition=batch
#SBATCH --mail-type=ALL #SBATCH --mail-type=ALL
#SBATCH --mail-user=me@somewhere.com #SBATCH --mail-user=me@somewhere.com
##Load basic Python module: Need input ######### Template identifier (don't remove) #########
#module load Python echo "Do not run the template scripts"
exit 99
######### Template identifier (don't remove) #########
##Create and activate a virtual environment: Need input
#VENV_NAME=<my_venv>
#Python -m venv ../virtual_envs/${VENV_NAME}
#source ../virtual_envs/${VENV_NAME}/bin/activate
## Install required packages
# set PYTHONPATH...
WORKING_DIR="$(pwd)"
BASE_DIR=="$(WORKING_DIR "$dir")"
export PYTHONPATH=${BASE_DIR}/virtual_envs/${VENV_NAME}/lib/python3.8/site-packages:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/utils:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/model_modules:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/postprocess:$PYTHONPATH
# ... install requirements
pip install --no-cache-dir -r ../env_setup/requirements.txt
# auxiliary variables
WORK_DIR="$(pwd)"
BASE_DIR=$(dirname "$WORK_DIR")
# Name of virtual environment # Name of virtual environment
VENV_NAME=venv_hdfml VIRT_ENV_NAME="my_venv"
# !!! ADAPAT DEPENDING ON USAGE OF CONTAINER !!!
# For container usage, comment in the follwoing lines
# Name of container image (must be available in working directory) # Name of container image (must be available in working directory)
CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif" CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif"
WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh" WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh"
...@@ -58,14 +46,32 @@ module purge ...@@ -58,14 +46,32 @@ module purge
source_dir=/my/path/to/pkl/files/ source_dir=/my/path/to/pkl/files/
destination_dir=/my/path/to/tfrecords/files destination_dir=/my/path/to/tfrecords/files
sequence_length=24 sequence_length=20
sequences_per_file=10 sequences_per_file=10
# run Preprocessing (step 2 where Tf-records are generated) # run Preprocessing (step 2 where Tf-records are generated)
# run postprocessing/generation of model results including evaluation metrics
export CUDA_VISIBLE_DEVICES=0 export CUDA_VISIBLE_DEVICES=0
## One node, single GPU ## One node, single GPU
srun --mpi=pspmix --cpu-bind=none \ srun --mpi=pspmix --cpu-bind=none \
singularity exec --nv "${CONTAINER_IMG}" "${WRAPPER}" ${VENV_NAME} \ singularity exec --nv "${CONTAINER_IMG}" "${WRAPPER}" ${VIRT_ENV_NAME} \
python3 ../main_scripts/main_preprocess_data_step2.py -source_dir ${source_dir} -dest_dir ${destination_dir} \ python3 ../main_scripts/main_preprocess_data_step2.py -source_dir ${source_dir} -dest_dir ${destination_dir} \
-sequence_length ${sequence_length} -sequences_per_file ${sequences_per_file} -sequence_length ${sequence_length} -sequences_per_file ${sequences_per_file}
# WITHOUT container usage, comment in the follwoing lines (and uncomment the lines above)
# Activate virtual environment if needed (and possible)
#if [ -z ${VIRTUAL_ENV} ]; then
# if [[ -f ../virtual_envs/${VIRT_ENV_NAME}/bin/activate ]]; then
# echo "Activating virtual environment..."
# source ../virtual_envs/${VIRT_ENV_NAME}/bin/activate
# else
# echo "ERROR: Requested virtual environment ${VIRT_ENV_NAME} not found..."
# exit 1
# fi
#fi
#
# Loading modules
#module purge
#source ../env_setup/modules_train.sh
#export CUDA_VISIBLE_DEVICES=0
#
# srun python3 ../main_scripts/main_preprocess_data_step2.py -source_dir ${source_dir} -dest_dir ${destination_dir} \
# -sequence_length ${sequence_length} -sequences_per_file ${sequences_per_file}
#!/bin/bash -x #!/bin/bash -x
#SBATCH --account=deepacf #SBATCH --account=<your_project>
#SBATCH --nodes=1 #SBATCH --nodes=1
#SBATCH --ntasks=13 #SBATCH --ntasks=1
##SBATCH --ntasks-per-node=13 ##SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=1 #SBATCH --cpus-per-task=1
#SBATCH --output=DataPreprocess_era5_step2-out.%j #SBATCH --output=DataPreprocess_moving_mnist-out.%j
#SBATCH --error=DataPreprocess_era5_step2-err.%j #SBATCH --error=DataPreprocess_moving_mnist-err.%j
#SBATCH --time=04:00:00 #SBATCH --time=04:00:00
#SBATCH --gres=gpu:0
#SBATCH --partition=batch #SBATCH --partition=batch
#SBATCH --mail-type=ALL #SBATCH --mail-type=ALL
#SBATCH --mail-user=me@somewhere.com #SBATCH --mail-user=me@somewhere.com
...@@ -17,11 +16,11 @@ echo "Do not run the template scripts" ...@@ -17,11 +16,11 @@ echo "Do not run the template scripts"
exit 99 exit 99
######### Template identifier (don't remove) ######### ######### Template identifier (don't remove) #########
# auxiliary variables
WORK_DIR="$(pwd)"
BASE_DIR=$(dirname "$WORK_DIR")
# Name of virtual environment # Name of virtual environment
VIRT_ENV_NAME="my_venv" VIRT_ENV_NAME="my_venv"
# !!! ADAPAT DEPENDING ON USAGE OF CONTAINER !!!
# For container usage, comment in the follwoing lines
# Name of container image (must be available in working directory) # Name of container image (must be available in working directory)
CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif" CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif"
WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh" WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh"
...@@ -40,18 +39,33 @@ fi ...@@ -40,18 +39,33 @@ fi
# clean-up modules to avoid conflicts between host and container settings # clean-up modules to avoid conflicts between host and container settings
module purge module purge
# declare directory-variables which will be modified by config_runscript.py # declare directory-variables which will be modified generate_runscript.py
source_dir=/my/path/to/pkl/files/ source_dir=/my/path/to/mnist/raw/data/
destination_dir=/my/path/to/tfrecords/files destination_dir=/my/path/to/mnist/tfrecords/
sequence_length=20
sequences_per_file=10
# run Preprocessing (step 2 where Tf-records are generated) # run Preprocessing (step 2 where Tf-records are generated)
# run postprocessing/generation of model results including evaluation metrics # run postprocessing/generation of model results including evaluation metrics
export CUDA_VISIBLE_DEVICES=0 export CUDA_VISIBLE_DEVICES=0
## One node, single GPU ## One node, single GPU
srun --mpi=pspmix --cpu-bind=none \ srun --mpi=pspmix --cpu-bind=none \
singularity exec --nv "${CONTAINER_IMG}" "${WRAPPER}" ${VIRT_ENV_NAME} \ singularity exec --nv "${CONTAINER_IMG}" "${WRAPPER}" ${VIRT_ENV_NAME} \
python3 ../main_scripts/main_preprocess_data_step2.py -source_dir ${source_dir} -dest_dir ${destination_dir} \ python3 ../video_prediction/datasets/moving_mnist.py ${source_dir} ${destination_dir}
-sequence_length ${sequence_length} -sequences_per_file ${sequences_per_file}
# WITHOUT container usage, comment in the follwoing lines (and uncomment the lines above)
# Activate virtual environment if needed (and possible)
#if [ -z ${VIRTUAL_ENV} ]; then
# if [[ -f ../virtual_envs/${VIRT_ENV_NAME}/bin/activate ]]; then
# echo "Activating virtual environment..."
# source ../virtual_envs/${VIRT_ENV_NAME}/bin/activate
# else
# echo "ERROR: Requested virtual environment ${VIRT_ENV_NAME} not found..."
# exit 1
# fi
#fi
#
# Loading modules
#module purge
#source ../env_setup/modules_train.sh
#export CUDA_VISIBLE_DEVICES=0
#
# srun python3 .../video_prediction/datasets/moving_mnist.py ${source_dir} ${destination_dir}
\ No newline at end of file
#!/bin/bash -x #!/bin/bash -x
## Controlling Batch-job: Need input #SBATCH --account=<your_project>
#SBATCH --account=<Project name>
#SBATCH --nodes=1 #SBATCH --nodes=1
#SBATCH --ntasks=13 #SBATCH --ntasks=1
##SBATCH --ntasks-per-node=13 #SBATCH --output=train_model_era5-out.%j
#SBATCH --cpus-per-task=1 #SBATCH --error=train_model_era5-err.%j
#SBATCH --output=Data_Preprocess_step1_era5-out.%j #SBATCH --time=24:00:00
#SBATCH --error=Data_Preprocess_step1era5-err.%j #SBATCH --gres=gpu:1
#SBATCH --time=04:20:00 #SBATCH --partition=some_partition
#SBATCH --partition=batch
#SBATCH --gres=gpu:0
#SBATCH --mail-type=ALL #SBATCH --mail-type=ALL
#SBATCH --mail-user=me@somewhere.com #SBATCH --mail-user=me@somewhere.com
##Load basic Python module: Need input ######### Template identifier (don't remove) #########
#module load Python echo "Do not run the template scripts"
exit 99
######### Template identifier (don't remove) #########
##Create and activate a virtual environment: Need input
#VENV_NAME=<my_venv>
#Python -m venv ../virtual_envs/${VENV_NAME}
#source ../virtual_envs/${VENV_NAME}/bin/activate
## Install required packages
# set PYTHONPATH...
WORKING_DIR="$(pwd)"
BASE_DIR=="$(WORKING_DIR "$dir")"
export PYTHONPATH=${BASE_DIR}/virtual_envs/${VENV_NAME}/lib/python3.8/site-packages:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/utils:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/model_modules:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/postprocess:$PYTHONPATH
# ... install requirements
pip install --no-cache-dir -r ../env_setup/requirements.txt
# auxiliary variables
WORK_DIR="$(pwd)"
BASE_DIR=$(dirname "$WORK_DIR")
# Name of virtual environment # Name of virtual environment
VENV_NAME=venv_hdfml VIRT_ENV_NAME="my_venv"
# !!! ADAPAT DEPENDING ON USAGE OF CONTAINER !!!
# For container usage, comment in the follwoing lines
# Name of container image (must be available in working directory) # Name of container image (must be available in working directory)
CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif" CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif"
WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh" WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh"
...@@ -54,7 +40,6 @@ fi ...@@ -54,7 +40,6 @@ fi
# clean-up modules to avoid conflicts between host and container settings # clean-up modules to avoid conflicts between host and container settings
module purge module purge
# declare directory-variables which will be modified by generate_runscript.py # declare directory-variables which will be modified by generate_runscript.py
source_dir=/my/path/to/tfrecords/files source_dir=/my/path/to/tfrecords/files
destination_dir=/my/model/output/path destination_dir=/my/model/output/path
...@@ -72,3 +57,22 @@ srun --mpi=pspmix --cpu-bind=none \ ...@@ -72,3 +57,22 @@ srun --mpi=pspmix --cpu-bind=none \
python3 "${BASE_DIR}"/main_scripts/main_train_models.py --input_dir ${source_dir} --datasplit_dict ${datasplit_dict} \ python3 "${BASE_DIR}"/main_scripts/main_train_models.py --input_dir ${source_dir} --datasplit_dict ${datasplit_dict} \
--dataset era5 --model ${model} --model_hparams_dict ${model_hparams} --output_dir ${destination_dir}/ --dataset era5 --model ${model} --model_hparams_dict ${model_hparams} --output_dir ${destination_dir}/
# WITHOUT container usage, comment in the follwoing lines (and uncomment the lines above)
# Activate virtual environment if needed (and possible)
#if [ -z ${VIRTUAL_ENV} ]; then
# if [[ -f ../virtual_envs/${VIRT_ENV_NAME}/bin/activate ]]; then
# echo "Activating virtual environment..."
# source ../virtual_envs/${VIRT_ENV_NAME}/bin/activate
# else
# echo "ERROR: Requested virtual environment ${VIRT_ENV_NAME} not found..."
# exit 1
# fi
#fi
#
# Loading modules
#module purge
#source ../env_setup/modules_train.sh
#export CUDA_VISIBLE_DEVICES=0
#
# srun python3 "${BASE_DIR}"/main_scripts/main_train_models.py --input_dir ${source_dir} --datasplit_dict ${datasplit_dict} \
# --dataset era5 --model ${model} --model_hparams_dict ${model_hparams} --output_dir ${destination_dir}/
\ No newline at end of file
#!/bin/bash -x #!/bin/bash -x
#SBATCH --account=deepacf #SBATCH --account=<your_project>
#SBATCH --nodes=1 #SBATCH --nodes=1
#SBATCH --ntasks=1 #SBATCH --ntasks=1
##SBATCH --ntasks-per-node=1 ##SBATCH --ntasks-per-node=1
...@@ -10,8 +10,7 @@ ...@@ -10,8 +10,7 @@
#SBATCH --gres=gpu:1 #SBATCH --gres=gpu:1
#SBATCH --partition=gpus #SBATCH --partition=gpus
#SBATCH --mail-type=ALL #SBATCH --mail-type=ALL
#SBATCH --mail-user=b.gong@fz-juelich.de #SBATCH --mail-user=me@somewhere.com
##jutil env activate -p cjjsc42
######### Template identifier (don't remove) ######### ######### Template identifier (don't remove) #########
echo "Do not run the template scripts" echo "Do not run the template scripts"
...@@ -23,6 +22,8 @@ WORK_DIR=`pwd` ...@@ -23,6 +22,8 @@ WORK_DIR=`pwd`
BASE_DIR=$(dirname "$WORK_DIR") BASE_DIR=$(dirname "$WORK_DIR")
# Name of virtual environment # Name of virtual environment
VIRT_ENV_NAME="my_venv" VIRT_ENV_NAME="my_venv"
# !!! ADAPAT DEPENDING ON USAGE OF CONTAINER !!!
# For container usage, comment in the follwoing lines
# Name of container image (must be available in working directory) # Name of container image (must be available in working directory)
CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif" CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif"
WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh" WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh"
...@@ -52,7 +53,6 @@ dataset=moving_mnist ...@@ -52,7 +53,6 @@ dataset=moving_mnist
model_hparams=../hparams/${dataset}/${model}/model_hparams.json model_hparams=../hparams/${dataset}/${model}/model_hparams.json
destination_dir=${destination_dir}/${model}/"$(date +"%Y%m%dT%H%M")_"$USER"" destination_dir=${destination_dir}/${model}/"$(date +"%Y%m%dT%H%M")_"$USER""
# rund training
# run training in container # run training in container
export CUDA_VISIBLE_DEVICES=0 export CUDA_VISIBLE_DEVICES=0
## One node, single GPU ## One node, single GPU
...@@ -60,3 +60,23 @@ srun --mpi=pspmix --cpu-bind=none \ ...@@ -60,3 +60,23 @@ srun --mpi=pspmix --cpu-bind=none \
singularity exec --nv "${CONTAINER_IMG}" "${WRAPPER}" ${VIRT_ENV_NAME} \ singularity exec --nv "${CONTAINER_IMG}" "${WRAPPER}" ${VIRT_ENV_NAME} \
python ../main_scripts/train.py --input_dir ${source_dir}/tfrecords/ --dataset ${dataset} --model ${model} \ python ../main_scripts/train.py --input_dir ${source_dir}/tfrecords/ --dataset ${dataset} --model ${model} \
--model_hparams_dict ${model_hparams} --output_dir "${destination_dir}"/ --model_hparams_dict ${model_hparams} --output_dir "${destination_dir}"/
# WITHOUT container usage, comment in the follwoing lines (and uncomment the lines above)
# Activate virtual environment if needed (and possible)
#if [ -z ${VIRTUAL_ENV} ]; then
# if [[ -f ../virtual_envs/${VIRT_ENV_NAME}/bin/activate ]]; then
# echo "Activating virtual environment..."
# source ../virtual_envs/${VIRT_ENV_NAME}/bin/activate
# else
# echo "ERROR: Requested virtual environment ${VIRT_ENV_NAME} not found..."
# exit 1
# fi
#fi
#
# Loading modules
#module purge
#source ../env_setup/modules_train.sh
#export CUDA_VISIBLE_DEVICES=0
#
# srun python3 ../main_scripts/train.py --input_dir ${source_dir}/tfrecords/ --dataset ${dataset} --model ${model} \
# --model_hparams_dict ${model_hparams} --output_dir "${destination_dir}"/
\ No newline at end of file
#!/bin/bash -x #!/bin/bash -x
## Controlling Batch-job: Need input #SBATCH --account=<your_project>
#SBATCH --account=<Project name>
#SBATCH --nodes=1 #SBATCH --nodes=1
#SBATCH --ntasks=13 #SBATCH --ntasks=1
##SBATCH --ntasks-per-node=13 ##SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=1 #SBATCH --cpus-per-task=1
#SBATCH --output=Data_Preprocess_step1_era5-out.%j #SBATCH --output=postprocess_era5-out.%j
#SBATCH --error=Data_Preprocess_step1era5-err.%j #SBATCH --error=postprocess_era5-err.%j
#SBATCH --time=04:20:00 #SBATCH --time=01:00:00
#SBATCH --partition=batch #SBATCH --gres=gpu:1
#SBATCH --gres=gpu:0 #SBATCH --partition=gpus
#SBATCH --mail-type=ALL #SBATCH --mail-type=ALL
#SBATCH --mail-user=me@somewhere.com #SBATCH --mail-user=me@somewhere.com
##Load basic Python module: Need input ######### Template identifier (don't remove) #########
#module load Python echo "Do not run the template scripts"
exit 99
######### Template identifier (don't remove) #########
##Create and activate a virtual environment: Need input
#VENV_NAME=<my_venv>
#Python -m venv ../virtual_envs/${VENV_NAME}
#source ../virtual_envs/${VENV_NAME}/bin/activate
## Install required packages
# set PYTHONPATH...
WORKING_DIR="$(pwd)"
BASE_DIR=="$(WORKING_DIR "$dir")"
export PYTHONPATH=${BASE_DIR}/virtual_envs/${VENV_NAME}/lib/python3.8/site-packages:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/utils:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/model_modules:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/postprocess:$PYTHONPATH
# ... install requirements
pip install --no-cache-dir -r ../env_setup/requirements.txt
# auxiliary variables
WORK_DIR="$(pwd)"
BASE_DIR=$(dirname "$WORK_DIR")
# Name of virtual environment # Name of virtual environment
VENV_NAME=venv_hdfml VIRT_ENV_NAME="my_venv"
# !!! ADAPAT DEPENDING ON USAGE OF CONTAINER !!!
# For container usage, comment in the follwoing lines
# Name of container image (must be available in working directory) # Name of container image (must be available in working directory)
CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif" CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif"
WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh" WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh"
...@@ -51,10 +39,11 @@ if [[ ! -f ${WRAPPER} ]]; then ...@@ -51,10 +39,11 @@ if [[ ! -f ${WRAPPER} ]]; then
exit 1 exit 1
fi fi
# clean-up modules to avoid conflicts between host and container settings
module purge
# declare directory-variables which will be modified by generate_runscript.py # declare directory-variables which will be modified by generate_runscript.py
# Note: source_dir is only needed for retrieving the base-directory # Note: source_dir is only needed for retrieving the base-directory
source_dir=/my/source/dir/
checkpoint_dir=/my/trained/model/dir checkpoint_dir=/my/trained/model/dir
results_dir=/my/results/dir results_dir=/my/results/dir
lquick="" lquick=""
...@@ -69,3 +58,24 @@ srun --mpi=pspmix --cpu-bind=none \ ...@@ -69,3 +58,24 @@ srun --mpi=pspmix --cpu-bind=none \
--num_stochastic_samples 1 ${lquick} \ --num_stochastic_samples 1 ${lquick} \
> postprocess_era5-out_all."${SLURM_JOB_ID}" > postprocess_era5-out_all."${SLURM_JOB_ID}"
# WITHOUT container usage, comment in the follwoing lines (and uncomment the lines above)
# Activate virtual environment if needed (and possible)
#if [ -z ${VIRTUAL_ENV} ]; then
# if [[ -f ../virtual_envs/${VIRT_ENV_NAME}/bin/activate ]]; then
# echo "Activating virtual environment..."
# source ../virtual_envs/${VIRT_ENV_NAME}/bin/activate
# else
# echo "ERROR: Requested virtual environment ${VIRT_ENV_NAME} not found..."
# exit 1
# fi
#fi
#
# Loading modules
#module purge
#source ../env_setup/modules_train.sh
#export CUDA_VISIBLE_DEVICES=0
#
# srun python3 ../main_scripts/main_visualize_postprocess.py --checkpoint ${checkpoint_dir} --mode test \
# --results_dir ${results_dir} --batch_size 4 \
# --num_stochastic_samples 1 ${lquick} \
# > postprocess_era5-out_all."${SLURM_JOB_ID}"
\ No newline at end of file
#!/bin/bash -x #!/bin/bash -x
#SBATCH --account=deepacf #SBATCH --account=<your_project>
#SBATCH --nodes=1 #SBATCH --nodes=1
#SBATCH --ntasks=1 #SBATCH --ntasks=1
##SBATCH --ntasks-per-node=1 ##SBATCH --ntasks-per-node=1
...@@ -10,8 +10,7 @@ ...@@ -10,8 +10,7 @@
#SBATCH --gres=gpu:1 #SBATCH --gres=gpu:1
#SBATCH --partition=develgpus #SBATCH --partition=develgpus
#SBATCH --mail-type=ALL #SBATCH --mail-type=ALL
#SBATCH --mail-user=b.gong@fz-juelich.de #SBATCH --mail-user=me@somewhere.com
##jutil env activate -p cjjsc42
######### Template identifier (don't remove) ######### ######### Template identifier (don't remove) #########
echo "Do not run the template scripts" echo "Do not run the template scripts"
...@@ -23,6 +22,8 @@ WORK_DIR="$(pwd)" ...@@ -23,6 +22,8 @@ WORK_DIR="$(pwd)"
BASE_DIR=$(dirname "$WORK_DIR") BASE_DIR=$(dirname "$WORK_DIR")
# Name of virtual environment # Name of virtual environment
VIRT_ENV_NAME="my_venv" VIRT_ENV_NAME="my_venv"
# !!! ADAPAT DEPENDING ON USAGE OF CONTAINER !!!
# For container usage, comment in the follwoing lines
# Name of container image (must be available in working directory) # Name of container image (must be available in working directory)
CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif" CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif"
WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh" WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh"
...@@ -57,3 +58,23 @@ srun --mpi=pspmix --cpu-bind=none \ ...@@ -57,3 +58,23 @@ srun --mpi=pspmix --cpu-bind=none \
--checkpoint ${checkpoint_dir}/${model} --mode test --model ${model} --results_dir ${results_dir}/${model} \ --checkpoint ${checkpoint_dir}/${model} --mode test --model ${model} --results_dir ${results_dir}/${model} \
--batch_size 2 --dataset era5 > generate_era5-out."${SLURM_JOB_ID}" --batch_size 2 --dataset era5 > generate_era5-out."${SLURM_JOB_ID}"
# WITHOUT container usage, comment in the follwoing lines (and uncomment the lines above)
# Activate virtual environment if needed (and possible)
#if [ -z ${VIRTUAL_ENV} ]; then
# if [[ -f ../virtual_envs/${VIRT_ENV_NAME}/bin/activate ]]; then
# echo "Activating virtual environment..."
# source ../virtual_envs/${VIRT_ENV_NAME}/bin/activate
# else
# echo "ERROR: Requested virtual environment ${VIRT_ENV_NAME} not found..."
# exit 1
# fi
#fi
#
# Loading modules
#module purge
#source ../env_setup/modules_train.sh
#export CUDA_VISIBLE_DEVICES=0
#
# srun python3 ../scripts/generate_movingmnist.py --input_dir ${source_dir}/ --dataset_hparams sequence_length=20 \
# --checkpoint ${checkpoint_dir}/${model} --mode test --model ${model} --results_dir ${results_dir}/${model} \
# --batch_size 2 --dataset era5 > generate_era5-out."${SLURM_JOB_ID}"
\ No newline at end of file
#!/bin/bash -x
## Controlling Batch-job
#SBATCH --account=deepacf
#SBATCH --nodes=1
#SBATCH --ntasks=13
##SBATCH --ntasks-per-node=13
#SBATCH --cpus-per-task=1
#SBATCH --output=data_extraction_era5-out.%j
#SBATCH --error=data_extraction_era5-err.%j
#SBATCH --time=04:20:00
#SBATCH --partition=batch
#SBATCH --gres=gpu:0
#SBATCH --mail-type=ALL
#SBATCH --mail-user=me@somewhere.com
######### Template identifier (don't remove) #########
echo "Do not run the template scripts"
exit 99
######### Template identifier (don't remove) #########
jutil env activate -p deepacf
# Name of virtual environment
VIRT_ENV_NAME="my_venv"
# Loading mouldes
source ../env_setup/modules_preprocess+extract.sh
# Activate virtual environment if needed (and possible)
if [ -z ${VIRTUAL_ENV} ]; then
if [[ -f ../virtual_envs/${VIRT_ENV_NAME}/bin/activate ]]; then
echo "Activating virtual environment..."
source ../virtual_envs/${VIRT_ENV_NAME}/bin/activate
else
echo "ERROR: Requested virtual environment ${VIRT_ENV_NAME} not found..."
exit 1
fi
fi
# Declare path-variables (dest_dir will be set and configured automatically via generate_runscript.py)
source_dir=/my/path/to/era5
destination_dir=/my/path/to/extracted/data
varmap_file=/my/path/to/varmapping/file
years=( "2015" )
# Run data extraction
for year in "${years[@]}"; do
echo "Perform ERA5-data extraction for year ${year}"
srun python ../main_scripts/main_data_extraction.py --source_dir ${source_dir} --target_dir ${destination_dir} \
--year ${year} --varslist_path ${varmap_file}
done
#!/bin/bash -x
## Controlling Batch-job
#SBATCH --account=deepacf
#SBATCH --nodes=1
#SBATCH --ntasks=1
##SBATCH --ntasks-per-node=13
#SBATCH --cpus-per-task=1
#SBATCH --output=meta_postprocess_era5-out.%j
#SBATCH --error=meta_postprocess_era5-err.%j
#SBATCH --time=00:20:00
#SBATCH --partition=batch
#SBATCH --gres=gpu:0
#SBATCH --mail-type=ALL
#SBATCH --mail-user=me@somewhere.com
######### Template identifier (don't remove) #########
echo "Do not run the template scripts"
exit 99
######### Template identifier (don't remove) #########
jutil env activate -p deepacf
# Declare input parameters
root_dir=/p/project/deepacf/deeprain/video_prediction_shared_folder/
analysis_config=video_prediction_tools/meta_postprocess_config/meta_config.json
metric=mse
exp_id=test
enable_skill_scores=True
srun python ../main_scripts/main_meta_postprocess.py --root_dir ${root_dir} --analysis_config ${analysis_config} \
--metric ${metric} --exp_id ${exp_id} --enable_skill_scores ${enable_skill_scores}
#!/bin/bash -x
## Controlling Batch-job
#SBATCH --account=deepacf
#SBATCH --nodes=1
#SBATCH --ntasks=13
##SBATCH --ntasks-per-node=12
#SBATCH --cpus-per-task=1
#SBATCH --output=DataPreprocess_era5_step1-out.%j
#SBATCH --error=DataPreprocess_era5_step1-err.%j
#SBATCH --time=04:20:00
#SBATCH --gres=gpu:0
#SBATCH --partition=batch
#SBATCH --mail-type=ALL
#SBATCH --mail-user=me@somewhere.com
######### Template identifier (don't remove) #########
echo "Do not run the template scripts"
exit 99
######### Template identifier (don't remove) #########
# Name of virtual environment
VIRT_ENV_NAME="my_venv"
# Activate virtual environment if needed (and possible)
if [ -z ${VIRTUAL_ENV} ]; then
if [[ -f ../virtual_envs/${VIRT_ENV_NAME}/bin/activate ]]; then
echo "Activating virtual environment..."
source ../virtual_envs/${VIRT_ENV_NAME}/bin/activate
else
echo "ERROR: Requested virtual environment ${VIRT_ENV_NAME} not found..."
exit 1
fi
fi
# Loading mouldes
source ../env_setup/modules_preprocess+extract.sh
# select years and variables for dataset and define target domain
years=( "2015" )
variables=( "t2" "t2" "t2" )
sw_corner=( -999.9 -999.9)
nyx=( -999 -999 )
# set some paths
# note, that destination_dir is adjusted during runtime based on the data
source_dir=/my/path/to/extracted/data/
destination_dir=/my/path/to/pickle/files
# execute Python-scripts
for year in "${years[@]}"; do
echo "start preprocessing data for year ${year}"
srun python ../main_scripts/main_preprocess_data_step1.py \
--source_dir ${source_dir} --destination_dir ${destination_dir} --years "${year}" \
--vars "${variables[0]}" "${variables[1]}" "${variables[2]}" \
--sw_corner "${sw_corner[0]}" "${sw_corner[1]}" --nyx "${nyx[0]}" "${nyx[1]}"
done
#srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_split_data_multi_years.py --destination_dir ${destination_dir} --varnames T2 MSL gph500
#!/bin/bash -x
#SBATCH --account=deepacf
#SBATCH --nodes=1
#SBATCH --ntasks=1
##SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=1
#SBATCH --output=DataPreprocess_moving_mnist-out.%j
#SBATCH --error=DataPreprocess_moving_mnist-err.%j
#SBATCH --time=04:00:00
#SBATCH --partition=batch
#SBATCH --mail-type=ALL
#SBATCH --mail-user=me@somewhere.com
######### Template identifier (don't remove) #########
echo "Do not run the template scripts"
exit 99
######### Template identifier (don't remove) #########
# Name of virtual environment
VIRT_ENV_NAME="my_venv"
# Loading mouldes
source ../env_setup/modules_train.sh
# Activate virtual environment if needed (and possible)
if [ -z ${VIRTUAL_ENV} ]; then
if [[ -f ../${VIRT_ENV_NAME}/bin/activate ]]; then
echo "Activating virtual environment..."
source ../${VIRT_ENV_NAME}/bin/activate
else
echo "ERROR: Requested virtual environment ${VIRT_ENV_NAME} not found..."
exit 1
fi
fi
# declare directory-variables which will be modified generate_runscript.py
source_dir=/my/path/to/mnist/raw/data/
destination_dir=/my/path/to/mnist/tfrecords/
# run Preprocessing (step 2 where Tf-records are generated)
# run postprocessing/generation of model results including evaluation metrics
export CUDA_VISIBLE_DEVICES=0
## One node, single GPU
srun --mpi=pspmix --cpu-bind=none \
singularity exec --nv "${CONTAINER_IMG}" "${WRAPPER}" ${VIRT_ENV_NAME} \
python3 ../video_prediction/datasets/moving_mnist.py ${source_dir} ${destination_dir}
...@@ -19,7 +19,26 @@ check_argin() { ...@@ -19,7 +19,26 @@ check_argin() {
if [[ $argin == *"-base_dir="* ]]; then if [[ $argin == *"-base_dir="* ]]; then
base_outdir=${argin#"-base_dir="} base_outdir=${argin#"-base_dir="}
fi fi
if [[ $argin == *"-tf_container="* ]]; then
TF_CONTAINER_NAME=${argin#"-tf_container="}
fi
if [[ $argin == *"-l_nocontainer"* ]]; then
bool_container=0
fi
if [[ $argin == *"-l_nohpc"* ]]; then
bool_hpc=0
fi
done done
if [[ -z "${bool_container}" ]]; then
bool_container=1
fi
if [[ -z "${bool_hpc}" ]]; then
bool_hpc=1
fi
# in case that no TF-container is set manually, set the default
if [[ -z "${TF_CONTAINER_NAME}" ]]; then
TF_CONTAINER_NAME="tensorflow_21.09-tf1-py3.sif"
fi
} }
# **************** Auxiliary functions **************** # **************** Auxiliary functions ****************
...@@ -38,9 +57,7 @@ if [[ -z "$1" ]]; then ...@@ -38,9 +57,7 @@ if [[ -z "$1" ]]; then
return return
fi fi
if [[ "$#" -gt 1 ]]; then check_argin ${@:2} # sets further variables
check_argin ${@:2} # sets base_outdir if provided
fi
# set some variables # set some variables
HOST_NAME="$(hostname)" HOST_NAME="$(hostname)"
...@@ -49,12 +66,15 @@ THIS_DIR="$(pwd)" ...@@ -49,12 +66,15 @@ THIS_DIR="$(pwd)"
WORKING_DIR="$(dirname "$THIS_DIR")" WORKING_DIR="$(dirname "$THIS_DIR")"
EXE_DIR="$(basename "$THIS_DIR")" EXE_DIR="$(basename "$THIS_DIR")"
ENV_DIR=${WORKING_DIR}/virtual_envs/${ENV_NAME} ENV_DIR=${WORKING_DIR}/virtual_envs/${ENV_NAME}
TF_CONTAINER=${WORKING_DIR}/HPC_scripts/tensorflow_21.09-tf1-py3.sif TF_CONTAINER=${WORKING_DIR}/HPC_scripts/${TF_CONTAINER_NAME}
if [[ ${bool_hpc} == 0 ]]; then
TF_CONTAINER=${WORKING_DIR}/no_HPC_scripts/${TF_CONTAINER_NAME}
fi
## perform sanity checks ## perform sanity checks
modules_purge="" modules_purge=""
if [[ ! -f ${TF_CONTAINER} ]]; then if [[ ! -f ${TF_CONTAINER} ]] && [[ ${bool_container} == 1 ]]; then
echo "ERROR: Cannot find required TF1.15 container image '${TF_CONTAINER}'." echo "ERROR: Cannot find required TF1.15 container image '${TF_CONTAINER}'."
return return
fi fi
...@@ -70,10 +90,15 @@ if [[ "${EXE_DIR}" != "env_setup" ]]; then ...@@ -70,10 +90,15 @@ if [[ "${EXE_DIR}" != "env_setup" ]]; then
fi fi
if ! [[ "${HOST_NAME}" == hdfml* || "${HOST_NAME}" == *jwlogin* ]]; then if ! [[ "${HOST_NAME}" == hdfml* || "${HOST_NAME}" == *jwlogin* ]]; then
echo "ERROR: AMBS-workflow is currently only supported on the Juelich HPC-systems HDF-ML, Juwels and Juwels Booster" if [[ ${bool_container} == 0 ]]; then
return echo "Execution without container. Please ensure that you fulfill the software requirements for Preprocessing."
# unset PYTHONPATH on every other machine that is not a known HPC-system if [[ ${bool_hpc} == 1 ]]; then
# unset PYTHONPATH echo "Make use of modules provided on your HPC-system if possible, i.e. adapt modules_preprocess.sh and modules_train.sh."
fi
fi
if [[ ${bool_hpc} == 0 ]]; then
echo "Running on a non-HPC system. Ensure that you fulfill the software requirements on your machine, e.g. CDO."
fi
fi fi
if [[ -d ${ENV_DIR} ]]; then if [[ -d ${ENV_DIR} ]]; then
...@@ -89,18 +114,37 @@ if [[ "$ENV_EXIST" == 0 ]]; then ...@@ -89,18 +114,37 @@ if [[ "$ENV_EXIST" == 0 ]]; then
# Activate virtual environment and install additional Python packages. # Activate virtual environment and install additional Python packages.
echo "Configuring and activating virtual environment on ${HOST_NAME}" echo "Configuring and activating virtual environment on ${HOST_NAME}"
if [[ ${bool_container} == 1 ]]; then
if [[ ${bool_hpc} == 1 ]]; then
module purge module purge
fi
singularity exec --nv "${TF_CONTAINER}" ./install_venv_container.sh "${ENV_DIR}" singularity exec --nv "${TF_CONTAINER}" ./install_venv_container.sh "${ENV_DIR}"
info_str="Virtual environment ${ENV_DIR} has been set up successfully." info_str="Virtual environment ${ENV_DIR} has been set up successfully."
else
if [[ ${bool_hpc} == 1 ]]; then
source ${THIS_DIR}/modules_train.sh
fi
unset PYTHONPATH
./install_venv.sh "${ENV_DIR}"
# Activate virtual environment again
source "${ENV_DIR}/bin/activate"
if [[ ${bool_hpc} == 0 ]]; then
pip3 install --no-cache-dir tensorflow==1.13.1
fi
fi
elif [[ "$ENV_EXIST" == 1 ]]; then elif [[ "$ENV_EXIST" == 1 ]]; then
info_str="Virtual environment ${ENV_DIR} already exists." info_str="Virtual environment ${ENV_DIR} already exists."
fi fi
## load modules (for running runscript-generator... ## load modules (for running runscript-generator...
echo "${info_str}" echo "${info_str}"
if [[ ${bool_hpc} == 1 ]]; then
echo "Load modules to enable running of runscript generator '${ENV_DIR}'." echo "Load modules to enable running of runscript generator '${ENV_DIR}'."
source ${THIS_DIR}/modules_preprocess+extract.sh source ${THIS_DIR}/modules_preprocess+extract.sh
fi
## ... and prepare runscripts ## ... and prepare runscripts
echo "Set up runscript template for user ${USER}..." echo "Set up runscript template for user ${USER}..."
......
#!/usr/bin/env bash
#
# __authors__ = Bing Gong
# __date__ = '2022_02_20'
unset PYTHONPATH
ENV_NAME=$1
THIS_DIR="$(pwd)"
WORKING_DIR="$(dirname "$THIS_DIR")"
VENV_BASE=${WORKING_DIR}/virtual_envs
VENV_DIR=${WORKING_DIR}/virtual_envs/${ENV_NAME}
ACT_VENV="${VENV_DIR}/bin/activate"
# check if directory to virtual environment is parsed
if [ -z "$1" ]; then
echo "ERROR: Provide a name to set up the virtual environment."
return
fi
#Create virtual enviornment
if ! [[ -d "${VENV_BASE}" ]]; then
mkdir "${VENV_BASE}"
echo "Installing virtualenv under ${VENV_BASE}..."
cd "${VENV_BASE}"
python3 -m virtualenv -p python3 ${ENV_NAME}
#activate source directory
source ${VENV_DIR}/bin/activate
fi
#Install site packages
pip install --no-cache-dir -r requirements_non_HPC.txt
echo "The site-packages is installed for non_HPC users"
## Add modules from the project
unset PYTHONPATH
export PYTHONPATH=${WORKING_DIR}:$PYTHONPATH
export PYTHONPATH=${WORKING_DIR}/utils:$PYTHONPATH
export PYTHONPATH=${WORKING_DIR}/model_modules:$PYTHONPATH
export PYTHONPATH=${WORKING_DIR}/postprocess:$PYTHONPATH
#ensure the PYTHONPATH is appended when activating the virtual enviornemnt
echo 'export PYTHONPATH='${WORKING_DIR}':$PYTHONPATH' >> ${ACT_VENV}
echo 'export PYTHONPATH='${WORKING_DIR}'/utils:$PYTHONPATH' >> ${ACT_VENV}
echo 'export PYTHONPATH='${WORKING_DIR}'/model_modules:$PYTHONPATH' >> ${ACT_VENV}
echo 'export PYTHONPATH='${WORKING_DIR}'/postprocess:$PYTHONPATH' >> ${ACT_VENV}
# get back to basic directory
cd "${WORKING_DIR}" || exit
#!/usr/bin/env bash
#
# __authors__ = Bing Gong, Michael Langguth
# __date__ = '2022_02_28'
# __last_update__ = '2022_02_28' by Michael Langguth
#
# **************** Description ****************
# This auxiliary script sets up the virtual environment OUTSIDE singularity container.
# **************** Description ****************
# set some basic variables
BASE_DIR="$(pwd)"
VENV_DIR=$1
VENV_NAME="$(basename "${VENV_DIR}")"
VENV_BASE="$(dirname "${VENV_DIR}")"
WORKING_DIR="$(dirname "${VENV_BASE}")"
VENV_REQ=${BASE_DIR}/requirements_nocontainer.txt
# sanity checks
# check if directory to virtual environment is parsed
if [ -z "$1" ]; then
echo "ERROR: Provide a name to set up the virtual environment."
return
fi
# check if virtual environment is not already existing
if [ -d "$1" ]; then
echo "ERROR: Target directory of virtual environment ${1} already exists. Chosse another directory path."
return
fi
# check for requirement-file
if [ ! -f "${VENV_REQ}" ]; then
echo "ERROR: Cannot find requirement-file '${VENV_REQ}' to set up virtual environment."
return
fi
# get Python-version
PYTHON_VERSION=$(python3 -c 'import sys; version=sys.version_info[:2]; print("{0}.{1}".format(*version))')
unset PYTHONPATH
# create or change to base directory for virtual environment (i.e. where the virtualenv-module is placed)
if ! [[ -d "${VENV_BASE}" ]]; then
mkdir "${VENV_BASE}"
# Install virtualenv in this directory
echo "Installing virtualenv under ${VENV_BASE}..."
pip3 install --target="${VENV_BASE}/" virtualenv
# Change into the base-directory of virtual environments...
cd "${VENV_BASE}" || return
else
# Change into the base-directory of virtual environments...
cd "${VENV_BASE}" || return
if ! python3 -m virtualenv --version >/dev/null; then
echo "WARNING: Base directory for virtual environment exists, but virtualenv-module is unavailable."
echo "Try installing virtualenv."
pip3 install --target="${VENV_BASE}/" virtualenv
fi
echo "Virtualenv is already installed."
fi
# Set-up virtual environment in base directory for virtual environments
python3 -m virtualenv "${VENV_NAME}"
# Activate virtual environment and install required packages
echo "Activating virtual environment ${VENV_NAME} to install required Python modules..."
ACT_VENV="${VENV_DIR}/bin/activate"
source "${VENV_DIR}/bin/activate"
# set PYTHONPATH...
export PYTHONPATH=""
export PYTHONPATH=${WORKING_DIR}/virtual_envs/${VENV_NAME}/lib/python${PYTHON_VERSION}/site-packages:$PYTHONPATH
export PYTHONPATH=${WORKING_DIR}:$PYTHONPATH
export PYTHONPATH=${WORKING_DIR}/utils:$PYTHONPATH
export PYTHONPATH=${WORKING_DIR}/model_modules:$PYTHONPATH
export PYTHONPATH=${WORKING_DIR}/postprocess:$PYTHONPATH
# ... also ensure that PYTHONPATH is appended when activating the virtual environment...
echo 'export PYTHONPATH='"" >> ${ACT_VENV}
echo 'export PYTHONPATH='${WORKING_DIR}'/virtual_envs/'${VENV_NAME}'/lib/python'${PYTHON_VERSION}'/site-packages:$PYTHONPATH' >> ${ACT_VENV}
echo 'export PYTHONPATH='${WORKING_DIR}':$PYTHONPATH' >> ${ACT_VENV}
echo 'export PYTHONPATH='${WORKING_DIR}'/utils:$PYTHONPATH' >> ${ACT_VENV}
echo 'export PYTHONPATH='${WORKING_DIR}'/model_modules:$PYTHONPATH' >> ${ACT_VENV}
echo 'export PYTHONPATH='${WORKING_DIR}'/postprocess:$PYTHONPATH' >> ${ACT_VENV}
# ... install requirements
pip3 install --no-cache-dir -r "${VENV_REQ}"
# get back to basic directory
cd "${BASE_DIR}" || exit
...@@ -41,6 +41,9 @@ if [ ! -f "${VENV_REQ}" ]; then ...@@ -41,6 +41,9 @@ if [ ! -f "${VENV_REQ}" ]; then
return return
fi fi
# get Python-version
PYTHON_VERSION=$(python3 -c 'import sys; version=sys.version_info[:2]; print("{0}.{1}".format(*version))')
# create or change to base directory for virtual environment (i.e. where the virtualenv-module is placed) # create or change to base directory for virtual environment (i.e. where the virtualenv-module is placed)
if ! [[ -d "${VENV_BASE}" ]]; then if ! [[ -d "${VENV_BASE}" ]]; then
mkdir "${VENV_BASE}" mkdir "${VENV_BASE}"
...@@ -53,8 +56,9 @@ else ...@@ -53,8 +56,9 @@ else
# Change into the base-directory of virtual environments... # Change into the base-directory of virtual environments...
cd "${VENV_BASE}" || return cd "${VENV_BASE}" || return
if ! python -m virtualenv --version >/dev/null; then if ! python -m virtualenv --version >/dev/null; then
echo "ERROR: Base directory for virtual environment exists, but virtualenv-module is unavailable." echo "WARNING: Base directory for virtual environment exists, but virtualenv-module is unavailable."
exit echo "Try installation."
pip3 install --target="${VENV_BASE}"/ virtualenv
fi fi
echo "Virtualenv is already installed." echo "Virtualenv is already installed."
fi fi
...@@ -67,8 +71,8 @@ echo "Actiavting virtual environment ${VENV_NAME} to install required Python mod ...@@ -67,8 +71,8 @@ echo "Actiavting virtual environment ${VENV_NAME} to install required Python mod
ACT_VENV="${VENV_DIR}/bin/activate" ACT_VENV="${VENV_DIR}/bin/activate"
source "${VENV_DIR}/bin/activate" source "${VENV_DIR}/bin/activate"
# set PYTHONPATH... # set PYTHONPATH...
export PYTHONPATH=/usr/local/lib/python3.8/dist-packages/:$PYTHONPATH export PYTHONPATH=/usr/local/lib/python${PYTHON_VERSION}/dist-packages/:$PYTHONPATH
export PYTHONPATH=${WORKING_DIR}/virtual_envs/${VENV_NAME}/lib/python3.8/site-packages:$PYTHONPATH export PYTHONPATH=${WORKING_DIR}/virtual_envs/${VENV_NAME}/lib/python${PYTHON_VERSION}/site-packages:$PYTHONPATH
export PYTHONPATH=${WORKING_DIR}:$PYTHONPATH export PYTHONPATH=${WORKING_DIR}:$PYTHONPATH
export PYTHONPATH=${WORKING_DIR}/utils:$PYTHONPATH export PYTHONPATH=${WORKING_DIR}/utils:$PYTHONPATH
export PYTHONPATH=${WORKING_DIR}/model_modules:$PYTHONPATH export PYTHONPATH=${WORKING_DIR}/model_modules:$PYTHONPATH
......
# Please populate this file when you run on an HPC-system without singularity containers
# You may refer to the file modules_preprocess+extract.sh as blue print which serves for the data extraction and the
# preprocessing step 1 on JSC's HPC_system.
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment