diff --git a/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh b/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh index 1f6d7c2b879283c8bfcc04785ef003c84ef0d997..518fe84998745905092c0d7a7e62e97119d8909e 100644 --- a/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh +++ b/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh @@ -33,7 +33,7 @@ export PYTHONPATH=${WORKING_DIR}/utils:$PYTHONPATH export PYTHONPATH=${WORKING_DIR}/model_modules:$PYTHONPATH export PYTHONPATH=${WORKING_DIR}/postprocess:$PYTHONPATH # ... install requirements -pip install --no-cache-dir -r ../env_setup/requirements_nonJSC.txt +pip install --no-cache-dir -r ../env_setup/requirements.txt # Declare path-variables (dest_dir will be set and configured automatically via generate_runscript.py) diff --git a/video_prediction_tools/HPC_scripts/meta_postprocess_era5_template.sh b/video_prediction_tools/HPC_scripts/meta_postprocess_era5_template.sh new file mode 100644 index 0000000000000000000000000000000000000000..7d9dcd10cabf0b44ae75ead14711059c5c167d3c --- /dev/null +++ b/video_prediction_tools/HPC_scripts/meta_postprocess_era5_template.sh @@ -0,0 +1,62 @@ +#!/bin/bash -x +## Controlling Batch-job: Need input +#SBATCH --account=<Project name> +#SBATCH --nodes=1 +#SBATCH --ntasks=13 +##SBATCH --ntasks-per-node=13 +#SBATCH --cpus-per-task=1 +#SBATCH --output=Data_Preprocess_step1_era5-out.%j +#SBATCH --error=Data_Preprocess_step1era5-err.%j +#SBATCH --time=04:20:00 +#SBATCH --partition=batch +#SBATCH --gres=gpu:0 +#SBATCH --mail-type=ALL +#SBATCH --mail-user=me@somewhere.com + +##Load basic Python module: Need input +#module load Python + + +##Create and activate a virtual environment: Need input +#VENV_NAME=<my_venv> +#Python -m venv ../virtual_envs/${VENV_NAME} +#source ../virtual_envs/${VENV_NAME}/bin/activate + +## Install required packages +# set PYTHONPATH... +WORKING_DIR="$(pwd)" +BASE_DIR=="$(WORKING_DIR "$dir")" +export PYTHONPATH=${BASE_DIR}/virtual_envs/${VENV_NAME}/lib/python3.8/site-packages:$PYTHONPATH +export PYTHONPATH=${BASE_DIR}:$PYTHONPATH +export PYTHONPATH=${BASE_DIR}/utils:$PYTHONPATH +export PYTHONPATH=${BASE_DIR}/model_modules:$PYTHONPATH +export PYTHONPATH=${BASE_DIR}/postprocess:$PYTHONPATH +# ... install requirements +pip install --no-cache-dir -r ../env_setup/requirements.txt + +# Name of virtual environment +VENV_NAME=venv_hdfml +# Name of container image (must be available in working directory) +CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif" +WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh" + +# sanity checks +if [[ ! -f ${CONTAINER_IMG} ]]; then + echo "ERROR: Cannot find required TF1.15 container image '${CONTAINER_IMG}'." + exit 1 +fi + +if [[ ! -f ${WRAPPER} ]]; then + echo "ERROR: Cannot find wrapper-script '${WRAPPER}' for TF1.15 container image." + exit 1 +fi + +# Declare input parameters +root_dir=/p/project/deepacf/deeprain/video_prediction_shared_folder/ +analysis_config=video_prediction_tools/meta_postprocess_config/meta_config.json +metric=mse +exp_id=test +enable_skill_scores=True + +srun python ../main_scripts/main_meta_postprocess.py --root_dir ${root_dir} --analysis_config ${analysis_config} \ + --metric ${metric} --exp_id ${exp_id} --enable_skill_scores ${enable_skill_scores} diff --git a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh index a27ef326b3c55f9366d483c4621136a786c950c9..cc500654c49e4b619399aa8685d51b7299836d42 100644 --- a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh +++ b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh @@ -33,7 +33,7 @@ export PYTHONPATH=${WORKING_DIR}/utils:$PYTHONPATH export PYTHONPATH=${WORKING_DIR}/model_modules:$PYTHONPATH export PYTHONPATH=${WORKING_DIR}/postprocess:$PYTHONPATH # ... install requirements -pip install --no-cache-dir -r ../env_setup/requirements_nonJSC.txt +pip install --no-cache-dir -r ../env_setup/requirements.txt # select years for dataset diff --git a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh index a4ee9a4c8a344e7e4713af6e2e90f9b7db45be40..1afb89088c008666c974adfc1bebe96e0c68f169 100644 --- a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh +++ b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh @@ -32,7 +32,7 @@ export PYTHONPATH=${BASE_DIR}/utils:$PYTHONPATH export PYTHONPATH=${BASE_DIR}/model_modules:$PYTHONPATH export PYTHONPATH=${BASE_DIR}/postprocess:$PYTHONPATH # ... install requirements -pip install --no-cache-dir -r ../env_setup/requirements_nonJSC.txt +pip install --no-cache-dir -r ../env_setup/requirements.txt # Name of virtual environment VENV_NAME=venv_hdfml diff --git a/video_prediction_tools/HPC_scripts/train_model_era5_template.sh b/video_prediction_tools/HPC_scripts/train_model_era5_template.sh index 30e524ae6f42c11bcad0d739c331cc9571133a38..0f7b054908d09087dc266751157959f906e33fd8 100644 --- a/video_prediction_tools/HPC_scripts/train_model_era5_template.sh +++ b/video_prediction_tools/HPC_scripts/train_model_era5_template.sh @@ -32,7 +32,7 @@ export PYTHONPATH=${BASE_DIR}/utils:$PYTHONPATH export PYTHONPATH=${BASE_DIR}/model_modules:$PYTHONPATH export PYTHONPATH=${BASE_DIR}/postprocess:$PYTHONPATH # ... install requirements -pip install --no-cache-dir -r ../env_setup/requirements_nonJSC.txt +pip install --no-cache-dir -r ../env_setup/requirements.txt # Name of virtual environment VENV_NAME=venv_hdfml diff --git a/video_prediction_tools/HPC_scripts/visualize_postprocess_era5_template.sh b/video_prediction_tools/HPC_scripts/visualize_postprocess_era5_template.sh index 09a16556cc4bb5ee417f34ed0cc33a2c4156c54b..e7f169337b5bddb47fc62116bce6b2af96991d7d 100644 --- a/video_prediction_tools/HPC_scripts/visualize_postprocess_era5_template.sh +++ b/video_prediction_tools/HPC_scripts/visualize_postprocess_era5_template.sh @@ -1,12 +1,56 @@ #!/bin/bash -x +## Controlling Batch-job: Need input +#SBATCH --account=<Project name> +#SBATCH --nodes=1 +#SBATCH --ntasks=13 +##SBATCH --ntasks-per-node=13 +#SBATCH --cpus-per-task=1 +#SBATCH --output=Data_Preprocess_step1_era5-out.%j +#SBATCH --error=Data_Preprocess_step1era5-err.%j +#SBATCH --time=04:20:00 +#SBATCH --partition=batch +#SBATCH --gres=gpu:0 +#SBATCH --mail-type=ALL +#SBATCH --mail-user=me@somewhere.com -######### Template identifier (don't remove) ######### -echo "Do not run the template scripts" -exit 99 -######### Template identifier (don't remove) ######### +##Load basic Python module: Need input +#module load Python + + +##Create and activate a virtual environment: Need input +#VENV_NAME=<my_venv> +#Python -m venv ../virtual_envs/${VENV_NAME} +#source ../virtual_envs/${VENV_NAME}/bin/activate + +## Install required packages +# set PYTHONPATH... +WORKING_DIR="$(pwd)" +BASE_DIR=="$(WORKING_DIR "$dir")" +export PYTHONPATH=${BASE_DIR}/virtual_envs/${VENV_NAME}/lib/python3.8/site-packages:$PYTHONPATH +export PYTHONPATH=${BASE_DIR}:$PYTHONPATH +export PYTHONPATH=${BASE_DIR}/utils:$PYTHONPATH +export PYTHONPATH=${BASE_DIR}/model_modules:$PYTHONPATH +export PYTHONPATH=${BASE_DIR}/postprocess:$PYTHONPATH +# ... install requirements +pip install --no-cache-dir -r ../env_setup/requirements.txt + +# Name of virtual environment +VENV_NAME=venv_hdfml +# Name of container image (must be available in working directory) +CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif" +WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh" + +# sanity checks +if [[ ! -f ${CONTAINER_IMG} ]]; then + echo "ERROR: Cannot find required TF1.15 container image '${CONTAINER_IMG}'." + exit 1 +fi + +if [[ ! -f ${WRAPPER} ]]; then + echo "ERROR: Cannot find wrapper-script '${WRAPPER}' for TF1.15 container image." + exit 1 +fi -# clean-up modules to avoid conflicts between host and container settings -module purge # declare directory-variables which will be modified by generate_runscript.py # Note: source_dir is only needed for retrieving the base-directory diff --git a/video_prediction_tools/env_setup/requirements.txt b/video_prediction_tools/env_setup/requirements.txt index 28b7c6f83865095745ccab685b08c60aba8a71f9..35f6eb2439047f0697b04a0bac87bfb361fa0790 100755 --- a/video_prediction_tools/env_setup/requirements.txt +++ b/video_prediction_tools/env_setup/requirements.txt @@ -6,4 +6,8 @@ basemap==1.3.0 numpy==1.17.3 # although this numpy-version is in the container, we set it here to avoid any further installation scikit-image==0.18.1 opencv-python-headless==4.2.0.34 -netcdf4 +netcdf4==1.5.8 +metadata==0.2 +normalization==0.4 +utils==1.0.1 +