From 28bc7d55e0d15d338a2f109c29e9070e50a08d2a Mon Sep 17 00:00:00 2001
From: Michael <m.langguth@fz-juelich.de>
Date: Fri, 28 Jan 2022 15:21:22 +0100
Subject: [PATCH] Harmonize modules for preprocessing and data extraction by
loading CDO-module and adapt runscripts and create_env.sh accordingly.
---
.../data_extraction_era5_template.sh | 2 +-
.../preprocess_data_era5_step1_template.sh | 2 +-
.../env_setup/create_env.sh | 4 +-
.../env_setup/modules_data_extraction.sh | 47 -------------------
...ocess.sh => modules_preprocess+extract.sh} | 0
.../config_preprocess_step1.py | 2 +-
6 files changed, 5 insertions(+), 52 deletions(-)
delete mode 100755 video_prediction_tools/env_setup/modules_data_extraction.sh
rename video_prediction_tools/env_setup/{modules_preprocess.sh => modules_preprocess+extract.sh} (100%)
diff --git a/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh b/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh
index 661eaccf..f856eb55 100644
--- a/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh
+++ b/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh
@@ -24,7 +24,7 @@ jutil env activate -p deepacf
VIRT_ENV_NAME="my_venv"
# Loading mouldes
-source ../env_setup/modules_data_extraction.sh
+source ../env_setup/modules_preprocess+extract.sh
# Activate virtual environment if needed (and possible)
if [ -z ${VIRTUAL_ENV} ]; then
if [[ -f ../${VIRT_ENV_NAME}/bin/activate ]]; then
diff --git a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh
index 9576eafc..a6da4643 100644
--- a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh
+++ b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh
@@ -32,7 +32,7 @@ if [ -z ${VIRTUAL_ENV} ]; then
fi
fi
# Loading mouldes
-source ../env_setup/modules_preprocess.sh
+source ../env_setup/modules_preprocess+extract.sh
# select years and variables for dataset and define target domain
diff --git a/video_prediction_tools/env_setup/create_env.sh b/video_prediction_tools/env_setup/create_env.sh
index fdfe3114..574a57e3 100755
--- a/video_prediction_tools/env_setup/create_env.sh
+++ b/video_prediction_tools/env_setup/create_env.sh
@@ -99,8 +99,8 @@ fi
## load modules (for running runscript-generator...
echo "${info_str}"
-echo "Load modules and activate virtual environment '${ENV_DIR}'"
-source ${THIS_DIR}/modules_preprocess.sh
+echo "Load modules to enable running of runscript generator '${ENV_DIR}'."
+source ${THIS_DIR}/modules_preprocess+extract.sh
## ... and prepare runscripts
echo "Set up runscript template for user ${USER}..."
diff --git a/video_prediction_tools/env_setup/modules_data_extraction.sh b/video_prediction_tools/env_setup/modules_data_extraction.sh
deleted file mode 100755
index 41a6e8af..00000000
--- a/video_prediction_tools/env_setup/modules_data_extraction.sh
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/env bash
-
-# __author__ = Bing Gong, Michael Langguth
-# __date__ = '2020_06_26'
-
-# This script loads the required modules for ambs on Juwels and HDF-ML.
-# Note that some other packages have to be installed into a venv (see create_env.sh and requirements.txt).
-
-HOST_NAME=`hostname`
-
-echo "Start loading modules on ${HOST_NAME} required for era5 data extraction..."
-echo "modules_data_etraction.sh is subject to: "
-echo "* data_extraction_era5_<exp_id>.sh"
-
-module purge
-# serialized version is not available on HFML
-# see https://gitlab.version.fz-juelich.de/haf/Wiki/-/wikis/HDF-ML%20System
-if [[ "${HOST_NAME}" == hdfml* ]]; then
- module use $OTHERSTAGES
- ml Stages/2019a
- ml GCC/8.3.0
- ml ParaStationMPI/5.2.2-1
- ml CDO/1.9.6
- ml mpi4py/3.0.1-Python-3.6.8
- ml SciPy-Stack/2019a-Python-3.6.8
- ml scikit/2019a-Python-3.6.8
- ml netcdf4-python/1.5.0.1-Python-3.6.8
-
-else
- module load Stages/2020
- ml GCC/9.3.0
- ml ParaStationMPI/5.4.7-1
- ml CDO/1.9.8
- ml mpi4py/3.0.3-Python-3.8.5
- echo "I am here"
- ml SciPy-Stack/2020-Python-3.8.5
- ml scikit/2020-Python-3.8.5
-fi
-#ml SciPy-Stack/2019a-Python-3.6.8
-#ml scikit/2019a-Python-3.6.8
-#ml netcdf4-python/1.5.0.1-Python-3.6.8
-
-# clean up if triggered via script argument
-if [[ $1 == purge ]]; then
- echo "Purge all modules after loading them..."
- module --force purge
-fi
diff --git a/video_prediction_tools/env_setup/modules_preprocess.sh b/video_prediction_tools/env_setup/modules_preprocess+extract.sh
similarity index 100%
rename from video_prediction_tools/env_setup/modules_preprocess.sh
rename to video_prediction_tools/env_setup/modules_preprocess+extract.sh
diff --git a/video_prediction_tools/utils/runscript_generator/config_preprocess_step1.py b/video_prediction_tools/utils/runscript_generator/config_preprocess_step1.py
index 025a203b..7a8f841e 100755
--- a/video_prediction_tools/utils/runscript_generator/config_preprocess_step1.py
+++ b/video_prediction_tools/utils/runscript_generator/config_preprocess_step1.py
@@ -10,7 +10,7 @@ try:
import xarray as xr
except:
raise ImportError("Loading preprocssing modules in advance is mandotory, " +
- "i.e. execute 'source modules_preprocess.sh' from env_setup-directory in terminal first.")
+ "i.e. execute 'source modules_preprocess+extract.sh' from env_setup-directory in terminal first.")
from netcdf_datahandling import NetcdfUtils
from general_utils import check_str_in_list
--
GitLab