Skip to content
Snippets Groups Projects
Commit cf736df2 authored by gong1's avatar gong1
Browse files

Prepare the templatae for other HPC scripts

parent fe08acbc
Branches bing_issue#155_Non_JSC_HPC_scripts_prepration
Tags
No related merge requests found
Pipeline #91575 failed
...@@ -33,7 +33,7 @@ export PYTHONPATH=${WORKING_DIR}/utils:$PYTHONPATH ...@@ -33,7 +33,7 @@ export PYTHONPATH=${WORKING_DIR}/utils:$PYTHONPATH
export PYTHONPATH=${WORKING_DIR}/model_modules:$PYTHONPATH export PYTHONPATH=${WORKING_DIR}/model_modules:$PYTHONPATH
export PYTHONPATH=${WORKING_DIR}/postprocess:$PYTHONPATH export PYTHONPATH=${WORKING_DIR}/postprocess:$PYTHONPATH
# ... install requirements # ... install requirements
pip install --no-cache-dir -r ../env_setup/requirements_nonJSC.txt pip install --no-cache-dir -r ../env_setup/requirements.txt
# Declare path-variables (dest_dir will be set and configured automatically via generate_runscript.py) # Declare path-variables (dest_dir will be set and configured automatically via generate_runscript.py)
......
#!/bin/bash -x
## Controlling Batch-job: Need input
#SBATCH --account=<Project name>
#SBATCH --nodes=1
#SBATCH --ntasks=13
##SBATCH --ntasks-per-node=13
#SBATCH --cpus-per-task=1
#SBATCH --output=Data_Preprocess_step1_era5-out.%j
#SBATCH --error=Data_Preprocess_step1era5-err.%j
#SBATCH --time=04:20:00
#SBATCH --partition=batch
#SBATCH --gres=gpu:0
#SBATCH --mail-type=ALL
#SBATCH --mail-user=me@somewhere.com
##Load basic Python module: Need input
#module load Python
##Create and activate a virtual environment: Need input
#VENV_NAME=<my_venv>
#Python -m venv ../virtual_envs/${VENV_NAME}
#source ../virtual_envs/${VENV_NAME}/bin/activate
## Install required packages
# set PYTHONPATH...
WORKING_DIR="$(pwd)"
BASE_DIR=="$(WORKING_DIR "$dir")"
export PYTHONPATH=${BASE_DIR}/virtual_envs/${VENV_NAME}/lib/python3.8/site-packages:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/utils:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/model_modules:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/postprocess:$PYTHONPATH
# ... install requirements
pip install --no-cache-dir -r ../env_setup/requirements.txt
# Name of virtual environment
VENV_NAME=venv_hdfml
# Name of container image (must be available in working directory)
CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif"
WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh"
# sanity checks
if [[ ! -f ${CONTAINER_IMG} ]]; then
echo "ERROR: Cannot find required TF1.15 container image '${CONTAINER_IMG}'."
exit 1
fi
if [[ ! -f ${WRAPPER} ]]; then
echo "ERROR: Cannot find wrapper-script '${WRAPPER}' for TF1.15 container image."
exit 1
fi
# Declare input parameters
root_dir=/p/project/deepacf/deeprain/video_prediction_shared_folder/
analysis_config=video_prediction_tools/meta_postprocess_config/meta_config.json
metric=mse
exp_id=test
enable_skill_scores=True
srun python ../main_scripts/main_meta_postprocess.py --root_dir ${root_dir} --analysis_config ${analysis_config} \
--metric ${metric} --exp_id ${exp_id} --enable_skill_scores ${enable_skill_scores}
...@@ -33,7 +33,7 @@ export PYTHONPATH=${WORKING_DIR}/utils:$PYTHONPATH ...@@ -33,7 +33,7 @@ export PYTHONPATH=${WORKING_DIR}/utils:$PYTHONPATH
export PYTHONPATH=${WORKING_DIR}/model_modules:$PYTHONPATH export PYTHONPATH=${WORKING_DIR}/model_modules:$PYTHONPATH
export PYTHONPATH=${WORKING_DIR}/postprocess:$PYTHONPATH export PYTHONPATH=${WORKING_DIR}/postprocess:$PYTHONPATH
# ... install requirements # ... install requirements
pip install --no-cache-dir -r ../env_setup/requirements_nonJSC.txt pip install --no-cache-dir -r ../env_setup/requirements.txt
# select years for dataset # select years for dataset
......
...@@ -32,7 +32,7 @@ export PYTHONPATH=${BASE_DIR}/utils:$PYTHONPATH ...@@ -32,7 +32,7 @@ export PYTHONPATH=${BASE_DIR}/utils:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/model_modules:$PYTHONPATH export PYTHONPATH=${BASE_DIR}/model_modules:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/postprocess:$PYTHONPATH export PYTHONPATH=${BASE_DIR}/postprocess:$PYTHONPATH
# ... install requirements # ... install requirements
pip install --no-cache-dir -r ../env_setup/requirements_nonJSC.txt pip install --no-cache-dir -r ../env_setup/requirements.txt
# Name of virtual environment # Name of virtual environment
VENV_NAME=venv_hdfml VENV_NAME=venv_hdfml
......
...@@ -32,7 +32,7 @@ export PYTHONPATH=${BASE_DIR}/utils:$PYTHONPATH ...@@ -32,7 +32,7 @@ export PYTHONPATH=${BASE_DIR}/utils:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/model_modules:$PYTHONPATH export PYTHONPATH=${BASE_DIR}/model_modules:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/postprocess:$PYTHONPATH export PYTHONPATH=${BASE_DIR}/postprocess:$PYTHONPATH
# ... install requirements # ... install requirements
pip install --no-cache-dir -r ../env_setup/requirements_nonJSC.txt pip install --no-cache-dir -r ../env_setup/requirements.txt
# Name of virtual environment # Name of virtual environment
VENV_NAME=venv_hdfml VENV_NAME=venv_hdfml
......
#!/bin/bash -x #!/bin/bash -x
## Controlling Batch-job: Need input
#SBATCH --account=<Project name>
#SBATCH --nodes=1
#SBATCH --ntasks=13
##SBATCH --ntasks-per-node=13
#SBATCH --cpus-per-task=1
#SBATCH --output=Data_Preprocess_step1_era5-out.%j
#SBATCH --error=Data_Preprocess_step1era5-err.%j
#SBATCH --time=04:20:00
#SBATCH --partition=batch
#SBATCH --gres=gpu:0
#SBATCH --mail-type=ALL
#SBATCH --mail-user=me@somewhere.com
######### Template identifier (don't remove) ######### ##Load basic Python module: Need input
echo "Do not run the template scripts" #module load Python
exit 99
######### Template identifier (don't remove) #########
##Create and activate a virtual environment: Need input
#VENV_NAME=<my_venv>
#Python -m venv ../virtual_envs/${VENV_NAME}
#source ../virtual_envs/${VENV_NAME}/bin/activate
## Install required packages
# set PYTHONPATH...
WORKING_DIR="$(pwd)"
BASE_DIR=="$(WORKING_DIR "$dir")"
export PYTHONPATH=${BASE_DIR}/virtual_envs/${VENV_NAME}/lib/python3.8/site-packages:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/utils:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/model_modules:$PYTHONPATH
export PYTHONPATH=${BASE_DIR}/postprocess:$PYTHONPATH
# ... install requirements
pip install --no-cache-dir -r ../env_setup/requirements.txt
# Name of virtual environment
VENV_NAME=venv_hdfml
# Name of container image (must be available in working directory)
CONTAINER_IMG="${WORK_DIR}/tensorflow_21.09-tf1-py3.sif"
WRAPPER="${BASE_DIR}/env_setup/wrapper_container.sh"
# sanity checks
if [[ ! -f ${CONTAINER_IMG} ]]; then
echo "ERROR: Cannot find required TF1.15 container image '${CONTAINER_IMG}'."
exit 1
fi
if [[ ! -f ${WRAPPER} ]]; then
echo "ERROR: Cannot find wrapper-script '${WRAPPER}' for TF1.15 container image."
exit 1
fi
# clean-up modules to avoid conflicts between host and container settings
module purge
# declare directory-variables which will be modified by generate_runscript.py # declare directory-variables which will be modified by generate_runscript.py
# Note: source_dir is only needed for retrieving the base-directory # Note: source_dir is only needed for retrieving the base-directory
......
...@@ -6,4 +6,8 @@ basemap==1.3.0 ...@@ -6,4 +6,8 @@ basemap==1.3.0
numpy==1.17.3 # although this numpy-version is in the container, we set it here to avoid any further installation numpy==1.17.3 # although this numpy-version is in the container, we set it here to avoid any further installation
scikit-image==0.18.1 scikit-image==0.18.1
opencv-python-headless==4.2.0.34 opencv-python-headless==4.2.0.34
netcdf4 netcdf4==1.5.8
metadata==0.2
normalization==0.4
utils==1.0.1
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment