From d096cf347cb8332afd33c5e6881fa19badd36dd4 Mon Sep 17 00:00:00 2001
From: Bing Gong <gongbing1112@gmail.com>
Date: Tue, 10 Mar 2020 12:15:34 +0100
Subject: [PATCH] add bash folder
---
bash/download_and_preprocess_dataset_era5.sh | 125 +++++++++++++++++++
bash/workflow_era5.sh | 56 +++++++++
2 files changed, 181 insertions(+)
create mode 100644 bash/download_and_preprocess_dataset_era5.sh
create mode 100755 bash/workflow_era5.sh
diff --git a/bash/download_and_preprocess_dataset_era5.sh b/bash/download_and_preprocess_dataset_era5.sh
new file mode 100644
index 00000000..e71d4763
--- /dev/null
+++ b/bash/download_and_preprocess_dataset_era5.sh
@@ -0,0 +1,125 @@
+#!/usr/bin/env bash
+
+# exit if any command fails
+set -e
+
+
+#if [ "$#" -eq 2 ]; then
+# if [ $1 = "bair" ]; then
+# echo "IMAGE_SIZE argument is only applicable to kth dataset" >&2
+# exit 1
+# fi
+#elif [ "$#" -ne 1 ]; then
+# echo "Usage: $0 DATASET_NAME [IMAGE_SIZE]" >&2
+# exit 1
+#fi
+#if [ $1 = "bair" ]; then
+# TARGET_DIR=./data/bair
+# mkdir -p ${TARGET_DIR}
+# TAR_FNAME=bair_robot_pushing_dataset_v0.tar
+# URL=http://rail.eecs.berkeley.edu/datasets/${TAR_FNAME}
+# echo "Downloading '$1' dataset (this takes a while)"
+# #wget ${URL} -O ${TARGET_DIR}/${TAR_FNAME} Bing: on MacOS system , use curl instead of wget
+# curl ${URL} -O ${TARGET_DIR}/${TAR_FNAME}
+# tar -xvf ${TARGET_DIR}/${TAR_FNAME} --strip-components=1 -C ${TARGET_DIR}
+# rm ${TARGET_DIR}/${TAR_FNAME}
+# mkdir -p ${TARGET_DIR}/val
+# # reserve a fraction of the training set for validation
+# mv ${TARGET_DIR}/train/traj_256_to_511.tfrecords ${TARGET_DIR}/val/
+#elif [ $1 = "kth" ]; then
+# if [ "$#" -eq 2 ]; then
+# IMAGE_SIZE=$2
+# TARGET_DIR=./data/kth_${IMAGE_SIZE}
+# else
+# IMAGE_SIZE=64
+# fi
+# echo ${TARGET_DIR} ${IMAGE_SIZE}
+# mkdir -p ${TARGET_DIR}
+# mkdir -p ${TARGET_DIR}/raw
+# echo "Downloading '$1' dataset (this takes a while)"
+ # TODO Bing: for save time just use walking, need to change back if all the data are needed
+ #for ACTION in walking jogging running boxing handwaving handclapping; do
+# for ACTION in walking; do
+# echo "Action: '$ACTION' "
+# ZIP_FNAME=${ACTION}.zip
+# URL=http://www.nada.kth.se/cvap/actions/${ZIP_FNAME}
+# # wget ${URL} -O ${TARGET_DIR}/raw/${ZIP_FNAME}
+# echo "Start downloading action '$ACTION' ULR '$URL' "
+# curl ${URL} -O ${TARGET_DIR}/raw/${ZIP_FNAME}
+# unzip ${TARGET_DIR}/raw/${ZIP_FNAME} -d ${TARGET_DIR}/raw/${ACTION}
+# echo "Action '$ACTION' data download and unzip "
+# done
+# FRAME_RATE=25
+# mkdir -p ${TARGET_DIR}/processed
+# # download files with metadata specifying the subsequences
+# TAR_FNAME=kth_meta.tar.gz
+# URL=http://rail.eecs.berkeley.edu/models/savp/data/${TAR_FNAME}
+# echo "Downloading '${TAR_FNAME}' ULR '$URL' "
+# #wget ${URL} -O ${TARGET_DIR}/processed/${TAR_FNAME}
+# curl ${URL} -O ${TARGET_DIR}/processed/${TAR_FNAME}
+# tar -xzvf ${TARGET_DIR}/processed/${TAR_FNAME} --strip 1 -C ${TARGET_DIR}/processed
+ # convert the videos into sequence of downscaled images
+# echo "Processing '$1' dataset"
+# #TODO Bing, just use walking for test
+# #for ACTION in walking jogging running boxing handwaving handclapping; do
+# #Todo Bing: remove the comments below after testing
+# for ACTION in walking; do
+# for VIDEO_FNAME in ${TARGET_DIR}/raw/${ACTION}/*.avi; do
+# FNAME=$(basename ${VIDEO_FNAME})
+# FNAME=${FNAME%_uncomp.avi}
+# echo "FNAME '$FNAME' "
+# # sometimes the directory is not created, so try until it is
+# while [ ! -d "${TARGET_DIR}/processed/${ACTION}/${FNAME}" ]; do
+# mkdir -p ${TARGET_DIR}/processed/${ACTION}/${FNAME}
+# done
+# ffmpeg -i ${VIDEO_FNAME} -r ${FRAME_RATE} -f image2 -s ${IMAGE_SIZE}x${IMAGE_SIZE} \
+# ${TARGET_DIR}/processed/${ACTION}/${FNAME}/image-%03d_${IMAGE_SIZE}x${IMAGE_SIZE}.png
+# done
+# done
+# python video_prediction/datasets/kth_dataset.py ${TARGET_DIR}/processed ${TARGET_DIR} ${IMAGE_SIZE}
+# rm -rf ${TARGET_DIR}/raw
+# rm -rf ${TARGET_DIR}/processed
+
+while [[ $# -gt 0 ]] #of the number of passed argument is greater than 0
+do
+key="$1"
+case $key in
+ -d|--data)
+ DATA="$2"
+ shift
+ shift
+ ;;
+ -i|--input_dir)
+ INPUT_DIR="$2"
+ shift
+ shift
+ ;;
+ -o|--output_dir)
+ OUTPUT_DIR="$2"
+ shift
+ shift
+ ;;
+esac
+done
+
+echo "DATA = ${DATA} "
+echo "OUTPUT_DIRECTORY = ${OUTPUT_DIR}"
+
+if [ -d $INPUT_DIR ]; then
+ echo "INPUT DIRECTORY = ${INPUT_DIR}"
+
+else
+ echo "INPUT DIRECTORY '$INPUT_DIR' DOES NOT EXIST"
+ exit 1
+fi
+
+
+if [ $DATA = "era5" ]; then
+ mkdir -p ${OUTPUT_DIR}
+ python3 video_prediction/datasets/era5_dataset.py ${INPUT_DIR} ${OUTPUT_DIR}
+else
+ echo "dataset name: '$DATA' (choose from 'era5')" >&2
+ exit 1
+fi
+
+echo "Succesfully finished downloading and preprocessing dataset '$DATA' "
\ No newline at end of file
diff --git a/bash/workflow_era5.sh b/bash/workflow_era5.sh
new file mode 100755
index 00000000..a96cb259
--- /dev/null
+++ b/bash/workflow_era5.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -e
+
+model="savp"
+#train_mode: end_to_end, pre_trained, fine_tune
+train_mode="end_to_end"
+is_pretrain=False
+exp_name="era5_size_64_64_3_3t_norm"
+
+if [ $model=="savp" ]
+then
+ method_dir="ours_savp"
+elif [ $model=="gan" ]
+then
+ method_dir="ours_gan"
+elif [ $model=="vae" ]
+then
+ method_dir="ours_vae"
+else
+ echo "model does not exist" 2>&1
+ exit 1
+fi
+
+raw_dataset_input=./splits/${exp_name}
+prep_data_input=./data/${exp_name}
+train_output=./logs/${exp_name}
+results_output=./results_test_samples/${exp_name}/${method_dir}
+
+##############Datat Preprocessing################
+#For parallel on HPC
+python ../workflow_video_prediction/DataPreprocess/benchmark/mpi_stager_v2_process_netCDF.py
+#On local machine
+#python ...
+#Change the .hkl data to .tfrecords data
+python ./video_prediction/datasets/era5_dataset_v2.py ${raw_dataset_input} ${prep_data_input}
+
+#########Train##########################
+python ./scripts/train_v2.py --input_dir ${prep_data_input} --dataset era5 \
+--model savp --model_hparams_dict hparams/kth/${method_dir}/model_hparams.json \
+--output_dir logs/era5_64_64_3_3t_norm/${train_mode}/${method_dir}\
+
+#--checkpoint pretrained_models/kth/ours_savp
+#########Generate results#################
+python ./scripts/generate_transfer_learning_finetune.py --input_dir ${prep_data_input} \
+--dataset_hparams sequence_length=20 --checkpoint ${train_output}\
+--mode test --results_dir ${results_output} \
+--batch_size 4 --dataset era5
+
+
+
+
+
+
+
+
+
--
GitLab