From c7062e764a656111f21d638b741b0dea713fcb51 Mon Sep 17 00:00:00 2001
From: Michael <m.langguth@fz-juelich.de>
Date: Mon, 8 Jun 2020 21:19:24 +0200
Subject: [PATCH] Implementation of 'dirty' workaround to get destinantion_dir
when data is splitted in 'mpi_split_data_multi_years.py' by setting env.
shell variable.
---
DataPreprocess/mpi_split_data_multi_years.py | 2 +-
DataPreprocess/mpi_stager_v2_process_netCDF.py | 12 ++++++++----
2 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/DataPreprocess/mpi_split_data_multi_years.py b/DataPreprocess/mpi_split_data_multi_years.py
index 363aa199..1494192c 100644
--- a/DataPreprocess/mpi_split_data_multi_years.py
+++ b/DataPreprocess/mpi_split_data_multi_years.py
@@ -12,7 +12,7 @@ parser.add_argument("--varnames","-vars",dest="varnames", nargs = '+')
# help="--partition allows to control the splitting of the processed data in training, test and validation data. Pass a dictionary-like string.")
args = parser.parse_args()
-target_dir = args.destination_dir
+target_dir = os.system("echo ${dest_dir_split}")#args.destination_dir
varnames = args.varnames
#partition = args.partition
diff --git a/DataPreprocess/mpi_stager_v2_process_netCDF.py b/DataPreprocess/mpi_stager_v2_process_netCDF.py
index ae93b982..b712218e 100755
--- a/DataPreprocess/mpi_stager_v2_process_netCDF.py
+++ b/DataPreprocess/mpi_stager_v2_process_netCDF.py
@@ -114,10 +114,14 @@ def main():
md = MetaData(suffix_indir=destination_dir,data_filename=data_files_list[0],slices=slices,variables=vars)
# modify Batch scripts
- md.write_dirs_to_batch_scripts(os.path.join(scr_dir,"DataPreprocess.sh"))
- #md.write_dirs_to_batch_scripts(os.path.join(scr_dir,"DataPreprocess_to_tf.sh"))
- #md.write_dirs_to_batch_scripts(os.path.join(scr_dir,"generate_era5.sh"))
- #md.write_dirs_to_batch_scripts(os.path.join(scr_dir,"train_era5.sh"))
+ md.write_dirs_to_batch_scripts(scr_dir+"/DataPreprocess.sh")
+ #md.write_dirs_to_batch_scripts(scr_dir+"DataPreprocess_to_tf.sh")
+ #md.write_dirs_to_batch_scripts(scr_dir+"generate_era5.sh")
+ #md.write_dirs_to_batch_scripts(scr_dir+"train_era5.sh")
+ # ML 2020/06/08: Dirty workaround as long as data-splitting is done with a seperate Python-script
+ # called from the same parent Shell-/Batch-script
+ # -> export env. variable to Shell which is read in
+ os.system("export dest_dir_split="+os.path.join(md.expdir,md.expname))
destination_dir= os.path.join(md.expdir,md.expname,years,"hickle")
--
GitLab