Implementation of 'dirty' workaround to get destinantion_dir when data is...

Implementation of 'dirty' workaround to get destinantion_dir when data is splitted in 'mpi_split_data_multi_years.py' by setting env. shell variable.

DataPreprocess/mpi_split_data_multi_years.py

@@ -12,7 +12,7 @@ parser.add_argument("--varnames","-vars",dest="varnames", nargs = '+')
 #                    help="--partition allows to control the splitting of the processed data in training, test and validation data. Pass a dictionary-like string.")
 
 args = parser.parse_args()
-target_dir = args.destination_dir
+target_dir = os.system("echo ${dest_dir_split}")#args.destination_dir
 varnames = args.varnames
 
 #partition = args.partition

DataPreprocess/mpi_stager_v2_process_netCDF.py

@@ -114,10 +114,14 @@ def main():
         
         md = MetaData(suffix_indir=destination_dir,data_filename=data_files_list[0],slices=slices,variables=vars)
         # modify Batch scripts
-        md.write_dirs_to_batch_scripts(os.path.join(scr_dir,"DataPreprocess.sh"))
-        #md.write_dirs_to_batch_scripts(os.path.join(scr_dir,"DataPreprocess_to_tf.sh"))
-        #md.write_dirs_to_batch_scripts(os.path.join(scr_dir,"generate_era5.sh"))
-        #md.write_dirs_to_batch_scripts(os.path.join(scr_dir,"train_era5.sh"))
+        md.write_dirs_to_batch_scripts(scr_dir+"/DataPreprocess.sh")
+        #md.write_dirs_to_batch_scripts(scr_dir+"DataPreprocess_to_tf.sh")
+        #md.write_dirs_to_batch_scripts(scr_dir+"generate_era5.sh")
+        #md.write_dirs_to_batch_scripts(scr_dir+"train_era5.sh")
+        # ML 2020/06/08: Dirty workaround as long as data-splitting is done with a seperate Python-script 
+        #                called from the same parent Shell-/Batch-script
+        #                -> export env. variable to Shell which is read in 
+        os.system("export dest_dir_split="+os.path.join(md.expdir,md.expname))
         
         
         destination_dir= os.path.join(md.expdir,md.expname,years,"hickle")