Dirty workaround with temproray json-file to communicate destination_dir to...

Dirty workaround with temproray json-file to communicate destination_dir to 'mpi_split_data_multi_years.py'.

DataPreprocess/mpi_split_data_multi_years.py

@@ -12,7 +12,11 @@ parser.add_argument("--varnames","-vars",dest="varnames", nargs = '+')
 #                    help="--partition allows to control the splitting of the processed data in training, test and validation data. Pass a dictionary-like string.")
 
 args = parser.parse_args()
-target_dir = os.system("echo ${dest_dir_split}")#args.destination_dir
+# ML 2020/06/08: Dirty workaround as long as data-splitting is done with this seperate Python-script 
+#                called from the same parent Shell-/Batch-script as 'mpi_stager_v2_process_netCDF.py'
+dict_dirty = json.load(os.system("pwd")"/temp")
+target_dir = dict_dirty["dest_dir_split"]
+print(target_dir)
 varnames = args.varnames
 
 #partition = args.partition

DataPreprocess/mpi_stager_v2_process_netCDF.py

@@ -120,8 +120,12 @@ def main():
         #md.write_dirs_to_batch_scripts(scr_dir+"train_era5.sh")
         # ML 2020/06/08: Dirty workaround as long as data-splitting is done with a seperate Python-script 
         #                called from the same parent Shell-/Batch-script
-        #                -> export env. variable to Shell which is read in 
-        os.system("export dest_dir_split="+os.path.join(md.expdir,md.expname))
+        #                -> temproary dictionary
+        dict_dirty = {"dest_dir_split": os.path.join(md.expdir,md.expname)}
+        print("Workaround for correct destination in data splitting: Write dictionary to json-file: temp'")
+        with open(os.system("pwd")"/temp",'w') as js_file:
+            json.dump(dict_dirty,js_file)
+        
         
         
         destination_dir= os.path.join(md.expdir,md.expname,years,"hickle")