Merge branch 'michael_issue050_adapt_stat' into test_zam347

DataPreprocess/mpi_split_data_multi_years.py

@@ -5,12 +5,15 @@ import json
 
 #add parser arguments
 parser = argparse.ArgumentParser()
+#parser.add_argument("--source_dir", type=str, default="/p/scratch/deepacf/bing/extractedData/")
 parser.add_argument("--destination_dir","-dest",dest="destination_dir",type=str, default="/p/scratch/deepacf/bing/processData_size_64_64_3_3t_norm")
+parser.add_argument("--varnames","-vars",dest="varnames", nargs = '+')
 #parser.add_argument("--partition","-part",dest="partition",type=json.loads)
 #                    help="--partition allows to control the splitting of the processed data in training, test and validation data. Pass a dictionary-like string.")
-#parser.add_argument("--partition", type=str, default="")
+
 args = parser.parse_args()
 target_dir = args.destination_dir
+varnames = args.varnames
 
 #partition = args.partition
 #all_keys  = partition.keys()
@@ -33,6 +36,6 @@ comm = MPI.COMM_WORLD
 my_rank = comm.Get_rank()  # rank of the node
 p = comm.Get_size()  # number of assigned nods
 if my_rank == 0:  # node is master
-    split_data_multiple_years(target_dir=target_dir,partition=partition)
+    split_data_multiple_years(target_dir=target_dir,partition=partition,varnames=varnames)
 else:
     pass

DataPreprocess/mpi_stager_v2_process_netCDF.py

@@ -105,7 +105,6 @@ def main():
             logging.critical('The Destination does not exist')
             logging.info('Create new destination dir')
             os.makedirs(destination_dir,exist_ok=True)
-            #if not os.path.exists(destination_dir): os.mkdir(destination_dir)
 
     if my_rank == 0:  # node is master:
         # ==================================== Master : Directory scanner ================================= #
@@ -151,10 +150,8 @@ def main():
             logging.info(message_in)
             message_counter = message_counter + 1
         #Bing
-        split_data(target_dir=destination_dir, partition = [0.6, 0.2, 0.2])
-        # ML 2020/04/06 S
-        create_stat_json_master(destination_dir,p-idle_counter,vars)
-        # ML 2020/04/06 E 
+        # ML 2020/05/19: Splitting now controlled from batch-script
+        # split_data(target_dir=destination_dir, partition = [0.6, 0.2, 0.2])
 
         # stamp the end of the runtime
         end = time.time()