Skip to content
Snippets Groups Projects
Commit c789dbb4 authored by Michael Langguth's avatar Michael Langguth
Browse files

Merge branch 'michael_issue050_adapt_stat' into test_zam347

parents 54f8a4ba 31f002cd
Branches
Tags
No related merge requests found
......@@ -5,12 +5,15 @@ import json
#add parser arguments
parser = argparse.ArgumentParser()
#parser.add_argument("--source_dir", type=str, default="/p/scratch/deepacf/bing/extractedData/")
parser.add_argument("--destination_dir","-dest",dest="destination_dir",type=str, default="/p/scratch/deepacf/bing/processData_size_64_64_3_3t_norm")
parser.add_argument("--varnames","-vars",dest="varnames", nargs = '+')
#parser.add_argument("--partition","-part",dest="partition",type=json.loads)
# help="--partition allows to control the splitting of the processed data in training, test and validation data. Pass a dictionary-like string.")
#parser.add_argument("--partition", type=str, default="")
args = parser.parse_args()
target_dir = args.destination_dir
varnames = args.varnames
#partition = args.partition
#all_keys = partition.keys()
......@@ -33,6 +36,6 @@ comm = MPI.COMM_WORLD
my_rank = comm.Get_rank() # rank of the node
p = comm.Get_size() # number of assigned nods
if my_rank == 0: # node is master
split_data_multiple_years(target_dir=target_dir,partition=partition)
split_data_multiple_years(target_dir=target_dir,partition=partition,varnames=varnames)
else:
pass
......@@ -105,7 +105,6 @@ def main():
logging.critical('The Destination does not exist')
logging.info('Create new destination dir')
os.makedirs(destination_dir,exist_ok=True)
#if not os.path.exists(destination_dir): os.mkdir(destination_dir)
if my_rank == 0: # node is master:
# ==================================== Master : Directory scanner ================================= #
......@@ -151,10 +150,8 @@ def main():
logging.info(message_in)
message_counter = message_counter + 1
#Bing
split_data(target_dir=destination_dir, partition = [0.6, 0.2, 0.2])
# ML 2020/04/06 S
create_stat_json_master(destination_dir,p-idle_counter,vars)
# ML 2020/04/06 E
# ML 2020/05/19: Splitting now controlled from batch-script
# split_data(target_dir=destination_dir, partition = [0.6, 0.2, 0.2])
# stamp the end of the runtime
end = time.time()
......
This diff is collapsed.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment