From 26dd4c95f235b9b67cafc4c5d83dafd3b78f92e4 Mon Sep 17 00:00:00 2001
From: Michael <m.langguth@fz-juelich.de>
Date: Tue, 9 Jun 2020 09:00:41 +0200
Subject: [PATCH] Introduction of status-attribute for MetaData-class and some
minor clarifications in the source code.
---
.../mpi_stager_v2_process_netCDF.py | 37 ++++++++-----------
1 file changed, 15 insertions(+), 22 deletions(-)
diff --git a/DataPreprocess/mpi_stager_v2_process_netCDF.py b/DataPreprocess/mpi_stager_v2_process_netCDF.py
index dcd17bb8..84ee9c19 100755
--- a/DataPreprocess/mpi_stager_v2_process_netCDF.py
+++ b/DataPreprocess/mpi_stager_v2_process_netCDF.py
@@ -113,22 +113,21 @@ def main():
if not data_files_list: raise ValueError("Could not find any data to be processed in '"+source_dir+"'")
md = MetaData(suffix_indir=destination_dir,data_filename=data_files_list[0],slices=slices,variables=vars)
- # modify Batch scripts
- #md.write_dirs_to_batch_scripts(scr_dir+"/DataPreprocess.sh")
- md.write_dirs_to_batch_scripts(scr_dir+"/DataPreprocess_to_tf.sh")
- #md.write_dirs_to_batch_scripts(scr_dir+"generate_era5.sh")
- #md.write_dirs_to_batch_scripts(scr_dir+"train_era5.sh")
- # ML 2020/06/08: Dirty workaround as long as data-splitting is done with a seperate Python-script
- # called from the same parent Shell-/Batch-script
- # -> temproary dictionary
- dict_dirty = {"dest_dir_split": os.path.join(md.expdir,md.expname)}
- print("Workaround for correct destination in data splitting: Write dictionary to json-file: temp'")
- with open(os.getcwd()+"/temp.json",'w') as js_file:
- print("Create: '"+os.getcwd()+"/temp.json'")
- print(dict_dirty)
- json.dump(dict_dirty,js_file)
-
-
+ # modify Batch scripts if metadata has been retrieved for the first time (md.status = "new")
+ if (md.status = "new"):
+ #md.write_dirs_to_batch_scripts(scr_dir+"/DataPreprocess.sh")
+ md.write_dirs_to_batch_scripts(scr_dir+"/DataPreprocess_to_tf.sh")
+ #md.write_dirs_to_batch_scripts(scr_dir+"generate_era5.sh")
+ #md.write_dirs_to_batch_scripts(scr_dir+"train_era5.sh")
+ # ML 2020/06/08: Dirty workaround as long as data-splitting is done with a seperate Python-script
+ # called from the same parent Shell-/Batch-script
+ # -> temproary json-file in working directory
+ dict_dirty = {"dest_dir_split": os.path.join(md.expdir,md.expname)}
+ print("Workaround for correct destination in data splitting: Write dictionary to json-file: temp'")
+ with open(os.getcwd()+"/temp.json",'w') as js_file:
+ print("Create: '"+os.getcwd()+"/temp.json'")
+ json.dump(dict_dirty,js_file)
+ #else: nothing to do
destination_dir= os.path.join(md.expdir,md.expname,years,"hickle")
@@ -140,12 +139,6 @@ def main():
# ML 2020/04/24 E
- if not os.path.exists(destination_dir): # check if the Destination dir. is existing
- if my_rank == 0:
- logging.critical('The Destination does not exist')
- logging.info('Create new destination dir')
- os.makedirs(destination_dir,exist_ok=True)
-
if my_rank == 0: # node is master:
# ==================================== Master : Directory scanner ================================= #
--
GitLab