diff --git a/DataPreprocess/mpi_split_data_multi_years.py b/DataPreprocess/mpi_split_data_multi_years.py index 1494192ca11a5467cae7c4291e9156291600ab1a..a3f25d5617d41b1e255ee7c3e631b5d953f4b80c 100644 --- a/DataPreprocess/mpi_split_data_multi_years.py +++ b/DataPreprocess/mpi_split_data_multi_years.py @@ -12,7 +12,11 @@ parser.add_argument("--varnames","-vars",dest="varnames", nargs = '+') # help="--partition allows to control the splitting of the processed data in training, test and validation data. Pass a dictionary-like string.") args = parser.parse_args() -target_dir = os.system("echo ${dest_dir_split}")#args.destination_dir +# ML 2020/06/08: Dirty workaround as long as data-splitting is done with this seperate Python-script +# called from the same parent Shell-/Batch-script as 'mpi_stager_v2_process_netCDF.py' +dict_dirty = json.load(os.system("pwd")"/temp") +target_dir = dict_dirty["dest_dir_split"] +print(target_dir) varnames = args.varnames #partition = args.partition diff --git a/DataPreprocess/mpi_stager_v2_process_netCDF.py b/DataPreprocess/mpi_stager_v2_process_netCDF.py index b712218e8f23dfb1407c53814658b066d10f9ba6..1b4b7b8b3c4e6c9d2d62e2eed8bbf7f668e9595c 100755 --- a/DataPreprocess/mpi_stager_v2_process_netCDF.py +++ b/DataPreprocess/mpi_stager_v2_process_netCDF.py @@ -120,8 +120,12 @@ def main(): #md.write_dirs_to_batch_scripts(scr_dir+"train_era5.sh") # ML 2020/06/08: Dirty workaround as long as data-splitting is done with a seperate Python-script # called from the same parent Shell-/Batch-script - # -> export env. variable to Shell which is read in - os.system("export dest_dir_split="+os.path.join(md.expdir,md.expname)) + # -> temproary dictionary + dict_dirty = {"dest_dir_split": os.path.join(md.expdir,md.expname)} + print("Workaround for correct destination in data splitting: Write dictionary to json-file: temp'") + with open(os.system("pwd")"/temp",'w') as js_file: + json.dump(dict_dirty,js_file) + destination_dir= os.path.join(md.expdir,md.expname,years,"hickle")