diff --git a/DataPreprocess/mpi_split_data_multi_years.py b/DataPreprocess/mpi_split_data_multi_years.py index 363aa19986db0821193be3aa8d1e31065b6b13fd..1494192ca11a5467cae7c4291e9156291600ab1a 100644 --- a/DataPreprocess/mpi_split_data_multi_years.py +++ b/DataPreprocess/mpi_split_data_multi_years.py @@ -12,7 +12,7 @@ parser.add_argument("--varnames","-vars",dest="varnames", nargs = '+') # help="--partition allows to control the splitting of the processed data in training, test and validation data. Pass a dictionary-like string.") args = parser.parse_args() -target_dir = args.destination_dir +target_dir = os.system("echo ${dest_dir_split}")#args.destination_dir varnames = args.varnames #partition = args.partition diff --git a/DataPreprocess/mpi_stager_v2_process_netCDF.py b/DataPreprocess/mpi_stager_v2_process_netCDF.py index ae93b98250afd0a58e4934ea5d08b30dd3e7a17d..b712218e8f23dfb1407c53814658b066d10f9ba6 100755 --- a/DataPreprocess/mpi_stager_v2_process_netCDF.py +++ b/DataPreprocess/mpi_stager_v2_process_netCDF.py @@ -114,10 +114,14 @@ def main(): md = MetaData(suffix_indir=destination_dir,data_filename=data_files_list[0],slices=slices,variables=vars) # modify Batch scripts - md.write_dirs_to_batch_scripts(os.path.join(scr_dir,"DataPreprocess.sh")) - #md.write_dirs_to_batch_scripts(os.path.join(scr_dir,"DataPreprocess_to_tf.sh")) - #md.write_dirs_to_batch_scripts(os.path.join(scr_dir,"generate_era5.sh")) - #md.write_dirs_to_batch_scripts(os.path.join(scr_dir,"train_era5.sh")) + md.write_dirs_to_batch_scripts(scr_dir+"/DataPreprocess.sh") + #md.write_dirs_to_batch_scripts(scr_dir+"DataPreprocess_to_tf.sh") + #md.write_dirs_to_batch_scripts(scr_dir+"generate_era5.sh") + #md.write_dirs_to_batch_scripts(scr_dir+"train_era5.sh") + # ML 2020/06/08: Dirty workaround as long as data-splitting is done with a seperate Python-script + # called from the same parent Shell-/Batch-script + # -> export env. variable to Shell which is read in + os.system("export dest_dir_split="+os.path.join(md.expdir,md.expname)) destination_dir= os.path.join(md.expdir,md.expname,years,"hickle")