From d8cd3a1fdfbcd9a734d36dd8d8d43587ffc12ed5 Mon Sep 17 00:00:00 2001
From: Michael <m.langguth@fz-juelich.de>
Date: Mon, 8 Jun 2020 22:09:16 +0200
Subject: [PATCH] Dirty workaround with temproray json-file to communicate
destination_dir to 'mpi_split_data_multi_years.py'.
---
DataPreprocess/mpi_split_data_multi_years.py | 6 +++++-
DataPreprocess/mpi_stager_v2_process_netCDF.py | 8 ++++++--
2 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/DataPreprocess/mpi_split_data_multi_years.py b/DataPreprocess/mpi_split_data_multi_years.py
index 1494192c..a3f25d56 100644
--- a/DataPreprocess/mpi_split_data_multi_years.py
+++ b/DataPreprocess/mpi_split_data_multi_years.py
@@ -12,7 +12,11 @@ parser.add_argument("--varnames","-vars",dest="varnames", nargs = '+')
# help="--partition allows to control the splitting of the processed data in training, test and validation data. Pass a dictionary-like string.")
args = parser.parse_args()
-target_dir = os.system("echo ${dest_dir_split}")#args.destination_dir
+# ML 2020/06/08: Dirty workaround as long as data-splitting is done with this seperate Python-script
+# called from the same parent Shell-/Batch-script as 'mpi_stager_v2_process_netCDF.py'
+dict_dirty = json.load(os.system("pwd")"/temp")
+target_dir = dict_dirty["dest_dir_split"]
+print(target_dir)
varnames = args.varnames
#partition = args.partition
diff --git a/DataPreprocess/mpi_stager_v2_process_netCDF.py b/DataPreprocess/mpi_stager_v2_process_netCDF.py
index b712218e..1b4b7b8b 100755
--- a/DataPreprocess/mpi_stager_v2_process_netCDF.py
+++ b/DataPreprocess/mpi_stager_v2_process_netCDF.py
@@ -120,8 +120,12 @@ def main():
#md.write_dirs_to_batch_scripts(scr_dir+"train_era5.sh")
# ML 2020/06/08: Dirty workaround as long as data-splitting is done with a seperate Python-script
# called from the same parent Shell-/Batch-script
- # -> export env. variable to Shell which is read in
- os.system("export dest_dir_split="+os.path.join(md.expdir,md.expname))
+ # -> temproary dictionary
+ dict_dirty = {"dest_dir_split": os.path.join(md.expdir,md.expname)}
+ print("Workaround for correct destination in data splitting: Write dictionary to json-file: temp'")
+ with open(os.system("pwd")"/temp",'w') as js_file:
+ json.dump(dict_dirty,js_file)
+
destination_dir= os.path.join(md.expdir,md.expname,years,"hickle")
--
GitLab