diff --git a/run_wrf_dh.py b/run_wrf_dh.py
index e769338b254a04de8cad50d29e70f2e9d22953a5..315ca5ae5ea388753e5d227c06003e508ba4b8bc 100644
--- a/run_wrf_dh.py
+++ b/run_wrf_dh.py
@@ -8,7 +8,8 @@ from mlair.workflows import DefaultWorkflow
from mlair.helpers import remove_items
from mlair.configuration.defaults import DEFAULT_PLOT_LIST
-from mlair.model_modules.model_class import IntelliO3TsArchitecture, MyLSTMModel, MyCNNModel, MyLuongAttentionLSTMModel
+from mlair.model_modules.model_class import IntelliO3TsArchitecture, MyLSTMModel, MyCNNModel, MyCNNModelSect, MyLuongAttentionLSTMModel
+
import os
@@ -16,6 +17,7 @@ def load_stations():
import json
try:
filename = 'supplement/station_list_north_german_plain_rural.json'
+ filename = 'supplement/WRF_coord_list_from_IntelliO3.json'
with open(filename, 'r') as jfile:
stations = json.load(jfile)
except FileNotFoundError:
@@ -25,15 +27,11 @@ def load_stations():
def main(parser_args):
plots = remove_items(DEFAULT_PLOT_LIST, "PlotConditionalQuantiles")
- plots.append("PlotBootstrapSkillScore")
- workflow = DefaultWorkflow( # stations=load_stations(),
+ workflow = DefaultWorkflow( stations=load_stations(),
# stations=["DEBW087","DEBW013", "DEBW107", "DEBW076"],
-
+ lazy_preprocessing=False,
train_model=False, create_new_model=True, network="UBA",
- evaluate_bootstraps=True, number_of_bootstraps=30, create_new_bootstraps=True, #
- plot_list=plots,
- model_name_for_plots='SecModel',
- time_zone="UTC",
+ evaluate_bootstraps=False, # plot_list=["PlotCompetitiveSkillScore"],
# competitors=["test_model", "test_model2"],
# competitor_path=os.path.join(os.getcwd(), "data", "comp_test"),
competitors=["baseline", "sector_baseline"],
@@ -41,45 +39,35 @@ def main(parser_args):
train_min_length=1, val_min_length=1, test_min_length=1,
# data_handler=DataHandlerSingleStation,
# data_handler=DataHandlerSingleGridColumn,
- upsampeling=False,
- epochs=20,
+ epochs=100,
window_lead_time=4,
window_history_size=6,
-
- # ('Germany', (5.98865807458, 47.3024876979, 15.0169958839, 54.983104153))
- stations=["coords__48_8479__10_0963", "coords__51_8376__14_1417",
- # "coords__50_7536__7_0827", "coords__51_4070__6_9656",
- # "coords__49_8421__7_8662", "coords__49_7410__7_1935",
- # "coords__51_1566__11_8182", "coords__51_4065__6_9660",
-
- # "coords__50_7536__7_0827", "coords__54_0000__10_000",
-
- # "coords__50_7333__7_1000", "coords__50_0000__8_0000",
- # "coords__48_7444__7_6000", "coords__51_0000__11_0000",
- # "coords__52_7555__8_1000", "coords__50_0000__2_0000",
- # "coords__51_7666__8_6000", "coords__50_0000__3_0000",
- # "coords__45_7777__9_1000", "coords__50_0000__4_0000",
- ],
-
- # data_handler=DataHandlerWRF,
- data_handler=DataHandlerMainSectWRF, #,
- # data_handler=DataHandlerMainMinorSectWRF,
- # data_path='/home/felix/Data/WRF-Chem/upload_aura_2021-02-24/2009/',
- # data_path='/home/felix/Data/WRF-Chem/test_cut_nc/',
- # data_path='/home/felix/Data/WRF-Chem/test_cut_nc_joint',
- # data_path="/home/felix/Data/WRF-Chem/test_cut_nc_joint/short_test",
+# stations=["coords__48_8479__10_0963", "coords__51_8376__14_1417",
+# "coords__50_7536__7_0827", "coords__51_4070__6_9656",
+# "coords__49_8421__7_8662", "coords__49_7410__7_1935",
+# "coords__51_1566__11_8182", "coords__51_4065__6_9660",
+# "coords__50_7333__7_1000", "coords__50_0000__8_0000",
+# "coords__48_7444__7_6000", "coords__51_0000__11_0000",
+# "coords__52_7555__8_1000", "coords__50_0000__2_0000",
+# "coords__51_7666__8_6000", "coords__50_0000__3_0000",
+# "coords__45_7777__9_1000", "coords__50_0000__4_0000",
+# ],
+ data_handler=DataHandlerWRF,
+# data_handler=DataHandlerMainSectWRF, #,
+ # data_path="/p/scratch/deepacf/kleinert1/IASS_proc_monthyl",
+ #data_path="/p/scratch/deepacf/kleinert1/IASS_proc",
+ #data_path="/p/project/deepacf/intelliaq/kleinert1/DATA/WRF_CHEM_soft_ln_small_test",
# data_path="/media/felix/INTENSO/WRF_CHEM/hourly/cdo_output_test/jan_test",
- # data_path = "/p/scratch/deepacf/kleinert1/IASS_proc_monthyl",
- data_path="/media/felix/INTENSO/WRF_CHEM/JFM_2009",
-
- # external data coords
- external_coords_file='/home/felix/Data/WRF-Chem/test_cut_nc/coords.nc',
- # external_coords_file = "/p/scratch/deepacf/kleinert1/IASS_proc/coords.nc",
-
- date_format_of_nc_file="%Y-%m",
+ # data_path="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly2009",
+ # data_path="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly_count_test",
+ data_path = "/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly2009_2010-03",
+ #data_path="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly_01-03",
common_file_starter="wrfout_d01",
+ date_format_of_nc_file="%Y-%m",
time_dim='XTIME',
-
+ #external_coords_file='/p/project/deepacf/inbound_data/IASS_upload/coords.nc',
+ external_coords_file="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/coords.nc",
+ # external_coords_file="/media/felix/INTENSO/WRF_CHEM/monthly/coords.nc",
transformation={
"T2": {"method": "standardise"},
"Q2": {"method": "standardise"},
@@ -104,8 +92,6 @@ def main(parser_args):
'PSFC': "average_values", 'PBLH': "average_values",
# 'CLDFRA': "average_values",
},
- # separate_vars=["o3", "o3Sect", "o3SectLeft", "o3SectRight"],
- separate_vars=["o3", "o3Sect"],
# variables=['T2', 'Q2', 'PBLH', 'U10ll', 'V10ll', 'wdir10ll', 'wspd10ll'],
# target_var=["T2"],
# statistics_per_var={'T2': None, 'Q2': None, 'PBLH': None,
@@ -114,62 +100,52 @@ def main(parser_args):
var_logical_z_coord_selector=0,
targetvar_logical_z_coord_selector=0,
aggregation_dim='bottom_top',
+
radius=200, # km
start='2009-01-01',
# end='2009-01-04',
-
#end='2009-01-31',
- end='2009-03-31',
+ end='2010-03-31',
+ #train_start='2009-01-01',
+ #train_end='2009-01-02',
train_start='2009-01-01',
- # train_end='2009-01-02',
- # train_start='2009-01-01',
#train_end='2009-01-15',
- # train_end='2009-02-15',
- # train_end="2009-01-31",
- train_end='2009-02-28',
+ train_end='2009-10-15',
- # val_start='2009-01-02',
- # val_end='2009-01-03',
+ #val_start='2009-01-02',
+ #val_end='2009-01-03',
###################################
#val_start='2009-01-15',
#val_end='2009-01-22',
###################################
- val_start='2009-03-01',
- val_end='2009-03-14',
- ###################################
- # val_start='2009-02-15',
- # val_end='2009-03-02',
- ###################################
- # val_start='2009-02-01',
- # val_end='2009-02-28',
-
- # test_start='2009-01-03',
- # test_end='2009-01-04',
- ###################################
- # test_start='2009-01-22',
- # test_end='2009-01-31',
+ val_start='2009-10-16',
+ val_end='2009-12-14',
+ #test_start='2009-01-03',
+ #test_end='2009-01-04',
###################################
- test_start='2009-03-15',
- test_end='2009-03-31',
+ #test_start='2009-01-22',
+ #test_end='2009-01-31',
###################################
- # test_start='2009-03-02',
- # test_end='2009-03-31',
+ test_start='2009-12-15',
+ test_end='2010-03-31',
# sampling='hourly',
sampling="daily",
input_output_sampling4toarstats=("hourly", "daily"),
-
+ time_zone="UTC",
+ target_time_type="solar_time",
+ use_multiprocessing=True,
+
+ batch_size=64*2*2,
interpolation_limit=0,
as_image_like_data_format=False,
- # model=MyLSTMModel,
- model=MyLuongAttentionLSTMModel,
- use_multiprocessing=True,
- # as_image_like_data_format=True,
- # model=MyLSTMModel,
- # model=MyCNNModel,
+# model=MyLSTMModel,
+ model=MyLuongAttentionLSTMModel,
+# model=MyCNNModelSect,
+# model=MyCNNModel,
**parser_args.__dict__)
workflow.run()
diff --git a/run_wrf_dh_sector.py b/run_wrf_dh_sector.py
index 7c7fd3c8b06f9568b17124252879ee39b80feaeb..ff7545088c7239a264e96e30c8e3ccfad18985b1 100644
--- a/run_wrf_dh_sector.py
+++ b/run_wrf_dh_sector.py
@@ -8,7 +8,7 @@ from mlair.workflows import DefaultWorkflow
from mlair.helpers import remove_items
from mlair.configuration.defaults import DEFAULT_PLOT_LIST
-from mlair.model_modules.model_class import IntelliO3TsArchitecture, MyLSTMModel, MyCNNModel, MyCNNModelSect
+from mlair.model_modules.model_class import IntelliO3TsArchitecture, MyLSTMModel, MyCNNModel, MyCNNModelSect, MyLuongAttentionLSTMModel
import os
@@ -17,6 +17,7 @@ def load_stations():
import json
try:
filename = 'supplement/station_list_north_german_plain_rural.json'
+ filename = 'supplement/WRF_coord_list_from_IntelliO3.json'
with open(filename, 'r') as jfile:
stations = json.load(jfile)
except FileNotFoundError:
@@ -26,9 +27,9 @@ def load_stations():
def main(parser_args):
plots = remove_items(DEFAULT_PLOT_LIST, "PlotConditionalQuantiles")
- workflow = DefaultWorkflow( # stations=load_stations(),
+ workflow = DefaultWorkflow( stations=load_stations(),
# stations=["DEBW087","DEBW013", "DEBW107", "DEBW076"],
-
+ lazy_preprocessing=False,
train_model=False, create_new_model=True, network="UBA",
evaluate_bootstraps=False, # plot_list=["PlotCompetitiveSkillScore"],
# competitors=["test_model", "test_model2"],
@@ -38,31 +39,35 @@ def main(parser_args):
train_min_length=1, val_min_length=1, test_min_length=1,
# data_handler=DataHandlerSingleStation,
# data_handler=DataHandlerSingleGridColumn,
- epochs=10,
+ epochs=100,
window_lead_time=4,
window_history_size=6,
- stations=["coords__48_8479__10_0963", "coords__51_8376__14_1417",
- # "coords__50_7536__7_0827", "coords__51_4070__6_9656",
- # "coords__49_8421__7_8662", "coords__49_7410__7_1935",
- # "coords__51_1566__11_8182", "coords__51_4065__6_9660",
- # "coords__50_7333__7_1000", "coords__50_0000__8_0000",
- # "coords__48_7444__7_6000", "coords__51_0000__11_0000",
- # "coords__52_7555__8_1000", "coords__50_0000__2_0000",
- # "coords__51_7666__8_6000", "coords__50_0000__3_0000",
- # "coords__45_7777__9_1000", "coords__50_0000__4_0000",
- ],
+# stations=["coords__48_8479__10_0963", "coords__51_8376__14_1417",
+# "coords__50_7536__7_0827", "coords__51_4070__6_9656",
+# "coords__49_8421__7_8662", "coords__49_7410__7_1935",
+# "coords__51_1566__11_8182", "coords__51_4065__6_9660",
+# "coords__50_7333__7_1000", "coords__50_0000__8_0000",
+# "coords__48_7444__7_6000", "coords__51_0000__11_0000",
+# "coords__52_7555__8_1000", "coords__50_0000__2_0000",
+# "coords__51_7666__8_6000", "coords__50_0000__3_0000",
+# "coords__45_7777__9_1000", "coords__50_0000__4_0000",
+# ],
# data_handler=DataHandlerWRF,
data_handler=DataHandlerMainSectWRF, #,
# data_path="/p/scratch/deepacf/kleinert1/IASS_proc_monthyl",
#data_path="/p/scratch/deepacf/kleinert1/IASS_proc",
#data_path="/p/project/deepacf/intelliaq/kleinert1/DATA/WRF_CHEM_soft_ln_small_test",
- data_path="/media/felix/INTENSO/WRF_CHEM/hourly/cdo_output_test/jan_test",
+ # data_path="/media/felix/INTENSO/WRF_CHEM/hourly/cdo_output_test/jan_test",
+ # data_path="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly2009",
+ # data_path="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly_count_test",
+ data_path = "/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly2009_2010-03",
+ #data_path="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly_01-03",
common_file_starter="wrfout_d01",
date_format_of_nc_file="%Y-%m",
time_dim='XTIME',
#external_coords_file='/p/project/deepacf/inbound_data/IASS_upload/coords.nc',
- # external_coords_file="/p/scratch/deepacf/kleinert1/IASS_proc/coords.nc",
- external_coords_file="/media/felix/INTENSO/WRF_CHEM/monthly/coords.nc",
+ external_coords_file="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/coords.nc",
+ # external_coords_file="/media/felix/INTENSO/WRF_CHEM/monthly/coords.nc",
transformation={
"T2": {"method": "standardise"},
"Q2": {"method": "standardise"},
@@ -100,43 +105,47 @@ def main(parser_args):
start='2009-01-01',
# end='2009-01-04',
- end='2009-01-31',
- # end='2009-03-31',
+ #end='2009-01-31',
+ end='2010-03-31',
#train_start='2009-01-01',
#train_end='2009-01-02',
train_start='2009-01-01',
- train_end='2009-01-15',
- # train_end='2009-02-28',
+ #train_end='2009-01-15',
+ train_end='2009-10-15',
#val_start='2009-01-02',
#val_end='2009-01-03',
###################################
- val_start='2009-01-15',
- val_end='2009-01-22',
+ #val_start='2009-01-15',
+ #val_end='2009-01-22',
###################################
- # val_start='2009-03-01',
- # val_end='2009-03-14',
+ val_start='2009-10-16',
+ val_end='2009-12-14',
#test_start='2009-01-03',
#test_end='2009-01-04',
###################################
- test_start='2009-01-22',
- test_end='2009-01-31',
+ #test_start='2009-01-22',
+ #test_end='2009-01-31',
###################################
- # test_start='2009-03-15',
- # test_end='2009-03-31',
+ test_start='2009-12-15',
+ test_end='2010-03-31',
# sampling='hourly',
sampling="daily",
input_output_sampling4toarstats=("hourly", "daily"),
+ time_zone="UTC",
+ target_time_type="solar_time",
use_multiprocessing=True,
+ batch_size=64*2*2,
interpolation_limit=0,
-# as_image_like_data_format=False,
+ as_image_like_data_format=False,
# model=MyLSTMModel,
+ model=MyLuongAttentionLSTMModel,
# model=MyCNNModelSect,
- model=MyCNNModel,
+# model=MyCNNModel,
**parser_args.__dict__)
workflow.run()
diff --git a/run_wrfdh_sector3_hdfml.bash b/run_wrfdh_sector3_hdfml.bash
new file mode 100644
index 0000000000000000000000000000000000000000..e73380cfbded22fd23b93578d6bf861a2990d1b9
--- /dev/null
+++ b/run_wrfdh_sector3_hdfml.bash
@@ -0,0 +1,18 @@
+#!/bin/bash -x
+#SBATCH --account=deepacf
+#SBATCH --nodes=1
+#SBATCH --output=HPC_logging/mlt-out.%j
+#SBATCH --error=HPC_logging/mlt-err.%j
+#SBATCH --time=06:00:00
+#SBATCH --gres=gpu:4
+#SBATCH --mail-type=ALL
+#SBATCH --mail-user=f.kleinert@fz-juelich.de
+
+source HPC_setup/mlt_modules_hdfml.sh
+source venv_hdfml/bin/activate
+
+timestamp=`date +"%Y-%m-%d_%H%M-%S"`
+
+export PYTHONPATH=${PWD}/venv_hdfml/lib/python3.6/site-packages:${PYTHONPATH}
+
+srun --cpu-bind=none python run_wrf_dh_sector3.py --experiment_date=${timestamp}_WRF_sector