Skip to content
Snippets Groups Projects
Commit 768ec348 authored by Felix Kleinert's avatar Felix Kleinert
Browse files

update wrf run_scripts

parent adf861d5
No related branches found
No related tags found
2 merge requests!333Merge run_scripts,!259Draft: Resolve "WRF-Datahandler should inherit from SingleStationDatahandler"
Hide whitespace changes
Inline Side-by-side
run_wrf_dh.py
+ 54
− 78
View file @ 768ec348
......@@ -8,7 +8,8 @@ from mlair.workflows import DefaultWorkflow
from mlair.helpers import remove_items
from mlair.configuration.defaults import DEFAULT_PLOT_LIST
from mlair.model_modules.model_class import IntelliO3TsArchitecture, MyLSTMModel, MyCNNModel, MyLuongAttentionLSTMModel
from mlair.model_modules.model_class import IntelliO3TsArchitecture, MyLSTMModel, MyCNNModel, MyCNNModelSect, MyLuongAttentionLSTMModel
import os
......@@ -16,6 +17,7 @@ def load_stations():
import json
try:
filename = 'supplement/station_list_north_german_plain_rural.json'
filename = 'supplement/WRF_coord_list_from_IntelliO3.json'
with open(filename, 'r') as jfile:
stations = json.load(jfile)
except FileNotFoundError:
......@@ -25,15 +27,11 @@ def load_stations():
def main(parser_args):
plots = remove_items(DEFAULT_PLOT_LIST, "PlotConditionalQuantiles")
plots.append("PlotBootstrapSkillScore")
workflow = DefaultWorkflow( # stations=load_stations(),
workflow = DefaultWorkflow( stations=load_stations(),
# stations=["DEBW087","DEBW013", "DEBW107", "DEBW076"],
lazy_preprocessing=False,
train_model=False, create_new_model=True, network="UBA",
evaluate_bootstraps=True, number_of_bootstraps=30, create_new_bootstraps=True, #
plot_list=plots,
model_name_for_plots='SecModel',
time_zone="UTC",
evaluate_bootstraps=False, # plot_list=["PlotCompetitiveSkillScore"],
# competitors=["test_model", "test_model2"],
# competitor_path=os.path.join(os.getcwd(), "data", "comp_test"),
competitors=["baseline", "sector_baseline"],
......@@ -41,45 +39,35 @@ def main(parser_args):
train_min_length=1, val_min_length=1, test_min_length=1,
# data_handler=DataHandlerSingleStation,
# data_handler=DataHandlerSingleGridColumn,
upsampeling=False,
epochs=20,
epochs=100,
window_lead_time=4,
window_history_size=6,
# ('Germany', (5.98865807458, 47.3024876979, 15.0169958839, 54.983104153))
stations=["coords__48_8479__10_0963", "coords__51_8376__14_1417",
# "coords__50_7536__7_0827", "coords__51_4070__6_9656",
# "coords__49_8421__7_8662", "coords__49_7410__7_1935",
# "coords__51_1566__11_8182", "coords__51_4065__6_9660",
# "coords__50_7536__7_0827", "coords__54_0000__10_000",
# "coords__50_7333__7_1000", "coords__50_0000__8_0000",
# "coords__48_7444__7_6000", "coords__51_0000__11_0000",
# "coords__52_7555__8_1000", "coords__50_0000__2_0000",
# "coords__51_7666__8_6000", "coords__50_0000__3_0000",
# "coords__45_7777__9_1000", "coords__50_0000__4_0000",
],
# data_handler=DataHandlerWRF,
data_handler=DataHandlerMainSectWRF, #,
# data_handler=DataHandlerMainMinorSectWRF,
# data_path='/home/felix/Data/WRF-Chem/upload_aura_2021-02-24/2009/',
# data_path='/home/felix/Data/WRF-Chem/test_cut_nc/',
# data_path='/home/felix/Data/WRF-Chem/test_cut_nc_joint',
# data_path="/home/felix/Data/WRF-Chem/test_cut_nc_joint/short_test",
# stations=["coords__48_8479__10_0963", "coords__51_8376__14_1417",
# "coords__50_7536__7_0827", "coords__51_4070__6_9656",
# "coords__49_8421__7_8662", "coords__49_7410__7_1935",
# "coords__51_1566__11_8182", "coords__51_4065__6_9660",
# "coords__50_7333__7_1000", "coords__50_0000__8_0000",
# "coords__48_7444__7_6000", "coords__51_0000__11_0000",
# "coords__52_7555__8_1000", "coords__50_0000__2_0000",
# "coords__51_7666__8_6000", "coords__50_0000__3_0000",
# "coords__45_7777__9_1000", "coords__50_0000__4_0000",
# ],
data_handler=DataHandlerWRF,
# data_handler=DataHandlerMainSectWRF, #,
# data_path="/p/scratch/deepacf/kleinert1/IASS_proc_monthyl",
#data_path="/p/scratch/deepacf/kleinert1/IASS_proc",
#data_path="/p/project/deepacf/intelliaq/kleinert1/DATA/WRF_CHEM_soft_ln_small_test",
# data_path="/media/felix/INTENSO/WRF_CHEM/hourly/cdo_output_test/jan_test",
# data_path = "/p/scratch/deepacf/kleinert1/IASS_proc_monthyl",
data_path="/media/felix/INTENSO/WRF_CHEM/JFM_2009",
# external data coords
external_coords_file='/home/felix/Data/WRF-Chem/test_cut_nc/coords.nc',
# external_coords_file = "/p/scratch/deepacf/kleinert1/IASS_proc/coords.nc",
date_format_of_nc_file="%Y-%m",
# data_path="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly2009",
# data_path="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly_count_test",
data_path = "/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly2009_2010-03",
#data_path="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly_01-03",
common_file_starter="wrfout_d01",
date_format_of_nc_file="%Y-%m",
time_dim='XTIME',
#external_coords_file='/p/project/deepacf/inbound_data/IASS_upload/coords.nc',
external_coords_file="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/coords.nc",
# external_coords_file="/media/felix/INTENSO/WRF_CHEM/monthly/coords.nc",
transformation={
"T2": {"method": "standardise"},
"Q2": {"method": "standardise"},
......@@ -104,8 +92,6 @@ def main(parser_args):
'PSFC': "average_values", 'PBLH': "average_values",
# 'CLDFRA': "average_values",
},
# separate_vars=["o3", "o3Sect", "o3SectLeft", "o3SectRight"],
separate_vars=["o3", "o3Sect"],
# variables=['T2', 'Q2', 'PBLH', 'U10ll', 'V10ll', 'wdir10ll', 'wspd10ll'],
# target_var=["T2"],
# statistics_per_var={'T2': None, 'Q2': None, 'PBLH': None,
......@@ -114,62 +100,52 @@ def main(parser_args):
var_logical_z_coord_selector=0,
targetvar_logical_z_coord_selector=0,
aggregation_dim='bottom_top',
radius=200, # km
start='2009-01-01',
# end='2009-01-04',
#end='2009-01-31',
end='2009-03-31',
end='2010-03-31',
#train_start='2009-01-01',
#train_end='2009-01-02',
train_start='2009-01-01',
# train_end='2009-01-02',
# train_start='2009-01-01',
#train_end='2009-01-15',
# train_end='2009-02-15',
# train_end="2009-01-31",
train_end='2009-02-28',
train_end='2009-10-15',
# val_start='2009-01-02',
# val_end='2009-01-03',
#val_start='2009-01-02',
#val_end='2009-01-03',
###################################
#val_start='2009-01-15',
#val_end='2009-01-22',
###################################
val_start='2009-03-01',
val_end='2009-03-14',
###################################
# val_start='2009-02-15',
# val_end='2009-03-02',
###################################
# val_start='2009-02-01',
# val_end='2009-02-28',
# test_start='2009-01-03',
# test_end='2009-01-04',
###################################
# test_start='2009-01-22',
# test_end='2009-01-31',
val_start='2009-10-16',
val_end='2009-12-14',
#test_start='2009-01-03',
#test_end='2009-01-04',
###################################
test_start='2009-03-15',
test_end='2009-03-31',
#test_start='2009-01-22',
#test_end='2009-01-31',
###################################
# test_start='2009-03-02',
# test_end='2009-03-31',
test_start='2009-12-15',
test_end='2010-03-31',
# sampling='hourly',
sampling="daily",
input_output_sampling4toarstats=("hourly", "daily"),
time_zone="UTC",
target_time_type="solar_time",
use_multiprocessing=True,
batch_size=64*2*2,
interpolation_limit=0,
as_image_like_data_format=False,
# model=MyLSTMModel,
model=MyLuongAttentionLSTMModel,
use_multiprocessing=True,
# as_image_like_data_format=True,
# model=MyLSTMModel,
# model=MyCNNModel,
# model=MyLSTMModel,
model=MyLuongAttentionLSTMModel,
# model=MyCNNModelSect,
# model=MyCNNModel,
**parser_args.__dict__)
workflow.run()
......
run_wrf_dh_sector.py
+ 40
− 31
View file @ 768ec348
......@@ -8,7 +8,7 @@ from mlair.workflows import DefaultWorkflow
from mlair.helpers import remove_items
from mlair.configuration.defaults import DEFAULT_PLOT_LIST
from mlair.model_modules.model_class import IntelliO3TsArchitecture, MyLSTMModel, MyCNNModel, MyCNNModelSect
from mlair.model_modules.model_class import IntelliO3TsArchitecture, MyLSTMModel, MyCNNModel, MyCNNModelSect, MyLuongAttentionLSTMModel
import os
......@@ -17,6 +17,7 @@ def load_stations():
import json
try:
filename = 'supplement/station_list_north_german_plain_rural.json'
filename = 'supplement/WRF_coord_list_from_IntelliO3.json'
with open(filename, 'r') as jfile:
stations = json.load(jfile)
except FileNotFoundError:
......@@ -26,9 +27,9 @@ def load_stations():
def main(parser_args):
plots = remove_items(DEFAULT_PLOT_LIST, "PlotConditionalQuantiles")
workflow = DefaultWorkflow( # stations=load_stations(),
workflow = DefaultWorkflow( stations=load_stations(),
# stations=["DEBW087","DEBW013", "DEBW107", "DEBW076"],
lazy_preprocessing=False,
train_model=False, create_new_model=True, network="UBA",
evaluate_bootstraps=False, # plot_list=["PlotCompetitiveSkillScore"],
# competitors=["test_model", "test_model2"],
......@@ -38,31 +39,35 @@ def main(parser_args):
train_min_length=1, val_min_length=1, test_min_length=1,
# data_handler=DataHandlerSingleStation,
# data_handler=DataHandlerSingleGridColumn,
epochs=10,
epochs=100,
window_lead_time=4,
window_history_size=6,
stations=["coords__48_8479__10_0963", "coords__51_8376__14_1417",
# "coords__50_7536__7_0827", "coords__51_4070__6_9656",
# "coords__49_8421__7_8662", "coords__49_7410__7_1935",
# "coords__51_1566__11_8182", "coords__51_4065__6_9660",
# "coords__50_7333__7_1000", "coords__50_0000__8_0000",
# "coords__48_7444__7_6000", "coords__51_0000__11_0000",
# "coords__52_7555__8_1000", "coords__50_0000__2_0000",
# "coords__51_7666__8_6000", "coords__50_0000__3_0000",
# "coords__45_7777__9_1000", "coords__50_0000__4_0000",
],
# stations=["coords__48_8479__10_0963", "coords__51_8376__14_1417",
# "coords__50_7536__7_0827", "coords__51_4070__6_9656",
# "coords__49_8421__7_8662", "coords__49_7410__7_1935",
# "coords__51_1566__11_8182", "coords__51_4065__6_9660",
# "coords__50_7333__7_1000", "coords__50_0000__8_0000",
# "coords__48_7444__7_6000", "coords__51_0000__11_0000",
# "coords__52_7555__8_1000", "coords__50_0000__2_0000",
# "coords__51_7666__8_6000", "coords__50_0000__3_0000",
# "coords__45_7777__9_1000", "coords__50_0000__4_0000",
# ],
# data_handler=DataHandlerWRF,
data_handler=DataHandlerMainSectWRF, #,
# data_path="/p/scratch/deepacf/kleinert1/IASS_proc_monthyl",
#data_path="/p/scratch/deepacf/kleinert1/IASS_proc",
#data_path="/p/project/deepacf/intelliaq/kleinert1/DATA/WRF_CHEM_soft_ln_small_test",
data_path="/media/felix/INTENSO/WRF_CHEM/hourly/cdo_output_test/jan_test",
# data_path="/media/felix/INTENSO/WRF_CHEM/hourly/cdo_output_test/jan_test",
# data_path="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly2009",
# data_path="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly_count_test",
data_path = "/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly2009_2010-03",
#data_path="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/monthly_01-03",
common_file_starter="wrfout_d01",
date_format_of_nc_file="%Y-%m",
time_dim='XTIME',
#external_coords_file='/p/project/deepacf/inbound_data/IASS_upload/coords.nc',
# external_coords_file="/p/scratch/deepacf/kleinert1/IASS_proc/coords.nc",
external_coords_file="/media/felix/INTENSO/WRF_CHEM/monthly/coords.nc",
external_coords_file="/p/scratch/deepacf/intelliaq/kleinert1/IASS_proc_monthly/coords.nc",
# external_coords_file="/media/felix/INTENSO/WRF_CHEM/monthly/coords.nc",
transformation={
"T2": {"method": "standardise"},
"Q2": {"method": "standardise"},
......@@ -100,43 +105,47 @@ def main(parser_args):
start='2009-01-01',
# end='2009-01-04',
end='2009-01-31',
# end='2009-03-31',
#end='2009-01-31',
end='2010-03-31',
#train_start='2009-01-01',
#train_end='2009-01-02',
train_start='2009-01-01',
train_end='2009-01-15',
# train_end='2009-02-28',
#train_end='2009-01-15',
train_end='2009-10-15',
#val_start='2009-01-02',
#val_end='2009-01-03',
###################################
val_start='2009-01-15',
val_end='2009-01-22',
#val_start='2009-01-15',
#val_end='2009-01-22',
###################################
# val_start='2009-03-01',
# val_end='2009-03-14',
val_start='2009-10-16',
val_end='2009-12-14',
#test_start='2009-01-03',
#test_end='2009-01-04',
###################################
test_start='2009-01-22',
test_end='2009-01-31',
#test_start='2009-01-22',
#test_end='2009-01-31',
###################################
# test_start='2009-03-15',
# test_end='2009-03-31',
test_start='2009-12-15',
test_end='2010-03-31',
# sampling='hourly',
sampling="daily",
input_output_sampling4toarstats=("hourly", "daily"),
time_zone="UTC",
target_time_type="solar_time",
use_multiprocessing=True,
batch_size=64*2*2,
interpolation_limit=0,
# as_image_like_data_format=False,
as_image_like_data_format=False,
# model=MyLSTMModel,
model=MyLuongAttentionLSTMModel,
# model=MyCNNModelSect,
model=MyCNNModel,
# model=MyCNNModel,
**parser_args.__dict__)
workflow.run()
......
run_wrfdh_sector3_hdfml.bash 0 → 100644
+ 18
− 0
View file @ 768ec348
#!/bin/bash -x
#SBATCH --account=deepacf
#SBATCH --nodes=1
#SBATCH --output=HPC_logging/mlt-out.%j
#SBATCH --error=HPC_logging/mlt-err.%j
#SBATCH --time=06:00:00
#SBATCH --gres=gpu:4
#SBATCH --mail-type=ALL
#SBATCH --mail-user=f.kleinert@fz-juelich.de
source HPC_setup/mlt_modules_hdfml.sh
source venv_hdfml/bin/activate
timestamp=`date +"%Y-%m-%d_%H%M-%S"`
export PYTHONPATH=${PWD}/venv_hdfml/lib/python3.6/site-packages:${PYTHONPATH}
srun --cpu-bind=none python run_wrf_dh_sector3.py --experiment_date=${timestamp}_WRF_sector
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment