diff --git a/requirements.txt b/requirements.txt
index 97fe3afaed96178cb2ed4c778c84ea2d1164d6b3..7d51370d2a46bbdd09ef133e079b48730e95df9e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,4 +7,5 @@ netCDF4
pandas
PILLOW
pyunicore
+toarstats @ git+https://gitlab.jsc.fz-juelich.de/esde/toar-public/toarstats.git@v0.6.0
xarray
diff --git a/src/mlair/time_series.py b/src/mlair/time_series.py
index 1f5b6a60420b7857ff505a73674f17b9d8c4c2f7..8c9986423d47c3c042da132b9ebc0bac1ecd5a3b 100644
--- a/src/mlair/time_series.py
+++ b/src/mlair/time_series.py
@@ -4,8 +4,9 @@ import xarray as xr
import pandas as pd
import datetime as dt
import pages.dashboard_translations as guitr
+from toarstats.metrics.interface import calculate_statistics
-def plot(station, language_id=0, species=0, start_date = "2018-07-18 00:00", forecast_length = 4, input_dir="static/data/MLAIR", output_dir=None, figsize=(6, 3)):
+def plot(station, language_id=0, species=0, start_date = "2018-07-18 00:00", forecast_length = 4, statistics = "dma8eu", input_dir="static/data/MLAIR", output_dir=None, figsize=(6, 3)):
# get this from outside!!!
units = "ppb"
@@ -20,16 +21,16 @@ def plot(station, language_id=0, species=0, start_date = "2018-07-18 00:00", for
station_code = station_code.strip()
station_label = station_label.strip()
start_date = start_date[0:10]
- hourly_start_date = (dt.datetime.strptime(start_date, '%Y-%m-%d') - dt.timedelta(days=forecast_length)).strftime('%Y-%m-%d')
- hourly_end_date = (dt.datetime.strptime(start_date, '%Y-%m-%d') - dt.timedelta(days=1)).strftime('%Y-%m-%d')
+ hourly_start_date = (dt.datetime.strptime(start_date, guitr.date_format2[0][:8]) - dt.timedelta(days=forecast_length)).strftime(guitr.date_format2[language_id][:8])
+ hourly_end_date = (dt.datetime.strptime(start_date, guitr.date_format2[0][:8]) - dt.timedelta(days=1)).strftime(guitr.date_format2[language_id][:8])
file_s = f"{input_dir}/forecasts_{station_code}_test.nc"
dataset = xr.open_dataset(file_s)
d4_fc = dataset['__xarray_dataarray_variable__'].sel(index=start_date,type='nn').data[:forecast_length]
# plot the values at the middle of the day --> offset 12:00
- d4_fc_times = [ (dt.datetime.strptime(start_date, '%Y-%m-%d') + dt.timedelta(days=i)).strftime('%Y-%m-%d 12:00') for i in range(forecast_length) ]
+ d4_fc_times = [(dt.datetime.strptime(start_date, guitr.date_format2[0][:8]) + dt.timedelta(days=i)).strftime(f'{guitr.date_format2[language_id][:8]} 12:00') for i in range(forecast_length)]
df_fc = pd.DataFrame({'fc_time': d4_fc_times, 'value': d4_fc})
# plot the values at the middle of the day --> give an hourly period
- df_fc['fc_time'] = pd.to_datetime(df_fc['fc_time']).dt.to_period('H')
+ df_fc['fc_time'] = pd.to_datetime(df_fc['fc_time'], dayfirst=True).dt.to_period('h')
file_s = f"{input_dir}/{station_code}_no_no2_o3.nc"
dataset = xr.open_dataset(file_s)
d4_hourly = dataset['__xarray_dataarray_variable__'].sel(datetime=slice(hourly_start_date,hourly_end_date),variables=species.lower()).data[0]
@@ -38,28 +39,47 @@ def plot(station, language_id=0, species=0, start_date = "2018-07-18 00:00", for
df_hourly['hourly_time'] = pd.to_datetime(df_hourly['hourly_time'])
fc_length_str = "{}{}{}".format(forecast_length, guitr.day_label[language_id], guitr.day_plural_label[language_id] if forecast_length > 1 else "")
output_file_name = f"mlair_{species}_{station_code}_{start_date}_{fc_length_str}.png"
+ df_hourly_with_index = df_hourly.set_index("hourly_time")
+ dma8eu = calculate_statistics("daily", statistics, df_hourly_with_index)
+ dma8eu.index = dma8eu.index + pd.Timedelta('12 hour')
+ df_ob = pd.DataFrame({'ob_time': list(dma8eu.index.to_pydatetime()), 'ob_value': dma8eu[statistics].values})
+ df_ob['ob_time'] = pd.to_datetime(df_ob['ob_time']).dt.to_period('h')
# plot
- plt.figure(figsize=figsize)
- ax = df_hourly.plot.line(x="hourly_time", y= "value", marker="o", label='Beobachtungen')
- df_fc.plot.line(x="fc_time", y= "value", linestyle=" ", marker="o", ax=ax, label='Vorhersagen')
+ plt.figure(figsize=figsize) #creates new figure
+ ax = df_hourly.plot.line(x="hourly_time", y= "value", marker="o", label=f'{guitr.mlair_legend_obs_label[language_id]}', color='lightGrey') #plot observations
+ df_ob.plot.line(x="ob_time", y="ob_value", linestyle=" ", marker="o", ax=ax, label=f"{guitr.mlair_legend_obs_label[language_id]} [{statistics}]", color='blue') #plot observations with dma8eu
+ df_fc.plot.line(x="fc_time", y="value", linestyle=" ", marker="o", ax=ax, label=f'{guitr.mlair_legend_fc_label[language_id]} [{statistics}]', color='orange') #plot fc
plt.title(f'{species}: {station_label}')
- plt.xlabel('Datum')
- locs, labels = plt.xticks()
- xticks = [locs[0] + i * 24 for i in range(8) ]
- xlabels = [(dt.datetime.strptime(hourly_start_date, '%Y-%m-%d') + dt.timedelta(days=i)).strftime('%Y-%m-%d') for i in range(8) ]
- plt.xticks(xticks, xlabels, rotation=45)
- plt.ylabel(f'Konzentration ({units})')
- ax.grid(True)
- plt.tight_layout()
+ plt.xlabel(f'{guitr.date_label[language_id]}')
+ locs, labels = plt.xticks() #get location and label of x-Axis
+ xticks = [locs[0] + i * 24 for i in range(8)] #set location of labels
+ xlabels = [(dt.datetime.strptime(hourly_start_date, guitr.date_format2[language_id][:8])
+ + dt.timedelta(days=i)).strftime(guitr.date_format2[language_id][:8]) for i in range(8)] #set labels
+ plt.xticks(xticks, xlabels, rotation=45) #apply location and labels of x-Axis
+ plt.ylabel(f'{guitr.mlair_conc_label[language_id]} ({units})')
+ ax.grid(True) #adds grids to plot
+ plt.tight_layout() #adjusts the space between subplots
if not output_dir:
output_dir = str(os.path.join("assets", "generated_plots", "mlair"))
- plt.savefig(f"{output_dir}/{output_file_name}")
+ plt.savefig(f"{output_dir}/{output_file_name}") #saves plot as file in `output_dir`
return os.path.join("assets", "generated_plots", "mlair", output_file_name)
if __name__ == "__main__":
- plot(station = "DENW067, Bielefeld Ost",
- input_dir = "/home/s.schroeder/mlworkflowinterface/src/static/data/MLAIR",
- output_dir = "/home/s.schroeder/mlworkflowinterface/src/mlair/assets/generated_plots")
+ # a plot in German
+ plot(language_id = 1, # a plot in German
+ station = "DENW067, Bielefeld Ost",
+ start_date = "2018-07-18 00:00",
+ statistics = "dma8eu",
+ input_dir = "./data",
+ output_dir = ".")
+
+ # a plot in English
+ plot(language_id = 0, # a plot in English
+ station = "DENW067, Bielefeld Ost",
+ start_date = "2018-07-18 00:00",
+ statistics = "dma8eu",
+ input_dir = "./data",
+ output_dir = ".")
diff --git a/src/pages/dashboard.py b/src/pages/dashboard.py
index 8254b122519acd83d413a19cab565a04fc3f015e..93bc4fa789a114c674eae72bc55589a7678fd382 100644
--- a/src/pages/dashboard.py
+++ b/src/pages/dashboard.py
@@ -616,7 +616,12 @@ def create_description_content(language_id):
)
-def get_job_status(jobid, convoc_status_dict):
+def get_job_status(jobid, application, convoc_status_dict):
+ application_settings = { 0 : { "ecf_host": "jrlogin05.jureca",
+ "ecf_port": 4960 },
+ 1 : { "ecf_host": "jrlogin09.jureca",
+ "ecf_port": 20932 } }
+
base_url = f"{UNICORE_BASE}JURECA/rest/core"
# Authentification/Authorization
@@ -629,8 +634,8 @@ def get_job_status(jobid, convoc_status_dict):
environment = [ "PYTHONPATH=$PYTHONPATH:/p/project/cjicg21/jicg2126/ecflow/ecFlow-5.11.0-Source/lib/python3.8/site-packages/ecflow",
"PATH=$PATH:/p/project/cjicg21/jicg2126/ecflow/ecFlow-5.11.0-Source/bin/",
"ECFLOW_DIR=/p/project/cjicg21/jicg2126/ecflow/ecFlow-5.11.0-Source",
- "ECF_HOST=jrlogin05.jureca",
- "ECF_PORT=4960" ]
+ f"ECF_HOST={application_settings[application]['ecf_host']}",
+ f"ECF_PORT={application_settings[application]['ecf_port']}" ]
arguments = [ f"--query dstate /{jobid}" ]
job_description = {"Executable": executable,
@@ -690,7 +695,7 @@ def get_my_jobs_from_db(user_id=None, language_id=0):
for job in data_rows_from_db:
job_status = job[1]
if (job_status == convoc_status_dict['active']) or (job_status == convoc_status_dict['waiting']):
- new_status = get_job_status(job[9].upper(), convoc_status_dict)
+ new_status = get_job_status(job[9].upper(), job[0], convoc_status_dict)
if job_status != new_status:
change_status_of_job(job[9], new_status)
job_status = new_status
@@ -994,20 +999,36 @@ def ml_fcast_job_run(run_button, region, startdate, forecast_length, species, me
user_id = json.loads(user_dict)["user_id"]
language_id = json.loads(user_dict)["language_id"]
new_job_dict = {}
- new_job_dict['id'] = get_random_string(12)
+ jobnr = get_random_string(12)
+ new_job_dict['id'] = jobnr
# application is MLAir -- should be taken from the controlled vocabulary!
new_job_dict['application'] = 1
new_job_dict['region'] = region
new_job_dict['start_date'] = startdate + ' 00:00'
new_job_dict['forecast_length'] = int(forecast_length.split()[0])
# also take the coding from the controlled vocabulary!
- new_job_dict['status'] = 0
+ # set it to "waiting" since it will just be submitted
+ new_job_dict['status'] = 2
# at the moment only ozone is available!
new_job_dict['species'] = species
# at the moment only dma8eu is available!
new_job_dict['metric'] = metrics
# MLAir does not have any emission scenario
new_job_dict['emis_scen'] = None
+
+ # submit job
+ base_url = f"{UNICORE_BASE}JURECA/rest/core"
+ credentials = uc_credentials.UsernamePassword(UNICORE_USER, UNICORE_PASSWORD)
+ client = uc_client.Client(credentials, base_url)
+ job_description = {'Executable': "/p/project/deepacf/intelliaq/schroeder5/ecflow_mlair/start_destine_mlair_demonstrator.sh", "Job type": "ON_LOGIN_NODE", 'Arguments':[jobnr], }
+ job = client.new_job(job_description)
+
+ # let's wait while the job is still running
+ # otherwise, the job status will not be able to be determined
+ job.poll()
+
+ # now create job in db (otherwise the status cannot be determined!)
+
create_db_job_entry(user_id, new_job_dict)
retval = ctx.triggered_id == "ml_fcast_open"
return retval,\
diff --git a/src/pages/dashboard_translations.py b/src/pages/dashboard_translations.py
index 52952c3ee15346363c774ba841adaa6c1ad4aa45..2984d469f3afa55b80895df4ed2d11b7d44d0bee 100644
--- a/src/pages/dashboard_translations.py
+++ b/src/pages/dashboard_translations.py
@@ -94,7 +94,7 @@ user_label = ["user:", "Benutzer:"]
run_label = ["run", "Starten"]
run2_label = ["Run", "Lauf"]
close_label = ["close", "Schließen"]
-date_label = ["date:", "Datum:"]
+date_label = ["date", "Datum"]
date_format = ["M/D/Y", "D.M.Y"]
date_format2 = ['%Y-%m-%d %H:%M', '%d.%m.%Y %H:%M']
first_day_of_week = [0, 1]
@@ -126,4 +126,6 @@ im_download_label = ["Download Plots", "Plots herunterladen"]
downscaling_label = ["Postprocessing with ML-Downscaling", "Postprocessing mit ML-Downscaling"]
show_downscaling_label = ["Show results with ML downscaling", "Ergebnisse mit ML-Downscaling anzeigen"]
out_option_label = ["output option", "Ausgabe-Option"]
-
+mlair_legend_obs_label = ["measurements", "Beobachtungen"]
+mlair_legend_fc_label = ["forecasts", "Vorhersagen"]
+mlair_conc_label = ["concentration", "Konzentration"]
diff --git a/utils/start_destine_mlair_demonstrator.sh b/utils/start_destine_mlair_demonstrator.sh
new file mode 100644
index 0000000000000000000000000000000000000000..6d76f5fe5b18be4cd0306c58b02c8cc6360f2c46
--- /dev/null
+++ b/utils/start_destine_mlair_demonstrator.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+#get setup from script arguments
+MODE=$1
+
+# set environment
+ml load Python/3.11.3
+export PATH=/p/scratch/deepacf/kreshpa1/tmp/ecflow_build/ecFlow-5.11.3-Source/build/bin:$PATH
+export PYTHONPATH=$PYTHONPATH:/p/scratch/deepacf/kreshpa1/tmp/ecflow_build/lib/python3.11/site-packages/ecflow/
+export ECF_HOST=jrlogin09.jureca
+export ECF_PORT=20932
+cd /p/scratch/deepacf/kreshpa1/mlair_application/
+
+# adapt configuration to actual setup
+EXP_NAME=${MODE^^}
+cp mlair_client.py ${MODE}_client.py
+sed -i -e 's!/MLAir!/'"${EXP_NAME}!"'' ${MODE}_client.py
+sed -i -e "s/MLAir/${EXP_NAME}/" ${MODE}_client.py
+sed -i -e 's/mlair.def/'"${MODE}"'.def/' ${MODE}_client.py
+cp mlair.def ${MODE}.def
+sed -i -e "s/MLAir/${EXP_NAME}/" ${MODE}.def
+sed -i -e "s/run_mlair/run_${MODE}/" ${MODE}.def
+mkdir $EXP_NAME
+cp MLAir/run_mlair.ecf ${EXP_NAME}/run_${MODE}.ecf
+# ...
+
+# run the script
+python ${MODE}_client.py
+
+# reset settings
+# ...