diff --git a/src/pages/dashboard.py b/src/pages/dashboard.py
index d2c6036e01763fb238ff48e53f6e0128b396f589..602f4f8791813e9b23658862987d0b5e9e93b9b0 100644
--- a/src/pages/dashboard.py
+++ b/src/pages/dashboard.py
@@ -23,11 +23,11 @@ from deployment_settings import (
UNICORE_BASE, UFTP_BASE,
UNICORE_USER, UNICORE_PASSWORD
)
+from utils import transfer_results_from_HPC
from mlair.time_series import plot as plot_ml_time_series
import pages.dashboard_translations as guitr
import pages.dashboard_constants as guiconst
-
# the following should be done with static files!
APP_HOME = Path.cwd()
IMAGE_PATH = APP_HOME.joinpath("static", "images")
@@ -78,21 +78,20 @@ def get_db_job_entry(jobid):
cur = conn.cursor()
cur.execute(f"SELECT application, status, start_date, forecast_length, region, species, metric, emis_scen FROM jobs \
WHERE id = '{jobid}'")
-# application, status, start_date, forecast_length, region, species, metric, emis_scen = cur.fetchall()[0]
jobs = cur.fetchall()
conn.close()
+ job_dict = {}
for job in jobs:
application, status, start_date, forecast_length, region, species, metric, emis_scen = job
- job_dict = {}
- job_dict['id'] = jobid
- job_dict['application'] = application
- job_dict['status'] = status
- job_dict['start_date'] = start_date
- job_dict['forecast_length'] = forecast_length
- job_dict['region'] = region
- job_dict['species'] = species
- job_dict['metric'] = metric
- job_dict['emis_scen'] = emis_scen
+ job_dict['id'] = jobid
+ job_dict['application'] = application
+ job_dict['status'] = status
+ job_dict['start_date'] = start_date
+ job_dict['forecast_length'] = forecast_length
+ job_dict['region'] = region
+ job_dict['species'] = species
+ job_dict['metric'] = metric
+ job_dict['emis_scen'] = emis_scen
return job_dict
@@ -309,14 +308,18 @@ def generate_eurad_im_body(language_id=0):
dbc.Row([
dbc.Col(dbc.Label(guitr.date_label[language_id]), width=3),
dbc.Col(dcc.DatePickerSingle(id="eurad_im_start_date",
- date=dt.date(2018, 7, 22),
+ date=dt.date(2018, 7, 18),
display_format=guitr.date_format[language_id],
first_day_of_week=guitr.first_day_of_week[language_id],
- min_date_allowed=dt.date(2018, 7, 22),
+ min_date_allowed=dt.date(2018, 7, 18),
max_date_allowed=dt.date(2018, 7, 31),
initial_visible_month=dt.date(2018, 7, 1))),
dbc.Col(dbc.Label(f"{guitr.forecast_length_label[language_id]}:")),
- dbc.Col(dcc.Dropdown(value=guitr.forecast_length_options[language_id][0], options=guitr.forecast_length_options[language_id],
+ dbc.Col(dcc.Dropdown(value=guitr.forecast_length_options[language_id][-1],
+ options=[{'label': guitr.forecast_length_options[language_id][i],
+ 'value': guitr.forecast_length_options[language_id][i],
+ 'disabled': True
+ } for i in range(len(guitr.forecast_length_options[language_id]))],
id="eurad_im_forecast_length"))
], class_name="row mt-3"),
dbc.Row([
@@ -368,23 +371,35 @@ def update_image(selected_time_step, selected_variable, selected_station, users_
-def generate_eurad_im_output_body(language_id, context):
+def generate_eurad_im_output_body(language_id, context, jobnr):
+
+ if jobnr and (jobnr != 'bla'):
+ infile = str(DATA_PATH.joinpath(f'{jobnr}.nc'))
+ job_props = get_db_job_entry(jobnr)
+ start_date = job_props['start_date']
- # This needs to come from outside!
- infile = str(DATA_PATH.joinpath('ctmout_wrf_cutcropped_199_h09.nc'))
+ if not os.path.isfile(infile):
+ trans_filename = transfer_results_from_HPC(jobnr, start_date, DATA_PATH)
+ os.symlink(trans_filename, infile)
- timestep_list = info.get_available_time_stamps(infile)
- timestep_strings = [np.datetime_as_string(ts, unit="m") for ts in timestep_list]
+ timestep_list = info.get_available_time_stamps(infile)
+ timestep_strings = [np.datetime_as_string(ts, unit="m") for ts in timestep_list]
# TODO: Zeit in Stunden seit Start
- start_date = pd.to_datetime(timestep_list[0]).strftime(guitr.date_format2[language_id])
- fc_length = (len(timestep_list)-1) // 24
- fc_length_str = "{} {}{}".format(fc_length, guitr.day_label[language_id], guitr.day_plural_label[language_id] if fc_length > 1 else "")
+ start_date = pd.to_datetime(timestep_list[0]).strftime(guitr.date_format2[language_id])
+ fc_length = (len(timestep_list)-1) // 24
+ fc_length_str = "{} {}{}".format(fc_length, guitr.day_label[language_id], guitr.day_plural_label[language_id] if fc_length > 1 else "")
- variables_list = info.get_available_variables(infile)
+ variables_list = info.get_available_variables(infile)
- stations_list = info.get_available_stations()
- stations_list = sorted(stations_list)
+ stations_list = info.get_available_stations()
+ stations_list = sorted(stations_list)
+ else:
+ start_date = '2018-07-19T00:00'
+ timestep_strings = '2018-07-19 00:00'
+ fc_length_str = '1'
+ variables_list = 'o3'
+ stations_list = "XX, XX"
return [
dbc.Row([dbc.Label(f"{guitr.start_date_label[language_id]}: {start_date}, {guitr.forecast_length_label[language_id]}: {fc_length_str}")]),
dbc.Row([
@@ -425,7 +440,7 @@ def generate_eurad_im_output_modal(jobnr=None, lisopen=False, language_id=0):
[
dbc.ModalTitle(f"{guitr.run2_label[language_id]} {jobnr}", style={"fontSize": 15}, id="eurad_im_output_modal_title"),
dbc.ModalHeader(dbc.ModalTitle(guitr.application_text[language_id][0])),
- dbc.ModalBody(generate_eurad_im_output_body(language_id, "output")),
+ dbc.ModalBody(generate_eurad_im_output_body(language_id, "output", jobnr)),
dbc.ModalFooter([dbc.Button(f"{guitr.close_label[language_id]}", id="eurad_im_output_close",
class_name="fzj_style_btn", n_clicks=0)
]),
@@ -442,7 +457,7 @@ def generate_eurad_im_result_modal(jobnr=None, lisopen=False, language_id=0):
[
dbc.ModalTitle(f"{guitr.run2_label[language_id]} {jobnr}", style={"fontSize": 15}),
dbc.ModalHeader(dbc.ModalTitle(guitr.application_text[language_id][0])),
- dbc.ModalBody(generate_eurad_im_output_body(language_id, "result")),
+ dbc.ModalBody(generate_eurad_im_output_body(language_id, "result", jobnr)),
dbc.ModalFooter(dbc.Button(f"{guitr.close_label[language_id]}", id="eurad_im_result_close",
class_name="fzj_style_btn", n_clicks=0)),
],
@@ -484,7 +499,11 @@ def generate_eurad_scen_body(language_id):
max_date_allowed=dt.date(2018, 7, 31),
initial_visible_month=dt.date(2018, 7, 1))),
dbc.Col(dbc.Label(f"{guitr.forecast_length_label[language_id]}:")),
- dbc.Col(dcc.Dropdown(value=guitr.forecast_length_options[language_id][0], options=guitr.forecast_length_options[language_id],
+ dbc.Col(dcc.Dropdown(value=guitr.forecast_length_options[language_id][-1],
+ options=[{'label': guitr.forecast_length_options[language_id][i],
+ 'value': guitr.forecast_length_options[language_id][i],
+ 'disabled': True
+ } for i in range(len(guitr.forecast_length_options[language_id]))],
id="eurad_scen_forecast_length"))
], class_name="row mt-3"),
dbc.Row([
@@ -628,9 +647,11 @@ def create_description_content(language_id):
def get_job_status(jobid, application, convoc_status_dict):
application_settings = { 0 : { "ecf_host": "jrlogin05.jureca",
- "ecf_port": 4960 },
+ "ecf_port": 4960 },
1 : { "ecf_host": "jrlogin09.jureca",
- "ecf_port": 20932 } }
+ "ecf_port": 20932 },
+ 2 : { "ecf_host": "jrlogin05.jureca",
+ "ecf_port": 4960 } }
base_url = f"{UNICORE_BASE}JURECA/rest/core"
@@ -965,7 +986,9 @@ def eurad_im_job_run(run_button, region, startdate, forecast_length, user_dict):
base_url = f"{UNICORE_BASE}JURECA/rest/core"
credentials = uc_credentials.UsernamePassword(UNICORE_USER, UNICORE_PASSWORD)
client = uc_client.Client(credentials, base_url)
- job_description = {'Executable': "/p/project/cjicg21/schroeder5/Destine_AQ/start_destine_demonstrator.sh", "Job type": "ON_LOGIN_NODE", 'Arguments':[jobnr, "0"], }
+ job_description = {'Executable': "/p/project/cjicg21/schroeder5/Destine_AQ/start_destine_demonstrator.sh",
+ 'Job type': "ON_LOGIN_NODE",
+ 'Arguments': [jobnr, "0", startdate, str(new_job_dict['forecast_length'])], }
job = client.new_job(job_description)
# let's wait while the job is still running
@@ -1067,7 +1090,7 @@ def eurad_scen_job_run(run_button, region, startdate, forecast_length, emi_scen,
jobnr = get_random_string(12)
new_job_dict['id'] = jobnr
# application is EURAD-IM -- should be taken from the controlled vocabulary!
- new_job_dict['application'] = 0
+ new_job_dict['application'] = 2
new_job_dict['region'] = region
new_job_dict['start_date'] = startdate + ' 00:00'
new_job_dict['forecast_length'] = int(forecast_length.split()[0])
@@ -1084,7 +1107,7 @@ def eurad_scen_job_run(run_button, region, startdate, forecast_length, emi_scen,
base_url = f"{UNICORE_BASE}JURECA/rest/core"
credentials = uc_credentials.UsernamePassword(UNICORE_USER, UNICORE_PASSWORD)
client = uc_client.Client(credentials, base_url)
- job_description = {'Executable': "/p/project/cjicg21/schroeder5/Destine_AQ/start_destine_demonstrator.sh", "Job type": "ON_LOGIN_NODE", 'Arguments':[jobnr, emi_scen], }
+ job_description = {'Executable': "/p/project/cjicg21/schroeder5/Destine_AQ/start_destine_demonstrator.sh", "Job type": "ON_LOGIN_NODE", 'Arguments':[jobnr, str(emi_scen)], }
job = client.new_job(job_description)
# let's wait while the job is still running
@@ -1106,11 +1129,10 @@ def eurad_scen_job_run(run_button, region, startdate, forecast_length, emi_scen,
@callback(
Output("eurad_im_download_result", "data"),
Input("eurad_im_output_download", "n_clicks"),
- [State("job-info", "data"),
- State("plot-info", "data")],
+ State("job-info", "data"),
prevent_initial_call=True
)
-def eurad_im_output_download(download_button, job_dict, user_dict):
+def eurad_im_output_download(download_button, job_dict):
jobid = json.loads(job_dict)["jobnr"]
infile = str(DATA_PATH.joinpath(f'{jobid}.nc'))
return dcc.send_file(
diff --git a/utils/utils.py b/utils/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..80606e83a35a62269c8296a0b2dd23bce3f7dc78
--- /dev/null
+++ b/utils/utils.py
@@ -0,0 +1,34 @@
+import os
+import json
+import pyunicore.client as uc_client
+import pyunicore.credentials as uc_credentials
+from datetime import date
+
+from deployment_settings import (
+ UNICORE_BASE,
+ UNICORE_USER, UNICORE_PASSWORD
+ )
+
+base_url = f"{UNICORE_BASE}JURECA/rest/core"
+
+
+def transfer_results_from_HPC(jobnr, model_date, data_path):
+
+ # Authentification/Authorization
+ credentials = uc_credentials.UsernamePassword(UNICORE_USER, UNICORE_PASSWORD)
+
+ yy,mm,dd=model_date.split('-')
+ dd=dd.split(' ')[0]
+ timestep_date = date(int(yy), int(mm), int(dd))
+ base_directory = f"{UNICORE_USER}/{jobnr.upper()}/CTM/{yy}{mm}/{dd}"
+ download_filename = f"ctmout_wrf_cutcropped_{timestep_date.strftime('%j')}_h09.nc"
+
+ # only transfer file if results are not already there (from another jobnr);
+ # in the latter case, a link to the already downloaded data is sufficient
+ if not os.path.isfile(str(data_path.joinpath(download_filename))):
+ transport = uc_client.Transport(credentials)
+ scratch = uc_client.Storage(transport, f"{base_url}/storages/SCRATCH")
+ dfile = scratch.stat(f"{base_directory}/{download_filename}")
+ dfile.download(str(data_path.joinpath(download_filename)))
+
+ return download_filename