diff --git a/mlair/configuration/defaults.py b/mlair/configuration/defaults.py index 5be0e7d3a3e5dd8b402230f281b528fb0a2476a2..9bb15068ce3a5ad934f7b0251b84cb19f37702f6 100644 --- a/mlair/configuration/defaults.py +++ b/mlair/configuration/defaults.py @@ -48,6 +48,7 @@ DEFAULT_TEST_END = "2017-12-31" DEFAULT_TEST_MIN_LENGTH = 90 DEFAULT_TRAIN_VAL_MIN_LENGTH = 180 DEFAULT_USE_ALL_STATIONS_ON_ALL_DATA_SETS = True +DEFAULT_COMPETITORS = ["ols"] DEFAULT_DO_UNCERTAINTY_ESTIMATE = True DEFAULT_UNCERTAINTY_ESTIMATE_BLOCK_LENGTH = "1m" DEFAULT_UNCERTAINTY_ESTIMATE_EVALUATE_COMPETITORS = True diff --git a/mlair/plotting/abstract_plot_class.py b/mlair/plotting/abstract_plot_class.py index 21e5d9413b490a4be5281c2a80308be558fe64c8..a26023bb6cb8772623479491ac8bcc731dd42223 100644 --- a/mlair/plotting/abstract_plot_class.py +++ b/mlair/plotting/abstract_plot_class.py @@ -72,7 +72,10 @@ class AbstractPlotClass: # pragma: no cover self._update_rc_params() def __del__(self): - plt.close('all') + try: + plt.close('all') + except ImportError: + pass def _plot(self, *args): """Abstract plot class needs to be implemented in inheritance.""" diff --git a/mlair/run_modules/experiment_setup.py b/mlair/run_modules/experiment_setup.py index 706f6169c756a1558eaf0177801a7f484fdca1d1..f89633cbe0f80f26dbb2481ca24a7fd294ee6888 100644 --- a/mlair/run_modules/experiment_setup.py +++ b/mlair/run_modules/experiment_setup.py @@ -24,7 +24,7 @@ from mlair.configuration.defaults import DEFAULT_STATIONS, DEFAULT_VAR_ALL_DICT, DEFAULT_FEATURE_IMPORTANCE_BOOTSTRAP_TYPE, DEFAULT_FEATURE_IMPORTANCE_BOOTSTRAP_METHOD, DEFAULT_OVERWRITE_LAZY_DATA, \ DEFAULT_UNCERTAINTY_ESTIMATE_BLOCK_LENGTH, DEFAULT_UNCERTAINTY_ESTIMATE_EVALUATE_COMPETITORS, \ DEFAULT_UNCERTAINTY_ESTIMATE_N_BOOTS, DEFAULT_DO_UNCERTAINTY_ESTIMATE, DEFAULT_CREATE_SNAPSHOT, \ - DEFAULT_EARLY_STOPPING_EPOCHS, DEFAULT_RESTORE_BEST_MODEL_WEIGHTS + DEFAULT_EARLY_STOPPING_EPOCHS, DEFAULT_RESTORE_BEST_MODEL_WEIGHTS, DEFAULT_COMPETITORS from mlair.data_handler import DefaultDataHandler from mlair.run_modules.run_environment import RunEnvironment from mlair.model_modules.fully_connected_networks import FCN_64_32_16 as VanillaModel @@ -417,7 +417,7 @@ class ExperimentSetup(RunEnvironment): raise IndexError(f"Given model_display_name {model_display_name} is also present in the competitors " f"variable {competitors}. To assure a proper workflow it is required to have unique names " f"for each model and competitor. Please use a different model display name or competitor.") - self._set_param("competitors", competitors, default=[]) + self._set_param("competitors", competitors, default=DEFAULT_COMPETITORS) competitor_path_default = os.path.join(self.data_store.get("data_path"), "competitors", "_".join(self.data_store.get("target_var"))) self._set_param("competitor_path", competitor_path, default=competitor_path_default) diff --git a/mlair/run_modules/post_processing.py b/mlair/run_modules/post_processing.py index 97d1817f5eb884e80c042d56f02dd7a61f88d935..b1a0bdc7738dc0ac6f5afd98290c07d6d5f91e0e 100644 --- a/mlair/run_modules/post_processing.py +++ b/mlair/run_modules/post_processing.py @@ -34,7 +34,7 @@ class PostProcessing(RunEnvironment): Perform post-processing for performance evaluation. Schedule of post-processing: - #. train a ordinary least squared model (ols) for reference + #. train an ordinary least squared model (ols) for reference #. create forecasts for nn, ols, and persistence #. evaluate feature importance with bootstrapped predictions #. calculate skill scores @@ -695,8 +695,13 @@ class PostProcessing(RunEnvironment): @TimeTrackingWrapper def train_ols_model(self): """Train ordinary least squared model on train data.""" - logging.info(f"start train_ols_model on train data") - self.ols_model = OrdinaryLeastSquaredModel(self.train_data) + if "ols" in map(lambda x: x.lower(), self.competitors): + logging.info(f"start train_ols_model on train data") + self.ols_model = OrdinaryLeastSquaredModel(self.train_data) + self.competitors = [e for e in self.competitors if e.lower() != "ols"] + else: + logging.info(f"Skip train ols model as it is not present in competitors.") + @TimeTrackingWrapper def make_prediction(self, subset): @@ -733,7 +738,11 @@ class PostProcessing(RunEnvironment): transformation_func, normalised) # ols - ols_prediction = self._create_ols_forecast(input_data, ols_prediction, transformation_func, normalised) + if self.ols_model is not None: + ols_prediction = self._create_ols_forecast(input_data, ols_prediction, transformation_func, + normalised) + else: + ols_prediction = None # observation observation = self._create_observation(target_data, observation, transformation_func, normalised) @@ -817,8 +826,8 @@ class PostProcessing(RunEnvironment): tmp_ols = self.ols_model.predict(input_data) target_shape = ols_prediction.values.shape if target_shape != tmp_ols.shape: - if len(target_shape)==2: - new_values = np.swapaxes(tmp_ols,1,0) + if len(target_shape) == 2: + new_values = np.swapaxes(tmp_ols, 1, 0) else: new_values = np.swapaxes(tmp_ols, 2, 0) else: @@ -922,6 +931,7 @@ class PostProcessing(RunEnvironment): :return: xarray of dimension 3: index, ahead_names, # predictions """ + kwargs = {k: v for k, v in kwargs.items() if v is not None} keys = list(kwargs.keys()) res = xr.DataArray(np.full((len(index.index), len(ahead_names), len(keys)), np.nan), coords=[index.index, ahead_names, keys], dims=[index_dim, ahead_dim, type_dim])