added min_length parameter to data handling, currently only supported for test set

src/data_handling/data_preparation.py

@@ -353,7 +353,9 @@ class DataPrep(object):
             non_nan_observation = self.observation.dropna(dim=dim)
             intersect = reduce(np.intersect1d, (non_nan_history.coords[dim].values, non_nan_label.coords[dim].values, non_nan_observation.coords[dim].values))
 
-        if len(intersect) == 0:
+        min_length = self.kwargs.get("min_length", 0)
+        length = len(intersect)
+        if len(intersect) < max(min_length, 1):
             self.history = None
             self.label = None
             self.observation = None

src/run_modules/experiment_setup.py

@@ -34,7 +34,8 @@ class ExperimentSetup(RunEnvironment):
                  limit_nan_fill=None, train_start=None, train_end=None, val_start=None, val_end=None, test_start=None,
                  test_end=None, use_all_stations_on_all_data_sets=True, trainable=None, fraction_of_train=None,
                  experiment_path=None, plot_path=None, forecast_path=None, overwrite_local_data=None, sampling="daily",
-                 create_new_model=None, bootstrap_path=None, permute_data_on_training=None, transformation=None):
+                 create_new_model=None, bootstrap_path=None, permute_data_on_training=None, transformation=None,
+                 test_min_length=None):
 
         # create run framework
         super().__init__()
@@ -107,6 +108,7 @@ class ExperimentSetup(RunEnvironment):
         # test set parameters
         self._set_param("start", test_start, default="2010-01-01", scope="general.test")
         self._set_param("end", test_end, default="2017-12-31", scope="general.test")
+        self._set_param("min_length", test_min_length, default=30, scope="general.test")
 
         # train_val set parameters
         self._set_param("start", self.data_store.get("start", "general.train"), scope="general.train_val")

src/run_modules/pre_processing.py

@@ -11,7 +11,7 @@ from src.join import EmptyQueryResult
 from src.run_modules.run_environment import RunEnvironment
 
 DEFAULT_ARGS_LIST = ["data_path", "network", "stations", "variables", "interpolate_dim", "target_dim", "target_var"]
-DEFAULT_KWARGS_LIST = ["limit_nan_fill", "window_history_size", "window_lead_time", "statistics_per_var",
+DEFAULT_KWARGS_LIST = ["limit_nan_fill", "window_history_size", "window_lead_time", "statistics_per_var", "min_length",
                        "station_type", "overwrite_local_data", "start", "end", "sampling", "transformation"]