Changed list_based storage to directly concatenating with self._X_extreme

mlair/data_handler/default_data_handler.py

@@ -168,13 +168,15 @@ class DefaultDataHandler(AbstractDataHandler):
 
     def apply_oversampling(self, bin_edges, oversampling_rates, timedelta: Tuple[int, str] = (1, 's'), timedelta2: Tuple[int, str] = (1, 'ms')):
         self._load()
+        self._X_extreme = None
+        self._X_extreme = None
         if (self._X is None) or (self._Y is None):
             logging.debug(f"{str(self.id_class)} has no data for X or Y, skip multiply extremes")
             return
         Y = self._Y
         X = self._X
-        complete_extremes_X_list = []
-        complete_extremes_Y_list = []
+        #complete_extremes_X_list = []
+        #complete_extremes_Y_list = []
         for i_bin in range(len(bin_edges)-1):
             bin_start = bin_edges[i_bin]
             if i_bin == len(bin_edges) - 1:
@@ -188,8 +190,8 @@ class DefaultDataHandler(AbstractDataHandler):
             extreme_idx = xr.concat([(Y >= bin_start).any(dim=other_dims[0]),
                                          (Y < bin_end).any(dim=other_dims[0])],
                                         dim=other_dims[0]).all(dim=other_dims[0])
-            extremes_X_list =[]
-            extremes_Y_list = []
+            #extremes_X_list =[]
+            #extremes_Y_list = []
             for i in range(np.ceil(rate).astype(int)):
                 sel = extreme_idx.coords[self.time_dim].values
                 if rate-i < 1:
@@ -200,12 +202,18 @@ class DefaultDataHandler(AbstractDataHandler):
                 self._add_timedelta(extremes_X, self.time_dim, (i_bin, timedelta2[1]))
                 extremes_Y = Y.sel(**{self.time_dim: sel})
                 extremes_Y.coords[self.time_dim] = extremes_Y.coords[self.time_dim].values + i*np.timedelta64(*timedelta) + i_bin*np.timedelta64(*timedelta2)
-                extremes_X_list.append(extremes_X)
-                extremes_Y_list.append(extremes_Y)
-
-            complete_extremes_X_list.append(extremes_X_list)
-            complete_extremes_Y_list.append(extremes_Y_list)
-
+                if (self._X_extreme is None) or (self._Y_extreme is None):
+                    self._X_extreme = extremes_X
+                    self._Y_extreme = extremes_Y
+                else:
+                    self._X_extreme = list(map(lambda x1, x2: xr.concat([x1, x2], dim=self.time_dim), self._X_extreme, extremes_X))
+                    self._Y_extreme = xr.concat([self._Y_extreme, extremes_Y], dim=self.time_dim)
+                #extremes_X_list.append(extremes_X)
+                #extremes_Y_list.append(extremes_Y)
+
+            #complete_extremes_X_list.append(extremes_X_list)
+            #complete_extremes_Y_list.append(extremes_Y_list)
+        test = 0
         #Convert complete_extremes_X_list (list of lists of xarrays) into xarray and give it to self._X_extreme
         #self._X_extreme = [[xr.concat(X_list, dim=self.time_dim) for X_list in complete_X_list] for complete_X_list in complete_extremes_X_list]
         #self._Y_extreme = [[xr.concat(Y_list, dim=self.time_dim) for Y_list in complete_Y_list] for complete_Y_list in complete_extremes_Y_list]