From 54d6e5918d71c32f5bda970a381b9a9f1792cf42 Mon Sep 17 00:00:00 2001
From: Simon Grasse <s.grasse@fz-juelich.de>
Date: Wed, 10 Jan 2024 14:45:06 +0100
Subject: [PATCH] fix: handle date inconsistencies at begin/end due to local
time
---
toargridding/metadata.py | 5 +++--
toargridding/toar_rest_client.py | 12 ++++++------
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/toargridding/metadata.py b/toargridding/metadata.py
index 2c60885..2ba4978 100644
--- a/toargridding/metadata.py
+++ b/toargridding/metadata.py
@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timedelta
from enum import Enum
from dataclasses import dataclass
@@ -33,7 +33,8 @@ class TimeSample:
@property
def daterange_option(self):
- return f"{self.start.isoformat()},{self.end.isoformat()}"
+ end_with_padding = self.end + timedelta(1)
+ return f"{self.start.isoformat()},{end_with_padding.isoformat()}"
@property
def frequency(self):
diff --git a/toargridding/toar_rest_client.py b/toargridding/toar_rest_client.py
index 26b3d3b..28fb9fc 100644
--- a/toargridding/toar_rest_client.py
+++ b/toargridding/toar_rest_client.py
@@ -154,16 +154,16 @@ class AnalysisService:
return coords[valid_coords]
def get_clean_timeseries(self, timeseries, metadata: Metadata):
+ # TODO maybe use cf-index here already ?
+ first, last = timeseries.columns[0], timeseries.columns[-1]
+ # remove data where utc -> sun/local ? time conversion leads to dateshift
+ timeseries.drop(columns=[first, last], inplace=True)
+ timeseries.columns = metadata.time.as_datetime_index()
+
all_na = timeseries.isna().all(axis=1)
timeseries = timeseries[~all_na]
timeseries = timeseries.fillna(0)
- response_timesteps = timeseries.columns
- expected_timesteps = metadata.time.as_datetime_index()
- if not (response_timesteps == expected_timesteps).all():
- raise ValueError("foo")
- # TODO maybe use cf-index here already ?
- timeseries.columns = metadata.time.as_datetime_index()
return timeseries
def load_data(
--
GitLab