data capture evaluations

prepare.sh

@@ -5,7 +5,7 @@
 # venv is created in this directory.
 
 # check if we are really in the ozone-mapping directory
-S="ozone-mapping"
+S="aq-bench"
 if [[ $(pwd) == *$S ]]
 then
     echo "Prepare..."

source/dataset_retrieval.py

@@ -419,6 +419,43 @@ class MetricsRow():
               + ' ' + print_metric + ': ' + str(self.row[print_metric]))
 
 
+def data_capture():
+    """
+    A simple function to look at the data capture of our metrics.
+    """
+    import pdb
+    # df = pd.read_csv(resources_dir+'yearly_metrics.csv')
+    # df['hourly_samples'] = [0] * len(df)
+    # df['capture'] = [0] * len(df)
+    df = pd.read_csv(resources_dir+'intermediate_at_cap5500.csv')
+    for idx, row in df.iterrows():
+        if idx > 5500:
+            id_tuple = row['o3_series_id']
+            id_string_list = [str(id_) for id_ in eval(id_tuple)]
+            query = f"""
+                SELECT
+                datetime, value
+                FROM o3_hourly
+                WHERE id IN ({','.join(id_string_list)})
+                AND datetime between '2010-01-01 00:00:00'
+                                 AND '2014-12-31 23:59:59';
+                """
+            result = query_db(query)
+            result.drop_duplicates(subset='datetime', inplace=True,
+                                      ignore_index=True)
+            count = len(result)
+            df.loc[idx, 'hourly_samples'] = count
+            df.loc[idx, 'capture'] = count / 43824
+            print(count, count/43824)
+            if (idx > 1) and (idx % 500 == 0):
+                df.to_csv(resources_dir+f'intermediate_at_cap{idx}.csv',
+                          index=False)
+
+    df.to_csv(resources_dir+f'yearly_metrics_cap.csv',
+              index=False)
+
+    pdb.set_trace()
+
 def full_aqbench():
     """
     start one retrieval.
@@ -458,4 +495,5 @@ if __name__ == '__main__':
                   logging.StreamHandler()])
 
     # start retrieval
-    full_aqbench()
+    # full_aqbench()
+    data_capture()