Added global vaiables such as src_dir_base to unit test and implemented check...

Added global vaiables such as src_dir_base to unit test and implemented check for processed number of files.

test/test_process_netCDF_v2.py

@@ -4,22 +4,26 @@ from data_preprocess.process_netCDF_v2 import *
 import pytest
 import numpy as np
 import json
+
+# some basic variables
+src_dir_base = "/p/project/deepacf/deeprain/video_prediction_shared_folder/extractedData/test/"
+target_dir_base = "/p/project/deepacf/deeprain/video_prediction_shared_folder/preprocessedData/test/" 
+year = "2017"
 slices = {"lat_s": 74,
           "lat_e": 202,
           "lon_s": 550,
           "lon_e": 710
           }
-
+job_name = "01"
 
 @pytest.fixture(scope="module")
-def preprocessData_case1(src_dir="/p/project/deepacf/deeprain/video_prediction_shared_folder/extractedData/test/",\
-                         target_dir="/p/project/deepacf/deeprain/video_prediction_shared_folder/preprocessedData/test/",\
-                         year="2017",job_name="01",slices=slices):
+def preprocessData_case1(src_dir=src_dir_base, target_dir=target_dir_base,\
+                         year=year,job_name=job_name,slices=slices):
     return PreprocessNcToPkl(src_dir,target_dir,job_name,year,slices)
 
 
 def test_directory_path(preprocessData_case1):
-    assert preprocessData_case1.directory_to_process == "/p/project/deepacf/deeprain/video_prediction_shared_folder/extractedData/test/2017/01" 
+    assert preprocessData_case1.directory_to_process == os.path.join(src_dir_base,str(year),job_name) 
 
 
 def test_get_image_list(preprocessData_case1):
@@ -39,7 +43,7 @@ def test_process_images_to_list_by_month(preprocessData_case1):
     preprocessData_case1.initia_list_and_stat()
     preprocessData_case1.process_images_to_list_by_month()
     #Get the first elemnt of imageList, which is ecmwf_era5_17010100.nc and check if the variables values are equal to the first element of EU_list
-    im_path = "/p/project/deepacf/deeprain/video_prediction_shared_folder/extractedData/test/2017/01/ecmwf_era5_17010100.nc" 
+    im_path = os.path.join(src_dir_base,str(year),job_name,"ecmwf_era5_17010100.nc") 
     with Dataset(im_path,"r") as data_file:
         times = data_file.variables["time"]
         time = num2date(times[:],units=times.units,calendar=times.calendar)
@@ -51,15 +55,15 @@ def test_process_images_to_list_by_month(preprocessData_case1):
 
 def test_save_stat_info(preprocessData_case1):
     # statistic file to be tested
-    path_test_dir = '/p/project/deepacf/deeprain/video_prediction_shared_folder/preprocessedData/test/pickle'
-    fstat2test = os.path.join(path_test_dir,'stat_01.json')
+    path_test_dir = os.path.join(target_dir_base,"pickle",str(year))
+    fstat2test = os.path.join(path_test_dir,'stat_'+job_name+'.json')
     # if statistic file is not present, create it
     if not os.path.isfile(fstat2test):
         preprocessData_case1()
         
-        l_stat_exists  = os.path.isfile(os.path.join(path_test_dir,'stat_01.json'))
-        l_pickle_exists= os.path.isfile(os.path.join(path_test_dir,'X_01.pkl')) and \
-                         os.path.isfile(os.path.join(path_test_dir,'T_01.pkl'))
+        l_stat_exists  = os.path.isfile(os.path.join(path_test_dir,'stat_'+job_name+'.json'))
+        l_pickle_exists= os.path.isfile(os.path.join(path_test_dir,'X_'+job_name+'.pkl')) and \
+                         os.path.isfile(os.path.join(path_test_dir,'T_'+job_name+'.pkl'))
         
         assert l_stat_exists  == True
         assert l_pickle_exists== True
@@ -77,6 +81,7 @@ def test_save_stat_info(preprocessData_case1):
     assert data["T2"][0]["min"] == pytest.approx(temp_min,0.001)
     assert data["T2"][0]["max"] == pytest.approx(temp_max,0.001)
     assert data["MSL"][0]["avg"] == pytest.approx(msl_mean,0.001) 
+    assert data["common_stat"][0]["nfiles"] == 70       
  
     #assert preprocessData_case1.save_stat_info.stat_obj["T2"]["min"] == temp_min
     #assert preprocessData_case1.save_stat_info.stat_obj["T2"]["max"] == temp_max