diff --git a/test/test_process_netCDF_v2.py b/test/test_process_netCDF_v2.py index cf07928d21e5a1754874f699aa0f5b184d0f4bdd..a7319d46bb5804f95820cbfd838360c6de17933c 100644 --- a/test/test_process_netCDF_v2.py +++ b/test/test_process_netCDF_v2.py @@ -4,22 +4,26 @@ from data_preprocess.process_netCDF_v2 import * import pytest import numpy as np import json + +# some basic variables +src_dir_base = "/p/project/deepacf/deeprain/video_prediction_shared_folder/extractedData/test/" +target_dir_base = "/p/project/deepacf/deeprain/video_prediction_shared_folder/preprocessedData/test/" +year = "2017" slices = {"lat_s": 74, "lat_e": 202, "lon_s": 550, "lon_e": 710 } - +job_name = "01" @pytest.fixture(scope="module") -def preprocessData_case1(src_dir="/p/project/deepacf/deeprain/video_prediction_shared_folder/extractedData/test/",\ - target_dir="/p/project/deepacf/deeprain/video_prediction_shared_folder/preprocessedData/test/",\ - year="2017",job_name="01",slices=slices): +def preprocessData_case1(src_dir=src_dir_base, target_dir=target_dir_base,\ + year=year,job_name=job_name,slices=slices): return PreprocessNcToPkl(src_dir,target_dir,job_name,year,slices) def test_directory_path(preprocessData_case1): - assert preprocessData_case1.directory_to_process == "/p/project/deepacf/deeprain/video_prediction_shared_folder/extractedData/test/2017/01" + assert preprocessData_case1.directory_to_process == os.path.join(src_dir_base,str(year),job_name) def test_get_image_list(preprocessData_case1): @@ -39,7 +43,7 @@ def test_process_images_to_list_by_month(preprocessData_case1): preprocessData_case1.initia_list_and_stat() preprocessData_case1.process_images_to_list_by_month() #Get the first elemnt of imageList, which is ecmwf_era5_17010100.nc and check if the variables values are equal to the first element of EU_list - im_path = "/p/project/deepacf/deeprain/video_prediction_shared_folder/extractedData/test/2017/01/ecmwf_era5_17010100.nc" + im_path = os.path.join(src_dir_base,str(year),job_name,"ecmwf_era5_17010100.nc") with Dataset(im_path,"r") as data_file: times = data_file.variables["time"] time = num2date(times[:],units=times.units,calendar=times.calendar) @@ -51,15 +55,15 @@ def test_process_images_to_list_by_month(preprocessData_case1): def test_save_stat_info(preprocessData_case1): # statistic file to be tested - path_test_dir = '/p/project/deepacf/deeprain/video_prediction_shared_folder/preprocessedData/test/pickle' - fstat2test = os.path.join(path_test_dir,'stat_01.json') + path_test_dir = os.path.join(target_dir_base,"pickle",str(year)) + fstat2test = os.path.join(path_test_dir,'stat_'+job_name+'.json') # if statistic file is not present, create it if not os.path.isfile(fstat2test): preprocessData_case1() - l_stat_exists = os.path.isfile(os.path.join(path_test_dir,'stat_01.json')) - l_pickle_exists= os.path.isfile(os.path.join(path_test_dir,'X_01.pkl')) and \ - os.path.isfile(os.path.join(path_test_dir,'T_01.pkl')) + l_stat_exists = os.path.isfile(os.path.join(path_test_dir,'stat_'+job_name+'.json')) + l_pickle_exists= os.path.isfile(os.path.join(path_test_dir,'X_'+job_name+'.pkl')) and \ + os.path.isfile(os.path.join(path_test_dir,'T_'+job_name+'.pkl')) assert l_stat_exists == True assert l_pickle_exists== True @@ -77,7 +81,8 @@ def test_save_stat_info(preprocessData_case1): assert data["T2"][0]["min"] == pytest.approx(temp_min,0.001) assert data["T2"][0]["max"] == pytest.approx(temp_max,0.001) assert data["MSL"][0]["avg"] == pytest.approx(msl_mean,0.001) - + assert data["common_stat"][0]["nfiles"] == 70 + #assert preprocessData_case1.save_stat_info.stat_obj["T2"]["min"] == temp_min #assert preprocessData_case1.save_stat_info.stat_obj["T2"]["max"] == temp_max