diff --git a/video_prediction_tools/data_preprocess/era5_varmapping.json b/video_prediction_tools/data_preprocess/era5_varmapping.json new file mode 100644 index 0000000000000000000000000000000000000000..bc32d9060e915a597e596730ed6275754c1a2260 --- /dev/null +++ b/video_prediction_tools/data_preprocess/era5_varmapping.json @@ -0,0 +1,9 @@ +{ +"surface": ["2t", "tcc","msl","10u","10v"], +"multi":{ + "t" : { + "pl": 85000 + } + + } +} diff --git a/video_prediction_tools/data_preprocess/era5_varmapping_template.json b/video_prediction_tools/data_preprocess/era5_varmapping_template.json index f8b9b0038b392fba07da8d71eda4471abf3c608c..e62aba65c85f46cb85a81adbd07b3e31e9501361 100644 --- a/video_prediction_tools/data_preprocess/era5_varmapping_template.json +++ b/video_prediction_tools/data_preprocess/era5_varmapping_template.json @@ -10,12 +10,8 @@ # !!! This file should be only adapted if you are familiar with the ERA5 grib files!!! { "surface":{ - "2t": "var167", - "tcc": "var164", - "msl": "var151", - "10u": "var165", - "10v": "var166" - }, + ["2t", "tcc","msl","10u","10v"] + }, "multi":{ "t" : { diff --git a/video_prediction_tools/data_preprocess/prepare_era5_data.py b/video_prediction_tools/data_preprocess/prepare_era5_data.py index 9de5b07079e5f6b26bd381a46b577b7084b98529..e34bf8fe3da5f378142005a5ac18dc1c131112ba 100644 --- a/video_prediction_tools/data_preprocess/prepare_era5_data.py +++ b/video_prediction_tools/data_preprocess/prepare_era5_data.py @@ -8,8 +8,8 @@ Functions required for extracting ERA5 data. import os import json __email__ = "b.gong@fz-juelich.de" -__author__ = "Bing Gong, Scarlet Stadtler, Michael Langguth,Yanji" -__date__ = "unknown" +__author__ = "Bing Gong,Michael Langguth,Yanji" +__update_date__ = "2022-02-15" # specify source and target directories @@ -64,21 +64,24 @@ class ERA5DataExtraction(object): temp_path = os.path.join(self.target_dir, self.year, month) os.makedirs(temp_path, exist_ok=True) - for var, value in self.varslist_surface.items(): + for value in self.varslist_surface: # surface variables infile = os.path.join(self.src_dir, self.year, month, self.year+month+day+hour+'_sf.grb') - outfile_sf = os.path.join(self.target_dir, self.year, month, self.year+month+day+hour+'_'+var+'.nc') - os.system('cdo -f nc copy -selname,%s %s %s' % (value, infile, outfile_sf)) - os.system('cdo -chname,%s,%s %s %s' % (value, var, outfile_sf, outfile_sf)) + outfile_sf = os.path.join(self.target_dir, self.year, month, self.year+month+day+hour+'_'+value+'.nc') + os.system('cdo --eccodes -f nc copy -selname,%s %s %s' % (value, infile, outfile_sf)) + # multi-level variables for var, pl_dic in self.varslist_multi.items(): for pl, pl_value in pl_dic.items(): infile = os.path.join(self.src_dir, self.year, month, self.year+month+day+hour+'_ml.grb') - outfile_sf = os.path.join(self.target_dir, self.year, month, self.year+month+day+hour+'_'+var + + outfile_sf_temp = os.path.join(self.target_dir, self.year, month, self.year+month+day+hour+'_'+var + str(pl_value) + '.nc') - os.system('cdo -f nc copy -selname,%s -ml2plx,%d %s %s' % (var,pl_value,infile,outfile_sf)) - os.system('cdo -chname,%s,%s %s %s' % (var, var+"_{0:d}".format(int(pl_value/100.)), outfile_sf, outfile_sf)) + outfile_sf = os.path.join(self.target_dir, self.year, month, self.year+month+day+hour+'_'+var + + str(int(pl_value/100.)) + '.nc') + os.system('cdo -f nc copy -selname,%s -ml2pl,%d %s %s' % (var,pl_value,infile,outfile_sf_temp)) + os.system('cdo -chname,%s,%s %s %s' % (var, var+"_{0:d}".format(int(pl_value/100.)), outfile_sf_temp, outfile_sf)) + os.system('rm %s' % (outfile_sf_temp)) # merge both variables infile = os.path.join(self.target_dir, self.year, month, self.year+month+day+hour+'*.nc') # change the output file name