Skip to content
Snippets Groups Projects
Commit 4838a8fc authored by stadtler1's avatar stadtler1
Browse files

Preliminary persistence forecast added. Still needs to be checked if working fine.

parent e5f89d0c
Branches
Tags
No related merge requests found
Pipeline #46156 failed
Stage: build
Stage: test
Stage: deploy
Show whitespace changes
Inline Side-by-side
video_prediction_savp/scripts/generate_transfer_learning_finetune.py
+ 55
− 53
View file @ 4838a8fc
......@@ -338,10 +338,12 @@ def get_persistence(ts, input_dir_pkl):
ts: output by generate_seq_timestamps(t_start,len_seq=sequence_length)
Is a list containing dateime objects
input_dir_pkl: input directory to pickle files
Ouputs:
ts_persistence: list containing the dates and times of the
time_persistence: list containing the dates and times of the
persistence forecast.
peristence_images: sequence of images corresponding to the times
var_peristence : sequence of images corresponding to the times
in ts_persistence
"""
ts_persistence = []
......@@ -354,26 +356,40 @@ def get_persistence(ts, input_dir_pkl):
month_start = t_persistence_start.month
month_end = t_persistence_end.month
# only one pickle file is needed (all hours during the same month)
if month_start == month_end:
pass
else: # case that we need to derive the data from two pickle files
# Open files to search for the indizes of the corresponding time
time_pickle = load_pickle_for_persistence(input_dir_pkl, year_start, month_start, 'T')
# Open file to search for the correspoding meteorological fields
var_pickle = load_pickle_for_persistence(input_dir_pkl, year_start, month_start, 'X')
# Retrieve starting index
ind = list(time_pickle).index(np.array(ts_persistence[0]))
#print('Scarlet, Original', ts_persistence)
#print('From Pickle', time_pickle[ind:ind+len(ts_persistence)])
var_persistence = var_pickle[ind:ind+len(ts_persistence)]
time_persistence = time_pickle[ind:ind+len(ts_persistence)].ravel()
print(' Scarlet Shape of time persistence',time_persistence.shape)
#print(' Scarlet Shape of var persistence',var_persistence.shape)
# case that we need to derive the data from two pickle files (changing month during the forecast periode)
else:
t_persistence_first_m = [] # should hold dates of the first month
t_persistence_second_m = [] # should hold dates of the second month
for t in range(len(ts)):
m = ts_persistence[t].month
if m == month_start:
t_persistence_first_m.append(ts_persistence[t])
if m == month_end:
t_persistence_second_m.append(ts_persistence[t])
# Open files to search for the indizes
#path_to_pickle = input_dir_pkl+'/'+str(year_start)+'/T_{:02}.pkl'.format(month_start)
#infile = open(path_to_pickle,'rb')
# Open files to search for the indizes of the corresponding time
time_pickle_first = load_pickle_for_persistence(input_dir_pkl, year_start, month_start, 'T')
time_pickle_second = load_pickle_for_persistence(input_dir_pkl, year_start, month_end, 'T')
# Open file to search for the correspoding meteorological fields
#path_to_pickle = input_dir_pkl+'/'+str(year_start)+'/X_{:02}.pkl'.format(month_start)
#infile = open(path_to_pickle,'rb')
var_pickle_first = load_pickle_for_persistence(input_dir_pkl, year_start, month_start, 'X')
var_pickle_second = load_pickle_for_persistence(input_dir_pkl, year_start, month_end, 'X')
......@@ -381,15 +397,24 @@ def get_persistence(ts, input_dir_pkl):
ind_first_m = list(time_pickle_first).index(np.array(t_persistence_first_m[0]))
ind_second_m = list(time_pickle_second).index(np.array(t_persistence_second_m[0]))
# construct following indices and store them in a list - DO I NEED THAT?
idx_fist = construct_index(ind_second_m, t_persistence_second_m)
idx_second = construct_index(ind_second_m, t_persistence_second_m)
print('Scarlet this is the index {} in month {}'.format(ind_first_m, month_start))
print('length sequence first month', len(t_persistence_first_m))
print('Scarlet this is the index {} in month {}'.format(ind_second_m, month_end))
print('length sequence second month', len(t_persistence_second_m))
print('Original', ts_persistence)
print('From Pickle', time_pickle_first[ind_first_m:ind_first_m+len(t_persistence_first_m)])
#print('Scarlet, Original', ts_persistence)
#print('From Pickle', time_pickle_first[ind_first_m:ind_first_m+len(t_persistence_first_m)], time_pickle_second[ind_second_m:ind_second_m+len(t_persistence_second_m)])
#print(' Scarlet before', time_pickle_first[ind_first_m:ind_first_m+len(t_persistence_first_m)].shape, time_pickle_second[ind_second_m:ind_second_m+len(t_persistence_second_m)].shape)
# append the sequence of the second month to the first month
var_persistence = np.concatenate((var_pickle_first[ind_first_m:ind_first_m+len(t_persistence_first_m)],
var_pickle_second[ind_second_m:ind_second_m+len(t_persistence_second_m)]),
axis=0)
time_persistence = np.concatenate((time_pickle_first[ind_first_m:ind_first_m+len(t_persistence_first_m)],
time_pickle_second[ind_second_m:ind_second_m+len(t_persistence_second_m)]),
axis=0).ravel() # ravel is needed to eliminate the unnecessary dimension (20,1) becomes (20,)
print(' Scarlet concatenate and ravel (time)', var_persistence.shape, time_persistence.shape)
# tolist() is needed for plotting
return var_persistence, time_persistence.tolist()
def load_pickle_for_persistence(input_dir_pkl, year_start, month_start, pkl_type):
"""Helper to get the content of the pickle files. There are two types in our workflow:
......@@ -402,37 +427,6 @@ def load_pickle_for_persistence(input_dir_pkl, year_start, month_start, pkl_type
var = pickle.load(infile)
return var
def construct_index(first_idx, sequence):
"""Helper function to construct the idx sequences from the starting index and the lenght
of the squence.
Wait do I even need a sequence????"""
idx_list = []
for i in range(first_idx, first_idx+len(sequence)):
#print(i)
idx_list.append(i)
return idx_list
# Oder stuff:
# Retrieve indizes
#print(list(time_pickle))
#for item in time_pickle:
# time_pickle_list.append(item)
#print(time_pickle_list)
#print(time_pickle)
#print(t_persistence_first_m)
#idx_first_m = time_pickle_list.index(t_persistence_first_m)
#print('Scarlet, indizes', idx_first_m)
#persistent_idx = list(test_temporal_pkl).index(np.array(persistent_ts))
#path_to_pickle = input_dir_pkl+'/'+str(year_start)+'/X_{:02}.pkl'.format(month_start)
#print('hypothetical path', path_to_pickle)
#infile = open(path_to_pickle,'rb')
#print('Scarlet start persistence and start regular', t_persistence_start, ts[0] )
#return ts_persistence, peristence_images
def main():
parser = argparse.ArgumentParser()
......@@ -582,9 +576,17 @@ def main():
#Generate forecast images
plot_seq_imgs(imgs=gen_images_denorm[context_frames:,:,:,0],lats=lats,lons=lons,ts=ts[context_frames+1:],label="Forecast by Model " + args.model,output_png_dir=args.results_dir)
#+++ Scarlet 20200828
get_persistence(ts, input_dir_pkl)
#--- Scarlet 20200528
#+++ Scarlet 20200922
print('Scarlet', type(ts[context_frames+1:]))
print('ts', ts[context_frames+1:])
print('context_frames:', context_frames)
persistence_images, ts_persistence = get_persistence(ts, input_dir_pkl)
print('Scarlet', type(ts_persistence))
# I am not sure about the number of frames given with context_frames and context_frames +1
plot_seq_imgs(imgs=persistence_images[context_frames+1:,:,:,0],lats=lats,lons=lons,ts=ts_persistence[context_frames+1:],
label="Persistence Forecast" + args.model,output_png_dir=args.results_dir)
#--- Scarlet 20200922
#in case of generate the images for all the input, we just generate the first 5 sampe_ind examples for visuliation
sample_ind += args.batch_size
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment