diff --git a/scripts/train_dummy.py b/scripts/train_dummy.py
index b89ca957aa4696f4ba6f4118a83bee10683c16ff..6ebdb70bf24ecc53fd9611a7af948842600cd0db 100644
--- a/scripts/train_dummy.py
+++ b/scripts/train_dummy.py
@@ -164,13 +164,13 @@ def main():
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.gpu_mem_frac, allow_growth=True)
config = tf.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True)
#global_step = tf.train.get_or_create_global_step()
+ #global_step = tf.Variable(0, name = 'global_step', trainable = False)
max_steps = model.hparams.max_steps
print ("max_steps",max_steps)
with tf.Session(config=config) as sess:
print("parameter_count =", sess.run(parameter_count))
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
-
#coord = tf.train.Coordinator()
#threads = tf.train.start_queue_runners(sess = sess, coord = coord)
print("Init done: {sess.run(tf.local_variables_initializer())}%")
@@ -188,20 +188,23 @@ def main():
# start at one step earlier to log everything without doing any training
# step is relative to the start_step
for step in range(-1, max_steps - start_step):
+ global_step = sess.run(model.global_step)
+ print ("global_step:", global_step)
val_handle_eval = sess.run(val_handle)
print ("val_handle_val",val_handle_eval)
if step == 1:
# skip step -1 and 0 for timing purposes (for warmstarting)
start_time = time.time()
-
- fetches = {"global_step": model.global_step}
+
+ fetches = {"global_step":model.global_step}
fetches["train_op"] = model.train_op
- fetches["latent_loss"] = model.latent_loss
+
+ # fetches["latent_loss"] = model.latent_loss
fetches["total_loss"] = model.total_loss
- if isinstance(model.learning_rate, tf.Tensor):
- fetches["learning_rate"] = model.learning_rate
+ # if isinstance(model.learning_rate, tf.Tensor):
+ # fetches["learning_rate"] = model.learning_rate
fetches["summary"] = model.summary_op
@@ -210,21 +213,25 @@ def main():
X = inputs["images"].eval(session=sess)
#results = sess.run(fetches,feed_dict={model.x:X}) #fetch the elements in dictinoary fetch
results = sess.run(fetches)
+ print ("results global step:",results["global_step"])
run_elapsed_time = time.time() - run_start_time
if run_elapsed_time > 1.5 and step > 0 and set(fetches.keys()) == {"global_step", "train_op"}:
print('running train_op took too long (%0.1fs)' % run_elapsed_time)
#Run testing results
- val_fetches = {"global_step": model.global_step}
- val_fetches["latent_loss"] = model.latent_loss
- val_fetches["total_loss"] = model.total_loss
+ #val_fetches = {"global_step":global_step}
+ val_fetches = {}
+ #val_fetches["latent_loss"] = model.latent_loss
+ #val_fetches["total_loss"] = model.total_loss
val_fetches["summary"] = model.summary_op
- val_results = sess.run(val_fetches)
-
- summary_writer.add_summary(results["summary"], results["global_step"])
- summary_writer.add_summary(val_results["summary"], val_results["global_step"])
-
-
+ val_results = sess.run(val_fetches,feed_dict={train_handle: val_handle_eval})
+
+ summary_writer.add_summary(results["summary"])
+ summary_writer.add_summary(val_results["summary"])
+
+ #print("results_global_step", results["global_step"])
+ #print("Val_results_global_step", val_results["global_step"])
+
val_datasets = [val_dataset]
val_models = [model]
@@ -244,8 +251,9 @@ def main():
# global_step will have the correct step count if we resume from a checkpoint
# global step is read before it's incremented
steps_per_epoch = train_dataset.num_examples_per_epoch() / batch_size
- train_epoch = results["global_step"] / steps_per_epoch
- print("progress global step %d epoch %0.1f" % (results["global_step"] + 1, train_epoch))
+ #train_epoch = results["global_step"] / steps_per_epoch
+ train_epoch = global_step/steps_per_epoch
+ print("progress global step %d epoch %0.1f" % (global_step + 1, train_epoch))
if step > 0:
elapsed_time = time.time() - start_time
average_time = elapsed_time / step
@@ -266,7 +274,7 @@ def main():
print(" Results_total_loss",results["total_loss"])
print("saving model to", args.output_dir)
- saver.save(sess, os.path.join(args.output_dir, "model"), global_step=model.global_step)
+ saver.save(sess, os.path.join(args.output_dir, "model"), global_step=step)##Bing: cheat here a little bit because of the global step issue
print("done")
#global_step = global_step + 1
if __name__ == '__main__':
diff --git a/video_prediction/layers/layer_def.py b/video_prediction/layers/layer_def.py
index 35a7c910e0b3ec12cb9fdc3cbb9ceda3a86922dd..6b7f4387001c9318507ad809d7176071312742d0 100644
--- a/video_prediction/layers/layer_def.py
+++ b/video_prediction/layers/layer_def.py
@@ -28,11 +28,11 @@ def _variable_on_cpu(name, shape, initializer):
Variable Tensor
"""
with tf.device('/cpu:0'):
- var = tf.get_variable(name, shape, initializer = initializer)
+ var = tf.get_variable(name, shape, initializer=initializer)
return var
-def _variable_with_weight_decay(name, shape, stddev, wd):
+def _variable_with_weight_decay(name, shape, stddev, wd,initializer=tf.contrib.layers.xavier_initializer()):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
@@ -45,8 +45,8 @@ def _variable_with_weight_decay(name, shape, stddev, wd):
Returns:
Variable Tensor
"""
- var = _variable_on_cpu(name, shape,tf.truncated_normal_initializer(stddev = stddev))
- #var = _variable_on_cpu(name, shape,tf.contrib.layers.xavier_initializer())
+ #var = _variable_on_cpu(name, shape,tf.truncated_normal_initializer(stddev = stddev))
+ var = _variable_on_cpu(name, shape, initializer)
if wd:
weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name = 'weight_loss')
weight_decay.set_shape([])
@@ -54,16 +54,16 @@ def _variable_with_weight_decay(name, shape, stddev, wd):
return var
-def conv_layer(inputs, kernel_size, stride, num_features, idx, activate="relu"):
+def conv_layer(inputs, kernel_size, stride, num_features, idx, initializer=tf.contrib.layers.xavier_initializer() , activate="relu"):
print("conv_layer activation function",activate)
with tf.variable_scope('{0}_conv'.format(idx)) as scope:
- print ("DEBUG input shape",inputs.get_shape())
+
input_channels = inputs.get_shape()[-1]
weights = _variable_with_weight_decay('weights',shape = [kernel_size, kernel_size,
input_channels, num_features],
stddev = 0.01, wd = weight_decay)
- biases = _variable_on_cpu('biases', [num_features], tf.contrib.layers.xavier_initializer())
+ biases = _variable_on_cpu('biases', [num_features], initializer)
conv = tf.nn.conv2d(inputs, weights, strides = [1, stride, stride, 1], padding = 'SAME')
conv_biased = tf.nn.bias_add(conv, biases)
if activate == "linear":
@@ -72,25 +72,25 @@ def conv_layer(inputs, kernel_size, stride, num_features, idx, activate="relu"):
conv_rect = tf.nn.relu(conv_biased, name = '{0}_conv'.format(idx))
elif activate == "elu":
conv_rect = tf.nn.elu(conv_biased, name = '{0}_conv'.format(idx))
+ elif activate == "leaky_relu":
+ conv_rect = tf.nn.leaky_relu(conv_biased, name = '{0}_conv'.format(idx))
else:
raise ("activation function is not correct")
-
return conv_rect
-def transpose_conv_layer(inputs, kernel_size, stride, num_features, idx, activate="relu"):
+def transpose_conv_layer(inputs, kernel_size, stride, num_features, idx, initializer=tf.contrib.layers.xavier_initializer(),activate="relu"):
with tf.variable_scope('{0}_trans_conv'.format(idx)) as scope:
input_channels = inputs.get_shape()[3]
input_shape = inputs.get_shape().as_list()
- print("input_channel",input_channels)
+
weights = _variable_with_weight_decay('weights',
shape = [kernel_size, kernel_size, num_features, input_channels],
stddev = 0.1, wd = weight_decay)
- biases = _variable_on_cpu('biases', [num_features], tf.contrib.layers.xavier_initializer())
+ biases = _variable_on_cpu('biases', [num_features],initializer)
batch_size = tf.shape(inputs)[0]
-# output_shape = tf.stack(
-# [tf.shape(inputs)[0], tf.shape(inputs)[1] * stride, tf.shape(inputs)[2] * stride, num_features])
+
output_shape = tf.stack(
[tf.shape(inputs)[0], input_shape[1] * stride, input_shape[2] * stride, num_features])
print ("output_shape",output_shape)
@@ -102,11 +102,15 @@ def transpose_conv_layer(inputs, kernel_size, stride, num_features, idx, activat
return tf.nn.elu(conv_biased, name = '{0}_transpose_conv'.format(idx))
elif activate == "relu":
return tf.nn.relu(conv_biased, name = '{0}_transpose_conv'.format(idx))
+ elif activate == "leaky_relu":
+ return tf.nn.leaky_relu(conv_biased, name = '{0}_transpose_conv'.format(idx))
+ elif activate == "sigmoid":
+ return tf.nn.sigmoid(conv_biased, name ='sigmoid')
else:
- return None
+ return conv_biased
-def fc_layer(inputs, hiddens, idx, flat=False, activate="relu",weight_init=0.01):
+def fc_layer(inputs, hiddens, idx, flat=False, activate="relu",weight_init=0.01,initializer=tf.contrib.layers.xavier_initializer()):
with tf.variable_scope('{0}_fc'.format(idx)) as scope:
input_shape = inputs.get_shape().as_list()
if flat:
@@ -118,7 +122,7 @@ def fc_layer(inputs, hiddens, idx, flat=False, activate="relu",weight_init=0.01)
weights = _variable_with_weight_decay('weights', shape = [dim, hiddens], stddev = weight_init,
wd = weight_decay)
- biases = _variable_on_cpu('biases', [hiddens],tf.contrib.layers.xavier_initializer())
+ biases = _variable_on_cpu('biases', [hiddens],initializer)
if activate == "linear":
return tf.add(tf.matmul(inputs_processed, weights), biases, name = str(idx) + '_fc')
elif activate == "sigmoid":
@@ -127,15 +131,30 @@ def fc_layer(inputs, hiddens, idx, flat=False, activate="relu",weight_init=0.01)
return tf.nn.softmax(tf.add(tf.matmul(inputs_processed, weights), biases, name = str(idx) + '_fc'))
elif activate == "relu":
return tf.nn.relu(tf.add(tf.matmul(inputs_processed, weights), biases, name = str(idx) + '_fc'))
+ elif activate == "leaky_relu":
+ return tf.nn.leaky_relu(tf.add(tf.matmul(inputs_processed, weights), biases, name = str(idx) + '_fc'))
else:
ip = tf.add(tf.matmul(inputs_processed, weights), biases)
return tf.nn.elu(ip, name = str(idx) + '_fc')
-def bn_layers(inputs,idx,epsilon = 1e-3):
+def bn_layers(inputs,idx,is_training=True,epsilon=1e-3,decay=0.99,reuse=None):
with tf.variable_scope('{0}_bn'.format(idx)) as scope:
- # Calculate batch mean and variance
- batch_mean, batch_var = tf.nn.moments(inputs,[0])
- tz1_hat = (inputs - batch_mean) / tf.sqrt(batch_var + epsilon)
- l1_BN = tf.nn.sigmoid(tz1_hat)
+ #Calculate batch mean and variance
+ shape = inputs.get_shape().as_list()
+ scale = tf.get_variable("gamma", shape[-1], initializer=tf.constant_initializer(1.0), trainable=is_training)
+ beta = tf.get_variable("beta", shape[-1], initializer=tf.constant_initializer(0.0), trainable=is_training)
+ pop_mean = tf.Variable(tf.zeros([inputs.get_shape()[-1]]), trainable=False)
+ pop_var = tf.Variable(tf.ones([inputs.get_shape()[-1]]), trainable=False)
- return l1_BN
\ No newline at end of file
+ if is_training:
+ batch_mean, batch_var = tf.nn.moments(inputs,[0])
+ train_mean = tf.assign(pop_mean,pop_mean * decay + batch_mean * (1 - decay))
+ train_var = tf.assign(pop_var, pop_var * decay + batch_var * (1 - decay))
+ with tf.control_dependencies([train_mean,train_var]):
+ return tf.nn.batch_normalization(inputs,batch_mean,batch_var,beta,scale,epsilon)
+ else:
+ return tf.nn.batch_normalization(inputs,pop_mean,pop_var,beta,scale,epsilon)
+
+def bn_layers_wrapper(inputs, is_training):
+ pass
+
\ No newline at end of file
diff --git a/video_prediction/models/__init__.py b/video_prediction/models/__init__.py
index ea1fa77f821827b61c3c2cbfa362014c1da20faf..4103a236ab6430d701bae28ee9b6ff6670b110fa 100644
--- a/video_prediction/models/__init__.py
+++ b/video_prediction/models/__init__.py
@@ -8,6 +8,7 @@ from .dna_model import DNAVideoPredictionModel
from .sna_model import SNAVideoPredictionModel
from .sv2p_model import SV2PVideoPredictionModel
from .vanilla_vae_model import VanillaVAEVideoPredictionModel
+from .vanilla_convLSTM_model import VanillaConvLstmVideoPredictionModel
def get_model_class(model):
model_mappings = {
@@ -18,6 +19,7 @@ def get_model_class(model):
'sna': 'SNAVideoPredictionModel',
'sv2p': 'SV2PVideoPredictionModel',
'vae': 'VanillaVAEVideoPredictionModel',
+ 'convLSTM': 'VanillaConvLstmVideoPredictionModel'
}
model_class = model_mappings.get(model, model)
model_class = globals().get(model_class)
diff --git a/video_prediction/models/vanilla_convLSTM_model.py b/video_prediction/models/vanilla_convLSTM_model.py
index de4a05dc1d7720e2bc5259fcdc90d1748a46ddd1..6cb07df7f7cb72fa0943299adbc18a7641636521 100644
--- a/video_prediction/models/vanilla_convLSTM_model.py
+++ b/video_prediction/models/vanilla_convLSTM_model.py
@@ -14,14 +14,14 @@ from video_prediction.utils import tf_utils
from datetime import datetime
from pathlib import Path
from video_prediction.layers import layer_def as ld
-from video_prediction.layers import BasicConvLSTMCell
+from video_prediction.layers.BasicConvLSTMCell import BasicConvLSTMCell
-class VanillaVAEVideoPredictionModel(BaseVideoPredictionModel):
- def __init__(self, mode='train', hparams_dict=None,
+class VanillaConvLstmVideoPredictionModel(BaseVideoPredictionModel):
+ def __init__(self, mode='train',aggregate_nccl=None, hparams_dict=None,
hparams=None, **kwargs):
- super(VanillaVAEVideoPredictionModel, self).__init__(mode, hparams_dict, hparams, **kwargs)
+ super(VanillaConvLstmVideoPredictionModel, self).__init__(mode, hparams_dict, hparams, **kwargs)
+ print ("Hparams_dict",self.hparams)
self.mode = mode
- self.hparams = hparams
self.learning_rate = self.hparams.lr
self.gen_images_enc = None
self.recon_loss = None
@@ -30,7 +30,8 @@ class VanillaVAEVideoPredictionModel(BaseVideoPredictionModel):
self.context_frames = 10
self.sequence_length = 20
self.predict_frames = self.sequence_length - self.context_frames
-
+ self.aggregate_nccl=aggregate_nccl
+
def get_default_hparams_dict(self):
"""
The keys of this dict define valid hyperparameters for instances of
@@ -56,23 +57,24 @@ class VanillaVAEVideoPredictionModel(BaseVideoPredictionModel):
`sequence_length - context_frames` future frames. Must be
specified during instantiation.
"""
- default_hparams = super(VanillaVAEVideoPredictionModel, self).get_default_hparams_dict()
+ default_hparams = super(VanillaConvLstmVideoPredictionModel, self).get_default_hparams_dict()
+ print ("default hparams",default_hparams)
hparams = dict(
batch_size=16,
lr=0.001,
end_lr=0.0,
+ nz=16,
decay_steps=(200000, 300000),
max_steps=350000,
)
+
return dict(itertools.chain(default_hparams.items(), hparams.items()))
def build_graph(self, x):
- global_step = tf.train.get_or_create_global_step()
- original_global_variables = tf.global_variables()
- tf.reset_default_graph()
+ self.x = x["images"]
+ #self.global_step = tf.train.get_or_create_global_step()
self.global_step = tf.Variable(0, name = 'global_step', trainable = False)
- self.increment_global_step = tf.assign_add(self.global_step, 1, name = 'increment_global_step')
-
+ original_global_variables = tf.global_variables()
# ARCHITECTURE
self.x_hat_context_frames, self.x_hat_predict_frames = self.convLSTM_network()
self.x_hat = tf.concat([self.x_hat_context_frames, self.x_hat_predict_frames], 1)
@@ -93,7 +95,7 @@ class VanillaVAEVideoPredictionModel(BaseVideoPredictionModel):
self.loss_summary = tf.summary.scalar("total_loss", self.total_loss)
self.summary_op = tf.summary.merge_all()
global_variables = [var for var in tf.global_variables() if var not in original_global_variables]
- self.saveable_variables = [global_step] + global_variables
+ self.saveable_variables = [self.global_step] + global_variables
return
@@ -108,8 +110,9 @@ class VanillaVAEVideoPredictionModel(BaseVideoPredictionModel):
print("Encode 3_shape, ", conv3.shape)
y_0 = conv3
# conv lstm cell
+ cell_shape = y_0.get_shape().as_list()
with tf.variable_scope('conv_lstm', initializer = tf.random_uniform_initializer(-.01, 0.1)):
- cell = BasicConvLSTMCell(shape = [16, 16], filter_size = [3, 3], num_features = 8)
+ cell = BasicConvLSTMCell(shape = [cell_shape[1], cell_shape[2]], filter_size = [3, 3], num_features = 8)
if hidden is None:
hidden = cell.zero_state(y_0, tf.float32)
print("hidden zero layer", hidden.shape)
@@ -133,7 +136,7 @@ class VanillaVAEVideoPredictionModel(BaseVideoPredictionModel):
def convLSTM_network(self):
network_template = tf.make_template('network',
- convLSTM.convLSTM_cell) # make the template to share the variables
+ VanillaConvLstmVideoPredictionModel.convLSTM_cell) # make the template to share the variables
# create network
x_hat_context = []
x_hat_predict = []
@@ -155,4 +158,4 @@ class VanillaVAEVideoPredictionModel(BaseVideoPredictionModel):
x_hat_predict = tf.stack(x_hat_predict)
self.x_hat_context = tf.transpose(x_hat_context, [1, 0, 2, 3, 4]) # change first dim with sec dim
self.x_hat_predict = tf.transpose(x_hat_predict, [1, 0, 2, 3, 4]) # change first dim with sec dim
- return self.x_hat_context, self.x_hat_predict
\ No newline at end of file
+ return self.x_hat_context, self.x_hat_predict
diff --git a/video_prediction/models/vanilla_vae_model.py b/video_prediction/models/vanilla_vae_model.py
index 1ae1eb06351dd11fa2fd8269f4966a682c9341fd..74280896dca61007c1b361ec4caff9ad5f718d26 100644
--- a/video_prediction/models/vanilla_vae_model.py
+++ b/video_prediction/models/vanilla_vae_model.py
@@ -79,11 +79,14 @@ class VanillaVAEVideoPredictionModel(BaseVideoPredictionModel):
#print ("self_x:",self.x)
#tf.reset_default_graph()
#self.x = tf.placeholder(tf.float32, [None,20,64,64,3])
+ tf.set_random_seed(12345)
self.x = x["images"]
- self.global_step = tf.train.get_or_create_global_step()
+
+ #self.global_step = tf.train.get_or_create_global_step()
+ self.global_step = tf.Variable(0, name = 'global_step', trainable = False)
original_global_variables = tf.global_variables()
- #self.global_step = tf.Variable(0, name = 'global_step', trainable = False)
- #self.increment_global_step = tf.assign_add(self.global_step, 1, name = 'increment_global_step')
+ self.increment_global_step = tf.assign_add(self.global_step, 1, name = 'increment_global_step')
+
self.x_hat, self.z_log_sigma_sq, self.z_mu = self.vae_arc_all()
# Loss
# Reconstruction loss
@@ -129,6 +132,7 @@ class VanillaVAEVideoPredictionModel(BaseVideoPredictionModel):
self.outputs["gen_images"] = self.x_hat
global_variables = [var for var in tf.global_variables() if var not in original_global_variables]
self.saveable_variables = [self.global_step] + global_variables
+ #train_op = tf.assign_add(global_step, 1)
return