Added debugging code to visualize network inputs

DeepTrackingV3.py

@@ -154,7 +154,7 @@
 # In[7]:
 
 # ATT VARIABLES
-alpha = 0.8
+alpha = 0.1
 scale = 1.0
 epsilon = 0.05
 
@@ -187,7 +187,7 @@
 class Builder(object):
 
     def build(self, input, modules):
-        out = modules["attention"].getModel()(input)
+        out = modules["cropper"].getModel()([input[0], modules["beeper"].getModel()(input[1])])
         out = modules["cnn"].getModel()(out)
         out = modules["rnn"].getModel()(out)
         out = modules["regressor"].getModel()(out)
@@ -197,7 +197,6 @@ def build(self, input, modules):
 step = SequenceProcessor()
 builder = Builder()
 
-
 # ## TRACKER CONFIGURATION
 
 # In[9]:
@@ -208,22 +207,22 @@ def build(self, input, modules):
 # INPUT CONFIGURATION
 frameInp = Input(shape=(None, ) + frameDims)
 positionInp = Input(shape=(None, 4))
-#thetaInp = Input(shape=(None, 3, 3))
+thetaInp = Input(shape=(None, 3, 3))
 inputs = [frameInp, positionInp]
 
 # CROP CONFIGURATION
-#beeper = Beeper(positionInp, distortion, minSide, context)
-#cropper = SpatialTransformer([frameInp, thetaInp], downsampleFactor)
+beeper = Beeper(positionInp, distortion, minSide, context)
+cropper = SpatialTransformer([frameInp, thetaInp], downsampleFactor)
 
 # ATTENTION CONFIGURATION
-att = SquareAttention(inputs, alpha, scale)
+#att = SquareAttention(inputs, alpha, scale)
 #att = SpatialTransformer(inputs, downsampleFactor=1)
 
 # INVERTER CONFIGURATION
-#inverter = Inverter(thetaInp)
+inverter = Inverter(thetaInp)
 
 # TRANSFORMER CONFIGURATION
-#transformer = Transformer([positionInp, thetaInp])
+transformer = Transformer([positionInp, thetaInp])
 
 # CNN CONFIGURATION
 frame = Input(shape=frameDims)
@@ -261,7 +260,8 @@ def build(self, input, modules):
 regressor = Regressor([reg])
 
 # TRACKER CONFIGURATION
-modules = {"attention":att, "cnn":cnn, "rnn":rnn, "regressor":regressor}
+#modules = {"attention":att, "cnn":cnn, "rnn":rnn, "regressor":regressor}
+modules = {"beeper":beeper, "cropper": cropper, "inverter":inverter, "cnn":cnn, "rnn":rnn, "regressor":regressor}
 tracker = Tracker(inputs, modules, builder, optimizer, loss, step, timeSize)
 tracker.build()
 
@@ -272,8 +272,8 @@ def build(self, input, modules):
 
 # In[10]:
 
-offEpochs = 12
-offBatches = 10
+offEpochs = 1
+offBatches = 3
 trackerModelPath = "//home/ubuntu/tracking/data/Models/"+ trackerName + ".pkl"
 votDataSetPaths = ["/home/ubuntu/tracking/data/Datasets/votDataset10Fms50Seqs.pkl", "/home/ubuntu/tracking/data/Datasets/votDataset10Fms100Seqs.pkl"]
 
@@ -288,6 +288,9 @@ def trainGenerator():
         frame, position = processor.preprocess(frame, position)
         cropPosition = NP.roll(position, 1, axis=1) # Shift the time
         cropPosition[:, 0, :] = cropPosition[:, 1, :] # First frame is ground truth
+        #import cPickle as pickle
+        #with open("/home/ubuntu/tracking/data/debug_batch.pkl", "wb") as out:
+        #    pickle.dump([frame, cropPosition], out)
 
         yield [frame, cropPosition], position
 

tracking/model/keras/Tracker.py

@@ -58,9 +58,11 @@ def build(self):
         model = Model(input=self.input, output=output)
         model.compile(optimizer=self.optimizer, loss=self.loss)
         self.model = model
+        print self.model.summary()
 
 
     def train(self, generator, epochs, batches, batchSize, validator):
+        self.generator = generator
         history = LossHistory(validator, self)
         spe = batches * batchSize
         self.model.fit_generator(generator, nb_epoch=epochs, samples_per_epoch=spe, verbose=0, callbacks=[history])
@@ -114,8 +116,26 @@ def __init__(self, validator, tracker):
     def on_batch_end(self, batch, logs={}):
         loss = logs.get('loss')
         logging.info("Batch Loss: Epoch = %d, batch = %d, loss = %f", self.epoch, batch, loss)
-
+        '''from keras import backend as K
+        get_activations = K.function([self.model.layers[0].input, self.model.layers[1].input],[self.model.layers[2].get_output_at(0)])
+        import cPickle as pickle
+        sample = pickle.load(open("/home/ubuntu/tracking/data/debug_batch.pkl","r"))
+        activations = get_activations(sample) 
+        with open("/home/ubuntu/tracking/data/debug_act.pkl", "wb") as out:
+            pickle.dump(activations, out)'''
+        from keras import backend as K
+        print self.model.layers
+        activations = K.function([self.model.layers[1].input, self.model.layers[0].input],[self.model.layers[3].get_output_at(1)])
+
+        import cPickle as pickle
+        sample = pickle.load(open("/home/ubuntu/tracking/data/debug_batch.pkl","r"))
+        activations = activations(sample)
+        with open("/home/ubuntu/tracking/data/debug_trans.pkl", "wb") as out:
+            pickle.dump(activations, out)
+
 
     def on_epoch_end(self, epoch, logs={}):
         self.validator.validateEpoch(self.tracker)
         self.epoch += 1
+        print sample[0].shape
+        print sample[0].shape