BN evaluation script

main.py

from theano import function
from blocks.bricks import MLP, Identity
from blocks.filter import VariableFilter
from blocks.initialization import Constant
from blocks.roles import INPUT
from blocks.utils import shared_floatx
from theano import function, tensor
from blocks.graph import ComputationGraph, apply_batch_normalization

import numpy
import theano

x_tr = numpy.ones((3, 2),dtype=theano.config.floatX)

linear = MLP([Identity(), Identity()], [2, 10, 2],
             weights_init=Constant(1), biases_init=Constant(2))
linear.initialize()

x = tensor.matrix('x')
gamma_1 = shared_floatx(numpy.ones(2), name='gamma_1')
gamma_2 = shared_floatx(numpy.ones(10), name='gamma_2')
beta_1 = shared_floatx(numpy.zeros(2), name='beta_1')
beta_2 = shared_floatx(numpy.zeros(10), name='beta_2')

#gamma / sqrt(Var[x] + epsilon)
#beta - (gamma . E[x]) / sqrt(Var[x] + epsilon).

epsilon=1e-7

y = linear.apply(x)
cg = ComputationGraph(y)
inputs = VariableFilter(
    roles=[INPUT],
    bricks=linear.linear_transformations)(cg.variables)

use_population = dict([(var,False) for var in inputs])

gammas = [gamma_1, gamma_2]
betas = [beta_1, beta_2]

for i in range(len(inputs)):
    f = function([x], inputs[i])
    value = f(x_tr)
    #print value
    mu = value.mean(axis = 0)
    var = value.var(axis = 0)

    gammas[i] = shared_floatx(gammas[i].get_value() / numpy.sqrt(var + epsilon))
    betas[i] = shared_floatx(betas[i].get_value() - mu*gammas[i].get_value())

    for j in range(0,i):
        use_population[inputs[j]] = True
    cg = apply_batch_normalization(
        cg, inputs, gammas, betas, use_population = use_population)

    inputs = VariableFilter(
    roles=[INPUT],
    bricks=linear.linear_transformations)(cg.variables)

fprop = function(cg.inputs, cg.outputs[0])

fprop(x_tr)