mode.py

class Mode(object):
    """Setup mode train or test.

    It is meant to be used in a context created by `train` or `test`.
    Crashes if used outside of context.
    
    An intended use is to create a brick with conditional apply method
    which checks mode inside. A brick should store the mode object in
    a field. Several bricks can use the same mode object.

    Examples
    --------
    ```python
    >>> mode = Mode()
    >>> def foo():
    ...    print(mode)
    ...
    >>> with mode.train:
    ...    foo()
    train
    >>> with mode.test:
    ...    foo()
    test
    >>> foo()
    AttributeError...
    ```
    """
    @property
    def train(self):
        self._mode_to_set = 'train'
        return self

    @property
    def test(self):
        self._mode_to_set = 'test'
        return self

    def __enter__(self):
        self.mode = self._mode_to_set

    def __exit__(self, exc_type, exc_val, exc_tb):
        del self.mode

    def __deepcopy__(self, memo):
        return self

    def __eq__(self, other):
        return self.mode == other

    def __repr__(self):
        return self.mode

noisy_lstm.py

class NoisyLSTM(LSTM, Random):
    def __init__(self, dim, activation=None, gate_activation=None, **kwargs):
        super(NoisyLSTM, self).__init__(dim, activation, gate_activation, **kwargs)

    @recurrent(sequences=['inputs', 'mask', 'rand_noise'],
               states=['states', 'cells'],
               contexts=[], outputs=['states', 'cells'])
    def do_apply(self, inputs, states, cells, mask=None, rand_noise=None):
        def slice_last(x, no):
            return x[:, no*self.dim: (no+1)*self.dim]

        nonlinearity = self.children[0].apply

        activation = tensor.dot(states, self.W_state) + inputs
        in_gate = self.gate_activation.apply(
            slice_last(activation, 0) +
            cells * self.W_cell_to_in, slice_last(rand_noise, 0))
        forget_gate = self.gate_activation.apply(
            slice_last(activation, 1) +
            cells * self.W_cell_to_forget, slice_last(rand_noise, 1))
        next_cells = (forget_gate * cells +
                      in_gate * nonlinearity(slice_last(activation, 2),
                                             slice_last(rand_noise, 2)))
        out_gate = self.gate_activation.apply(
            slice_last(activation, 3) +
            next_cells * self.W_cell_to_out, slice_last(rand_noise, 3))
        next_states = out_gate * nonlinearity(next_cells, slice_last(rand_noise, 3))

        if mask:
            next_states = (mask[:, None] * next_states +
                           (1 - mask[:, None]) * states)
            next_cells = (mask[:, None] * next_cells +
                          (1 - mask[:, None]) * cells)

        return next_states, next_cells

    @application
    def apply(self, inputs, states, cells, mask=None, **kwargs):
        noise = self.theano_rng.normal(size=inputs.shape, avg=0, std=1,
                                       dtype='float32')
        return self.do_apply(inputs, states=states, cells=cells, mask=mask,
                             rand_noise=noise, **kwargs)

    @apply.delegate
    def apply_delegate(self):
        # when someone calls apply.sequences it will return do_apply.sequences
        return self.do_apply

    def get_dim(self, name):
        if name == 'rand_noise':
            return self.get_dim('inputs')
        return super(NoisyLSTM, self).get_dim(name)