NormalizedLinearNoBias.lua

local Linear, parent = torch.class('nn.NormalizedLinearNoBias', 'nn.Linear')
--[[
    This module creates a Linear layer, but with no bias component.
    In training mode, it constantly self-normalizes it's weights to
    be of unit norm.
    Authors: Mark Tygert, Soumith Chintala
]]--

function Linear:__init(inputSize, outputSize)
    parent.__init(self, inputSize, outputSize)
    self.bias:zero()
end

function Linear:updateOutput(input)
    if self.train then
        -- in training mode, renormalize the weights
        -- before every forward call
        self.weight:div(self.weight:norm())
        local scale = math.sqrt(self.weight:size(1))
        self.weight:mul(scale)
    end
    return parent.updateOutput(self, input)
end

function Linear:accGradParameters(input, gradOutput, scale)
    scale = scale or 1
    if input:dim() == 1 then
        self.gradWeight:addr(scale, gradOutput, input)
    elseif input:dim() == 2 then
        self.gradWeight:addmm(scale, gradOutput:t(), input)
    end
end