module implementation

gistfile1.txt

# forward(x) -> y
# backward(dy, x) -> dx

class Sigmoid(dl.Module):
    """
    Will take x, return an array with the same shape and values

        1 / (1 + e^(-x))
    """

    def forward(self, x):
        # takes x, returns something
        self.y = 1 / (1 + np.exp(-x))
        return self.y

    def backward(self, dy):
        # I want to return dL/dx, where I am given dy = dL/dy,
        # and y = self.forward(x)
        #
        # I want to compute
        #   dy / dx
        # where
        #   y = 1 / (1 + exp(-x))
        # because then
        #   dL/dx = dy/dx dL/dy.
        return self.y * (1 - self.y)  # this works because of a trick!