Feed Forward Neural Net Classifier using Autograd

ff_neural_classifyer_ag.py

## ext requirements
import autograd.numpy as anp 
from autograd import grad


# - - - - - - - - - - - - - - - - - -

    # -- Model --

# - - - - - - - - - - - - - - - - - -

## produces model outputs
def forward(params: dict, inputs: anp.ndarray = None, hps: anp.ndarray = None) -> list:
    hidden_activation = hps['hidden_activation'](
        anp.add(
            anp.matmul(
                inputs,
                params['input']['hidden']['weights']
            ),
            params['input']['hidden']['bias']
        )
    )

    output_activation = hps['output_activation'](
        anp.add(
            anp.matmul(
                hidden_activation,
                params['hidden']['output']['weights'],
            ),
            params['hidden']['output']['bias'],
        )
    )
    return [hidden_activation, output_activation]


## cross entropy loss function
def loss(params: dict, inputs: anp.ndarray = None, targets: anp.ndarray = None, hps: dict = None) -> float:
    return -anp.sum(
        anp.multiply(
            targets,
            anp.log(
                forward(params, inputs = inputs, hps = hps)[-1]
            )
        )
    ) / inputs.shape[0]


## optimization function
loss_grad = grad(loss)



# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



# - - - - - - - - - - - - - - - - - -

    # -- Convenience Functions --

# - - - - - - - - - - - - - - - - - -

def build_params(num_features: int, num_hidden_nodes: int, num_categories: int, weight_range: tuple = (-.1, .1)) -> dict:
    '''
    num_features <-- (numeric) number of feature in the dataset
    num_hidden_nodes <-- (numeric)
    num_categories <-- (list) list of category labels to use as keys for decode -- output connections
    weight_range = [-.1,.1] <-- (list of numeric)
    '''
    return {
        'input': {
            'hidden': {
                'weights': anp.random.uniform(*weight_range, [num_features, num_hidden_nodes]),
                'bias': anp.random.uniform(*weight_range, [1, num_hidden_nodes]),
            },
        },
        'hidden': {
            'output': {
                'weights': anp.random.uniform(*weight_range, [num_hidden_nodes, num_categories]),
                'bias': anp.random.uniform(*weight_range, [1, num_categories]),
            } 
        },
    }

def update_params(params: dict, gradients: dict, lr: float) -> dict:
    for layer in params:
        for connection in params[layer]:
            params[layer][connection]['weights'] -= lr * gradients[layer][connection]['weights']
            params[layer][connection]['bias'] -= lr * gradients[layer][connection]['bias']
    return params


def response(params: dict, inputs: anp.ndarray = None, hps: dict = None) -> anp.ndarray:
    return anp.argmax(
        forward(params, inputs = inputs, hps = hps)[-1],
        axis = 1
    )


# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



# - - - - - - - - - - - - - - - - - -

    # -- Run --

# - - - - - - - - - - - - - - - - - -

if __name__ == '__main__':
    
    # makeup a random dataset
    inputs = anp.array([
        [.2, .3],
        [.3, .4],
        [.4, .5],
        [.5, .6],
        [.6, .7],
        [.7, .8],
        [.8, .9],

        [.2, .1],
        [.3, .2],
        [.4, .3],
        [.5, .4],
        [.6, .5],
        [.7, .6],
        [.8, .7],
    ])

    labels = anp.array([
        [0,1],
        [0,1],
        [0,1],
        [0,1],
        [0,1],
        [0,1],
        [0,1],

        [1,0],
        [1,0],
        [1,0],
        [1,0],
        [1,0],
        [1,0],
        [1,0],
    ])

    hps = {
        'lr': .5,  # <-- learning rate
        'wr': [-.1, .1], # <-- weight range
        'num_hidden_nodes': 10,

        'hidden_activation': lambda x: 1 / (1 + anp.exp(-x)), # <-- sigmoid activation function
        'output_activation': lambda x: (anp.exp(x - anp.max(x)).T / anp.sum(anp.exp(x - anp.max(x)),axis=1)).T, # <-- softmax activation function
    }

    params = build_params(
        inputs.shape[1],  # <-- num features
        hps['num_hidden_nodes'],
        labels.shape[1]
    )

    num_epochs = 1000

    print('loss initially: ', loss(params, inputs = inputs, targets = labels, hps = hps))
    
    for epoch in range(num_epochs):
        gradients = loss_grad(params, inputs = inputs, targets = labels, hps = hps)
        params = update_params(params, gradients, hps['lr'])
    
    print('loss after training: ', loss(params, inputs = inputs, targets = labels, hps = hps))