softmax_regression.py

import numpy as np

# DEMO for my article: https://medium.com/@lilychencodes/implementing-a-multiclass-classification-machine-learning-algorithm-ea320518ea5d

# dataset = [
#     [4, 82, [0, 1, 0]],
#     [3, 80, [0, 0, 1]],
#     [2.5, 75, [0, 0, 1]],
#     [3.4, 90, [0, 1, 0]],
#     [4.2, 88, [1, 0, 0]],
#     [5, 92, [1, 0, 0]],
#     [2.7, 99, [0, 0, 1]],
#     [3.3, 85, [0, 0, 1]],
#     [4.2, 72, [0, 1, 0]],
#     [3.6, 80, [0, 1, 0]],
#     [2.9, 85, [0, 0, 1]],
#     [3.9, 85, [0, 1, 0]],
#     [4.5, 99, [1, 0, 0]],
#     [4.7, 90, [1, 0, 0]],
#     [4.6, 80, [1, 0, 0]],
#     [4.6, 75, [0, 1, 0]],
#     [3.4, 64, [0, 0, 1]],
#     [4, 95, [1, 0, 0]],
#     [4.1, 86, [0, 1, 0]],
#     [4.5, 90, [1, 0, 0]],
#     [3.7, 70, [0, 0, 1]],
#     [3.2, 81, [0, 0, 1]]
# ]


class SoftmaxRegression:
    def __init__(self, dataset):
        self.dataset = dataset
        self.alpha = 0.001  # alpha is "learning rate"

    def softmax(self, datapoint, thetas, class_idx):
        nominator_theta = thetas[class_idx]
        transformed_datapoint = [1, datapoint[0], datapoint[1]]
        nominator = np.exp(np.dot(nominator_theta, transformed_datapoint))
        denominator = 0
        for theta in thetas:
            denominator += np.exp(np.dot(theta, transformed_datapoint))

        return nominator / denominator

    # find cross entropy of one datapoint
    # datapoint is in format [GPA, exam score, label]
    # thetas = [theta_admitted, theta_waitlisted, theta_rejected]
    def calc_cross_entropy(self, datapoint, thetas):
        cross_ent = 0

        num_class = len(thetas)

        for j in range(0, num_class):
            # compute new theta for class
            label = datapoint[-1]
            class_label = [0] * len(thetas)
            class_label[j] = 1
            label_is_class = label == class_label
            zero_or_one = 1 if label_is_class else 0
            if zero_or_one == 0:
                cross_ent += 0
            else:
                softmax = self.softmax(datapoint, thetas, j)
                cross_ent += np.log(softmax)

        return -1 * cross_ent

    # calculate total cross entropy, which is the average of all cross-entropies over n training samples
    def calc_total_loss(self, thetas):
        total_cross_ent = 0
        for datapoint in self.dataset:
            cross_ent = self.calc_cross_entropy(datapoint, thetas)
            total_cross_ent += cross_ent

        return total_cross_ent

    # get theta for particular class
    def get_theta(self, thetas, class_idx):
        theta_class = thetas[class_idx]
        m = len(self.dataset)

        class_label = [0] * len(thetas)
        class_label[class_idx] = 1

        sum_loss = np.array([0.0] * len(thetas))
        for datapoint in self.dataset:
            softmax_value = self.softmax(datapoint, thetas, class_idx)
            transformed_datapoint = [1, datapoint[0], datapoint[1]]
            label = datapoint[-1]
            label_is_class = label == class_label
            zero_or_one = 1 if label_is_class else 0
            prod = np.array(transformed_datapoint) * (zero_or_one - softmax_value)
            # print(f'datapoint: {datapoint}, class label: {class_label}/{class_idx} softmax val: {softmax_value}')
            sum_loss += prod

        new_gradients = self.alpha * (1/m) * sum_loss

        new_theta = np.sum([new_gradients, theta_class], axis=0)

        return new_theta

    # get new thetas for all classes according to gradient descent formula
    def get_thetas(self, thetas):
        num_class = len(thetas)

        new_thetas = []

        for j in range(0, num_class):
            # compute new theta for class
            theta_class = self.get_theta(thetas, j)
            new_thetas.append(theta_class)

        return new_thetas

    def iterate(self):
        num_iteration = 0
        theta_admitted = [-250, 40, 1]
        theta_waitlisted = [-220, 40, 1]
        theta_rejected = [-220, 40, 1]
        thetas = [theta_admitted, theta_waitlisted, theta_rejected]

        loss = None
        losses = []

        while num_iteration < 10000:
            if num_iteration % 500 == 0:
                print('iteration:', num_iteration)
                print('loss:', loss)
                print('thetas:', thetas)
                losses.append({'iteration': num_iteration, 'loss': loss})

            loss = self.calc_total_loss(thetas)
            new_thetas = self.get_thetas(thetas)
            thetas = new_thetas
            num_iteration += 1

        print(f'After {num_iteration}, total loss is {loss}. Theta admitted is {thetas[0]}. Theta waitlisted is {thetas[1]}. Theta rejected is {thetas[2]}.')
        return losses