TARS - Go MLP Neural Network Library

example.go

package main

import (
	"encoding/csv"
	"fmt"
	"math/rand"
	"os"
	"strconv"
	"time"

	"github.com/wassupduck/tars"
)

func main() {
	rand.Seed(time.Now().UnixNano())

	// Load data
	inputs, outputs, err := loadBreastCancerDataSet()
	//fmt.Printf("%v %v\n", inputs, outputs)
	if err != nil {
		fmt.Println("Error: ", err)
		return
	}

	// Initialize neural network
	layerSizes := []int{9, 9, 1}
	net, err := tars.NewNeuralNetwork(layerSizes, tars.DefaultNeuralNetworkParams)
	if err != nil {
		fmt.Println("Error: ", err)
		return
	}

	// Train neural network
	datasetSize := len(inputs)
	trainingSetSize := int(float64(datasetSize) * 0.7)

	trainingInputs := inputs[:trainingSetSize]
	trainingOutputs := outputs[:trainingSetSize]

	trainingParams := tars.NewNeuralNetworkTrainParams()
	trainingParams.NumEpochs = 20
	trainingParams.MiniBatchSize = 30
	trainingParams.RegularizationTerm = 0
	trainingParams.LearningRate = 3

	net.Train(trainingInputs, trainingOutputs, *trainingParams)

	// Test neural network
	testingInputs := inputs[trainingSetSize:]
	testingOutputs := outputs[trainingSetSize:]
	testSetSize := datasetSize - trainingSetSize

	errorCount := 0
	correct := 0
	incorrect := 0
	for t := 0; t < testSetSize; t++ {
		testInput := testingInputs[t]
		testOutput := testingOutputs[t]

		netOutput, err := net.Predict(testInput)
		if err != nil {
			fmt.Println("Error: ", err)
			return
		}
		//fmt.Printf("%v %v\n", testOutput[0], netOutput[0])
		predictedClass := 0
		if netOutput[0] > 0.5 {
			predictedClass = 1
		}

		actualClass := int(testOutput[0])
		if predictedClass != actualClass {
			errorCount++
			incorrect++
		} else {
			correct++
		}
		fmt.Printf("%v %v\n", actualClass, netOutput[0])
	}
	fmt.Printf("correct: %v, incorrect: %v\n", correct, incorrect)

	testError := float64(errorCount) / float64(testSetSize)
	fmt.Printf("Misclassification Error: %v\n", testError)
}

func loadBreastCancerDataSet() ([][]float64, [][]float64, error) {
	// For more information on the datset
	// see https://archive.ics.uci.edu/ml/machine-learning-databases ...
	// /breast-cancer-wisconsin/breast-cancer-wisconsin.names
	file, err := os.Open("data/breast-cancer-wisconsin.data.txt")
	if err != nil {
		return nil, nil, err
	}
	defer file.Close()

	csvReader := csv.NewReader(file)
	records, err := csvReader.ReadAll()
	if err != nil {
		return nil, nil, err
	}

	// There are 16 examples in the dataset that
	// are missing features, remove these records
	records = filterRecords(records)
	inputs, outputs, err := formatData(records)
	if err != nil {
		return nil, nil, err
	}
	return inputs, outputs, nil
}

func filterRecords(records [][]string) [][]string {
	// There are 16 instances in Groups 1 to 6 that contain a single
	// missing (i.e., unavailable) attribute value, now denoted by "?".
	results := [][]string{}
	for _, record := range records {
		missingFeature := false
		for _, feature := range record {
			if feature == "?" {
				missingFeature = true
			}
		}
		if !missingFeature {
			results = append(results, record)
		}
	}
	return results
}

func formatData(records [][]string) ([][]float64, [][]float64, error) {
	recordCount := len(records)
	inputs := make([][]float64, recordCount)
	outputs := make([][]float64, recordCount)

	floatRecord := make([]float64, 10)
	for r, record := range records {
		// Drop first feature
		for j := 1; j < 11; j++ {
			feature := record[j]
			val, err := strconv.ParseFloat(feature, 64)
			if err != nil {
				return nil, nil, err
			}
			floatRecord[j-1] = val
		}
		inputs[r] = make([]float64, 9)
		copy(inputs[r], floatRecord[:9])
		output := floatRecord[9:]
		if output[0] == 4.0 { // malignant
			output[0] = 1.0
		} else {
			output[0] = 0.0
		}
		outputs[r] = make([]float64, 1)
		copy(outputs[r], output)
	}
	return inputs, outputs, nil
}

tars.go

package tars

import (
	"errors"
	"fmt"
	"math"
	"math/rand"
)

type ActivationFunc interface {
	Apply(float64) float64
	Derivative(float64) float64
}

type SigmoidActivationFunc struct{}

func (s SigmoidActivationFunc) Apply(z float64) float64 {
	return 1.0 / (1.0 + math.Exp(-z))
}

func (s SigmoidActivationFunc) Derivative(z float64) float64 {
	return z * (1 - z)
}

var SigmoidActivation SigmoidActivationFunc

type HyperbolicTangentActivationFunc struct{}

func (t HyperbolicTangentActivationFunc) Apply(z float64) float64 {
	return math.Tanh(z)
}

func (t HyperbolicTangentActivationFunc) Derivative(z float64) float64 {
	return 1 - math.Pow(math.Tanh(z), 2)
}

var HyperbolicTangentActivation HyperbolicTangentActivationFunc

type NeuralNetworkParams struct {
	InitWeightEpsilon float64
	ActivationFunc    ActivationFunc
}

var DefaultNeuralNetworkParams NeuralNetworkParams = NeuralNetworkParams{
	InitWeightEpsilon: 0.12,
	ActivationFunc:    SigmoidActivation,
}

func NewNeuralNetworkParams() *NeuralNetworkParams {
	params := DefaultNeuralNetworkParams
	return &params
}

type NeuralNetworkTrainParams struct {
	NumEpochs          int
	MiniBatchSize      int
	RegularizationTerm float64
	LearningRate       float64
}

var DefaultNeuralNetworkTrainParams NeuralNetworkTrainParams = NeuralNetworkTrainParams{
	NumEpochs:          5,
	MiniBatchSize:      10,
	RegularizationTerm: 1,
	LearningRate:       0.03,
}

func NewNeuralNetworkTrainParams() *NeuralNetworkTrainParams {
	params := DefaultNeuralNetworkTrainParams
	return &params
}

type NeuralNetwork struct {
	numLayers             int
	inputLayerSize        int
	outputLayerSize       int
	totalUnitCount        int
	layerSizes            []int
	weights               [][][]float64
	biases                [][]float64
	deltas                [][]float64
	outputsBuffer         [][]float64
	weightGradientsBuffer [][][]float64
	biasGradientsBuffer   [][]float64
	params                NeuralNetworkParams
	trainParams           NeuralNetworkTrainParams
}

func NewNeuralNetwork(layerSizes []int, params NeuralNetworkParams) (*NeuralNetwork, error) {
	net := new(NeuralNetwork)

	numLayers := len(layerSizes)
	if numLayers < 2 {
		return nil, errors.New("Number of layers must be >= 2")
	}
	for _, size := range layerSizes {
		if size < 1 {
			return nil, errors.New("All layers must have at least 1 unit")
		}
	}
	net.layerSizes = layerSizes
	net.params = params
	net.trainParams = DefaultNeuralNetworkTrainParams
	net.initialize()
	return net, nil
}

func (net *NeuralNetwork) initialize() {
	net.numLayers = len(net.layerSizes)
	net.inputLayerSize = net.layerSizes[0]
	net.outputLayerSize = net.layerSizes[net.numLayers-1]
	for _, size := range net.layerSizes {
		net.totalUnitCount += size
	}
	net.initializeWeights()
	net.initializeBiases()
	net.initializeDeltas()
	net.initializeOutputsBuffer()
	net.initializeWeightGradientsBuffer()
	net.initializeBiasGradientsBuffer()
}

func (net *NeuralNetwork) initializeWeights() {
	weights := make([][][]float64, net.numLayers)
	// Calculate total weights
	totalNetWeights := 0
	for layer := 1; layer < net.numLayers; layer++ {
		layerTotalWeights := net.layerSizes[layer-1] * net.layerSizes[layer]
		totalNetWeights += layerTotalWeights
	}
	// Allocate contiguous memory for weights
	weightsArr := make([]float64, totalNetWeights)
	// Initialize weights with random values
	// where -initWeightEps <= weight <= +initWeightEps
	initWeightEps := net.params.InitWeightEpsilon
	for k := range weightsArr {
		weightsArr[k] = rand.Float64()*(2*initWeightEps) - initWeightEps
	}
	// Assign weights array slice for each layer
	for layer := 1; layer < net.numLayers; layer++ {
		prevLayerSize := net.layerSizes[layer-1]
		layerSize := net.layerSizes[layer]
		weights[layer] = make([][]float64, layerSize)
		for unit := 0; unit < layerSize; unit++ {
			weights[layer][unit], weightsArr = weightsArr[:prevLayerSize], weightsArr[prevLayerSize:]
		}
	}
	net.weights = weights
}

func (net *NeuralNetwork) initializeBiases() {
	biases := make([][]float64, net.numLayers)
	// Calculate total biases
	totalNetBiases := net.totalUnitCount - net.inputLayerSize
	// Allocate contiguous memory for biases
	biasesArr := make([]float64, totalNetBiases)
	// Initialize biases with random values
	// where -initWeightEps <= bias <= +initWeightEps
	initWeightEps := net.params.InitWeightEpsilon
	for k := range biasesArr {
		biasesArr[k] = rand.Float64()*(2*initWeightEps) - initWeightEps
	}
	// Assign biases slice for each layer
	for layer := 1; layer < net.numLayers; layer++ {
		layerSize := net.layerSizes[layer]
		biases[layer], biasesArr = biasesArr[:layerSize], biasesArr[layerSize:]
	}
	net.biases = biases
}

func (net *NeuralNetwork) initializeDeltas() {
	deltas := make([][]float64, net.numLayers)
	// One delta per unit in network
	totalNetDeltas := net.totalUnitCount - net.inputLayerSize
	// Allocate contiguous memory for deltas
	deltasArr := make([]float64, totalNetDeltas)
	// Assign deltas slice for each layer
	for layer := 1; layer < net.numLayers; layer++ {
		layerSize := net.layerSizes[layer]
		deltas[layer], deltasArr = deltasArr[:layerSize], deltasArr[layerSize:]
	}
	net.deltas = deltas
}

func (net *NeuralNetwork) initializeOutputsBuffer() {
	outputs := make([][]float64, net.numLayers)
	// Calculate total outputs
	totalNetOutputs := net.totalUnitCount - net.inputLayerSize
	// Allocate contiguous memory for outputs
	outputsArr := make([]float64, totalNetOutputs)
	// Assign outputs slice for each layer
	for layer := 1; layer < net.numLayers; layer++ {
		layerSize := net.layerSizes[layer]
		outputs[layer], outputsArr = outputsArr[:layerSize], outputsArr[layerSize:]
	}
	net.outputsBuffer = outputs
}

func (net *NeuralNetwork) initializeWeightGradientsBuffer() {
	weightGradients := make([][][]float64, net.numLayers)
	// Calculate total weights
	totalNetWeights := 0
	for layer := 1; layer < net.numLayers; layer++ {
		layerTotalWeights := net.layerSizes[layer-1] * net.layerSizes[layer]
		totalNetWeights += layerTotalWeights
	}
	// Allocate contiguous memory for weight gradients
	weightGradientsArr := make([]float64, totalNetWeights)
	// Assign weight gradients slice for each layer
	for layer := 1; layer < net.numLayers; layer++ {
		prevLayerSize := net.layerSizes[layer-1]
		layerSize := net.layerSizes[layer]
		weightGradients[layer] = make([][]float64, layerSize)
		for unit := 0; unit < layerSize; unit++ {
			weightGradients[layer][unit], weightGradientsArr = weightGradientsArr[:prevLayerSize], weightGradientsArr[prevLayerSize:]
		}
	}
	net.weightGradientsBuffer = weightGradients
}

func (net *NeuralNetwork) initializeBiasGradientsBuffer() {
	biasGradients := make([][]float64, net.numLayers)
	// Calculate total biases
	totalNetBiases := net.totalUnitCount - net.inputLayerSize
	// Allocate contiguous memory for bias gradients
	biasGradientsArr := make([]float64, totalNetBiases)
	// Assign bias gradients slice for each layer
	for layer := 1; layer < net.numLayers; layer++ {
		layerSize := net.layerSizes[layer]
		biasGradients[layer], biasGradientsArr = biasGradientsArr[:layerSize], biasGradientsArr[layerSize:]
	}
	net.biasGradientsBuffer = biasGradients
}

func (net *NeuralNetwork) feedForward(input []float64) []float64 {
	var output []float64
	for layer := 1; layer < net.numLayers; layer++ {
		output = net.outputsBuffer[layer]
		for unit := 0; unit < net.layerSizes[layer]; unit++ {
			// Calculate activation of unit
			weights := net.weights[layer][unit]
			sum := net.biases[layer][unit]
			for k, inVal := range input {
				sum += weights[k] * inVal
			}
			output[unit] = net.params.ActivationFunc.Apply(sum)
		}
		input = output
	}
	return output
}

func (net *NeuralNetwork) calculateDeltas(target []float64) {
	outputLayer := net.numLayers - 1
	for layer := outputLayer; layer > 0; layer-- {
		for unit := 0; unit < net.layerSizes[layer]; unit++ {
			activation := net.outputsBuffer[layer][unit]
			delta := 0.0
			if layer == outputLayer {
				delta = activation - target[unit]
			} else {
				forwardLayer := layer + 1
				forwardLayerWeights := net.weights[forwardLayer]
				forwardLayerDeltas := net.deltas[forwardLayer]
				for fUnit := 0; fUnit < net.layerSizes[forwardLayer]; fUnit++ {
					delta += forwardLayerDeltas[fUnit] * forwardLayerWeights[fUnit][unit]
				}
				delta *= net.params.ActivationFunc.Derivative(activation)
			}
			net.deltas[layer][unit] = delta
		}
	}
}

func (net *NeuralNetwork) calculateAndAccumulateGradients() {
	for layer := 1; layer < net.numLayers; layer++ {
		incoming := net.outputsBuffer[layer-1]
		for unit := 0; unit < net.layerSizes[layer]; unit++ {
			delta := net.deltas[layer][unit]
			for k := range incoming {
				net.weightGradientsBuffer[layer][unit][k] += incoming[k] * delta
			}
			net.biasGradientsBuffer[layer][unit] += delta
		}
	}
}

func (net *NeuralNetwork) backprop(input []float64, target []float64) {
	// Forward propagate
	net.feedForward(input)
	// Back propagate
	net.calculateDeltas(target)
	// Calculate gradients
	net.calculateAndAccumulateGradients()
}

func (net *NeuralNetwork) zeroGradients() {
	for layer := 1; layer < net.numLayers; layer++ {
		for unit := 0; unit < net.layerSizes[layer]; unit++ {
			weightGradients := net.weightGradientsBuffer[layer][unit]
			for k := range weightGradients {
				weightGradients[k] = 0
			}
			net.biasGradientsBuffer[layer][unit] = 0
		}
	}
}

func (net *NeuralNetwork) miniBatchUpdate(inputs [][]float64, outputs [][]float64) {
	miniBatchSize := len(inputs)
	// Zero gradients
	net.zeroGradients()
	for i := 0; i < miniBatchSize; i++ {
		input := inputs[i]
		target := outputs[i]
		// Calculate and accumulate weight gradients
		// for each example in mini-batch
		net.backprop(input, target)
	}
	// Update weights
	alpha := net.trainParams.LearningRate
	lambda := net.trainParams.RegularizationTerm
	for layer := 1; layer < net.numLayers; layer++ {
		for unit := 0; unit < net.layerSizes[layer]; unit++ {
			weights := net.weights[layer][unit]
			weightGradients := net.weightGradientsBuffer[layer][unit]
			for k, weight := range weights {
				weightGradient := (weightGradients[k] + (lambda * weight)) / float64(miniBatchSize)
				weights[k] -= alpha * weightGradient
			}
			biasGradient := net.biasGradientsBuffer[layer][unit] / float64(miniBatchSize)
			net.biases[layer][unit] -= alpha * biasGradient
		}
	}
}

func (net *NeuralNetwork) shuffleTrainingData(inputs [][]float64, outputs [][]float64) {
	// Sattolo's shuffle algorithm
	inputsSize := len(inputs)
	for i := inputsSize - 1; i > 1; i-- {
		r := rand.Intn(i - 1)
		inputs[r], inputs[i] = inputs[i], inputs[r]
		outputs[r], outputs[i] = outputs[i], outputs[r]
	}
}

func (net *NeuralNetwork) Train(inputs [][]float64, outputs [][]float64, trainParams NeuralNetworkTrainParams) error {
	inputsSize := len(inputs)
	outputsSize := len(outputs)

	if inputsSize != outputsSize {
		return errors.New("Inputs size does not match outputs size")
	}
	if inputsSize == 0 {
		if inputs == nil {
			return errors.New("Inputs slice not initialized")
		}
		if outputs == nil {
			return errors.New("Outputs slice not initialized")
		}
		return nil
	}

	net.trainParams = trainParams
	miniBatchSize := net.trainParams.MiniBatchSize
	miniBatchCount := 0
	if inputsSize%miniBatchSize == 0 {
		miniBatchCount = inputsSize / miniBatchSize
	} else {
		miniBatchCount = (inputsSize / miniBatchSize) + 1
	}

	for epoch := 0; epoch < net.trainParams.NumEpochs; epoch++ {
		net.shuffleTrainingData(inputs, outputs)

		var miniBatchInputs [][]float64
		var miniBatchOutputs [][]float64

		inputsTmp := inputs
		outputsTmp := outputs

		lastMiniBatch := miniBatchCount - 1
		for miniBatch := 0; miniBatch < miniBatchCount; miniBatch++ {
			if miniBatch == lastMiniBatch {
				miniBatchInputs = inputsTmp
				miniBatchOutputs = outputsTmp
			} else {
				miniBatchInputs, inputsTmp = inputsTmp[:miniBatchSize], inputsTmp[miniBatchSize:]
				miniBatchOutputs, outputsTmp = outputsTmp[:miniBatchSize], outputsTmp[miniBatchSize:]
			}
			net.miniBatchUpdate(miniBatchInputs, miniBatchOutputs)
		}
	}
	return nil
}

func (net *NeuralNetwork) Predict(input []float64) ([]float64, error) {
	if len(input) != net.inputLayerSize {
		return nil, errors.New(
			fmt.Sprintf("Expected input vector of size: %v", net.inputLayerSize))
	}

	output := make([]float64, net.outputLayerSize)
	netOutput := net.feedForward(input)
	copy(output, netOutput)
	return output, nil
}

Breast Cancer Wisconsin (Diagnostic) Data Set
https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+%28Diagnostic%29