Created
May 18, 2015 08:34
-
-
Save wassupduck/c5751af874cc7979c403 to your computer and use it in GitHub Desktop.
TARS - Go MLP Neural Network Library
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"encoding/csv" | |
"fmt" | |
"math/rand" | |
"os" | |
"strconv" | |
"time" | |
"github.com/wassupduck/tars" | |
) | |
func main() { | |
rand.Seed(time.Now().UnixNano()) | |
// Load data | |
inputs, outputs, err := loadBreastCancerDataSet() | |
//fmt.Printf("%v %v\n", inputs, outputs) | |
if err != nil { | |
fmt.Println("Error: ", err) | |
return | |
} | |
// Initialize neural network | |
layerSizes := []int{9, 9, 1} | |
net, err := tars.NewNeuralNetwork(layerSizes, tars.DefaultNeuralNetworkParams) | |
if err != nil { | |
fmt.Println("Error: ", err) | |
return | |
} | |
// Train neural network | |
datasetSize := len(inputs) | |
trainingSetSize := int(float64(datasetSize) * 0.7) | |
trainingInputs := inputs[:trainingSetSize] | |
trainingOutputs := outputs[:trainingSetSize] | |
trainingParams := tars.NewNeuralNetworkTrainParams() | |
trainingParams.NumEpochs = 20 | |
trainingParams.MiniBatchSize = 30 | |
trainingParams.RegularizationTerm = 0 | |
trainingParams.LearningRate = 3 | |
net.Train(trainingInputs, trainingOutputs, *trainingParams) | |
// Test neural network | |
testingInputs := inputs[trainingSetSize:] | |
testingOutputs := outputs[trainingSetSize:] | |
testSetSize := datasetSize - trainingSetSize | |
errorCount := 0 | |
correct := 0 | |
incorrect := 0 | |
for t := 0; t < testSetSize; t++ { | |
testInput := testingInputs[t] | |
testOutput := testingOutputs[t] | |
netOutput, err := net.Predict(testInput) | |
if err != nil { | |
fmt.Println("Error: ", err) | |
return | |
} | |
//fmt.Printf("%v %v\n", testOutput[0], netOutput[0]) | |
predictedClass := 0 | |
if netOutput[0] > 0.5 { | |
predictedClass = 1 | |
} | |
actualClass := int(testOutput[0]) | |
if predictedClass != actualClass { | |
errorCount++ | |
incorrect++ | |
} else { | |
correct++ | |
} | |
fmt.Printf("%v %v\n", actualClass, netOutput[0]) | |
} | |
fmt.Printf("correct: %v, incorrect: %v\n", correct, incorrect) | |
testError := float64(errorCount) / float64(testSetSize) | |
fmt.Printf("Misclassification Error: %v\n", testError) | |
} | |
func loadBreastCancerDataSet() ([][]float64, [][]float64, error) { | |
// For more information on the datset | |
// see https://archive.ics.uci.edu/ml/machine-learning-databases ... | |
// /breast-cancer-wisconsin/breast-cancer-wisconsin.names | |
file, err := os.Open("data/breast-cancer-wisconsin.data.txt") | |
if err != nil { | |
return nil, nil, err | |
} | |
defer file.Close() | |
csvReader := csv.NewReader(file) | |
records, err := csvReader.ReadAll() | |
if err != nil { | |
return nil, nil, err | |
} | |
// There are 16 examples in the dataset that | |
// are missing features, remove these records | |
records = filterRecords(records) | |
inputs, outputs, err := formatData(records) | |
if err != nil { | |
return nil, nil, err | |
} | |
return inputs, outputs, nil | |
} | |
func filterRecords(records [][]string) [][]string { | |
// There are 16 instances in Groups 1 to 6 that contain a single | |
// missing (i.e., unavailable) attribute value, now denoted by "?". | |
results := [][]string{} | |
for _, record := range records { | |
missingFeature := false | |
for _, feature := range record { | |
if feature == "?" { | |
missingFeature = true | |
} | |
} | |
if !missingFeature { | |
results = append(results, record) | |
} | |
} | |
return results | |
} | |
func formatData(records [][]string) ([][]float64, [][]float64, error) { | |
recordCount := len(records) | |
inputs := make([][]float64, recordCount) | |
outputs := make([][]float64, recordCount) | |
floatRecord := make([]float64, 10) | |
for r, record := range records { | |
// Drop first feature | |
for j := 1; j < 11; j++ { | |
feature := record[j] | |
val, err := strconv.ParseFloat(feature, 64) | |
if err != nil { | |
return nil, nil, err | |
} | |
floatRecord[j-1] = val | |
} | |
inputs[r] = make([]float64, 9) | |
copy(inputs[r], floatRecord[:9]) | |
output := floatRecord[9:] | |
if output[0] == 4.0 { // malignant | |
output[0] = 1.0 | |
} else { | |
output[0] = 0.0 | |
} | |
outputs[r] = make([]float64, 1) | |
copy(outputs[r], output) | |
} | |
return inputs, outputs, nil | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package tars | |
import ( | |
"errors" | |
"fmt" | |
"math" | |
"math/rand" | |
) | |
type ActivationFunc interface { | |
Apply(float64) float64 | |
Derivative(float64) float64 | |
} | |
type SigmoidActivationFunc struct{} | |
func (s SigmoidActivationFunc) Apply(z float64) float64 { | |
return 1.0 / (1.0 + math.Exp(-z)) | |
} | |
func (s SigmoidActivationFunc) Derivative(z float64) float64 { | |
return z * (1 - z) | |
} | |
var SigmoidActivation SigmoidActivationFunc | |
type HyperbolicTangentActivationFunc struct{} | |
func (t HyperbolicTangentActivationFunc) Apply(z float64) float64 { | |
return math.Tanh(z) | |
} | |
func (t HyperbolicTangentActivationFunc) Derivative(z float64) float64 { | |
return 1 - math.Pow(math.Tanh(z), 2) | |
} | |
var HyperbolicTangentActivation HyperbolicTangentActivationFunc | |
type NeuralNetworkParams struct { | |
InitWeightEpsilon float64 | |
ActivationFunc ActivationFunc | |
} | |
var DefaultNeuralNetworkParams NeuralNetworkParams = NeuralNetworkParams{ | |
InitWeightEpsilon: 0.12, | |
ActivationFunc: SigmoidActivation, | |
} | |
func NewNeuralNetworkParams() *NeuralNetworkParams { | |
params := DefaultNeuralNetworkParams | |
return ¶ms | |
} | |
type NeuralNetworkTrainParams struct { | |
NumEpochs int | |
MiniBatchSize int | |
RegularizationTerm float64 | |
LearningRate float64 | |
} | |
var DefaultNeuralNetworkTrainParams NeuralNetworkTrainParams = NeuralNetworkTrainParams{ | |
NumEpochs: 5, | |
MiniBatchSize: 10, | |
RegularizationTerm: 1, | |
LearningRate: 0.03, | |
} | |
func NewNeuralNetworkTrainParams() *NeuralNetworkTrainParams { | |
params := DefaultNeuralNetworkTrainParams | |
return ¶ms | |
} | |
type NeuralNetwork struct { | |
numLayers int | |
inputLayerSize int | |
outputLayerSize int | |
totalUnitCount int | |
layerSizes []int | |
weights [][][]float64 | |
biases [][]float64 | |
deltas [][]float64 | |
outputsBuffer [][]float64 | |
weightGradientsBuffer [][][]float64 | |
biasGradientsBuffer [][]float64 | |
params NeuralNetworkParams | |
trainParams NeuralNetworkTrainParams | |
} | |
func NewNeuralNetwork(layerSizes []int, params NeuralNetworkParams) (*NeuralNetwork, error) { | |
net := new(NeuralNetwork) | |
numLayers := len(layerSizes) | |
if numLayers < 2 { | |
return nil, errors.New("Number of layers must be >= 2") | |
} | |
for _, size := range layerSizes { | |
if size < 1 { | |
return nil, errors.New("All layers must have at least 1 unit") | |
} | |
} | |
net.layerSizes = layerSizes | |
net.params = params | |
net.trainParams = DefaultNeuralNetworkTrainParams | |
net.initialize() | |
return net, nil | |
} | |
func (net *NeuralNetwork) initialize() { | |
net.numLayers = len(net.layerSizes) | |
net.inputLayerSize = net.layerSizes[0] | |
net.outputLayerSize = net.layerSizes[net.numLayers-1] | |
for _, size := range net.layerSizes { | |
net.totalUnitCount += size | |
} | |
net.initializeWeights() | |
net.initializeBiases() | |
net.initializeDeltas() | |
net.initializeOutputsBuffer() | |
net.initializeWeightGradientsBuffer() | |
net.initializeBiasGradientsBuffer() | |
} | |
func (net *NeuralNetwork) initializeWeights() { | |
weights := make([][][]float64, net.numLayers) | |
// Calculate total weights | |
totalNetWeights := 0 | |
for layer := 1; layer < net.numLayers; layer++ { | |
layerTotalWeights := net.layerSizes[layer-1] * net.layerSizes[layer] | |
totalNetWeights += layerTotalWeights | |
} | |
// Allocate contiguous memory for weights | |
weightsArr := make([]float64, totalNetWeights) | |
// Initialize weights with random values | |
// where -initWeightEps <= weight <= +initWeightEps | |
initWeightEps := net.params.InitWeightEpsilon | |
for k := range weightsArr { | |
weightsArr[k] = rand.Float64()*(2*initWeightEps) - initWeightEps | |
} | |
// Assign weights array slice for each layer | |
for layer := 1; layer < net.numLayers; layer++ { | |
prevLayerSize := net.layerSizes[layer-1] | |
layerSize := net.layerSizes[layer] | |
weights[layer] = make([][]float64, layerSize) | |
for unit := 0; unit < layerSize; unit++ { | |
weights[layer][unit], weightsArr = weightsArr[:prevLayerSize], weightsArr[prevLayerSize:] | |
} | |
} | |
net.weights = weights | |
} | |
func (net *NeuralNetwork) initializeBiases() { | |
biases := make([][]float64, net.numLayers) | |
// Calculate total biases | |
totalNetBiases := net.totalUnitCount - net.inputLayerSize | |
// Allocate contiguous memory for biases | |
biasesArr := make([]float64, totalNetBiases) | |
// Initialize biases with random values | |
// where -initWeightEps <= bias <= +initWeightEps | |
initWeightEps := net.params.InitWeightEpsilon | |
for k := range biasesArr { | |
biasesArr[k] = rand.Float64()*(2*initWeightEps) - initWeightEps | |
} | |
// Assign biases slice for each layer | |
for layer := 1; layer < net.numLayers; layer++ { | |
layerSize := net.layerSizes[layer] | |
biases[layer], biasesArr = biasesArr[:layerSize], biasesArr[layerSize:] | |
} | |
net.biases = biases | |
} | |
func (net *NeuralNetwork) initializeDeltas() { | |
deltas := make([][]float64, net.numLayers) | |
// One delta per unit in network | |
totalNetDeltas := net.totalUnitCount - net.inputLayerSize | |
// Allocate contiguous memory for deltas | |
deltasArr := make([]float64, totalNetDeltas) | |
// Assign deltas slice for each layer | |
for layer := 1; layer < net.numLayers; layer++ { | |
layerSize := net.layerSizes[layer] | |
deltas[layer], deltasArr = deltasArr[:layerSize], deltasArr[layerSize:] | |
} | |
net.deltas = deltas | |
} | |
func (net *NeuralNetwork) initializeOutputsBuffer() { | |
outputs := make([][]float64, net.numLayers) | |
// Calculate total outputs | |
totalNetOutputs := net.totalUnitCount - net.inputLayerSize | |
// Allocate contiguous memory for outputs | |
outputsArr := make([]float64, totalNetOutputs) | |
// Assign outputs slice for each layer | |
for layer := 1; layer < net.numLayers; layer++ { | |
layerSize := net.layerSizes[layer] | |
outputs[layer], outputsArr = outputsArr[:layerSize], outputsArr[layerSize:] | |
} | |
net.outputsBuffer = outputs | |
} | |
func (net *NeuralNetwork) initializeWeightGradientsBuffer() { | |
weightGradients := make([][][]float64, net.numLayers) | |
// Calculate total weights | |
totalNetWeights := 0 | |
for layer := 1; layer < net.numLayers; layer++ { | |
layerTotalWeights := net.layerSizes[layer-1] * net.layerSizes[layer] | |
totalNetWeights += layerTotalWeights | |
} | |
// Allocate contiguous memory for weight gradients | |
weightGradientsArr := make([]float64, totalNetWeights) | |
// Assign weight gradients slice for each layer | |
for layer := 1; layer < net.numLayers; layer++ { | |
prevLayerSize := net.layerSizes[layer-1] | |
layerSize := net.layerSizes[layer] | |
weightGradients[layer] = make([][]float64, layerSize) | |
for unit := 0; unit < layerSize; unit++ { | |
weightGradients[layer][unit], weightGradientsArr = weightGradientsArr[:prevLayerSize], weightGradientsArr[prevLayerSize:] | |
} | |
} | |
net.weightGradientsBuffer = weightGradients | |
} | |
func (net *NeuralNetwork) initializeBiasGradientsBuffer() { | |
biasGradients := make([][]float64, net.numLayers) | |
// Calculate total biases | |
totalNetBiases := net.totalUnitCount - net.inputLayerSize | |
// Allocate contiguous memory for bias gradients | |
biasGradientsArr := make([]float64, totalNetBiases) | |
// Assign bias gradients slice for each layer | |
for layer := 1; layer < net.numLayers; layer++ { | |
layerSize := net.layerSizes[layer] | |
biasGradients[layer], biasGradientsArr = biasGradientsArr[:layerSize], biasGradientsArr[layerSize:] | |
} | |
net.biasGradientsBuffer = biasGradients | |
} | |
func (net *NeuralNetwork) feedForward(input []float64) []float64 { | |
var output []float64 | |
for layer := 1; layer < net.numLayers; layer++ { | |
output = net.outputsBuffer[layer] | |
for unit := 0; unit < net.layerSizes[layer]; unit++ { | |
// Calculate activation of unit | |
weights := net.weights[layer][unit] | |
sum := net.biases[layer][unit] | |
for k, inVal := range input { | |
sum += weights[k] * inVal | |
} | |
output[unit] = net.params.ActivationFunc.Apply(sum) | |
} | |
input = output | |
} | |
return output | |
} | |
func (net *NeuralNetwork) calculateDeltas(target []float64) { | |
outputLayer := net.numLayers - 1 | |
for layer := outputLayer; layer > 0; layer-- { | |
for unit := 0; unit < net.layerSizes[layer]; unit++ { | |
activation := net.outputsBuffer[layer][unit] | |
delta := 0.0 | |
if layer == outputLayer { | |
delta = activation - target[unit] | |
} else { | |
forwardLayer := layer + 1 | |
forwardLayerWeights := net.weights[forwardLayer] | |
forwardLayerDeltas := net.deltas[forwardLayer] | |
for fUnit := 0; fUnit < net.layerSizes[forwardLayer]; fUnit++ { | |
delta += forwardLayerDeltas[fUnit] * forwardLayerWeights[fUnit][unit] | |
} | |
delta *= net.params.ActivationFunc.Derivative(activation) | |
} | |
net.deltas[layer][unit] = delta | |
} | |
} | |
} | |
func (net *NeuralNetwork) calculateAndAccumulateGradients() { | |
for layer := 1; layer < net.numLayers; layer++ { | |
incoming := net.outputsBuffer[layer-1] | |
for unit := 0; unit < net.layerSizes[layer]; unit++ { | |
delta := net.deltas[layer][unit] | |
for k := range incoming { | |
net.weightGradientsBuffer[layer][unit][k] += incoming[k] * delta | |
} | |
net.biasGradientsBuffer[layer][unit] += delta | |
} | |
} | |
} | |
func (net *NeuralNetwork) backprop(input []float64, target []float64) { | |
// Forward propagate | |
net.feedForward(input) | |
// Back propagate | |
net.calculateDeltas(target) | |
// Calculate gradients | |
net.calculateAndAccumulateGradients() | |
} | |
func (net *NeuralNetwork) zeroGradients() { | |
for layer := 1; layer < net.numLayers; layer++ { | |
for unit := 0; unit < net.layerSizes[layer]; unit++ { | |
weightGradients := net.weightGradientsBuffer[layer][unit] | |
for k := range weightGradients { | |
weightGradients[k] = 0 | |
} | |
net.biasGradientsBuffer[layer][unit] = 0 | |
} | |
} | |
} | |
func (net *NeuralNetwork) miniBatchUpdate(inputs [][]float64, outputs [][]float64) { | |
miniBatchSize := len(inputs) | |
// Zero gradients | |
net.zeroGradients() | |
for i := 0; i < miniBatchSize; i++ { | |
input := inputs[i] | |
target := outputs[i] | |
// Calculate and accumulate weight gradients | |
// for each example in mini-batch | |
net.backprop(input, target) | |
} | |
// Update weights | |
alpha := net.trainParams.LearningRate | |
lambda := net.trainParams.RegularizationTerm | |
for layer := 1; layer < net.numLayers; layer++ { | |
for unit := 0; unit < net.layerSizes[layer]; unit++ { | |
weights := net.weights[layer][unit] | |
weightGradients := net.weightGradientsBuffer[layer][unit] | |
for k, weight := range weights { | |
weightGradient := (weightGradients[k] + (lambda * weight)) / float64(miniBatchSize) | |
weights[k] -= alpha * weightGradient | |
} | |
biasGradient := net.biasGradientsBuffer[layer][unit] / float64(miniBatchSize) | |
net.biases[layer][unit] -= alpha * biasGradient | |
} | |
} | |
} | |
func (net *NeuralNetwork) shuffleTrainingData(inputs [][]float64, outputs [][]float64) { | |
// Sattolo's shuffle algorithm | |
inputsSize := len(inputs) | |
for i := inputsSize - 1; i > 1; i-- { | |
r := rand.Intn(i - 1) | |
inputs[r], inputs[i] = inputs[i], inputs[r] | |
outputs[r], outputs[i] = outputs[i], outputs[r] | |
} | |
} | |
func (net *NeuralNetwork) Train(inputs [][]float64, outputs [][]float64, trainParams NeuralNetworkTrainParams) error { | |
inputsSize := len(inputs) | |
outputsSize := len(outputs) | |
if inputsSize != outputsSize { | |
return errors.New("Inputs size does not match outputs size") | |
} | |
if inputsSize == 0 { | |
if inputs == nil { | |
return errors.New("Inputs slice not initialized") | |
} | |
if outputs == nil { | |
return errors.New("Outputs slice not initialized") | |
} | |
return nil | |
} | |
net.trainParams = trainParams | |
miniBatchSize := net.trainParams.MiniBatchSize | |
miniBatchCount := 0 | |
if inputsSize%miniBatchSize == 0 { | |
miniBatchCount = inputsSize / miniBatchSize | |
} else { | |
miniBatchCount = (inputsSize / miniBatchSize) + 1 | |
} | |
for epoch := 0; epoch < net.trainParams.NumEpochs; epoch++ { | |
net.shuffleTrainingData(inputs, outputs) | |
var miniBatchInputs [][]float64 | |
var miniBatchOutputs [][]float64 | |
inputsTmp := inputs | |
outputsTmp := outputs | |
lastMiniBatch := miniBatchCount - 1 | |
for miniBatch := 0; miniBatch < miniBatchCount; miniBatch++ { | |
if miniBatch == lastMiniBatch { | |
miniBatchInputs = inputsTmp | |
miniBatchOutputs = outputsTmp | |
} else { | |
miniBatchInputs, inputsTmp = inputsTmp[:miniBatchSize], inputsTmp[miniBatchSize:] | |
miniBatchOutputs, outputsTmp = outputsTmp[:miniBatchSize], outputsTmp[miniBatchSize:] | |
} | |
net.miniBatchUpdate(miniBatchInputs, miniBatchOutputs) | |
} | |
} | |
return nil | |
} | |
func (net *NeuralNetwork) Predict(input []float64) ([]float64, error) { | |
if len(input) != net.inputLayerSize { | |
return nil, errors.New( | |
fmt.Sprintf("Expected input vector of size: %v", net.inputLayerSize)) | |
} | |
output := make([]float64, net.outputLayerSize) | |
netOutput := net.feedForward(input) | |
copy(output, netOutput) | |
return output, nil | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Breast Cancer Wisconsin (Diagnostic) Data Set
https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+%28Diagnostic%29