CNTK Logistic

version1.fsx

// C# original: https://github.com/Microsoft/CNTK/blob/master/Examples/TrainingCSharp/Common/LogisticRegression.cs

// I assume CNTK.CPUOnly has been installed via Paket
// required dependencies:
// packages\CNTK.CPUOnly\lib\net45\x64
// packages\CNTK.CPUOnly\support\x64\Dependency
// packages\CNTK.CPUOnly\support\x64\Dependency\Release
// packages\CNTK.CPUOnly\support\x64\Release

// loading native dependencies in a script is a bit annoying,
// see this great reference:
// http://christoph.ruegg.name/blog/loading-native-dlls-in-fsharp-interactive.html
// we add all the required folders to the path

open System
open System.IO

let dependencies = [
        "packages/CNTK.CPUOnly/lib/net45/x64/"
        "packages/CNTK.CPUOnly/support/x64/Dependency/"
        "packages/CNTK.CPUOnly/support/x64/Dependency/Release/"
        "packages/CNTK.CPUOnly/support/x64/Release/"    
    ]

let packagesPath = Path.Combine(__SOURCE_DIRECTORY__,"./")

dependencies 
|> Seq.iter (fun dep -> 
    let path = Path.Combine(packagesPath,dep)
    Environment.SetEnvironmentVariable("Path",
        Environment.GetEnvironmentVariable("Path") + ";" + path)
    )    

#r "./packages/CNTK.CPUOnly/lib/net45/x64/Cntk.Core.Managed-2.2.dll"
open System
open System.Collections.Generic

open CNTK

let inputDim = 3
let numOutputClasses = 2
    
let CreateLinearModel(input:Variable, outputDim:int, device:DeviceDescriptor) =
        
    let inputDim = input.Shape.[0]

    let weightShape = NDShape.CreateNDShape [ outputDim; inputDim ]
    let biasShape = NDShape.CreateNDShape [ outputDim ]

    let weightParam = new Parameter(weightShape, DataType.Float, 1.0, device, "w");
    let biasParam = new Parameter(biasShape, DataType.Float, 0.0, device, "b");
        
    Variable(CNTKLib.Times(weightParam, input)) + biasParam

let GenerateGaussianNoise(mean, stdDev, random:Random) =
        
    let u1 = 1.0 - random.NextDouble()
    let u2 = 1.0 - random.NextDouble()
    let stdNormalRandomValue = sqrt(-2.0 * log(u1)) * sin(2.0 * Math.PI * u2)

    mean + stdDev * stdNormalRandomValue
        
let random = Random(0)

let GenerateRawDataSamples(sampleSize,inputDim,numOutputClasses) =
        
    let features = Array.init (sampleSize * inputDim) (fun _ -> float32 0.)
    let oneHotLabels = Array.init (sampleSize * numOutputClasses) (fun _ -> float32 0.)

    for sample in 0 .. sampleSize - 1 do

        let label = random.Next(numOutputClasses)
        for i in 0 .. numOutputClasses - 1 do             
            oneHotLabels.[sample * numOutputClasses + i] <- if label = i then float32 1.0 else float32 0.0
                
        for i in 0 .. inputDim - 1 do               
            features.[sample * inputDim + i] <- float32 (GenerateGaussianNoise(3.0, 1.0, random)) * float32 (label + 1)
            
    features, oneHotLabels
            
let GenerateValueData(sampleSize:int, inputDim:int, numOutputClasses:int, device:DeviceDescriptor) =
        
    let features, oneHotLabels = GenerateRawDataSamples(sampleSize, inputDim, numOutputClasses)
            
    let featureValue = Value.CreateBatch (NDShape.CreateNDShape [ inputDim ], features, device)
    let labelValue = Value.CreateBatch (NDShape.CreateNDShape [ numOutputClasses ], oneHotLabels, device)

    featureValue, labelValue
    
let PrintTrainingProgress(trainer:Trainer, minibatchIdx:int, outputFrequencyInMinibatches:int) =
        
    if ((minibatchIdx % outputFrequencyInMinibatches) = 0 && trainer.PreviousMinibatchSampleCount() <> (uint32 0))
    then  
        let trainLossValue = trainer.PreviousMinibatchLossAverage() |> float32
        let evaluationValue = trainer.PreviousMinibatchEvaluationAverage() |> float32
        printfn "Minibatch: %i CrossEntropyLoss = %f, EvaluationCriterion = %f" minibatchIdx trainLossValue evaluationValue
            
let device = DeviceDescriptor.CPUDevice

let inputShape = NDShape.CreateNDShape [inputDim]
let labelShape = NDShape.CreateNDShape [numOutputClasses]

let featureVariable = Variable.InputVariable(inputShape, DataType.Float)
let labelVariable = Variable.InputVariable(labelShape, DataType.Float)
let classifierOutput = CreateLinearModel(featureVariable, numOutputClasses, device)
let loss = CNTKLib.CrossEntropyWithSoftmax(Variable(classifierOutput), labelVariable)
let evalError = CNTKLib.ClassificationError(Variable(classifierOutput), labelVariable)

let learningRatePerSample = CNTK.TrainingParameterScheduleDouble(0.02, uint32 1)

let parameterLearners =
    ResizeArray<Learner>([ Learner.SGDLearner(classifierOutput.Parameters(), learningRatePerSample) ])
        
let trainer = Trainer.CreateTrainer(classifierOutput, loss, evalError, parameterLearners)

let minibatchSize = 64
let numMinibatchesToTrain = 1000
let updatePerMinibatches = 50
        
for minibatchCount in 0 .. (numMinibatchesToTrain - 1) do
        
    let features, labels = GenerateValueData(minibatchSize, inputDim, numOutputClasses, device)
    let foo = 
        [ (featureVariable, features); (labelVariable, labels) ] |> dict
            
    let _ = trainer.TrainMinibatch(foo, device)
            
    PrintTrainingProgress(trainer, minibatchCount, updatePerMinibatches)
        

version2.fsx

// C# original: https://github.com/Microsoft/CNTK/blob/master/Examples/TrainingCSharp/Common/LogisticRegression.cs

// I assume CNTK.CPUOnly has been installed via Paket
// required dependencies:
// packages\CNTK.CPUOnly\lib\net45\x64
// packages\CNTK.CPUOnly\support\x64\Dependency
// packages\CNTK.CPUOnly\support\x64\Dependency\Release
// packages\CNTK.CPUOnly\support\x64\Release

// loading native dependencies in a script is a bit annoying,
// see this great reference:
// http://christoph.ruegg.name/blog/loading-native-dlls-in-fsharp-interactive.html
// we add all the required folders to the path

open System
open System.IO

let dependencies = [
        "packages/CNTK.CPUOnly/lib/net45/x64/"
        "packages/CNTK.CPUOnly/support/x64/Dependency/"
        "packages/CNTK.CPUOnly/support/x64/Dependency/Release/"
        "packages/CNTK.CPUOnly/support/x64/Release/"    
    ]

let packagesPath = Path.Combine(__SOURCE_DIRECTORY__,"./")

dependencies 
|> Seq.iter (fun dep -> 
    let path = Path.Combine(packagesPath,dep)
    Environment.SetEnvironmentVariable("Path",
        Environment.GetEnvironmentVariable("Path") + ";" + path)
    )    

#r "./packages/CNTK.CPUOnly/lib/net45/x64/Cntk.Core.Managed-2.2.dll"

open System
open System.Collections.Generic

open CNTK

(*
Helpers to simplify model creation from F#
Credit goes to Kevin for most of this:
https://gist.github.com/kevmal/9661bd1f32beb0785649cbc5ab619b3b
*)

let shape (dims:int seq) = NDShape.CreateNDShape dims

type VarOrFunc =
    | Var of Variable
    | Fun of Function
    member x.Variable = 
        match x with
        | Var v -> v
        | Fun f -> new Variable(f)
    member x.Function = 
        match x with
        | Var v -> failwith "var"
        | Fun f -> f

    static member ( *. )(a : VarOrFunc, b : VarOrFunc) = CNTKLib.ElementTimes(a.Variable, b.Variable) |> Fun
    static member (*)(a : VarOrFunc, b : VarOrFunc) = CNTKLib.Times(a.Variable, b.Variable) |> Fun
    static member (+)(a : VarOrFunc, b : VarOrFunc) = (a.Variable + b.Variable) |> Fun

    static member log (x : VarOrFunc) = CNTKLib.Log(x.Variable) |> Fun
    static member exp (x : VarOrFunc) = CNTKLib.Exp(x.Variable) |> Fun
    static member sigmoid (x : VarOrFunc) = CNTKLib.Sigmoid(x.Variable) |> Fun
    static member tanh (x : VarOrFunc) = CNTKLib.Tanh(x.Variable) |> Fun

(*
C# Example, converted
*)

// Creating a synthetic dataset

let gaussianNoise (random:Random) (mean, stdDev) =
        
    let u1 = 1.0 - random.NextDouble()
    let u2 = 1.0 - random.NextDouble()
    let stdNormalRandomValue = sqrt(-2.0 * log(u1)) * sin(2.0 * Math.PI * u2)

    mean + stdDev * stdNormalRandomValue
        
let random = Random(0)
let noise = gaussianNoise random

let generateRawDataSamples(sampleSize, inputDim, numOutputClasses) =
        
    let features = Array.init (sampleSize * inputDim) (fun _ -> float32 0.)
    let oneHotLabels = Array.init (sampleSize * numOutputClasses) (fun _ -> float32 0.)

    for sample in 0 .. sampleSize - 1 do

        let label = random.Next(numOutputClasses)
        for i in 0 .. numOutputClasses - 1 do             
            oneHotLabels.[sample * numOutputClasses + i] <- if label = i then float32 1.0 else float32 0.0
                
        for i in 0 .. inputDim - 1 do               
            features.[sample * inputDim + i] <- float32 (noise (3.0, 1.0)) * float32 (label + 1)
            
    features, oneHotLabels
            
let generateValueData(sampleSize, inputDim, numOutputClasses, device:DeviceDescriptor) =
        
    let features, oneHotLabels = generateRawDataSamples(sampleSize, inputDim, numOutputClasses)
            
    let featureValue = Value.CreateBatch (shape [ inputDim ], features, device)
    let labelValue = Value.CreateBatch (shape [ numOutputClasses ], oneHotLabels, device)

    featureValue, labelValue
    
let printTrainingProgress(trainer:Trainer, minibatchIdx, outputFrequencyInMinibatches) =
        
    if 
        (minibatchIdx % outputFrequencyInMinibatches) = 0 && 
        trainer.PreviousMinibatchSampleCount() <> (uint32 0)
    then  
        let trainLossValue = trainer.PreviousMinibatchLossAverage() |> float32
        let evaluationValue = trainer.PreviousMinibatchEvaluationAverage() |> float32
        printfn "Minibatch: %i CrossEntropyLoss = %f, EvaluationCriterion = %f" minibatchIdx trainLossValue evaluationValue

// creating and training the model

let linearModel(input:Variable, outputDim:int, device:DeviceDescriptor) =
        
    let inputDim = input.Shape.[0]
    
    let weight = Parameter(shape [ outputDim; inputDim ], DataType.Float, 1.0, device, "w")
    let bias = Parameter(shape [ outputDim ], DataType.Float, 0.0, device, "b")
    
    (Fun(weight * input) + Var(bias)).Function
                
let device = DeviceDescriptor.CPUDevice

let inputDim = 3
let numOutputClasses = 2

let featureVariable = Variable.InputVariable(shape [inputDim], DataType.Float)
let labelVariable = Variable.InputVariable(shape [numOutputClasses], DataType.Float)

let classifierOutput = linearModel(featureVariable, numOutputClasses, device)
let loss = CNTKLib.CrossEntropyWithSoftmax(Variable(classifierOutput), labelVariable)
let evalError = CNTKLib.ClassificationError(Variable(classifierOutput), labelVariable)

let learningRatePerSample = CNTK.TrainingParameterScheduleDouble(0.02, uint32 1)

let parameterLearners =
    ResizeArray<Learner>(
        [ 
            Learner.SGDLearner(classifierOutput.Parameters(), learningRatePerSample) 
        ]
        )
        
let trainer = Trainer.CreateTrainer(classifierOutput, loss, evalError, parameterLearners)

let minibatchSize = 64
let numMinibatchesToTrain = 1000
let updatePerMinibatches = 50
        
for minibatchCount in 0 .. (numMinibatchesToTrain - 1) do
        
    let features, labels = generateValueData(minibatchSize, inputDim, numOutputClasses, device)
    let params = 
        [ (featureVariable, features); (labelVariable, labels) ] |> dict
            
    let _ = trainer.TrainMinibatch(params, device)
            
    printTrainingProgress(trainer, minibatchCount, updatePerMinibatches)
        

just to see what a CE could look like

// C# original: https://github.com/Microsoft/CNTK/blob/master/Examples/TrainingCSharp/Common/LogisticRegression.cs

let path = System.Environment.GetEnvironmentVariable "PATH"
System.Environment.SetEnvironmentVariable("PATH",@"C:\Users\Mathias Brandewinder\Documents\Visual Studio 2015\Projects\CNTK-FS\CNTK-FS\bin\Debug\" + string System.IO.Path.PathSeparator + path )

#r @"C:\Users\Mathias Brandewinder\documents\visual studio 2015\Projects\CNTK-FS\packages\CNTK.CPUOnly\lib\net45\x64\Cntk.Core.Managed-2.2.dll"

open System
open System.Collections.Generic

open CNTK


(*
Helpers to simplify model creation from F#
*)

let shape (dims:int seq) = NDShape.CreateNDShape dims

type VarOrFunc =
    | Var of Variable
    | Fun of Function
    member x.Variable = 
        match x with
        | Var v -> v
        | Fun f -> new Variable(f)
    member x.Function = 
        match x with
        | Var v -> v.ToFunction()
        | Fun f -> f

    static member ( *. )(a : VarOrFunc, b : VarOrFunc) = CNTKLib.ElementTimes(a.Variable, b.Variable) |> Fun
    static member (*)(a : VarOrFunc, b : VarOrFunc) = CNTKLib.Times(a.Variable, b.Variable) |> Fun
    static member (+)(a : VarOrFunc, b : VarOrFunc) = (a.Variable + b.Variable) |> Fun

    static member log (x : VarOrFunc) = CNTKLib.Log(x.Variable) |> Fun
    static member exp (x : VarOrFunc) = CNTKLib.Exp(x.Variable) |> Fun
    static member sigmoid (x : VarOrFunc) = CNTKLib.Sigmoid(x.Variable) |> Fun
    static member tanh (x : VarOrFunc) = CNTKLib.Tanh(x.Variable) |> Fun

let (|AsVar|) (v : Variable) = Var v
let (|AsFun|) (v : Function) = Fun v


(* Quick codegen
let t = typeof<Parameter>
t.GetConstructors()
|> Seq.iter 
    (fun i ->
        let allParameters = i.GetParameters()
        let args,ps = 
            allParameters 
            |> Seq.toList 
            |> List.partition (fun p -> p.ParameterType = typeof<DeviceDescriptor> || p.ParameterType = typeof<DataType>)
        let inParams = 
            ps 
            |> Seq.map (fun i -> sprintf "%s : %s" i.Name i.ParameterType.Name) 
            |> String.concat ", "
        let outParams =
            allParameters 
            |> Seq.map (fun i -> sprintf "%s" i.Name) 
            |> String.concat ", "
        if List.isEmpty args then
            printfn "static member Def(%s) = new Parameter(%s)" inParams outParams
        else
            let fArgs = 
                args
                |> Seq.sortBy (fun i -> i.ParameterType.Name) 
                |> Seq.map (fun i -> sprintf "(%s : %s)" i.Name i.ParameterType.Name) 
                |> String.concat " "
            printfn "static member Def(%s) = fun %s -> new Parameter(%s)" inParams fArgs outParams 
    )

*)
type Parameter with
    //static member Def(shape : NDShape, initValue : Single, name : String) = fun (device : DeviceDescriptor) -> new Parameter(shape, initValue, device, name)
    //static member Def(shape : NDShape, initValue : Double, name : String) = fun (device : DeviceDescriptor) -> new Parameter(shape, initValue, device, name)
    static member Def(value : NDArrayView, name : String) = new Parameter(value, name)
    static member Def(value : NDArrayView) = new Parameter(value)
    static member Def(shape : NDShape, initValue : Double, name : String) = fun (dataType : DataType) (device : DeviceDescriptor) -> new Parameter(shape, dataType, initValue, device, name)
    static member Def(shape : NDShape, initValue : Double) = fun (dataType : DataType) (device : DeviceDescriptor) -> new Parameter(shape, dataType, initValue, device)
    //static member Def(shape : NDShape, initValue : Double) = fun (dataType : DataType) -> new Parameter(shape, dataType, initValue)
    static member Def(shape : NDShape, initializer : CNTKDictionary, name : String) = fun (dataType : DataType) (device : DeviceDescriptor) -> new Parameter(shape, dataType, initializer, device, name)
    static member Def(shape : NDShape, initializer : CNTKDictionary) = fun (dataType : DataType) (device : DeviceDescriptor) -> new Parameter(shape, dataType, initializer, device)
    //static member Def(shape : NDShape, initializer : CNTKDictionary) = fun (dataType : DataType) -> new Parameter(shape, dataType, initializer)
    static member Def(variable : Variable) = new Parameter(variable)

type OnDevice(device : DeviceDescriptor, dataType : DataType) = 
    member __.Bind(v : Variable, f) = f(Var v)
    member __.Bind(v : DataType -> #Variable, f) = f(Var(v dataType))
    member __.Bind(v : DataType -> Function, f) = f(Fun(v dataType))
    member __.Bind(v : DataType -> DeviceDescriptor -> #Variable, f) = f(Var(v dataType device))
    member __.Bind(v : DataType -> DeviceDescriptor -> Function, f) = f(Fun(v dataType device))
    member __.Bind(v : DeviceDescriptor -> #Variable, f) = f(Var(v device))
    member __.Bind(v : DeviceDescriptor -> Function, f) = f(Fun(v device))
    member __.Return(x) = x

(*
C# Example, converted
*)

// Creating a synthetic dataset

let gaussianNoise (random:Random) (mean, stdDev) =
        
    let u1 = 1.0 - random.NextDouble()
    let u2 = 1.0 - random.NextDouble()
    let stdNormalRandomValue = sqrt(-2.0 * log(u1)) * sin(2.0 * Math.PI * u2)

    mean + stdDev * stdNormalRandomValue
        
let random = Random(0)
let noise = gaussianNoise random

let generateRawDataSamples(sampleSize, inputDim, numOutputClasses) =
        
    let features = Array.create (sampleSize * inputDim) 0.f
    let oneHotLabels = Array.create (sampleSize * numOutputClasses) 0.f

    for sample in 0 .. sampleSize - 1 do

        let label = random.Next(numOutputClasses)
        for i in 0 .. numOutputClasses - 1 do             
            oneHotLabels.[sample * numOutputClasses + i] <- if label = i then 1.f else 0.f
                
        for i in 0 .. inputDim - 1 do               
            features.[sample * inputDim + i] <- float32 (noise (3.0, 1.0)) * float32 (label + 1)
            
    features, oneHotLabels
            
let generateValueData(sampleSize, inputDim, numOutputClasses, device:DeviceDescriptor) =
        
    let features, oneHotLabels = generateRawDataSamples(sampleSize, inputDim, numOutputClasses)
            
    let featureValue = Value.CreateBatch (shape [ inputDim ], features, device)
    let labelValue = Value.CreateBatch (shape [ numOutputClasses ], oneHotLabels, device)

    featureValue, labelValue
    
let printTrainingProgress(trainer:Trainer, minibatchIdx, outputFrequencyInMinibatches) =
        
    if 
        (minibatchIdx % outputFrequencyInMinibatches) = 0 && 
        trainer.PreviousMinibatchSampleCount() <> (uint32 0)
    then  
        let trainLossValue = trainer.PreviousMinibatchLossAverage() |> float32
        let evaluationValue = trainer.PreviousMinibatchEvaluationAverage() |> float32
        printfn "Minibatch: %i CrossEntropyLoss = %f, EvaluationCriterion = %f" minibatchIdx trainLossValue evaluationValue

// creating and training the model

let linearModel(AsVar input, outputDim:int, device:DeviceDescriptor) =
        
    let inputDim = input.Variable.Shape.[0]
    OnDevice(device, DataType.Float){
        let! weight = Parameter.Def(shape [ outputDim; inputDim ], 1.0, "w")
        let! bias = Parameter.Def(shape [ outputDim ], 0.0, "b")
        return (weight * input) + bias
    }
    
                
let device = DeviceDescriptor.CPUDevice

let inputDim = 3
let numOutputClasses = 2

let featureVariable = Variable.InputVariable(shape [inputDim], DataType.Float)
let labelVariable = Variable.InputVariable(shape [numOutputClasses], DataType.Float)

let classifierOutput = linearModel(featureVariable, numOutputClasses, device).Function
let loss = CNTKLib.CrossEntropyWithSoftmax(Variable(classifierOutput), labelVariable)
let evalError = CNTKLib.ClassificationError(Variable(classifierOutput), labelVariable)

let learningRatePerSample = CNTK.TrainingParameterScheduleDouble(0.02, uint32 1)

let parameterLearners =
    ResizeArray<Learner>(
        [ 
            Learner.SGDLearner(classifierOutput.Parameters(), learningRatePerSample) 
        ]
        )
        
let trainer = Trainer.CreateTrainer(classifierOutput, loss, evalError, parameterLearners)

let minibatchSize = 64
let numMinibatchesToTrain = 1000
let updatePerMinibatches = 50
        
for minibatchCount in 0 .. (numMinibatchesToTrain - 1) do
        
    let features, labels = generateValueData(minibatchSize, inputDim, numOutputClasses, device)
    let params = 
        [ (featureVariable, features); (labelVariable, labels) ] |> dict
            
    let _ = trainer.TrainMinibatch(params, device)
            
    printTrainingProgress(trainer, minibatchCount, updatePerMinibatches)```