neural network

NeuralNetwork.fs

open System

type Class = Virus | Noise
type Datum = { Image : float[]; Class : Class }

type NeuralNetwork(inputSize, hiddenSize, outputSize, patternSize, maxT, eps, alpha, beta, W0, high, low) =
    let mutable eta = 0.005

    let inputLayer = Array.init (inputSize+1) (fun _ -> 0.0)
    let hiddenLayer = Array.init (hiddenSize+1) (fun _ -> 0.0)
    let outputLayer = Array.init outputSize (fun _ -> 0.0)

    let weight1 = Array2D.init hiddenSize (inputSize+1) (fun _ _ -> 0.0)
    let weight2 = Array2D.init outputSize (hiddenSize+1) (fun _ _ -> 0.0)

    let dWeight1 = Array2D.init hiddenSize (inputSize+1) (fun _ _ -> 0.0)
    let dWeight2 = Array2D.init outputSize (hiddenSize+1) (fun _ _ -> 0.0)

    let preDWeight1 = Array2D.init hiddenSize (inputSize+1) (fun _ _ -> 0.0)
    let preDWeight2 = Array2D.init outputSize (hiddenSize+1) (fun _ _ -> 0.0)

    let random = new Random()

    let load_data() = 
        let head = "C:\\virus\\"
        let raw = ".raw"
        let isNoise i = 1 <= i && i <= 400
        (*ここで学習する画像を指定する virusフォルダにある画像は1-200がノイズ１  201-400がノイズ２  401-700がウイルス１  701-900がウイルス２*)
        [|
          for v = 1 to 905 do
            let fileName = sprintf "%s%d%s" head v raw
            if System.IO.File.Exists(fileName)  then
                let image = System.IO.File.ReadAllBytes(fileName)
                let data = [| 
                  for k in 0 .. inputSize - 1 do
                     yield float(image.[k]) / 256.0 |]
                yield { Image = data; Class = if isNoise v then Noise else Virus }
        |] |> Array.sortBy (fun _ -> random.NextDouble())
           |> Seq.take patternSize
           |> Array.ofSeq

    let originalData = load_data()

    let zeta = function
      | Noise -> 0.0
      | Virus -> 1.0

    let sigmoid u = 1.0 / (1.0 + System.Math.Exp(-beta*u))

    let loop x y f = 
      for i in 0 .. x do
        for j in 0 .. y do
          f i j
          
    let loopSum x y f g =
      let mutable sum = 0.0
      for i in 0 .. x do
        sum <- 0.0
        for j in 0 .. y do
          sum <- sum + f i j
        g i sum
  
    member this.WeightInit() =
      let ranran() = random.NextDouble() * 2.0 * W0 - W0
      loop (hiddenSize-1) inputSize <| fun j k ->
        weight1.[j, k] <- ranran()
        dWeight1.[j, k] <- 0.0
      loop (outputSize-1) hiddenSize <| fun i j ->
        weight2.[i, j] <- ranran()
        dWeight2.[i, j] <- 0.0

    member this.DWeightInit() =
      loop (hiddenSize-1) inputSize <| fun j k ->
        preDWeight1.[j, k] <- dWeight1.[j, k]
        dWeight1.[j, k] <- 0.0
      loop (outputSize-1) hiddenSize <| fun i j ->
        preDWeight2.[i, j] <- dWeight2.[i, j]
        dWeight2.[i, j] <- 0.0

    member this.InputLayerSet(datum : Datum) =
      for k in 0 .. inputSize-1 do
        inputLayer.[k] <- datum.Image.[k]
      inputLayer.[inputSize] <- 1.0

    member this.Forward() =
      loopSum (hiddenSize-1) inputSize (fun j k ->
        inputLayer.[k] * weight1.[j, k]) <| fun j sum ->
        hiddenLayer.[j] <- sigmoid sum
  
      hiddenLayer.[hiddenSize] <- 1.0

      loopSum (outputSize-1) hiddenSize (fun i j ->
        hiddenLayer.[j] * weight2.[i, j]) <| fun i sum ->
        outputLayer.[i] <- sigmoid sum

    member this.Backward(datum : Datum) =
      let delta1 = Array.init (hiddenSize+1) (fun _ -> 0.0)
      let delta2 = Array.init outputSize (fun i -> beta * outputLayer.[i] * (1.0-outputLayer.[i]) * (zeta datum.Class - outputLayer.[i]))

      loopSum (hiddenSize-1) (outputSize-1) (fun j i ->
        weight2.[i, j] * delta2.[i]) <| fun j sum ->
        delta1.[j] <- beta * hiddenLayer.[j] * (1.0 - hiddenLayer.[j]) * sum

      loop (outputSize-1) hiddenSize <| fun i j ->
        dWeight2.[i, j] <- dWeight2.[i, j] + delta2.[i] * hiddenLayer.[j]

      loop (hiddenSize-1) inputSize <| fun j k ->
        dWeight1.[j, k] <- dWeight1.[j, k] + delta1.[j] * inputLayer.[k]

    member this.CalcError(datum: Datum) =
      seq { 
        for i = 0 to outputSize - 1 do
          yield (zeta datum.Class - outputLayer.[i]) * (zeta datum.Class - outputLayer.[i]) }
      |> Seq.sum

    member this.ModifyWeight() =
      loop (outputSize-1) hiddenSize <| fun i j ->
        dWeight2.[i, j] <- eta * dWeight2.[i,j] + alpha * preDWeight2.[i,j]
        weight2.[i,j] <- weight2.[i,j] + dWeight2.[i,j]

      loop (hiddenSize-1) inputSize <| fun j k ->  
        dWeight1.[j,k] <- eta * dWeight1.[j,k] + alpha * preDWeight1.[j,k]
        weight1.[j,k] <- weight1.[j,k] + dWeight1.[j,k]

    member this.Verification(testData : Datum[]) =
      let a = ref 0
      let b = ref 0
      let c = ref 0
      let d = ref 0
      
      testData
      |> Array.iteri (fun i ({Image=image; Class=cls}as datum) ->
        this.InputLayerSet(datum)
        this.Forward()
        printfn "%A" (cls, outputLayer.[0])
        match cls with
        | Noise ->
          if outputLayer.[0] <= 0.4 then
            incr a
          else
            incr c
        | Virus ->
          if outputLayer.[0] >= 0.6 then
            incr d
          else
            incr b
      )
  
      (float !a, float !b, float !c, float !d)

    member this.BackPropagation(data : Datum[]) =
      this.WeightInit()
      let mutable E = 0.0

      let mutable t = 0
      let mutable broken = false

      while not broken && t < maxT do    
        printfn "%A" (t, E)
        this.DWeightInit()
        let Esum = 
          seq {
            for datum in data do
              this.InputLayerSet(datum)
              this.Forward()
              this.Backward(datum)
              yield this.CalcError(datum) }
          |> Seq.sum
        eta <- 
          if E - Esum/float(outputSize * data.Length) > 0.0 then
            eta * 1.05
          else 
            eta * 0.7
        this.ModifyWeight()

        E <- Esum / float(outputSize * data.Length)

        if E < eps then
          broken <- true
        else
          t <- t + 1
      t

    member this.CrossValidation(n : int) =
        let size = originalData.Length
        let splited = 
          originalData
          |> Seq.windowed (size / n)
          |> Seq.mapi(fun i x -> (i, x))
        let results =
          [for (i, testData) in splited do
            let trainData =
              splited
              |> Seq.filter (fun (j, _) -> j <> i)
              |> Seq.map snd
              |> Seq.concat
              |> Array.ofSeq
            let t = this.BackPropagation(trainData)
            let (a, b, c, d) = this.Verification(testData)
            yield (t, (a+d)/(a+b+c+d), a/(a+b), a/(a+c))]
        let t = (results |> Seq.map (fun (t,_,_,_) -> t) |> Seq.sum) / (results.Length)
        let x = (results |> Seq.map (fun (_,x,_,_) -> x) |> Seq.sum) / (float(results.Length))
        let y = (results |> Seq.map (fun (_,_,y,_) -> y) |> Seq.sum) / (float(results.Length))
        let z = (results |> Seq.map (fun (_,_,_,z) -> z) |> Seq.sum) / (float(results.Length))
        (t, x, y, z)

[<EntryPoint>]
let main _ =
  let hiddenSize = 7
  let high = 1.05
  let low = 0.7
  let iterationSize = 1
  let now = System.DateTime.Now
  let logFileName = sprintf "%02d%02d%02d%02d.csv" (now.Month+1) now.Day now.Hour now.Minute

  let xs = Array.Parallel.init iterationSize (fun i ->
    let nn = new NeuralNetwork(32*32, hiddenSize, 1, 100, 1000, 1.0e-4, 0.8, 1.0, 0.5, high, low)
    let result = nn.CrossValidation(10)
    printfn "%d done" i
    result
  )
//  use writer = new System.IO.StreamWriter(name)
//  Printf.fprintf writer "中間%d,学習率上%f,学習率下%f\n" HIDDEN high low
  for i in 0 .. iterationSize - 1 do
    let (time, accuracy, precision, recall) = xs.[i]
//    Printf.fprintf writer "%d,%f\n" time pa2
    printf "%d回目  %d回  accuracy:%f％ precision:%f recall:%f f-value:%f\n" (i+1) time accuracy precision recall (2.0*precision*recall/(precision+recall))
  0

RejectionMethod.fs

type RejectionMethod(f, rangeMin, rangeMax, maxValue) =
    let random = new Random()  
    let rangeSize = rangeMax - rangeMin
    member this.RandomSeq() =
        seq { 
            let x = random.NextDouble()
            let y = random.NextDouble()
            let z = rangeSize * x + rangeMin
            if y < f(z) / maxValue then 
                yield z
            yield! this.RandomSeq() }

let f x = 1.0 / (1.0 + System.Math.Exp(-x))
let rangeMin = -10.0
let rangeMax = 10.0

let sigmoid = new RejectionMethod(f, rangeMin, rangeMax, 1.0)

sigmoid.RandomSeq() 
|> Seq.take 1000 
|> Seq.iter (printfn "%f")

VirusVisualizer.fs

open System.Drawing
open System.Windows.Forms

let drawDot (g : Graphics) a x y =
  g.DrawLine(new Pen(Color.FromArgb(a, a, a)), x, y, x+1, y+1)
 
type VirusVisualizerForm() as this =
    inherit Form()
    do
    this.Size <- new Size(100, 100)
    this.Text <- "VirusVisualizer"
    let label1 = new Label()
    label1.Size <- new Size(100, 100)
    let bytes = System.IO.File.ReadAllBytes(@"C:\virus\897.raw")
    label1.Paint.Add(fun e -> 
      let g = e.Graphics
      for i in 0 .. 32-1 do
        for j in 0 .. 32-1 do
          drawDot g (int bytes.[i*32+j]) (30+i) (30+j)
      )
    this.Controls.Add(label1)
 
Application.Run(new VirusVisualizerForm())

todo:
1.10-foldぐらいのcross validationをやる。

2.accuracyしか見ていないのでprecisionやrecallも見る。あとF値も。

3.画像ナンバー400でウィルスとノイズを分類しているが、これが正しくないかもしれない。分類を間違えている可能性有り。要検証。

4.rawファイルだと面倒なのでbmpへのコンバータが必要。元画像のサイズは？→32*32だった。

5.遅いので並列化すべき。
6.weight decay項を付ける等。メールに書いた奴を追加していく。
7.BackPropagation内でEがeps以上なのにも関わらず一定値に収束してしまう現象をなんとかする。たぶんSAとかGAのたまにジャンプする感じでやればおk。→ボルツマンマシンを調べる。
8.そもそももっと簡単に解けるのでは…？　パーセプトロンでいいのでは？

ウィルスとノイズの特徴量の違いが大きいのでNNだとオーバースペックかも。
PCAしてからパーセプトロンとかでいい予感。

SVMはローカルミニマムが無い。2クラス分類用なので他クラス分類が面倒っぽい。
RVMは確率モデルなので事後確率が出せる。→半教師付き学習でも使える？

deep learningとはニューラル・ネットワークのような多層構造の生成モデルを教師なし+教師ありで学習する手法。特徴抽出と識別を合わせたような動作をして、SVMよりも性能がよい(？)らしい。
http://deeplearning.net/