CNTK inference

Program.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Drawing.Imaging;
using CNTK;
using CNTKExtension;

namespace ConsoleApp2
{
    class Program
    {
        public static void Main(string[] args)
        {
            Console.WriteLine("======== Evaluate model using C# CPUOnly Build ========");
            // Evaluate a single image.
            EvaluationSingleImage(DeviceDescriptor.CPUDevice, @"D:\git_projects\CrackDetection\Images\crack\crack.jpg");
            Console.ReadLine();
        }
        /// <summary>
        /// Extracts image pixels in CHW using parallelization
        /// </summary>
        /// <param name="image">The bitmap image to extract features from</param>
        /// <returns>A list of pixels in CHW order</returns>
        public static List<float> ParallelExtractCHW(Bitmap image)
        {
            // We use local variables to avoid contention on the image object through the multiple threads.
            int channelStride = image.Width * image.Height;
            int imageWidth = image.Width;
            int imageHeight = image.Height;

            var features = new byte[imageWidth * imageHeight * 3];
            var bitmapData = image.LockBits(new System.Drawing.Rectangle(0, 0, imageWidth, imageHeight), ImageLockMode.ReadOnly, image.PixelFormat);
            IntPtr ptr = bitmapData.Scan0;
            int bytes = Math.Abs(bitmapData.Stride) * bitmapData.Height;
            byte[] rgbValues = new byte[bytes];

            int stride = bitmapData.Stride;

            // Copy the RGB values into the array.
            System.Runtime.InteropServices.Marshal.Copy(ptr, rgbValues, 0, bytes);

            // The mapping depends on the pixel format
            // The mapPixel lambda will return the right color channel for the desired pixel
            Func<int, int, int, int> mapPixel = GetPixelMapper(image.PixelFormat, stride);

            Parallel.For(0, imageHeight, (int h) =>
            {
                Parallel.For(0, imageWidth, (int w) =>
                {
                    Parallel.For(0, 3, (int c) =>
                    {
                        features[channelStride * c + imageWidth * h + w] = rgbValues[mapPixel(h, w, c)];
                    });
                });
            });

            image.UnlockBits(bitmapData);

            return features.Select(b => (float)b).ToList();
        }
        /// <summary>
        /// Returns a function for extracting the R-G-B values properly from an image based on its pixel format
        /// </summary>
        /// <param name="pixelFormat">The image's pixel format</param>
        /// <param name="heightStride">The stride (row byte count)</param>
        /// <returns>A function with signature (height, width, channel) returning the corresponding color value</returns>
        private static Func<int, int, int, int> GetPixelMapper(PixelFormat pixelFormat, int heightStride)
        {
            switch (pixelFormat)
            {
                case PixelFormat.Format32bppArgb:
                    return (h, w, c) => h * heightStride + w * 4 + c;  // bytes are B-G-R-A
                case PixelFormat.Format24bppRgb:
                default:
                    return (h, w, c) => h * heightStride + w * 3 + c;  // bytes are B-G-R
            }
        }

         /// <summary>
        /// The example shows
        /// - how to load model.
        /// - how to prepare input data for a single sample.
        /// - how to prepare input and output data map.
        /// - how to evaluate a model.
        /// - how to retrieve evaluation result and retrieve output data in dense format.
        /// </summary>
        /// <param name="device">Specify on which device to run the evaluation.</param>
        public static void EvaluationSingleImage(DeviceDescriptor device, string imgPath, string modelFilePath = @"D:\git_projects\CrackDetection\vgg16-crack.onnx")
        {
            try
            {
                Console.WriteLine("\n===== Evaluate single image =====");

                // Load the model.
                // The model resnet20.dnn is trained by <CNTK>/Examples/Image/Classification/ResNet/Python/TrainResNet_CIFAR10.py
                // Please see README.md in <CNTK>/Examples/Image/Classification/ResNet about how to train the model.
                ThrowIfFileNotExist(modelFilePath, string.Format("Error: The model '{0}' does not exist. Please follow instructions in README.md in <CNTK>/Examples/Image/Classification/ResNet to create the model.", modelFilePath));
                Function modelFunc = Function.Load(modelFilePath, device, ModelFormat.ONNX);

                // Get input variable. The model has only one single input.
                // The same way described above for output variable can be used here to get input variable by name.
                Variable inputVar = modelFunc.Arguments.Single();

                // Get shape data for the input variable
                NDShape inputShape = inputVar.Shape;
                // inputShape[0] is 3 => for each of the color channels
                int imageWidth = inputShape[1];
                int imageHeight = inputShape[2];

                // Image preprocessing to match input requirements of the model.
                // This program uses images from the CIFAR-10 dataset for evaluation.
                // Please see README.md in <CNTK>/Examples/Image/DataSets/CIFAR-10 about how to download the CIFAR-10 dataset.
                ThrowIfFileNotExist(imgPath, string.Format("Image not found", imgPath));
                Bitmap bmp = new Bitmap(Bitmap.FromFile(imgPath), new Size(imageWidth, imageHeight));
                List<float> resizedCHW = ParallelExtractCHW(bmp);
                var normalizedCHW = new List<float>();
                foreach (var entry in resizedCHW)
                {   
                    normalizedCHW.Add(entry / 255);
                }
                Console.WriteLine($"{normalizedCHW[0]}, {normalizedCHW[1]}, {normalizedCHW[2]}");
                Console.WriteLine($"{normalizedCHW[50176]}, {normalizedCHW[50177]}, {normalizedCHW[50178]}");
                Console.WriteLine($"{normalizedCHW[100352]}, {normalizedCHW[100353]}, {normalizedCHW[100354]}");

                // Create input data map
                var inputDataMap = new Dictionary<Variable, Value>();
                var inputVal = Value.CreateBatch(inputShape, normalizedCHW, device);
                inputDataMap.Add(inputVar, inputVal);

                // The model has only one output.
                // You can also use the following way to get output variable by name:
                // Variable outputVar = modelFunc.Outputs.Where(variable => string.Equals(variable.Name, outputName)).Single();
                var outputs = modelFunc.Outputs;
                Variable outputVar = outputs.Where(_ => _.Name == "Sigmoid").Single();

                // Create output data map. Using null as Value to indicate using system allocated memory.
                // Alternatively, create a Value object and add it to the data map.
                var outputDataMap = new Dictionary<Variable, Value>();
                outputDataMap.Add(outputVar, null);

                // Start evaluation on the device
                modelFunc.Evaluate(inputDataMap, outputDataMap, device);

                // Get evaluate result as dense output
                var outputVal = outputDataMap[outputVar];
                var outputData = outputVal.GetDenseData<float>(outputVar);

                Console.WriteLine("Evaluation result for image " + imgPath);
                PrintOutput(outputVar.Shape.TotalSize, outputData);
            }
            catch (Exception ex)
            {
                Console.WriteLine("Error: {0}\nCallStack: {1}\n Inner Exception: {2}", ex.Message, ex.StackTrace, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
                throw ex;
            }
        }

        /// <summary>
        /// Checks whether the file exists. If not, write the error message on the console and throw FileNotFoundException.
        /// </summary>
        /// <param name="filePath">The file to check.</param>
        /// <param name="errorMsg">The message to write on console if the file does not exist.</param>
        internal static void ThrowIfFileNotExist(string filePath, string errorMsg)
        {
            if (!File.Exists(filePath))
            {
                if (!string.IsNullOrEmpty(errorMsg))
                {
                    Console.WriteLine(errorMsg);
                }
                throw new FileNotFoundException(string.Format("File '{0}' not found.", filePath));
            }
        }
        /// <summary>
        /// Print out the evaluation results.
        /// </summary>
        /// <typeparam name="T">The data value type</typeparam>
        /// <param name="sampleSize">The size of each sample.</param>
        /// <param name="outputBuffer">The evaluation result data.</param>
        internal static void PrintOutput<T>(int sampleSize, IList<IList<T>> outputBuffer)
        {
            Console.WriteLine("The number of sequences in the batch: " + outputBuffer.Count);
            int seqNo = 0;
            int outputSampleSize = sampleSize;
            foreach (var seq in outputBuffer)
            {
                if (seq.Count % outputSampleSize != 0)
                {
                    throw new ApplicationException("The number of elements in the sequence is not a multiple of sample size");
                }

                Console.WriteLine(String.Format("Sequence {0} contains {1} samples.", seqNo++, seq.Count / outputSampleSize));
                int i = 0;
                int sampleNo = 0;
                foreach (var element in seq)
                {
                    if (i++ % outputSampleSize == 0)
                    {
                        Console.Write(String.Format("    sample {0}: ", sampleNo));
                    }
                    Console.Write(element);
                    if (i % outputSampleSize == 0)
                    {
                        Console.WriteLine(".");
                        sampleNo++;
                    }
                    else
                    {
                        Console.Write(",");
                    }
                }
            }
        }
    }
}

train.py

import tensorflow as tf
from IPython.display import display
from PIL import Image
print(tf.__version__)
from keras.preprocessing.image import ImageDataGenerator

# Data Generators
img_height, img_width = 224, 224
batch_size = 8
train_data_dir = "./dataset"
train_datagen = ImageDataGenerator(rescale=1./255,
                                   shear_range=0.2,
                                   zoom_range=0.2,
                                   horizontal_flip=True,
                                    #rotation_range=20,
                                   validation_split=0.2) # set validation split

train_generator = train_datagen.flow_from_directory(train_data_dir,
                                                   target_size=(img_height, img_width),
                                                   batch_size=batch_size,
                                                   class_mode='binary',
                                                   subset='training') # set as training data
validation_generator = train_datagen.flow_from_directory(train_data_dir,
                                                   target_size=(img_height, img_width),
                                                   batch_size=batch_size,
                                                   class_mode='binary',
                                                   subset='validation') # set as validation data
# Load Pretrained model
from keras import applications
# build the VGG16 network
base_model = applications.VGG16(weights='imagenet', include_top=True, input_shape=(img_height, img_width, 3))
print('Base model loaded. {}'.format(base_model.summary()))

from keras.models import Sequential
from keras.layers import Dropout, Flatten, Dense
from keras.regularizers import l2
model = Sequential()
# took off last two layers
for layer in base_model.layers[:-1]:
    model.add(layer)
model.add(Dense(256, activation='relu', kernel_regularizer=l2(0.001)))
model.add(Dropout(0.5))
model.add(Dense(64, activation='relu', kernel_regularizer=l2(0.001)))
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
# set the first 15 layers (up to the last conv block)
# Experiment with retraining 3rd and 4th layers to better fit the dataset
# to non-trainable (weights will not be updated)
for layer in model.layers[:11]:
    layer.trainable = False
for i, layer in enumerate(model.layers):
    print('{} is trainable={}'.format(i, layer.trainable))
    
from keras import optimizers
# setup optimization strategy
model.compile(loss='binary_crossentropy',
             optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),
             metrics=['accuracy'])
from keras.callbacks import ModelCheckpoint, EarlyStopping
epochs=60
epochs_to_wait_for_improve=25
early_stopping_callback = EarlyStopping(monitor='val_loss', patience=epochs_to_wait_for_improve)
# 2 was ~1 gig, trained for 1 epoch and got ~97% (used dropout and sgd with lr=1e-4)
# 3 was ~.1 gig, (used dropout and l2 reg with)
# TODO: update steps per epoch and validation steps to new size for dataset
checkpoint_callback = ModelCheckpoint('CrackDetectionVGG16_4.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min')
# train model
model.fit_generator(
    train_generator,
    steps_per_epoch=3601/batch_size,
    epochs=epochs,
    validation_data=validation_generator,
    validation_steps=899/batch_size, verbose=True,
    callbacks=[early_stopping_callback, checkpoint_callback])

# Test trained model
model.load_weights('CrackDetectionVGG16_4.h5')
crack_test_images = ["crack.jpg", "crack1.jpg", "crack2.jpg", "crack3.jpg",
                "crack4.jpg", "crack5.jpg", "crack6.jpg", "crack7.jpg",
                "crack8.jpg", "crack9.jpg", "crack10.jpg", "crack11.jpg",
                "crack12.jpg", "crack13.jpg"]
nocrack_test_images = ["nocrack1.jpg", "nocrack2.jpg", "nocrack3.jpg",
                      "nocrack4.jpg", "nocrack5.jpg", "nocrack6.jpg",
                      "nocrack7.jpg", "nocrack8.jpg", "nocrack9.jpg"]
from keras.applications.vgg16 import preprocess_input
from keras.preprocessing.image import load_img
from keras.preprocessing.image import img_to_array
i = 0
for crack_test_image in crack_test_images:
    # load an image from file
    image = load_img('D:/git_projects/CrackDetection/Images/crack/'+crack_test_image, target_size=(224, 224))
    # convert the image pixels to a numpy array
    image = img_to_array(image)
    # reshape data for the model
    image = image.reshape((1, image.shape[0], image.shape[1], image.shape[2]))
    # normalize input image
    image = image / 255
    print(image.shape)
    #print(image)
    print("{}, {}, {}".format(image[0][0][0], image[0][0][1], image[0][0][2]))
    # predict the probability across all output classes
    #print(image)
    yhat = model.predict(image)
    print("{}: {}".format(i, yhat))
    i += 1
    break
print("=============================")
i = 0
for nocrack_test_image in nocrack_test_images:
    break
    # load an image from file
    image = load_img('D:/git_projects/CrackDetection/Images/nocrack/'+nocrack_test_image, target_size=(224, 224))
    # convert the image pixels to a numpy array
    image = img_to_array(image)
    # reshape data for the model
    image = image.reshape((1, image.shape[0], image.shape[1], image.shape[2]))
    image = image / 255
    # predict the probability across all output classes
    yhat = model.predict(image)
    print("{}: {}".format(i, yhat))
    i += 1
    
import winmltools
# investigate effect of changeing opset
convert_model = winmltools.convert_keras(model, 8) 
winmltools.save_model(convert_model, "vgg16-crack.onnx")