Skip to content

Instantly share code, notes, and snippets.

@rlabrecque
Created December 31, 2016 08:54
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rlabrecque/feb714e279911f5c15276921345cca68 to your computer and use it in GitHub Desktop.
Save rlabrecque/feb714e279911f5c15276921345cca68 to your computer and use it in GitHub Desktop.
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Windows.Forms;
using System.Xml;
using System.Xml.Serialization; //these imports are for writing Matrix objects to file, see end of program
using System.IO;
using Emgu.CV;
using Emgu.CV.CvEnum;
using Emgu.CV.Structure;
using Emgu.CV.Util;
using Emgu.CV.UI;
namespace SomeNamspace {
public partial class GenData : Form {
const int MIN_CONTOUR_AREA = 100;
const int RESIZED_IMAGE_WIDTH = 20;
const int RESIZED_IMAGE_HEIGHT = 30;
public GenData() {
InitializeComponent();
}
private void btnOpenTrainingImage_Click(object sender, EventArgs e) {
DialogResult drChosenFile;
drChosenFile = ofdOpenFile.ShowDialog(); // open file dialog
if (drChosenFile != DialogResult.OK || ofdOpenFile.FileName == "") { // if user chose Cancel or filename is blank . . .
lblChosenFile.Text = "file not chosen"; // show error message on label
return; // and exit function
}
Mat imgTrainingNumbers;
try {
imgTrainingNumbers = new Mat(ofdOpenFile.FileName);
}
catch (Exception ex) { // if error occurred
lblChosenFile.Text = "unable to open image, error: " + ex.Message; // show error message on label
return; // and exit function
}
if (imgTrainingNumbers == null) { // if image could not be opened
lblChosenFile.Text = "unable to open image"; // show error message on label
return; // and exit function
}
lblChosenFile.Text = ofdOpenFile.FileName; //update label with file name
Mat imgGrayscale = new Mat();
Mat imgBlurred = new Mat(); // declare various images
Mat imgThresh = new Mat();
Mat imgThreshCopy = new Mat();
VectorOfVectorOfPoint contours = new VectorOfVectorOfPoint();
//Matrix<Single> mtxClassifications = new Matrix<Single>();
//Matrix<Single> mtxTrainingImages = new Matrix<Single>();
Mat matTrainingImagesAsFlattenedFloats = new Mat();
//possible chars we are interested in are digits 0 through 9 and capital letters A through Z, put these in list intValidChars
var intValidChars = new List<int>(new int[] {
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
'U', 'V', 'W', 'X', 'Y', 'Z' });
CvInvoke.CvtColor(imgTrainingNumbers, imgGrayscale, ColorConversion.Bgr2Gray); //convert to grayscale
CvInvoke.GaussianBlur(imgGrayscale, imgBlurred, new Size(5, 5), 0); //blur
//threshold image from grayscale to black and white
CvInvoke.AdaptiveThreshold(imgBlurred, imgThresh, 255.0, AdaptiveThresholdType.GaussianC, ThresholdType.BinaryInv, 11, 2);
CvInvoke.Imshow("imgThresh", imgThresh); //show threshold image for reference
imgThreshCopy = imgThresh.Clone(); //make a copy of the thresh image, this in necessary b/c findContours modifies the image
//get external countours only
CvInvoke.FindContours(imgThreshCopy, contours, null, RetrType.External, ChainApproxMethod.ChainApproxSimple);
int intNumberOfTrainingSamples = contours.Size;
Matrix<Single> mtxClassifications = new Matrix<Single>(intNumberOfTrainingSamples, 1); //this is our classifications data structure
//this is our training images data structure, note we will have to perform some conversions to write to this later
Matrix<Single> mtxTrainingImages = new Matrix<Single>(intNumberOfTrainingSamples, RESIZED_IMAGE_WIDTH * RESIZED_IMAGE_HEIGHT);
//this keeps track of which row we are on in both classifications and training images,
int intTrainingDataRowToAdd = 0; //note that each sample will correspond to one row in
//both the classifications XML file and the training images XML file
for (int i = 0; i <= contours.Size - 1; ++i) { //for each contour
if (CvInvoke.ContourArea(contours[i]) > MIN_CONTOUR_AREA) { //if contour is big enough to consider
Rectangle boundingRect = CvInvoke.BoundingRectangle(contours[i]); //get the bounding rect
CvInvoke.Rectangle(imgTrainingNumbers, boundingRect, new MCvScalar(0.0, 0.0, 255.0), 2); //draw red rectangle around each contour as we ask user for input
Mat imgROItoBeCloned = new Mat(imgThresh, boundingRect); //get ROI image of current char
Mat imgROI = imgROItoBeCloned.Clone(); //make a copy so we do not change the ROI area of the original image
Mat imgROIResized = new Mat();
//resize image, this is necessary for recognition and storage
CvInvoke.Resize(imgROI, imgROIResized, new Size(RESIZED_IMAGE_WIDTH, RESIZED_IMAGE_HEIGHT));
CvInvoke.Imshow("imgROI", imgROI); //show ROI image for reference
CvInvoke.Imshow("imgROIResized", imgROIResized); //show resized ROI image for reference
CvInvoke.Imshow("imgTrainingNumbers", imgTrainingNumbers); //show training numbers image, this will now have red rectangles drawn on it
int intChar = CvInvoke.WaitKey(0); //get key press
if (intChar == 27) { //if esc key was pressed
CvInvoke.DestroyAllWindows();
return; //exit the function
}
else if (intValidChars.Contains(intChar)) { //else if the char is in the list of chars we are looking for . . .
mtxClassifications[intTrainingDataRowToAdd, 0] = Convert.ToSingle(intChar); //write classification char to classifications Matrix
//now add the training image (some conversion is necessary first) . . .
//note that we have to covert the images to Matrix(Of Single) type, this is necessary to pass into the KNearest object call to train
Matrix<Single> mtxTemp = new Matrix<Single>(imgROIResized.Size);
Matrix<Single> mtxTempReshaped = new Matrix<Single>(1, RESIZED_IMAGE_WIDTH * RESIZED_IMAGE_HEIGHT);
imgROIResized.ConvertTo(mtxTemp, DepthType.Cv32F); //convert Image to a Matrix of Singles with the same dimensions
for (int intRow = 0; intRow <= RESIZED_IMAGE_HEIGHT - 1; ++intRow) { //flatten Matrix into one row by RESIZED_IMAGE_WIDTH * RESIZED_IMAGE_HEIGHT number of columns
for (int intCol = 0; intCol <= RESIZED_IMAGE_WIDTH - 1; ++intCol) {
mtxTempReshaped[0, (intRow * RESIZED_IMAGE_WIDTH) + intCol] = mtxTemp[intRow, intCol];
}
}
for (int intCol = 0; intCol <= (RESIZED_IMAGE_WIDTH * RESIZED_IMAGE_HEIGHT) - 1; ++intCol) { //write flattened Matrix into one row of training images Matrix
mtxTrainingImages[intTrainingDataRowToAdd, intCol] = mtxTempReshaped[0, intCol];
}
intTrainingDataRowToAdd = intTrainingDataRowToAdd + 1; //increment which row, i.e. sample we are on
}
}
}
txtInfo.Text = txtInfo.Text + "training complete !!" + "\n" + "\n";
//save classifications to file
XmlSerializer xmlSerializer = new XmlSerializer(mtxClassifications.GetType());
StreamWriter streamWriter;
try {
streamWriter = new StreamWriter("classifications.xml"); //attempt to open classifications file
}
catch (Exception ex) { //if error is encountered, show error and return
txtInfo.Text = "\n" + txtInfo.Text + "unable to open 'classifications.xml', error:" + "\n";
txtInfo.Text = txtInfo.Text + ex.Message + "\n" + "\n";
return;
}
xmlSerializer.Serialize(streamWriter, mtxClassifications);
streamWriter.Close();
//save training images to file
xmlSerializer = new XmlSerializer(mtxTrainingImages.GetType());
try {
streamWriter = new StreamWriter("images.xml"); // attempt to open images file
}
catch (Exception ex) { // if error is encountered, show error and return
txtInfo.Text = "\n" + txtInfo.Text + "unable to open 'images.xml', error:" + "\n";
txtInfo.Text = txtInfo.Text + ex.Message + "\n" + "\n";
return;
}
xmlSerializer.Serialize(streamWriter, mtxTrainingImages);
streamWriter.Close();
txtInfo.Text = "\n" + txtInfo.Text + "file writing done" + "\n";
MessageBox.Show("Training complete, file writing done !!");
}
}
}
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Windows.Forms;
using System.Xml;
using System.Xml.Serialization; //these imports are for writing Matrix objects to file, see end of program
using System.IO;
using Emgu.CV;
using Emgu.CV.CvEnum;
using Emgu.CV.Structure;
using Emgu.CV.Util;
using Emgu.CV.UI;
using Emgu.CV.ML;
namespace SolitaireAI {
public partial class TrainAndTest : Form {
public TrainAndTest() {
InitializeComponent();
}
const int RESIZED_IMAGE_WIDTH = 20;
const int RESIZED_IMAGE_HEIGHT = 30;
private void btnOpenTestImage_Click(object sender, EventArgs e) {
//note: we effectively have to read the first XML file twice
//first, we read the file to get the number of rows (which is the same as the number of samples)
//the first time reading the file we can't get the data yet, since we don't know how many rows of data there are
//next, reinstantiate our classifications Matrix and training images Matrix with the correct number of rows
//then, read the file again and this time read the data into our resized classifications Matrix and training images Matrix
Matrix<Single> mtxClassifications = new Matrix<Single>(1, 1); //for the first time through, declare these to be 1 row by 1 column
Matrix<Single> mtxTrainingImages = new Matrix<Single>(1, 1); //we will resize these when we know the number of rows (i.e. number of training samples)
var intValidChars = new List<int>(new int[] {
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
'U', 'V', 'W', 'X', 'Y', 'Z' });
XmlSerializer xmlSerializer = new XmlSerializer(mtxClassifications.GetType()); //these variables are for
StreamReader streamReader; //reading from the XML files
try {
streamReader = new StreamReader("classifications.xml"); //attempt to open classifications file
}
catch (Exception ex) { //if error is encountered, show error and return
txtInfo.AppendText("\n" + "unable to open 'classifications.xml', error: ");
txtInfo.AppendText(ex.Message + "\n");
return;
}
//read from the classifications file the 1st time, this is only to get the number of rows, not the actual data
mtxClassifications = (Matrix<Single>)xmlSerializer.Deserialize(streamReader);
streamReader.Close(); //close the classifications XML file
int intNumberOfTrainingSamples = mtxClassifications.Rows; //get the number of rows, i.e. the number of training samples
//now that we know the number of rows, reinstantiate classifications Matrix and training images Matrix with the actual number of rows
mtxClassifications = new Matrix<Single>(intNumberOfTrainingSamples, 1);
mtxTrainingImages = new Matrix<Single>(intNumberOfTrainingSamples, RESIZED_IMAGE_WIDTH * RESIZED_IMAGE_HEIGHT);
try {
streamReader = new StreamReader("classifications.xml"); //reinitialize the stream reader, attempt to open classifications file again
}
catch (Exception ex) { //if error is encountered, show error and return
txtInfo.AppendText("\n" + "unable to open 'classifications.xml', error:" + "\n");
txtInfo.AppendText(ex.Message + "\n" + "\n");
return;
}
//read from the classifications file again, this time we can get the actual data
mtxClassifications = (Matrix<Single>)xmlSerializer.Deserialize(streamReader);
streamReader.Close(); //close the classifications XML file
xmlSerializer = new XmlSerializer(mtxTrainingImages.GetType()); //reinstantiate file reading variable
try {
streamReader = new StreamReader("images.xml"); //attempt to open classifications file
}
catch (Exception ex) { //if error is encountered, show error and return
txtInfo.AppendText("unable to open 'images.xml', error:" + "\n");
txtInfo.AppendText(ex.Message + "\n" + "\n");
}
mtxTrainingImages = (Matrix<Single>)xmlSerializer.Deserialize(streamReader); //read from training images file
streamReader.Close(); //close the training images XML file
// train
KNearest kNearest = new KNearest();
kNearest.DefaultK = 1;
kNearest.Train(mtxTrainingImages, Emgu.CV.ML.MlEnum.DataLayoutType.RowSample, mtxClassifications);
// test
DialogResult drChosenFile;
drChosenFile = ofdOpenFile.ShowDialog(); //open file dialog
if (drChosenFile != DialogResult.OK || ofdOpenFile.FileName == "") {
lblChosenFile.Text = "file not chosen"; //show error message on label
return;
}
Mat imgTestingNumbers; //declare the input image
try {
imgTestingNumbers = CvInvoke.Imread(ofdOpenFile.FileName); //open image
}
catch (Exception ex) { //if error occurred
lblChosenFile.Text = "unable to open image, error: " + ex.Message; //show error message on label
return; //and exit function
}
if (imgTestingNumbers == null) { //if image could not be opened
lblChosenFile.Text = "unable to open image"; //show error message on label
return; //and exit function
}
if (imgTestingNumbers.IsEmpty) {
lblChosenFile.Text = "unable to open image";
return;
}
lblChosenFile.Text = ofdOpenFile.FileName; //update label with file name
Mat imgGrayscale = new Mat(); //
Mat imgBlurred = new Mat(); //declare various images
Mat imgThresh = new Mat(); //
Mat imgThreshCopy = new Mat(); //
CvInvoke.CvtColor(imgTestingNumbers, imgGrayscale, ColorConversion.Bgr2Gray); //convert to grayscale
CvInvoke.GaussianBlur(imgGrayscale, imgBlurred, new Size(5, 5), 0); //blur
//threshold image from grayscale to black and white
CvInvoke.AdaptiveThreshold(imgBlurred, imgThresh, 255.0, AdaptiveThresholdType.GaussianC, ThresholdType.BinaryInv, 11, 2.0);
imgThreshCopy = imgThresh.Clone(); //make a copy of the thresh image, this in necessary b/c findContours modifies the image
VectorOfVectorOfPoint contours = new VectorOfVectorOfPoint();
//get external countours only
CvInvoke.FindContours(imgThreshCopy, contours, null, RetrType.External, ChainApproxMethod.ChainApproxSimple);
List<ContourWithData> listOfContoursWithData = new List<ContourWithData>(); //declare a list of contours with data
//populate list of contours with data
for (int i = 0; i <= contours.Size - 1; ++i) { //for each contour
ContourWithData contourWithData = new ContourWithData(); //declare new contour with data
contourWithData.contour = contours[i]; //populate contour member variable
contourWithData.boundingRect = CvInvoke.BoundingRectangle(contourWithData.contour); //calculate bounding rectangle
contourWithData.dblArea = CvInvoke.ContourArea(contourWithData.contour); //calculate area
if (contourWithData.checkIfContourIsValid()) { //if contour with data is valis
listOfContoursWithData.Add(contourWithData); //add to list of contours with data
}
}
//sort contours with data from left to right
listOfContoursWithData.Sort(
(oneContourWithData, otherContourWithData) => {
return oneContourWithData.boundingRect.X.CompareTo(otherContourWithData.boundingRect.X);
});
string strFinalString = ""; //declare final string, this will have the final number sequence by the end of the program
foreach (ContourWithData contourWithData in listOfContoursWithData) {//for each contour in list of valid contours
CvInvoke.Rectangle(imgTestingNumbers, contourWithData.boundingRect, new MCvScalar(0.0, 255.0, 0.0), 2); //draw green rect around the current char
Mat imgROItoBeCloned = new Mat(imgThresh, contourWithData.boundingRect); //get ROI image of bounding rect
Mat imgROI = imgROItoBeCloned.Clone(); //clone ROI image so we don't change original when we resize
Mat imgROIResized = new Mat();
//resize image, this is necessary for char recognition
CvInvoke.Resize(imgROI, imgROIResized, new Size(RESIZED_IMAGE_WIDTH, RESIZED_IMAGE_HEIGHT));
//declare a Matrix of the same dimensions as the Image we are adding to the data structure of training images
Matrix<Single> mtxTemp = new Matrix<Single>(imgROIResized.Size);
//declare a flattened (only 1 row) matrix of the same total size
Matrix<Single> mtxTempReshaped = new Matrix<Single>(1, RESIZED_IMAGE_WIDTH * RESIZED_IMAGE_HEIGHT);
imgROIResized.ConvertTo(mtxTemp, DepthType.Cv32F); //convert Image to a Matrix of Singles with the same dimensions
for (int intRow = 0; intRow <= RESIZED_IMAGE_HEIGHT - 1; ++intRow) { //flatten Matrix into one row by RESIZED_IMAGE_WIDTH * RESIZED_IMAGE_HEIGHT number of columns
for (int intCol = 0; intCol <= RESIZED_IMAGE_WIDTH - 1; ++intCol) {
mtxTempReshaped[0, (intRow * RESIZED_IMAGE_WIDTH) + intCol] = mtxTemp[intRow, intCol];
}
}
Single sngCurrentChar;
sngCurrentChar = kNearest.Predict(mtxTempReshaped); //finally we can call Predict !!!
strFinalString = strFinalString + (char)(Convert.ToInt32(sngCurrentChar)); //append current char to full string of chars
}
txtInfo.AppendText("\n" + "\n" + "characters read from image = " + strFinalString + "\n");
CvInvoke.Imshow("imgTestingNumbers", imgTestingNumbers);
}
}
public class ContourWithData {
const int MIN_CONTOUR_AREA = 100;
public VectorOfPoint contour; //contour
public Rectangle boundingRect; //bounding rect for contour
public Double dblArea; //area of contour
public bool checkIfContourIsValid() { //this is oversimplified, for a production grade program better validity checking would be necessary
if (dblArea < MIN_CONTOUR_AREA) {
return false;
}
else {
return true;
}
}
}
}
@rlabrecque
Copy link
Author

Please note that this is pretty bad code, I wouldn't really recommend using it or starting from it, but hopefully it helps someone figure something out one day.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment