Receipt Scanner OpenCV C#

ReceiptScanner.cs

using OpenCvSharp;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using static OpenCvSharp.Cv2;
using static System.Math;

namespace ReceiptScannerCSharp
{
    // https://github.com/agrawalamod/Receipt-Scanner-in-OpenCV/blob/master/ReceiptScanner.cpp
    // Required Package: <package id="OpenCvSharp3-AnyCPU" version="3.2.0.20170911" targetFramework="net40" />

    internal static class ReceiptScanner
    {
        private static Mat FFT(Mat I)
        {
            Mat padded = new Mat(); //expand input image to optimal size
            int m = GetOptimalDFTSize(I.Rows);
            int n = GetOptimalDFTSize(I.Cols); // on the border add zero values
            CopyMakeBorder(I, padded, 0, m - I.Rows, 0, n - I.Cols, BorderTypes.Constant, Scalar.All(0));
            padded.ConvertTo(padded, MatType.CV_32F);
            for (int i = 0; i < padded.Size().Height; i++)
            {
                for (int j = 0; j < padded.Size().Width; j++)
                {
                    padded.Set(i, j, padded.At<float>(i, j) * (float)Pow(-1, i + j));
                }
            }

            Mat[] planes = { padded, Mat.Zeros(padded.Size(), MatType.CV_32F) };
            Mat complexI = new Mat();
            Merge(planes, complexI); // Add to the expanded another plane with zeros

            Dft(complexI, complexI); // this way the result may fit in the source matrix

            return complexI;
        }

        private static Mat IDFT(Mat complexI)
        {
            Mat inverseTransform = new Mat();
            Dft(complexI, inverseTransform, DftFlags.Inverse | DftFlags.RealOutput);
            for (int i = 0; i < inverseTransform.Size().Height; i++)
            {
                for (int j = 0; j < inverseTransform.Size().Width; j++)
                {
                    inverseTransform.Set(i, j, inverseTransform.At<float>(i, j) / Pow(-1, i + j));
                }
            }

            Normalize(inverseTransform, inverseTransform, 0, 1, NormTypes.MinMax);

            if (Debugger.IsAttached)
            {
                NamedWindow("Inverse Fourier Transform", WindowMode.AutoSize);
                ImShow("Inverse Fourier Transform", inverseTransform);
                WaitKey(0);
                DestroyWindow("Inverse Fourier Transform");
            }

            return inverseTransform;
        }

        private static Mat DFTMagnitude(Mat complexI, Size s)
        {
            Mat[] planes = { Mat.Zeros(s, MatType.CV_32F), Mat.Zeros(s, MatType.CV_32F) };
            Split(complexI, out planes); // planes[0] = Re(DFT(I), planes[1] = Im(DFT(I))
            Magnitude(planes[0], planes[1], planes[0]); // planes[0] = magnitude
            Mat magI = planes[0];

            magI += Scalar.All(1); // switch to logarithmic scale
            Log(magI, magI);
            
            // crop the spectrum, if it has an odd number of rows or columns
            magI = new Mat(magI, new Rect(0, 0, magI.Cols & -2, magI.Rows & -2));
            Normalize(magI, magI, 0, 1, NormTypes.MinMax); // Transform the matrix with float values into a

            if (Debugger.IsAttached)
            {
                // viewable image form (float between values 0 and 1).
                NamedWindow("DFT Magnitude", WindowMode.AutoSize);
                ImShow("DFT Magnitude", magI);
                WaitKey(0);
                DestroyWindow("DFT Magnitude");
            }

            return magI;
        }

        private static Mat Convolution_fourier(Mat FFT_img, Mat filter)
        {
            Size s = filter.Size();
            Mat resultComplex = new Mat();
            Mat[] planes = { Mat.Zeros(s, MatType.CV_32F), Mat.Zeros(s, MatType.CV_32F) };
            Split(FFT_img, out planes);
            Mat real = planes[0];
            Mat im = planes[1];

            for (int i = 0; i < real.Size().Height; i++)
            {
                for (int j = 0; j < real.Size().Width; j++)
                {
                    planes[0].Set(i, j, real.At<float>(i, j) * filter.At<float>(i, j));
                    planes[1].Set(i, j, im.At<float>(i, j) * filter.At<float>(i, j));
                }
            }

            Merge(planes, resultComplex);
            return resultComplex;
        }

        private static Mat InitializeMat(int n, byte[,] arr /*[5][5]*/)
        {
            Mat A = Mat.Zeros(n, n, MatType.CV_8UC1);

            for (int i = 0; i < n; i++)
            {
                for (int j = 0; j < n; j++)
                {
                    A.Set(i, j, arr[i, j]);
                }
            }

            return A;
        }

        private static Mat GaussianLPF(float D, int height, int width)
        {
            Mat filter = Mat.Zeros(rows: height, cols: width, type: MatType.CV_32F);
            int origin_x = width / 2;
            int origin_y = height / 2;
            double Dtemp, value;

            for (int i = 0; i < height; i++)
            {
                for (int j = 0; j < width; j++)
                {
                    Dtemp = Sqrt(Pow(j - origin_x, 2) + Pow(i - origin_y, 2));

                    double numerator = -Pow(Dtemp, 2);
                    double denominator = 2 * Pow(D, 2);

                    value = Pow(E, numerator / denominator);
                    filter.Set(i, j, value);
                }
            }

            if (Debugger.IsAttached)
            {
                ImShow("Gaussian LPF", filter);
                WaitKey(0);
                DestroyWindow("Gaussian LPF");
            }

            return filter;
        }

        private static Mat GaussianHPF(float D, int height, int width)
        {
            Mat filter = Mat.Zeros(rows: height, cols: width, type: MatType.CV_32F);
            int origin_x = width / 2;
            int origin_y = height / 2;
            double Dtemp, value;

            for (int i = 0; i < height; i++)
            {
                for (int j = 0; j < width; j++)
                {
                    Dtemp = Sqrt(Pow(j - origin_x, 2) + Pow(i - origin_y, 2));
                    
                    double numerator = -Pow(Dtemp, 2);
                    double denominator = 2 * Pow(D, 2);
                    value = 1 - Pow(E, numerator / denominator);
                    
                    filter.Set(i, j, value);
                }
            }

            if (Debugger.IsAttached)
            {
                ImShow("Gaussian HPF", filter);
                WaitKey(0);
                DestroyWindow("Gaussian HPF");
            }

            return filter;
        }

        private static Mat ButterworthLPF(float D, int height, int width, int n)
        {
            Mat filter = Mat.Zeros(rows: height, cols: width, type: MatType.CV_32F);
            int origin_x = width / 2;
            int origin_y = height / 2;
            double Dtemp, value;

            for (int i = 0; i < height; i++)
            {
                for (int j = 0; j < width; j++)
                {
                    Dtemp = Sqrt(Pow(j - origin_x, 2) + Pow(i - origin_y, 2));
                    
                    float ratio = (float)Pow((Dtemp / D), 2 * n);
                    value = 1 / (1 + ratio);
                    filter.Set(i, j, value);
                }
            }

            if (Debugger.IsAttached)
            {
                ImShow("Butterworth LPF", filter);
                WaitKey(0);
                DestroyWindow("Butterworth LPF");
            }

            return filter;
        }

        private static Mat ButterworthHPF(float D, int height, int width, int n)
        {
            Mat filter = Mat.Zeros(rows: height, cols: width, type: MatType.CV_32F);
            int origin_x = width / 2;
            int origin_y = height / 2;
            double Dtemp, value;

            for (int i = 0; i < height; i++)
            {
                for (int j = 0; j < width; j++)
                {
                    Dtemp = Sqrt(Pow(j - origin_x, 2) + Pow(i - origin_y, 2));
                    
                    float ratio = (float)Pow((D / Dtemp), 2 * n);
                    value = 1 / (1 + ratio);
                    filter.Set(i, j, value);
                }
            }

            if (Debugger.IsAttached)
            {
                ImShow("Butterworth HPF", filter);
                WaitKey(0);
                DestroyWindow("Butterworth HPF");
            }

            return filter;
        }

        private static Mat CannyThreshold(int lowThreshold, int highThreshold, Mat img1)
        {
            Mat detected_edges = new Mat();
            int kernel_size = 3;
            
            // Reduce noise with a kernel 3x3
            Blur(img1, detected_edges, new Size(3, 3));

            // Canny detector
            Canny(detected_edges, detected_edges, lowThreshold, highThreshold, kernel_size);

            if (Debugger.IsAttached)
            {
                // Using Canny's output as a mask, we display our result
                ImShow("Canny Output", detected_edges);
                ImWrite("canny_result.jpg", detected_edges);
                WaitKey(0);
                DestroyWindow("Canny Output");
            }

            return detected_edges;
        }

        private static Mat Erosion(Mat src, int erosion_size, int erosion_elem)
        {
            MorphShapes erosion_type = default(MorphShapes);
            Mat erosion_dst = new Mat();
            if (erosion_elem == 0) { erosion_type = MorphShapes.Rect; }
            else if (erosion_elem == 1) { erosion_type = MorphShapes.Cross; }
            else if (erosion_elem == 2) { erosion_type = MorphShapes.Ellipse; }

            Mat element = GetStructuringElement(erosion_type,
                                                 new Size(2 * erosion_size + 1, 2 * erosion_size + 1),
                                                 new Point(erosion_size, erosion_size));

            // Apply the erosion operation
            Erode(src, erosion_dst, element);
            //Console.Out.WriteLine("Erosion done! ");

            if (Debugger.IsAttached)
            {
                ImShow("Erosion Demo", erosion_dst);
                WaitKey(0);
                DestroyWindow("Erosion Demo");
            }

            return erosion_dst;
        }

        /** @function Dilation */
        private static Mat Dilation(Mat src, int dilation_size, int dilation_elem)
        {
            MorphShapes dilation_type = default(MorphShapes);
            Mat dilation_dst = new Mat();
            if (dilation_elem == 0) { dilation_type = MorphShapes.Rect; }
            else if (dilation_elem == 1) { dilation_type = MorphShapes.Cross; }
            else if (dilation_elem == 2) { dilation_type = MorphShapes.Ellipse; }

            Mat element = GetStructuringElement(dilation_type,
                                                 new Size(2 * dilation_size + 1, 2 * dilation_size + 1),
                                                 new Point(dilation_size, dilation_size));
            // Apply the dilation operation
            Dilate(src, dilation_dst, element);
            Console.Out.WriteLine("Dilation done! ");

            if (Debugger.IsAttached)
            {
                ImShow("Dilation Demo", dilation_dst);
                WaitKey(0);
                DestroyWindow("Dilation Demo");
            }

            return dilation_dst;
        }

        private static Mat WarpImage(Mat src, List<Point2f> corners)
        {
            float x1, x2, x3, x4, y1, y2, y3, y4;
            x1 = corners[0].X;
            y1 = corners[0].Y;
            x2 = corners[1].X;
            y2 = corners[1].Y;
            x3 = corners[2].X;
            y3 = corners[2].Y;
            x4 = corners[3].X;
            y4 = corners[3].Y;

            float minx, maxx, miny, maxy;
            minx = x1 > x4 ? x1 : x4;
            maxx = x2 < x3 ? x2 : x3;
            miny = y1 > y2 ? y1 : y2;
            maxy = y3 < y4 ? y3 : y4;

            Console.Out.WriteLine($"{minx} {maxx} {miny} {maxy}");

            // Define the destination image
            Mat quad = Mat.Zeros(rows: (int)(maxy - miny), cols: (int)(maxx - minx), type: MatType.CV_8UC1);

            // Corners of the destination image
            List<Point2f> quad_pts = new List<Point2f>
            {
                new Point2f(0, 0),
                new Point2f(quad.Cols, 0),
                new Point2f(quad.Cols, quad.Rows),
                new Point2f(0, quad.Rows)
            };

            // Get transformation matrix
            Mat transmtx = GetPerspectiveTransform(corners, quad_pts);

            // Apply perspective transformation
            WarpPerspective(src, quad, transmtx, quad.Size());

            if (Debugger.IsAttached)
            {
                ImShow("quadrilateral", quad);
                ImWrite("warpedImage.jpg", quad);
                WaitKey(0);
                DestroyWindow("quadrilateral");
            }

            return quad;
        }

        private static List<Point> FindLargestContours(Mat src)
        {
            int largest_area = 0;
            int largest_contour_index = 0;
            Rect bounding_rect;
            Mat thr = new Mat(src.Rows, src.Cols, MatType.CV_8UC1);
            Mat dst = new Mat(src.Rows, src.Cols, MatType.CV_8UC1, Scalar.All(0));
            Threshold(src, src, 25, 255, ThresholdTypes.Binary); //Threshold the gray

            // Vector for storing contour
            FindContours(src, out Point[][] contours, out HierarchyIndex[] hierarchy, RetrievalModes.CComp, ContourApproximationModes.ApproxSimple); // Find the contours in the image

            for (int i = 0; i < contours.Length; i++) // iterate through each contour. 
            {
                double a = ContourArea(contours[i], false);  //  Find the area of contour

                if (a > largest_area)
                {
                    largest_area = (int)a;
                    largest_contour_index = i; //Store the index of largest contour
                    bounding_rect = BoundingRect(contours[i]); // Find the bounding rectangle for biggest contour
                }
            }

            Console.Out.Write($"{largest_area} {src.Rows * src.Cols}");
            Scalar color = new Scalar(255, 255, 255);
            Mat x = Mat.Zeros(src.Rows, src.Cols, MatType.CV_8UC1);
            Mat contour_image = new Mat();
            src.CopyTo(contour_image);
            DrawContours(contour_image, contours, largest_contour_index, color, 5, LineTypes.Link8, hierarchy);

            List<List<Point>> contours_poly = new List<List<Point>>
            {
                new List<Point>()
            };

            ApproxPolyDP(InputArray.Create(contours[largest_contour_index]), OutputArray.Create(contours_poly[0]), 10, true);

            if (Debugger.IsAttached)
            {
                ImShow("Largest Contour", contour_image);
                ImWrite("largest_contour.jpg", contour_image);
                WaitKey(0);
                DestroyWindow("Largest Contour");
            }

            return contours_poly[0];
        }

        private static List<Point2f> DetectCorners(Mat src, List<Point> contours_poly)
        {
            List<Point2f> corners = Distance(src, contours_poly);
            Mat all_points = new Mat();
            Mat corners_points = new Mat();
            src.CopyTo(all_points);
            src.CopyTo(corners_points);

            for (int i = 0; i < contours_poly.Count; i++)
            {
                Circle(all_points, contours_poly[i], 20, new Scalar(255, 255, 255), -1);
            }

            for (int i = 0; i < corners.Count; i++)
            {
                Circle(corners_points, corners[i], 20, new Scalar(255, 255, 255), -1);
            }

            //ImWrite("all_points.jpg", all_points);
            //ImWrite("corners_points.jpg", corners_points);

            return corners;
        }

        private static List<Point2f> Distance(Mat img, List<Point> polygon)
        {
            List<Point2f> coord = new List<Point2f>();
            Console.Out.WriteLine($"{img.Rows} {img.Cols}");

            double min = Sqrt(Pow(img.Cols, 2) + Pow(img.Rows, 2));
            int index = 0;

            for (int i = 0; i < polygon.Count; i++)
            {
                double dist = Sqrt(Pow(polygon[i].X, 2) + Pow(polygon[i].Y, 2));

                if (dist <= min)
                {
                    min = dist;
                    index = i;
                }
            }

            coord.Add(new Point2f(polygon[index].X, polygon[index].Y));

            min = Sqrt(Pow(img.Cols, 2) + Pow(img.Rows, 2));

            for (int i = 0; i < polygon.Count; i++)
            {
                double dist = Sqrt(Pow(polygon[i].X - img.Cols, 2) + Pow((polygon[i].Y), 2));

                if (dist <= min)
                {
                    min = dist;
                    index = i;
                }
            }

            coord.Add(new Point2f(polygon[index].X, polygon[index].Y));
            min = Sqrt(Pow(img.Cols, 2) + Pow(img.Rows, 2));

            for (int i = 0; i < polygon.Count; i++)
            {
                double dist = Sqrt(Pow((polygon[i].X - img.Cols), 2) + Pow((polygon[i].Y - img.Rows), 2));

                if (dist <= min)
                {
                    min = dist;
                    index = i;
                }
            }

            coord.Add(new Point2f(polygon[index].X, polygon[index].Y));
            min = Sqrt(Pow(img.Cols, 2) + Pow(img.Rows, 2));

            for (int i = 0; i < polygon.Count; i++)
            {
                double dist = Sqrt(Pow(polygon[i].X, 2) + Pow((polygon[i].Y - img.Rows), 2));
                Console.Out.WriteLine($"{polygon[i]}: {dist}");

                if (dist <= min)
                {
                    min = dist;
                    index = i;
                }
            }

            coord.Add(new Point2f(polygon[index].X, polygon[index].Y));
            Console.Out.Write(coord);
            return coord;
        }

        [STAThread]
        private static int Main()
        {
            string[] argv = Environment.GetCommandLineArgs();
            int argc = argv.Length;

            if (argc != 2)
            {
                Console.Out.Write(" Program_name <JPEG image>");
                return -1;
            }

            Mat img1;
            string filePath = Path.GetFullPath(argv[1]);
            img1 = ImRead(filePath, ImreadModes.GrayScale); //read the image data in the file "MyPic.JPG" and store it in 'img'

            if (img1.Empty()) //check whether the image is loaded or not
            {
                Console.Out.WriteLine("Error : Image cannot be loaded!");
                return -1;
            }

            if (Debugger.IsAttached)
            {
                NamedWindow("Image Selected", WindowMode.AutoSize); //create a window with the name "MyWindow"
                ImShow("Image Selected", img1); //display the image which is stored in the 'img' in the "MyWindow" window
                WaitKey(0);
                DestroyWindow("Image Selected");
            }
            
            Mat CannyResult = CannyThreshold(75, 200, img1);
            List<Point> polygon = FindLargestContours(CannyResult);
            List<Point2f> corners = DetectCorners(CannyResult, polygon);

            Mat warpedImage = WarpImage(img1, corners);

            Mat dst = Mat.Zeros(warpedImage.Rows, warpedImage.Cols, MatType.CV_8UC1);

            AdaptiveThreshold(warpedImage, dst, 255, AdaptiveThresholdTypes.GaussianC, ThresholdTypes.Binary, 51, 30);

            if (Debugger.IsAttached)
            {
                ImShow("dst", dst);
                WaitKey(0);
                DestroyWindow("dst");
            }

            var outputPath = Path.GetFullPath($"threshold_{Path.GetFileName(filePath)}.jpg");
            ImWrite(outputPath, dst);

            Console.Out.WriteLine($"Output file: `{outputPath}` created.");
            return 0;
        }
    }
}

This doesn't work! Mat quad = Mat.Zeros( rows: ( int ) ( maxy - miny ), cols: ( int ) ( minx - maxx ), type: MatType.CV_8UC1 ); This line breaks as the columns is a negative. I switched it to maxx-minx which produces a positive number but the it then outputs a blank white image