smart_capture.cpp

//
//  smart_capture.cpp
//  Runner
//
//  Created by Kim Henneken on 28.04.21.
//

#include "smart_capture.hpp"
#include <opencv2/opencv.hpp>
#include <opencv2/imgproc/types_c.h>
#include "opencv2/calib3d.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/features2d.hpp"
#include <thread>
#include <future>
#include <iostream>
#include <string>
#include <iostream>

using namespace cv;
using namespace std;

Mat filter_image(Mat image)
{
    cvtColor(image, image, COLOR_RGB2GRAY);
    return image;
}

ExtractedFeatures extract_features(Mat mask_template, Mat Template, int nfeatures)
{
    Mat template_filtered = filter_image(Template);

    Ptr<SIFT> detector = SIFT::create(nfeatures);

    std::vector<KeyPoint> kp2;
    Mat des2;
    detector->detectAndCompute(template_filtered, mask_template, kp2, des2);

    ExtractedFeatures extractedFeatures;

    extractedFeatures.des2 = des2;
    extractedFeatures.kp2 = kp2;

    return extractedFeatures;
}

PreparedTemplate prepare_templates(int nfeatures, double ro, Mat MaskTemplate, Mat Template)
{

    int h = MaskTemplate.size().height;
    int w = MaskTemplate.size().width;

    cv::resize(MaskTemplate, MaskTemplate, cv::Size(w * ro, h * ro));
    cv::resize(Template, Template, cv::Size(w * ro, h * ro));

    Size template_shape = Template.size();

    //    Mat template_image = Template.setTo(Scalar().all(255.0));

    auto future = std::async(extract_features, MaskTemplate, Template, nfeatures);

    ExtractedFeatures extracted_features = future.get();

    PreparedTemplate preparedTemplate;

    preparedTemplate.template_shape = template_shape;
    preparedTemplate.des2 = extracted_features.des2;
    preparedTemplate.kp2 = extracted_features.kp2;
    
//    preparedTemplate.Template = Template;
//    preparedTemplate.mask_template = MaskTemplate;

    return preparedTemplate;
}

StabilisedCard stabilise_card(Mat img1,
                              int min_matches,
                              int nfeatures,
                              std::vector<KeyPoint> kp2, Mat des2,
                              Size template_shape)
{

    Mat mask1 = img1.clone().setTo(Scalar().all(255.0));

    mask1.convertTo(mask1, CV_8U);

    double S = mask1.size().height * mask1.size().width;

    double height = template_shape.height;
    double width = template_shape.width;

    double standard = width * height;

    double H = img1.size().height;
    double W = img1.size().width;

    double ratio = pow((S / standard), 0.5);

    Size sz = Size(W / ratio, H / ratio);

    cv::resize(img1, img1, sz);
    cv::resize(mask1, mask1, sz);
    mask1.setTo(Scalar().all(10));

    Mat filtered_img1 = filter_image(img1);

    //    cv::imwrite(path, mask1);
    Ptr<SIFT> detector = SIFT::create(nfeatures);

    std::vector<KeyPoint> kp1;
    Mat des1;
    detector->detectAndCompute(filtered_img1, noArray(), kp1, des1);

    Ptr<DescriptorMatcher> matcher = DescriptorMatcher::create(DescriptorMatcher::BRUTEFORCE);

    std::vector<std::vector<DMatch>> knn_matches;
    matcher->knnMatch(des1, des2, knn_matches, 2);

    std::vector<DMatch> good_matches;
    for (size_t i = 0; i < knn_matches.size(); i++)
    {
        if (knn_matches[i][0].distance < 0.7 * knn_matches[i][1].distance)
        {
            good_matches.push_back(knn_matches[i][0]);
        }
    }

    StabilisedCard stabilisiedCard;

    if (good_matches.size() > min_matches)
    {
        stabilisiedCard.alignment_quality = (double)good_matches.size() / (double)kp2.size();
        return stabilisiedCard;
    }

    stabilisiedCard.alignment_quality = 0;
    return stabilisiedCard;
}

void tokenize(string &str, char delim, vector<string> &out)
{
    size_t start;
    size_t end = 0;

    while ((start = str.find_first_not_of(delim, end)) != string::npos)
    {
        end = str.find(delim, start);
        out.push_back(str.substr(start, end - start));
    }
}

Mat parseDescriptorsFromCSVString(char* descriptorLines){
    std::string descriptorsString(descriptorLines);

    std::vector<std::string> descriptorsArr;
    std::vector<std::string> firstLineDescriptorArr;
    
    tokenize(descriptorsString, '\n', descriptorsArr);
    tokenize(descriptorsArr[0], ',', firstLineDescriptorArr);

    Mat descriptors(descriptorsArr.size(),firstLineDescriptorArr.size(),CV_32F);
    
    for (int i = 0; i < descriptorsArr.size(); i++) {
        std::string rowString = descriptorsArr[i];
        std::vector<std::string> rowArr;
        tokenize(rowString, ',', rowArr);
        for (int j = 0; j < rowArr.size(); j++) {
            descriptors.row(i).col(j).setTo(Scalar(stof(rowArr[j])));
        }
    }
    return descriptors;
}

void parseKeyPointsFromCSVString(char* keyPointLines, std::vector<KeyPoint>& kps){
    std::string keyPointString(keyPointLines);
    std::vector<std::string> keyPointsArr;
    tokenize(keyPointString, '\n', keyPointsArr);
    for (int i = 0; i < keyPointsArr.size(); i++) {
        std::string keyPointLine = keyPointsArr[i];
        std::vector<std::string> keyPointArr;
        tokenize(keyPointLine, ',', keyPointArr);
        float x = stof(keyPointArr[0]);
        float y = stof(keyPointArr[1]);
        float size = stof(keyPointArr[2]);
        float angle = stof(keyPointArr[3]);
        float response = stof(keyPointArr[4]);
        int octave = stoi(keyPointArr[5]);
        int class_id = stoi(keyPointArr[6]);
        
        KeyPoint kp = KeyPoint(x, y, size,angle,response,octave,class_id);
    
        kps.push_back(kp);
   }
}

double SmartCapture::smart_capture(Mat card, String min_matches, String card_nfeatures,
                                   char* descriptorLines,char* keyPointLines,int templateWidth,int templateHeight)
{
    int min_matches2 = stoi(min_matches);
    int card_nfeatures2 = stoi(card_nfeatures);
    
    std::vector<KeyPoint> kps;
    Mat descriptors = parseDescriptorsFromCSVString(descriptorLines);
    parseKeyPointsFromCSVString(keyPointLines, kps);

    PreparedTemplate Template{kps,descriptors,cv::Size(templateWidth,templateHeight)};

    auto future = std::async(stabilise_card, card, min_matches2, card_nfeatures2, Template.kp2, Template.des2, Template.template_shape);
    StabilisedCard stabilised_card = future.get();

    return stabilised_card.alignment_quality;
}