OCR on browser screengrab in openframeworks, ofxTesseract + ofxScreebGrab

CvUtils.h

#pragma once

// small excerpt from forthcoming of-cv utility/addon

#include "ofConstants.h"
#include "ofImage.h"
#include "cv.h"

namespace cv {
    
    inline int getCvImageType(const ofImageType& ofType) {
        switch(ofType) {
            case OF_IMAGE_GRAYSCALE: return CV_8UC1;
            case OF_IMAGE_COLOR: return CV_8UC3;
            case OF_IMAGE_COLOR_ALPHA: return CV_8UC4;
            default: return CV_8UC1;
        }
    }
    
    inline Mat getMat(ofImage& img) {
        int cvImageType = getCvImageType(img.getPixelsRef().getImageType());
        return Mat(img.getHeight(), img.getWidth(), cvImageType, img.getPixels(), 0);
    }
    
    inline int forceOdd(int x) {
        return (x / 2) * 2 + 1;
    }
    
    inline void medianBlur(ofImage& img, int size) {
        size = forceOdd(size);
        Mat mat = getMat(img);
        medianBlur(mat, mat, size);
    }
    
};

ofApp.cpp

#include "ofApp.h"

void ofApp::setup(){
    ofSetWindowShape(1280, 960);
    ofBackground(255);

    grabber.setup(ofGetWidth(), ofGetHeight(), true);
    fbo.allocate(ofGetWidth(), ofGetHeight());

    processText.addListener(this,&ofApp::updateText);
    size.addListener(this, &ofApp::changeSize);
    
    gui.setup();
    gui.add(topLeft.set("topleft", ofVec2f(0, 0), ofVec2f(0, 0), ofVec2f(ofGetScreenWidth(), ofGetScreenHeight()) ));
    gui.add(size.set("size", ofVec2f(800, 600), ofVec2f(0, 0), ofVec2f(ofGetScreenWidth(), ofGetScreenHeight()) ));
    gui.add(fboDisplayWidth.set("display", 800, 200, 1280));
    gui.add(processText.setup("process"));

    tess.setup();
    tess.setWhitelist("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890:-=!@#$%^&*()_+[]\{}|;'`~:\",.<>?");
    tess.setAccuracy(ofxTesseract::ACCURATE);
    tess.setMode(ofxTesseract::AUTO);
}

void ofApp::changeSize(ofVec2f &p)
{
    grabber.setup((int)size->x, (int)size->y, true);
    fbo.allocate((int)size->x, (int)size->y);
}

void ofApp::updateText()
{
    fbo.readToPixels(pix);
    ocrResult = tess.findText(pix);
}

void ofApp::update(){
    grabber.grabScreen(topLeft->x, topLeft->y);
    
    fbo.begin();
    ofSetColor(255);
    ofRect(0, 0, fbo.getWidth(), fbo.getHeight());
    grabber.draw(0, 0, fbo.getWidth(), fbo.getHeight());
    fbo.end();
}

void ofApp::draw(){
    ofSetColor(255);
    fbo.draw(ofGetWidth() - fboDisplayWidth, 0, fboDisplayWidth, fbo.getHeight() * fboDisplayWidth / fbo.getWidth());
    gui.draw();
    
    ofPushMatrix();
    ofTranslate(4, 140);
    ofSetColor(0);
    vector<string> lines = ofSplitString(ocrResult, "\n");
    for(int i = 0; i < lines.size(); i++) {
        ofDrawBitmapString(lines[i], 10, 20 + i * 12);
    }
    ofPopMatrix();
}

ofApp.h

#pragma once

#include "ofMain.h"
#include "ofxGui.h"
#include "ofxTesseract.h"
#include "CvUtils.h"
#include "ofxScreenGrab.h"

using namespace cv;

class ofApp : public ofBaseApp{
    
public:
    void setup();
    void update();
    void draw();
    
    void updateText();
    void changeSize(ofVec2f &p);

    ofxTesseract tess;
    string ocrResult;
    
    ofxPanel gui;
    ofxButton processText;
    
    ofParameter<ofVec2f> topLeft;
    ofParameter<ofVec2f> size;
    ofParameter<int> fboDisplayWidth;
    
    ofxScreenGrab grabber;
    ofFbo fbo;
    ofPixels pix;
    
};