Skip to content

Instantly share code, notes, and snippets.

@elliotwoods
Created September 16, 2014 09:26
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save elliotwoods/bcc566e22b80d24ab9f7 to your computer and use it in GitHub Desktop.
Save elliotwoods/bcc566e22b80d24ab9f7 to your computer and use it in GitHub Desktop.
#include "ofApp.h"
//--------------------------------------------------------------
void ofApp::setup(){
glDepthFunc(GL_LEQUAL);
this->payload.init(1920, 1080);
for(int i=0; i<PROJECTOR_COUNT; i++) {
this->decoder[i].init(this->payload);
}
this->scene.init();
this->scene.setThreadSet(this->threads);
gui.init();
auto worldPanel = gui.addWorld("World");
worldPanel->onDrawWorld += [this](ofCamera&){
this->scene.draw(this->roomEnabled, this->meshEnabled);
this->debugNode.draw();
};
gui.add(fboProjectorView, "Projection Space lines");
this->debugCameraPanel = gui.add(this->debugCameraMedian, "Camera Median");
this->debugProjectorPanel = gui.add(this->debugProjectorMedian, "Projector Median");
// gui.add(scene.viewCamera, "Camera View");
// gui.add(scene.viewProjector0, "Projector0 View");
// gui.add(scene.viewProjector1, "Projector1 View");
// gui.add(scene.imageCamera0, "Camera0 Median");
// gui.add(scene.imageCamera1, "Camera1 Median");
auto & camera = worldPanel->getCamera();
camera.setPosition(2.0f, 2.0f, -1.0f);
camera.lookAt(ofVec3f(0,0,-6.0f));
camera.setFov(90.0f);
camera.setNearClip(0.01f);
camera.setFarClip(100.0f);
ofBackground(50);
ofSetVerticalSync(true);
this->roomEnabled = false;
this->meshEnabled = false;
}
//--------------------------------------------------------------
void ofApp::update(){
}
//--------------------------------------------------------------
void ofApp::draw(){
ofPushStyle();
ofNoFill();
ofPushView();
glMatrixMode(GL_PROJECTION);
glLoadIdentity();
glMatrixMode(GL_MODELVIEW);
ofViewport(this->debugCameraPanel->getBounds());
ofCircle(debugCameraCoord, 0.1f);
ofViewport(this->debugProjectorPanel->getBounds());
ofCircle(debugProjectorCoord, 0.1f);
ofPopView();
ofPopStyle();
}
//--------------------------------------------------------------
void ofApp::keyPressed(int key){
if (key == 'c') {
for(int i=0; i<PROJECTOR_COUNT; i++) {
scene.points[i].clear();
}
scene.camera.setView(ofMatrix4x4());
scene.camera.setProjection(ofMatrix4x4());
scene.camera.distortion.clear();
for(int i=0; i<PROJECTOR_COUNT; i++) {
scene.projector[i].setView(ofMatrix4x4());
scene.projector[i].setProjection(ofMatrix4x4());
}
threads.clear();
for(int i=0; i<PROJECTOR_COUNT; i++) {
this->decoder[i].reset();
}
}
if (key == 's') {
ofstream saveFilesDragged;
saveFilesDragged.open(ofToDataPath("filesDragged.txt").c_str(), ios::out | ios::trunc);
for(auto filename : filesDragged) {
saveFilesDragged << filename << endl;
}
saveFilesDragged.close();
}
if (key == 'l') {
ifstream loadFilesDragged;
loadFilesDragged.open(ofToDataPath("filesDragged.txt").c_str(), ios::in);
if (loadFilesDragged.is_open()) {
string filename;
for( std::string line; getline( loadFilesDragged, line ); ) {
ofDragInfo dragInfo;
dragInfo.files = vector<string>(1, line);
this->dragEvent(dragInfo);
}
loadFilesDragged.close();
}
triangulateLine3D();
threads.save();
}
if (key == 'o') {
threads.save();
}
if (key == 't') {
triangulateLine3D();
this->keyPressed('o');
}
if (key == 'p') {
//debug view
bool found = false;
int cameraIndex = 0;
int projectorIndex;
auto & decoder = this->decoder[0];
auto & projector = this->scene.projector[0];
auto & dataSet = decoder.getDataSet();
while(!found) {
cameraIndex = ofRandom(0, decoder.getWidth() * decoder.getHeight() - 1.0f);
if (dataSet.getActive().getPixels()[cameraIndex]) {
found = true;
projectorIndex = dataSet.getData()[cameraIndex];
}
}
this->debugCameraCoord = scene.camera.getCoordinateFromIndex(cameraIndex);
this->debugProjectorCoord = projector.getCoordinateFromIndex(projectorIndex);
this->debugNode.cameraRay = scene.camera.castCoordinate(this->debugCameraCoord);
this->debugNode.projectorRay = projector.castCoordinate(this->debugProjectorCoord);
this->debugNode.intersectRay = this->debugNode.cameraRay.intersect(this->debugNode.projectorRay);
this->debugNode.cameraRay.color = ofColor(255,0,0);
this->debugNode.projectorRay.color = ofColor(0,255,0);
this->debugNode.intersectRay.color = ofColor(0,0,255);
ofLogNotice() << "Debug viewing camera pixel:" << cameraIndex << ", projector pixel:" << projectorIndex << ", finds point:" << this->debugNode.intersectRay.getMidpoint();
ofLogNotice() << "Camera coordinate:" << this->debugCameraCoord;
ofLogNotice() << "Projector coordinate:" << this->debugProjectorCoord;
}
if (key == 'r') {
this->roomEnabled = ! this->roomEnabled;
}
if (key == 'm') {
this->meshEnabled = !this->meshEnabled;
}
}
//--------------------------------------------------------------
void ofApp::keyReleased(int key){
}
//--------------------------------------------------------------
void ofApp::mouseMoved(int x, int y ){
}
//--------------------------------------------------------------
void ofApp::mouseDragged(int x, int y, int button){
}
//--------------------------------------------------------------
void ofApp::mousePressed(int x, int y, int button){
}
//--------------------------------------------------------------
void ofApp::mouseReleased(int x, int y, int button){
}
//--------------------------------------------------------------
void ofApp::windowResized(int w, int h){
}
//--------------------------------------------------------------
void ofApp::gotMessage(ofMessage msg){
}
//--------------------------------------------------------------
vector<double> loadDoubles(string& filename) {
ifstream file;
file.open(ofToDataPath(filename).c_str(), ios::in | ios::binary);
vector<double> result;
if (file.is_open()) {
while(!file.eof()) {
double peek;
file.read((char*)&peek, sizeof(double));
result.push_back(peek);
}
result.pop_back();
} else {
ofLogError() << "Failed to load file " << filename;
}
return result;
}
ofMatrix4x4 loadMatrix(string& filename) {
auto buffer = loadDoubles(filename);
if (buffer.size() >= 16) {
ofMatrix4x4 m;
auto floatBuffer = m.getPtr();
for(int i=0; i<16; i++) {
floatBuffer[i] = buffer[i];
}
ofMatrix4x4 reverseZ;
reverseZ.scale(1.0f, 1.0f, -1.0f);
return reverseZ * m * reverseZ;
} else {
return ofMatrix4x4();
}
}
vector<float> loadDistortion(string& filename) {
auto doublesBuffer = loadDoubles(filename);
vector<float> floatBuffer(doublesBuffer.size());
for(int i=0; i<doublesBuffer.size(); i++) {
floatBuffer[i] = (float) doublesBuffer[i];
}
return floatBuffer;
}
int findNumber(string filename) {
filename = ofFilePath::getBaseName(filename);
for(int i=0; i<PROJECTOR_COUNT; i++) {
if (filename.find(ofToString(i)) != string::npos) {
return i;
}
}
return -1;
}
void ofApp::dragEvent(ofDragInfo dragInfo){
for(auto entry : dragInfo.files) {
entry = ofToLower(entry);
ofLogNotice() << "Adding file " << entry;
filesDragged.push_back(entry);
bool isLineSet = entry.find("lines") != string::npos;
bool isGraycode = ofToLower(ofFilePath::getFileExt(entry)) == "sl";
bool isProjector = !isGraycode && entry.find("projector") != string::npos;
bool isCamera = !isProjector;
bool isDistortion = entry.find("distortion") != string::npos;
bool isView = entry.find("view") != string::npos;
bool isProjection = !(isView || isDistortion);
int iProjector = findNumber(entry);
if (isGraycode) {
auto & decoder = this->decoder[iProjector];
decoder.loadDataSet(entry);
decoder.setThreshold(10);
auto & projector = scene.projector[iProjector];
auto & mesh = scene.points[iProjector];
ofLogNotice() << "Added dataset for projector " << iProjector;
auto & imageProjector = scene.imageProjector[iProjector];
imageProjector = decoder.getDataSet().getMedianInverse();
imageProjector.update();
auto & imageCamera = scene.imageCamera[iProjector];
imageCamera = decoder.getDataSet().getMedian();
imageCamera.update();
ofxTriangulate::Triangulate(decoder.getDataSet(), scene.camera, projector, mesh, TRIANGULATE_MAX_DISTANCE);
if (iProjector==0) {
this->debugCameraMedian = this->decoder[0].getDataSet().getMedian();
this->debugCameraMedian.update();
this->debugProjectorMedian = this->decoder[0].getDataSet().getMedianInverse();
this->debugProjectorMedian.update();
}
} else if (isDistortion) {
scene.camera.distortion = loadDistortion(entry);
} else if (isLineSet) {
threads.load(entry);
ofLogNotice() << "Drawing lines into projector space";
fboProjectorView.allocate(1920, 1080);
fboProjectorView.begin();
ofClear(0);
threads.drawProjectionSpace();
fboProjectorView.end();
} else {
auto matrix = loadMatrix(entry);
if (isView) {
if (isCamera) {
scene.camera.setView(matrix);
} else {
scene.projector[iProjector].setView(matrix);
}
} else {
ofMatrix4x4 reverseZ;
cout << matrix << endl;
if (isCamera) {
scene.camera.setProjection(matrix);
} else {
scene.projector[iProjector].setProjection(matrix);
}
}
}
}
}
//---------
void ofApp::triangulateLine3D() {
ofxPolyFitf fit;
fit.init(1, 1, 3, BASIS_SHAPE_TRIANGLE);
for(int iProjector=0; iProjector<PROJECTOR_COUNT; iProjector++) {
auto & mapping = scene.activePixelsMapProjectorToCamera[iProjector];
auto & camera = scene.camera;
auto & projector = scene.projector[iProjector];
mapping.clear();
auto & decoder = this->decoder[iProjector];
auto projectorInCamera = decoder.getDataSet().getData().getPixels();
auto active = decoder.getDataSet().getActive().getPixels();
for(uint32_t cameraPixel=0; cameraPixel<decoder.getWidth() * decoder.getHeight(); cameraPixel++) {
if (active[cameraPixel]) {
uint32_t projectorPixel = projectorInCamera[cameraPixel];
if (projectorPixel != 0) {
mapping.insert(std::pair<uint32_t, uint32_t>(cameraPixel, projectorPixel));
}
}
}
// for(auto map : mapping) {
// cout << map.first << "->" << map.second << endl;
// }
const int searchLocations = 30;
const int searchSize = 2; // kernel size in projector space
int currentIndex = 0;
for (auto & thread : threads) {
if (thread.projector != iProjector) {
currentIndex++;
continue;
}
cout << "Triangulating thread " << currentIndex++ << "/" << threads.size() << endl;
cout << "Projector : " << iProjector << "/" << PROJECTOR_COUNT << endl;
vector<Find> finds;
for(int iSearchLocation=0; iSearchLocation<searchLocations; iSearchLocation++) {
float x = (float) iSearchLocation / (float) (searchLocations - 1);
const ofVec2f searchLocation = x * (thread.projectorEnd - thread.projectorStart) + thread.projectorStart;
const ofVec2f searchPixelLocation = projector.getIndexFromCoordinate(searchLocation);
for(auto correspondence : mapping) {
const int projector_x = correspondence.second % this->payload.getWidth();
const int projector_y = correspondence.second / this->payload.getWidth();
if (projector_y < searchPixelLocation.y - searchSize) {
continue;
} else if (projector_y > searchPixelLocation.y + searchSize) {
continue; //this can be break if we're organising mapping by projector pixels
}
const ofVec2f projectorPixelLocation = ofVec2f(projector_x, projector_y);
ofVec3f worldXYZ;
if ((projectorPixelLocation - searchPixelLocation).lengthSquared() <= searchSize * searchSize) {
if (ofxTriangulate::Triangulate(correspondence.first, correspondence.second, camera, projector, worldXYZ, TRIANGULATE_MAX_DISTANCE)) {
Find find;
find.x = x;
find.projectorXY.x = projector_x;
find.projectorXY.y = projector_y;
find.worldXYZ = worldXYZ;
find.cameraIndex = correspondence.first;
find.projectorIndex = correspondence.second;
finds.push_back(find);
}
}
}
}
if (finds.size() == 0) {
continue;
}
// cout << "Finds for thread at " << thread.projectorStart << " = {" << endl;
// for(auto find : finds) {
// cout << "[" << find.x << "] : \t" << find.worldXYZ << " \t->\t" << find.projectorXY << "\t..." << find.cameraIndex << "<>" << find.projectorIndex << endl;
// }
// cout << "}" << endl;
pfitDataSetf fitDataSet;
fitDataSet.init(1, 3);
fitDataSet.resize(finds.size());
pfitDataPoint<float> point = fitDataSet.begin();
for(auto find : finds) {
*point.getInput() = find.x;
*(ofVec3f*) point.getOutput() = find.worldXYZ;
++point;
}
const float selectionProbability = 0.3f;
const int maxIterations = 200;
const float searchWidth = 0.3f;
const float inclusionThreshold = 0.5f;
fitDataSet.setActiveAll();
cout << fitDataSet.toString();
//--
//check we've got a range of points
//--
//
bool hasRange = false;
bool firstFind = true;
float seenX;
for(auto find : finds) {
if (firstFind) {
seenX = find.x;
firstFind = false;
} else {
if(find.x != seenX) {
hasRange = true;
break;
}
}
}
//
//--
if (hasRange) {
fit.correlate(fitDataSet);
fit.RANSAC(fitDataSet, maxIterations, selectionProbability, searchWidth, inclusionThreshold);
pfitDataPointf evaluatePoint(1, 3);
*evaluatePoint.getInput() = 0.0f;
fit.evaluate(evaluatePoint);
thread.s = * (const ofVec3f*) evaluatePoint.getOutput();
*evaluatePoint.getInput() = 1.0f;
fit.evaluate(evaluatePoint);
thread.t = * (const ofVec3f*) evaluatePoint.getOutput() - thread.s;
cout << "Thread set to: " << thread.s << "\t->\t" << (thread.s + thread.t) << endl << endl;
} else {
cout << "We don't have a range of points so can't triangulate thread" << endl;
}
}
}
}
#pragma once
#include "ofMain.h"
#include "ofxGraycode.h"
#include "ofxGrabCam.h"
#include "ofxCvGui.h"
#include "ofxTriangulate.h"
#include "ofxPolyFit.h"
#include "Scene.h"
#include "ThreadSet.h"
#define TRIANGULATE_MAX_DISTANCE 3.0f
struct Find {
float x;
ofVec2f projectorXY;
ofVec3f worldXYZ;
uint32_t cameraIndex;
uint32_t projectorIndex;
};
class DebugNode : public ofNode {
public:
void customDraw() {
this->cameraRay.draw();
this->projectorRay.draw();
this->intersectRay.draw();
}
ofxRay::Ray cameraRay;
ofxRay::Ray projectorRay;
ofxRay::Ray intersectRay;
};
class ofApp : public ofBaseApp{
public:
void setup();
void update();
void draw();
void keyPressed(int key);
void keyReleased(int key);
void mouseMoved(int x, int y );
void mouseDragged(int x, int y, int button);
void mousePressed(int x, int y, int button);
void mouseReleased(int x, int y, int button);
void windowResized(int w, int h);
void dragEvent(ofDragInfo dragInfo);
void gotMessage(ofMessage msg);
void triangulateLine3D();
ofxCvGui::Builder gui;
ofxGraycode::Decoder decoder[PROJECTOR_COUNT];
ofxGraycode::PayloadGraycode payload;
Scene scene;
ThreadSet threads;
ofFbo fboProjectorView;
vector<string> filesDragged;
ofVec2f debugProjectorCoord;
ofVec2f debugCameraCoord;
DebugNode debugNode;
ofxCvGui::PanelPtr debugCameraPanel;
ofxCvGui::PanelPtr debugProjectorPanel;
ofImage debugCameraMedian;
ofImage debugProjectorMedian;
bool roomEnabled;
bool meshEnabled;
};
#include "Scene.h"
const ofVec3f roomMin(0.0f, 0.0f, 0.0f);
const ofVec3f roomMax(11.0f, 5.0f, 7.5f);
const ofVec3f roomScale = roomMax - roomMin;
const ofVec3f roomCenter = (roomMin + roomMax) / 2.0f;
//----------
Scene::Scene() {
this->grid = false;
this->threadSet = 0;
}
//----------
void Scene::init() {
room.set(roomScale.x, roomScale.y, roomScale.z, 10, 10, 10);
room.setPosition(roomCenter);
light.setPosition(roomMax * 1.2f);
//spin sides
auto & roomMesh = this->room.getMesh();
for(auto & normal : roomMesh.getNormals()) {
normal *= -1.0f;
}
camera.setProjection(ofMatrix4x4());
for(int i=0; i<PROJECTOR_COUNT; i++) {
projector[i].setProjection(ofMatrix4x4());
}
for(int i=0; i<PROJECTOR_COUNT; i++) {
projector[i].setWidth(1920);
projector[i].setHeight(1080);
}
camera.setWidth(5184);
camera.setHeight(3456);
viewCamera.allocate(1024.0f * 1.5f, 1024.0f);
for(int i=0; i<PROJECTOR_COUNT; i++) {
viewProjector[i].allocate(1920, 1080);
}
}
//----------
void Scene::updateViews() {
drawSceneTo(camera, viewCamera);
for(int i=0; i<PROJECTOR_COUNT; i++) {
drawSceneTo(projector[i], viewProjector[i]);
}
}
//----------
void Scene::setThreadSet(ThreadSet & threadSet) {
this->threadSet = &threadSet;
}
//----------
void Scene::draw(bool roomEnabled, bool pointsEnabled) {
if (roomEnabled) {
//--
//draw room
//
ofPushStyle();
ofFill();
glEnable(GL_CULL_FACE);
glCullFace(GL_FRONT);
this->light.enable();
ofSetColor(100);
room.draw();
this->light.disable();
glCullFace(GL_BACK);
glDisable(GL_CULL_FACE);
ofDisableLighting();
ofPopStyle();
//
//--
}
this->customDraw();
}
//----------
void Scene::drawSceneTo(ofProjector& device, ofFbo& fbo) {
fbo.begin();
ofSetColor(0);
device.beginAsCamera();
ofClear(0);
this->customDraw();
device.endAsCamera();
fbo.end();
}
//----------
void Scene::drawPoints() {
for(int i=0; i<PROJECTOR_COUNT; i++) {
imageCamera[i].getTextureReference().bind();
points[i].drawVertices();
imageCamera[i].getTextureReference().unbind();
}
}
//----------
void drawDevice(ofxRay::Projector& device, string label) {
ofPushMatrix();
ofTranslate(device.getPosition());
ofDrawBitmapString(label, ofVec3f());
ofPopMatrix();
}
void Scene::customDraw() {
ofDrawGrid(1.0f, 1.0f, true);
//--
//draw grid
//
if (this->grid) {
ofPushMatrix();
ofRotate(90, 0.0f, 0.0f, 1.0f);
ofDrawGridPlane(7.0f, 7);
ofPopMatrix();
ofPushStyle();
ofSetColor(0);
ofNoFill();
for(float x = roomMin.x; x<= roomMax.x; x++) {
ofPushMatrix();
ofTranslate(x, 0, 0);
ofRotate(-90, 0, 1.0f, 0);
ofRect(roomMin.z, roomMin.y, roomMax.z - roomMin.z, roomMax.y - roomMin.y);
ofPopMatrix();
}
for(float z = roomMin.z; z<= roomMax.z; z++) {
ofPushMatrix();
ofTranslate(0, 0, z);
ofRect(roomMin.x, roomMin.y, roomMax.x - roomMin.x, roomMax.y - roomMin.y);
ofPopMatrix();
}
for(float y = roomMin.y; y<= roomMax.y; y++) {
ofPushMatrix();
ofTranslate(0, y, 0);
ofRotate(90, 1.0f, 0, 0);
ofRect(roomMin.x, roomMin.z, roomMax.x - roomMin.x, roomMax.z - roomMin.z);
ofPopMatrix();
}
ofPopStyle();
}
//--
this->camera.draw();
for(int i=0; i<PROJECTOR_COUNT; i++) {
this->projector[i].draw();
}
ofPushStyle();
ofSetDrawBitmapMode(OF_BITMAPMODE_MODEL_BILLBOARD);
drawDevice(this->camera, "camera");
for(int i=0; i<PROJECTOR_COUNT; i++) {
drawDevice(this->projector[i], "projector" + ofToString(i));
}
ofPopStyle();
this->drawPoints();
for(int i=0; i<PROJECTOR_COUNT; i++) {
if (imageCamera[i].isAllocated()) {
camera.drawOnNearPlane(imageCamera[i], true);
break;
}
}
for(int i=0; i<PROJECTOR_COUNT; i++) {
projector[i].drawOnNearPlane(imageProjector[i], true);
}
if (threadSet != 0) {
threadSet->drawWorldSpace();
}
}
#pragma once
#include "ofMain.h"
#include "ofxRay.h"
#include "ThreadSet.h"
#define PROJECTOR_COUNT 4
class Scene {
public:
public:
Scene();
void init();
void updateViews();
ofxRay::Camera camera;
ofxRay::Projector projector[PROJECTOR_COUNT];
ofVboMesh points[PROJECTOR_COUNT];
ofFbo viewCamera;
ofFbo viewProjector[PROJECTOR_COUNT];
ofImage imageCamera[PROJECTOR_COUNT];
ofImage imageProjector[PROJECTOR_COUNT];
map<uint32_t, uint32_t> activePixelsMapProjectorToCamera[PROJECTOR_COUNT];
void setThreadSet(ThreadSet&);
void draw(bool roomEnabled, bool pointsEnabled);
protected:
void drawSceneTo(ofProjector&, ofFbo&);
void drawPoints();
void customDraw();
ofBoxPrimitive room;
ofLight light;
bool grid;
ThreadSet* threadSet;
};
#include "ThreadSet.h"
//---------
void ThreadSet::load(string filename) {
ofXml xml;
xml.load(filename);
if (xml.setTo("lines_combined")) {
if (xml.setTo("MyTable_0[0]")) {
do {
auto dataItems = xml.getAttributes();
Thread newThread;
ofVec2f & start = newThread.projectorStart;
ofVec2f & end = newThread.projectorEnd;
start.x = xml.getFloatValue("x1");
start.y = xml.getFloatValue("_x0020_y1");
end.x = xml.getFloatValue("_x0020_x2");
end.y = xml.getFloatValue("_x0020_y2");
newThread.projector = xml.getIntValue("_x0020_iProjector");
newThread.ID = xml.getIntValue("_x0020_ID");
this->push_back(newThread);
} while (xml.setToSibling());
}
xml.setToParent();
}
}
//---------
void ThreadSet::save(string filename) {
if (filename == "") {
auto result = ofSystemSaveDialog("lines_processed.xml", "Save xml of lines");
if (!result.bSuccess) {
ofLogError() << "Didn't select a filename";
return;
} else {
filename = result.filePath;
}
}
ofLogNotice() << "Saving all lines to " << filename;
ofXml xml;
xml.addChild("Lines");
xml.setTo("Lines");
for (auto thread : *this) {
ofXml innerXml = thread.getXml();
xml.addXml(innerXml);
}
xml.save(filename);
ofLogNotice() << "Saving complete";
}
//---------
void ThreadSet::drawProjectionSpace() {
ofPushView();
glMatrixMode(GL_PROJECTION);
glLoadIdentity();
glMatrixMode(GL_MODELVIEW);
glLoadIdentity();
for(auto thread : *this) {
thread.drawProjectionSpace();
}
ofPopView();
}
//----------
void ThreadSet::drawWorldSpace() {
for(auto thread : *this) {
thread.drawWorldSpace(3.0f);
}
}
#pragma once
#include "Thread.h"
class ThreadSet : public vector<Thread> {
public:
void load(string filename);
void save(string filename="");
void drawProjectionSpace();
void drawWorldSpace();
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment