Run video and 3D data in parallel

run_video_3d.pde

/**
 *  fjenett - 2015
 */

import org.piecemaker2.api.*;
import org.piecemaker2.models.*;

import java.util.Properties;
import java.util.Date;

import processing.video.*;

/* INSERT YOUR API KEY HERE */

String apiKey = "";

/* INSERT GROUP ID HERE */

int groupId = 140;

String apiHost = "http://piecemaker2-api-public.herokuapp.com";

PieceMakerApi api;
org.piecemaker2.models.Event videoEvent, dataEvent;
float eventTimeOffset, eventTimeOffsetScore;
ArrayList<HashMap<String, ArrayList<Float>>> travelPath = new ArrayList<HashMap<String, ArrayList<Float>>>();
ArrayList<HashMap<String, ArrayList<Float>>> score = new ArrayList<HashMap<String, ArrayList<Float>>>();
Movie video;

//float[][] trail = new float[20][2];

boolean loaded = false, loadedScore = false;
float minX=10000000, maxX=-10000000, minY=10000000, maxY=-10000000, minZ=10000000, maxZ=-10000000;

void setup ()
{
  size( 1760, 720 );

  // initialize the client with host and API-Key
  api = new PieceMakerApi( this, apiHost, apiKey );

  // trigger loading of group
  api.listEventsOfType( groupId, "video", api.createCallback( "groupLoaded" ) );

  fill( 255 );
  textSize( 24 );
  textAlign( CENTER );
}

void draw ()
{
  if ( loaded && loadedScore ) 
  {
    background( 255 );

    image( video, 0, 0, 1280, 720 );
    fill( 255 );
    noStroke();
    rect( 1280-240, 0, 720, 720 );

    float fps = 30.0 / 2;
    int frameNum = int( (video.time()-eventTimeOffset) * fps );
    if ( frameNum >= 0 && frameNum < travelPath.size() )
    {
      for (String key : travelPath.get(frameNum).keySet()) {
        ArrayList<Float> joint = travelPath.get(frameNum).get(key);
        float x = map(joint.get(0), min(minX, minY), max(maxX, maxY), 0, 680), 
          y = map(joint.get(1), min(minX, minY), max(maxX, maxY), 0, 680);

        fill( 0 );
        noStroke();
        ellipse( 1280 - 240 + x, y, 5, 5 );
      }
    }

    fps = 30.0 / 2;
    frameNum = int( (video.time()-eventTimeOffsetScore) * fps );
    if ( frameNum >= 0 && frameNum < score.size() )
    {
      float r = score.get(frameNum).get("Rotation").get(0);
      float tx = score.get(frameNum).get("Translation").get(0);
      float ty = score.get(frameNum).get("Translation").get(1);
      float sx = score.get(frameNum).get("Scaling").get(0);
      float sy = score.get(frameNum).get("Scaling").get(1);
      float tri = score.get(frameNum).get("Triangle").get(0);
      pushMatrix();

      translate(1280 - 240 + 360, 360);
      scale(360 / 16, 360 / 16);
      translate(tx, ty);
      rotate(r);

      stroke(0);
      strokeWeight(0.25);
      drawRect(sx, sy, tri, POINTS);

      noFill();
      strokeWeight(0.1);
      drawRect(sx, sy, tri, QUADS);
      fill(255);

      popMatrix();
    }
  } else {
    background( #994433 );
  }
}

void drawRect(float x, float y, float tri, int mode) {
  beginShape(mode);
  vertex(map(tri, 0, 1, x, 0), map(tri, 0, 1, y, 0));
  vertex(-x, y);
  vertex(-x, -y);
  vertex(x, -y);
  endShape(CLOSE);
}

// group loaded callback
void groupLoaded ( org.piecemaker2.models.Event[] videos )
{
  for ( org.piecemaker2.models.Event v : videos )
  {
    if ( v.fields.get("title").equals("170829 Pathfinder 003") )
    {
      videoEvent = v;
      api.listEventsForTimespan( groupId, 
        v.utc_timestamp, 
        new Date( v.utc_timestamp.getTime() + (long)(v.duration * 1000) ), 
        api.INTERSECTING, 
        api.createCallback("eventsLoaded") );
      break;
    }
  }
}

// context events loaded callback
void eventsLoaded ( org.piecemaker2.models.Event[] events ) 
{
  for ( org.piecemaker2.models.Event e : events ) 
  {
    if ( e.type.equals("scene") )
    {
      dataEvent = e;
      println( "ID: " + e.id + " / Title: " + e.fields.get("title") );

      String host = (String)e.fields.get("pma-server"), 
        uuid = (String)e.fields.get("pma-channel-uuid");

      if (e.fields.get("title").equals("Kinect Tracking"))
        loadChannelAndStreams( host, uuid );
      else if (e.fields.get("title").equals("Pathfinder Score"))
        loadChannelAndStreamsScore( host, uuid );

      //break;
    }
  }
}

// loading the actual data from Piecemeta
void loadChannelAndStreams ( String host, String channelUUID )
{
  JSONObject channel = loadJSONObject( "http://"+host+"/channels/"+channelUUID+".json" );
  JSONArray streams = loadJSONArray( "http://"+host+"/channels/"+channelUUID+"/streams.json" );

  println( "channel \"" + channel.getString("title") + "\" / streams: " + streams.size() );

  for ( int i = 0, k = streams.size(); i < k; i++ )
  {
    JSONObject streamInfo = streams.getJSONObject(i);
    String streamUUID = streamInfo.getString( "uuid" );
    JSONObject stream = loadJSONObject( "http://"+host+"/streams/"+streamUUID+".json" + "?skip=2" );

    String group = stream.getString("group");
    //println(group.equals("SpineMid"));
    //if(group.equals("SpineMid") == false) continue;

    String title = stream.getString("title");
    int dimension = -1;
    if (title.equals("X")) dimension = 0;
    else if (title.equals("Y")) dimension = 1;
    else if (title.equals("Z")) dimension = 2;
    else continue;
    JSONArray frames = stream.getJSONArray("frames");
    println( "stream \"" + group + "/" + title + "\" / frames: " + frames.size() );

    while (travelPath.size() < frames.size()) {
      HashMap<String, ArrayList<Float>> hm = new HashMap<String, ArrayList<Float>>();
      travelPath.add(hm);
    }

    for ( int ii = 0, kk = frames.size(); ii < kk; ii++ )
    {
      if (travelPath.get(ii).get(group) == null) {
        ArrayList<Float> p = new ArrayList<Float>();
        p.add(-10000.0);
        p.add(-10000.0);
        p.add(-10000.0);
        travelPath.get(ii).put(group, p);
      }
      if (frames.isNull(ii) == false) {
        float v = frames.getFloat( ii );
        travelPath.get(ii).get(group).set(dimension, v);
        if (dimension == 0 && v < minX) minX = v;
        if (dimension == 0 && v > maxX) maxX = v;
        if (dimension == 1 && v < minY) minY = v;
        if (dimension == 1 && v > maxY) maxY = v;
        if (dimension == 2 && v < minZ) minZ = v;
        if (dimension == 2 && v > maxZ) maxZ = v;
      }
    }
  }
  video = new Movie( this, "170829 Pathfinder 003.mp4" );
  //video = new Movie( this, "170824 Pathfinder 001 Yi.mp4" );
  video.play();
  video.jump(50.0);

  eventTimeOffset = (dataEvent.utc_timestamp.getTime() - videoEvent.utc_timestamp.getTime()) / 1000.0;

  loaded = true;
}

void loadChannelAndStreamsScore ( String host, String channelUUID )
{
  JSONObject channel = loadJSONObject( "http://"+host+"/channels/"+channelUUID+".json" );
  JSONArray streams = loadJSONArray( "http://"+host+"/channels/"+channelUUID+"/streams.json" );

  println( "channel \"" + channel.getString("title") + "\" / streams: " + streams.size() );

  for ( int i = 0, k = streams.size(); i < k; i++ )
  {
    JSONObject streamInfo = streams.getJSONObject(i);
    String streamUUID = streamInfo.getString( "uuid" );
    JSONObject stream = loadJSONObject( "http://"+host+"/streams/"+streamUUID+".json" + "?skip=2" );

    String group = stream.getString("group");

    String title = stream.getString("title");
    int dimension = -1;
    if (title.equals("X")) dimension = 0;
    else if (title.equals("Y")) dimension = 1;
    else if (title.equals("Z")) dimension = 2;
    else continue;
    JSONArray frames = stream.getJSONArray("frames");
    println( "stream \"" + group + "/" + title + "\" / frames: " + frames.size() );

    while (score.size() < frames.size()) {
      HashMap<String, ArrayList<Float>> hm = new HashMap<String, ArrayList<Float>>();
      score.add(hm);
    }

    for ( int ii = 0, kk = frames.size(); ii < kk; ii++ )
    {
      if (score.get(ii).get(group) == null) {
        ArrayList<Float> p = new ArrayList<Float>();
        p.add(-10000.0);
        p.add(-10000.0);
        p.add(-10000.0);
        score.get(ii).put(group, p);
      }
      if (frames.isNull(ii) == false) {
        float v = frames.getFloat( ii );
        score.get(ii).get(group).set(dimension, v);
      }
    }
  }
  eventTimeOffsetScore = (dataEvent.utc_timestamp.getTime() - videoEvent.utc_timestamp.getTime()) / 1000.0;

  loadedScore = true;
}

void movieEvent ( Movie mov )
{
  mov.read();
}