InteractionLab_SingingBird_InteractiveGame

FinalProject.pde

//co-coding With GuangyuWu        
boolean serialMode = true;
boolean debugMode = false;
boolean testMode = false;
boolean xbeeOn = false;
boolean showData = false;
boolean gameEnd = false;
int timeFromEnd = 0;
//int _endFrameCount = 0;

//background music
import ddf.minim.*;
Minim minim;
AudioPlayer groove;


//music def.
Table table;
ArrayList<MusicNote> music = new ArrayList<MusicNote>();

int timing = 0;

int _j; // store header temporarily and give it to the specific cloud

import processing.sound.*;
SoundFile[] soundfiles = new SoundFile[10];
//..




// kinect def.
import org.openkinect.freenect.*;
import org.openkinect.freenect2.*;
import org.openkinect.processing.*;

Kinect2 kinect2;


PImage depthImg;
float depthMin = 500;
float depthMax = 1200;
float posX;
float posY;
//..


// Debouncing 
boolean stickState = false;
//..

// opening effect
Circles c;
ArrayList <PVector> circle;
ArrayList <PVector> b;
boolean startTrigger = false;
int birdJumpOut = 0;

//..

// serial initializing
import processing.serial.*;
String myString = null;
Serial myPort;
int[] sensorValues = new int[3]; // 3 is number of sensors
int nx, ny, nz; // new acceleration values
int px, py, pz; // previous ones
int cx, cy, cz, cSum;  // change of acceleration, namely the difference between new and previous
float accumulatedChange;
//..

//count the change of birdImg
PImage[] birds = new PImage[4]; // four stages of flying
boolean birdFlying = false;
int l = 0; 
//..

//initializing clouds
int numOfClouds = 10;
ArrayList<Cloud> clouds = new ArrayList<Cloud>();
PImage[] cloudImgs = new PImage[numOfClouds];

//..

//bird position
float birdX;
float birdY;
// mapped bird height to full height to compare the user control point and cloud point
float fullY;
//..

// game feature
int score = 0;
//..



void setup() {



  size(1000, 512);

  background(255);
  //bgm
  
  minim = new Minim(this);
  groove = minim.loadFile("Edelweiss.mp3", 1024);
  groove.loop();

  imageMode(CENTER);
  // opening effect settings
  c=new Circles();
  //..

  if (serialMode) {
    setupSerial();
  }



  //kinect settings
  kinect2 = new Kinect2(this);
  //kinect2.initVideo();
  kinect2.initDepth();
  //kinect2.initRegistered();
  // Start all data
  kinect2.initDevice();
  // create a blank image, creating a memeory space
  depthImg = createImage(512, 424, ARGB);
  //..


  // load bird imgs
  for (int k=0; k<birds.length; k++) {
    String fileName = "bird" + k+ ".png";
    birds[k]=loadImage(fileName);
  }
  //..


  // load music score informtion
  table = loadTable("music.csv", "header");
  println(table.getRowCount() + " total rows in table");
  println();
  for (TableRow row : table.rows()) {
    int numOfNotes = row.getColumnCount();
    int[] tNotes = new int[numOfNotes]; 
    for (int i=0; i<numOfNotes; i++) {
      tNotes[i] = row.getInt(i);
      print(tNotes[i]);
    }
    music.add( new MusicNote(tNotes) );
    println();
  }
  //..

  // load music soundFiles
  for (int i=0; i<soundfiles.length; i++) {
    String filename = i + ".mp3";
    soundfiles[i] = new SoundFile(this, filename);
  }

  //..
}


void draw() {







  // serial part
  int ax, ay, az;
  if (serialMode) {
    updateSerial();
    xbeeOn = true;
    //printArray(sensorValues);
    ax = sensorValues[0];// it seems this definition has to stay in this local scope, but why?
    ay = sensorValues[1];
    az = sensorValues[2];

    //_ax= ax; //debug tool

    //.. serial part ends
  } else {
    xbeeOn = false;
    ax = int(map(mouseY, 0, height, -10, 10));
    ay = int(map(mouseY, 0, height, -10, 10));
    az = int(map(mouseY, 0, height, -10, 10));
  }

  //background(0);

  if (mousePressed||birdJumpOut>200) {
    startTrigger= true;
  }


  //
  if (startTrigger== false) {
    groove.loop();
    translate(width/2, height/2);
    smooth();
    c.display();

    textSize(birdJumpOut*0.7);
    textAlign(CENTER, CENTER);
    text("WELCOME", width/4, height/4);

    birdJumpOut++;
    if (birdJumpOut> 0) {
      image(birds[0], 0, 0, 91*(birdJumpOut)/100, 135*(birdJumpOut)/100 );
      tint(255, 255, 255, constrain((255)*(birdJumpOut-50)/100, 0, 255));
    }
  } 
  // start

  else {
    background(255);
    groove.pause();
    fill(255);
    textAlign(CENTER, CENTER);
    textSize(20);
    fill(0, constrain(255-frameCount, 0, 255));
    text("move your body horizontally and flap your arms", width/2, 50);
    text("to control the bird", width/2, 80);
    text("try to hit the cloud when it turns to be gold", width/2, 110);

    pushStyle();
    colorMode(HSB);
    int index = 0;
    int gridSize = 8;
    for ( int h=0; h<=height; h+= gridSize) {
      fill((frameCount+index)%256, (frameCount*0.5)%40, 255-(frameCount)%255, 255);
      rect(0, h, width, gridSize);
      index++;
    }
    popStyle();

    //kinect part
    int[] rawDepth = kinect2.getRawDepth();

    int w = kinect2.getDepthImage().width;
    int h = kinect2.getDepthImage().height;

    depthImg.loadPixels();

    // reset after each big loop
    float avgX = 0;
    float avgY = 0;
    float sumX = 0; 
    float sumY = 0; 
    int count = 0; 

    for (int i = 0; i < rawDepth.length; i++) {
      int x = i%w;
      int y = i/w;
      int depth = rawDepth[i];
      if (depth != 0 && depth > depthMin && depth < depthMax) {
        // this is our sensing area!!
        sumX += x;
        sumY += y;
        count++;

        float r = map(depth, depthMin, depthMax, 255, 0);
        float b = map(depth, depthMin, depthMax, 0, 255);
        depthImg.pixels[i] = color (r, 0, b);
      } else {
        depthImg.pixels[i] = color (0, 0);
      }
    }
    // the i-loop above gets the sum, the we calculate the average
    depthImg.updatePixels();
    if (count!= 0) {
      avgX = sumX / count;
      avgY = sumY / count;
    }
    // lerp to smooth the values
    posX = lerp(posX, avgX, 0.1);
    posY = lerp(posY, avgY, 0.1);

    if (debugMode) {
      pushStyle();
      imageMode(CORNER);
      image(kinect2.getDepthImage(), 0, 0);
      image(depthImg, 0, 0);
      fill(255, 255, 0);
      ellipse(posX, posY, 10, 10);
      popStyle();
    }

    //// kinect ends


    // calculation part
    nx = ax; // ax is changing every loop
    cx = nx - px; // calculate the change, the actual acceleration
    px = nx; // after use, the new gravity value become the old one

    ny = ay; 
    cy = ny - py; 
    py = ny;

    nz = az; 
    cz = nz - pz; 
    pz = nz;

    // the sum of change of x,y,z/ absolute value
    cSum = abs(cx) + abs(cy) + abs(cz);
    accumulatedChange = accumulatedChange + cSum*0.5;
    //accumulatedChange
    accumulatedChange = constrain(accumulatedChange, -50, 110);

    //println(accumulatedChange);
    accumulatedChange -= 3;
    //accumulatedChange = constrain(accumulatedChange, 0, 120);

    float mappedY = map(accumulatedChange, 0, 100, height-100, 100);
    //fullY = map(accumulatedChange, 0, 100, height, 0);

    birdY = int(mappedY);
    birdY = mappedY;

    //birdY = 2*height/3;
    // println(birdY); // we have data, because birdY is defined in this scope
    // out of it, system assume it is default value zero



    //*******************************************************// 
    // actual drawing parts


    //bird part
    if (debugMode) {
      birdX = map(posX, 0, 512, 0, width);
    } else {
      birdX = map(posX, 0, 512, 0, width);
    }

    if (testMode) {
      birdX = mouseX;
    } 

    if (accumulatedChange > 20) {
      int r = (l/3)%3;
      image(birds[r+1], birdX, birdY, 140, 91);
      birdFlying = true;
      l++;
    } else {
      image(birds[0], birdX, birdY, 60, 91);
      birdFlying = false;
    }



    //cloud part

    // generate cloud
    if (frameCount % 50 == 0) {
      if (gameEnd) {
      } else {
        for (int j=0; j<music.get(timing).notes.length; j++) { 

          if (music.get(timing).notes[j] == 1) {
            //soundfiles[j].play();
            clouds.add(new Cloud(cloudImgs[j], j*100-50, 0, 40, 2, j));// last two parameters: size and speed
          }
        }

        timing++; //next row, next beat

        if (timing > music.size()-1) {
          gameEnd = true; // restart the song
        }
      }
    }


    for (int i=clouds.size()-1; i>=0; i--) {
      Cloud c =clouds.get(i);
      c.checkCloseness();
      c.cloudFall();
      c.display();

      if (c.y>height) {
        clouds.remove(i);
      }
    }

    /*
    for (int z=1; z<clouds.size(); z++) {
     stroke(255, 0, 0);
     strokeWeight(2);
     line(z*width/clouds.size(), 0, z*width/clouds.size(), height);
     }
     
     */
    line(0, 2*height/3-height/6, width, 2*height/3-height/6);
    line(0, 2*height/3, width, 2*height/3);
    line(0, 2*height/3+height/6, width, 2*height/3+height/6);



    // other feedbacks for user
  }

  if (showData) {
    noStroke();
    //indication lights
    if (debugMode) {
      fill(0, 255, 0);
    } else {
      fill(255, 0, 0);
    }
    ellipse(width-50, 20, 10, 10);

    if (testMode) {
      fill(0, 255, 0);
    } else {
      fill(255, 0, 0);
    }
    ellipse(width-50, 40, 10, 10);

    if (stickState) {
      fill(0, 255, 0);
    } else {
      fill(255, 0, 0);
    }
    ellipse(width-50, 60, 10, 10);

    if (xbeeOn) {
      fill(0, 255, 0);
    } else {
      fill(255, 0, 0);
    }
    ellipse(width-50, 80, 10, 10);

    fill (255);

    textSize(20);
    //text("score: " + score, width-120, 120);

    //text("_ax: " + _ax, width-120, 160);
    //text("_j: " + _j, width-10, 120);
    //text("y: " + y, width-120, 150);

    text("frameRate: " +frameRate, 100, 10);
    text("timing" + timing, 100, 40);
    text("clouds.size" + clouds.size(), 100, 80);
    text("accumulCh" + accumulatedChange, 100, 120);

    // debug tool: see where are bird center and cloud center
    // bird center
    fill(0, 0, 255);
    ellipse(birdX, birdY, 10, 10);
    // cloud center
    for (int i=0; i<clouds.size(); i++) {
      fill(0, 255, 0);
      ellipse(clouds.get(i).x, clouds.get(i).y, 10, 10);
    }
  }


  if (gameEnd) {
    fill(#0A70F2);
    textSize(50);
    textAlign(CENTER, CENTER);
    text("TOTAL SCORE " + score, width/2, height/2-50);
    comment();
    timeFromEnd ++;
    if (timeFromEnd>300) {
      reset();
    }
  } else {
    fill(255);
    textSize(20);
    text("score: " + score, width-120, 120);
  }
} // end of void draw()



void keyPressed() {

  //background(0);
  if (key == 'd') {
    debugMode = !debugMode;
  } else if (key == 't') {
    testMode = !testMode;
  } else if (key == ' ') {
    showData = !showData;
  }
}


// serial part accessory
void setupSerial() {
  // serial settings
  printArray(Serial.list());
  myPort = new Serial(this, Serial.list()[ 3 ], 9600);
  // check the list of the ports,
  // find the port "/dev/cu.usbmodem----" or "/dev/tty.usbmodem----" 
  // and replace PORT_INDEX above with the index of the port
  myPort.clear();
  // Throw out the first reading,
  // in case we started reading in the middle of a string from the sender.
  myString = myPort.readStringUntil( 10 );  // 10 = '\n'  Linefeed in ASCII
  myString = null;
  //..
}

void updateSerial() {
  while (myPort.available() > 0) {

    myString = myPort.readStringUntil( 10 ); // 10 = '\n'  Linefeed in ASCII
    if (myString != null) {
      String[] serialInArray = split(trim(myString), ",");
      if (serialInArray.length == 3) {
        for (int i=0; i<serialInArray.length; i++) {
          sensorValues[i] = int(serialInArray[i]);
        }
      }
    }
  }
}

void comment() {
  fill(#0A70F2);
  textSize(50);
  textAlign(CENTER, CENTER);
  if (score< music.size()/3) { 
    text ("NICE TRY!!", width/2, height/2);
    text ("TRY AGAIN:D", width/2, height/2+50);
  } else if (score< 2*music.size()/3 && music.size()/3<score) { 
    text ("GOOD GAME!!", width/2, height/2);
    text ("TRY AGAIN :D", width/2, height/2+50);
  } else if (score> 2*music.size()/3 ) { 
    text ("CONGRATS!YOU ARE THE BEST!", width/2, height/2);
    text ("TRY AGAIN :D", width/2, height/2+50);
  }
}

void reset() {
  background(255);
  gameEnd = false;
  startTrigger = false;
  birdJumpOut = 0;
  timeFromEnd = 0;
  timing = 0;
  score=0;
}



class Circles {

  Circles() {

    circle = new ArrayList <PVector>();
    b = new ArrayList <PVector>();
    for (int i=0; i<150; i++) {
      circle.add(new PVector(random(-20, 30), random(-20, 30)));
      b.add(new PVector(random(10, 20), 10));
    }
  }


  void display() {

    for (int i=1; i<50; i++) {

      PVector a= circle.get(i-1);
      PVector c= b.get(i);

      //fill(7, 0, 153);
      stroke(#0A70F2);//ocean theme

      //fill(255);
      //stroke(0); //illusion theme
      noFill();
      strokeWeight(i/2);
      rotate(birdJumpOut /(10*i*i*1.0)*(birdJumpOut)/100);
      ellipse(0, 0, 7+birdJumpOut *8/i, 7+birdJumpOut *9/i);

      fill(#0A70F2);
      ellipse(0, 0, 5+birdJumpOut *20/49, 5+birdJumpOut *19/49); //uncomment if you want a black center
    }
  }
}


class Cloud {


  float x; // already defined as global variable, actually no need
  float y; // but y is used in the form of clouds.y, so it need to be defined here
  float size;
  float speed;
  int header;
  boolean scored = false;


  color c1;// color of cloud
  color c2 = color(255, 0, 0);
  PImage img;
  //int r1,g1,b1=255;
  //int r2,g2,b2;
  int rA = 255;// rgb with A: light golden, start to get golden
  int gA = 245;
  int bA = 121;
  int rB = 255;// rgb with B: golden, right time!!! 
  int gB = 235;
  int bB = 0;
  int rC = 70; // rgb with C: dark blue
  int gC = 66;
  int bC = 81;

  // debouncing: check bird come near cloud and leave it
  int prevState = 0;
  int newState = 0;


  // this is called "Constructor Function"
  // and this works like setup() in this class
  Cloud() {
    // this is a default constructor
  }
  Cloud(PImage _img, float _x, float _y, float _size, float _speed, int _header) {
    img = _img;
    x = _x;
    y = _y;
    size = _size;
    speed = _speed;
    header = _header;
  }

  // let's add functions at the end
  void display() {
    fill(255);
    //ellipse(x,y,size,size);

    pushMatrix();
    translate(x-50, y-50);

    // the following code creates gradient color effect using mapping
    if (y < height/3) {
      c1 = color (map(y, 0, height/3, 255, rA), map(y, 0, height/3, 255, gA), map(y, 0, height/3, 255, bA));
    } else if (y < 2*height/3) {
      c1 = color (map(y, height/3, 2*height/3, rA, rB), map(y, height/3, 2*height/3, gA, gB), map(y, height/3, 2*height/3, bA, bB));
    } else if (y < 7*height/8 ) {
      c1 = color (map(y, 2*height/3, 7*height/8, rB, rC), map(y, 2*height/3, 7*height/8, gB, gC), map(y, 2*height/3, 7*height/8, bB, bC));
    } else {
      c1 = color (map(y, 7*height/8, height, rC, 0), map(y, 7*height/8, height, gC, 0), map(y, 7*height/8, height, gC, 0), map(y, 7*height/8, height-40, 255, 0));
    }

    if (stickState) {
      //fill(c2);
      fill(c1);
    } else {
      fill(c1);
    }

    noStroke();

    scale(0.4);
    beginShape();
    vertex(50, 180);
    //bezierVertex(50, 150, 80, 120, 132, 150);
    //bezierVertex(150, 115, 210, 135, 200, 160);
    //bezierVertex(270, 175, 230, 235, 170, 220);
    //bezierVertex(170, 250, 80, 255, 70, 220);
    //bezierVertex(20, 240, 25, 170, 50, 180);
    bezierVertex(50*sin(frameCount*0.02)+50, 150, 80, 20*sin(frameCount*0.03)+ 120, 132, 150);
    bezierVertex(150, 115, 210, 50*sin(frameCount*0.003)+135, 20*sin(frameCount*0.02)+200, 160);
    bezierVertex(270+50*sin(frameCount*0.001), 175, 50*sin(frameCount*0.03)+230, 235, 170, 20*sin(frameCount*0.02)+ 220);
    bezierVertex(20*sin(frameCount*0.02)+170, 250, 20*sin(frameCount*0.01)+ 80, 255+50*sin(frameCount*0.02), 70, 220);
    bezierVertex(20, 20*sin(frameCount*0.03)+240, 25, 20*sin(frameCount*0.01)+170, 50, 180);
    endShape();
    popMatrix();
  }



  void cloudFall() {
    y = y + speed;
  }

  //### problem: if remove height check, it sometimes stops recieving data,namely stuck
  // with height check, the first if statement, it recieves no data (in the serial monitor in Processing)
  void checkCloseness() {

    //println(birdY); all zero , because birdY is defined in one of the scopes in main sketch
    // out of it, system assume it is default value zero
    // this is causing probelm in all previous versions


    // check if bird is close enough to the golden cloud, give score
    // println(y);

    if (y>(2*height/3 - height/4) && y<(2*height/3 + height/4)) {

      int area = 60;
      if ( dist(birdX, birdY, x, y-20)<area) {
        newState = 1;
      } else {
        newState = 0;
      }

      // to check the distance between the bird and this cloud
      //pushStyle();
      //stroke(255,0,0);
      //strokeWeight(2);
      //line(birdX, birdY, x, y);
      //ellipse(x, y-20, area,area);
      //popStyle();
      if (gameEnd) {
      } else {


        if (scored) {
        } else {
          if (newState-prevState == 1) {
            score++;
            scored = true;
            soundfiles[header].play();
            stickState = true;
          } else {
            stickState = false;
          }
          prevState = newState;
        }
      }
    }
  }
}


class MusicNote {
  int[] notes;

  MusicNote(int[] _notes) {
    notes = _notes;
  }
}