leap WebAudio API

_.md

[ Launch: leap Y input ] 4748207 by enjalot
[ Launch: leap Y input ] 4737320 by enjalot
[ Launch: leap HUD ] 4737158 by enjalot
[ Launch: leap setup ] 4736524 by enjalot

audio.js

if(!tributary.audio) { tributary.audio = {} };

try {
  tributary.audio.context = new webkitAudioContext();
}
catch(e) {
  console.error('Web Audio API is not supported in this browser');
}


tributary.audio.play = function(song, time) {
  if(!time) time = 0;
  console.log("time", time);
  song.source = tributary.audio.context.createBufferSource(); // creates a sound source
  song.source.buffer = song.buffer;
  song.source.connect(tributary.audio.context.destination);       // connect the source to the context's destination (the speakers)
  song.source.noteOn(time); // play the source now
}

tributary.audio.stop = function(song) {
  song.source.noteOff(0);
}


//load a song async;
tributary.loadSong = function(url, callback) {
  
  var request = new XMLHttpRequest();
  request.open("GET", url, true);
  request.responseType = "arraybuffer";
   
  // Our asynchronous callback
  request.onload = function() {
      tributary.audio.context.decodeAudioData(request.response, function(buffer) {
        var source = tributary.audio.context.createBufferSource(); // creates a sound source
        source.buffer = buffer;
        var song = {
          source: source,
          buffer: buffer
        }
        callback(null, song);
    }, callback);
      
  };
  request.send();
}



//CODE below this line could be pulled out and used in other files


//var song = "https://s3.amazonaws.com/tributary-sounds/12+No+Lie.mp3";
var song = "https://s3.amazonaws.com/tributary-sounds/11+Lil+Wayne+-+No+Lie+(Instrumental)+%5B2+Chainz+Ft.+Drake%5D.mp3";
//we check for existance of the song so we don't rerequest every keystroke
//delete tributary.song;
if(!tributary.song) {
  tributary.loadSong(song, function(error, song) {
    tributary.song = song;
  })
}

tributary.audio.filtered = function(song) {
  song.source = tributary.audio.context.createBufferSource(); // creates a sound source
  song.source.buffer = song.buffer;

  // Create a volume (gain) node
  volumeNode = tributary.audio.context.createGainNode();
  //Set the volume
  volumeNode.gain.value = 1.0;

  


  

  filterNode = tributary.audio.context.createBiquadFilter();
  // Specify this is a lowpass filter
  filterNode.type = 0;
  // Quieten sounds over 220Hz
  filterNode.frequency.value = 220;
  
  var osc = tributary.audio.context.createOscillator();
  osc.frequency.value=600;
  osc.type=1;
  //song.source.connect(osc)
  //osc.connect(tributary.audio.context.destination);
  //osc.connect(filterNode);

  
  song.source.connect(volumeNode)

  volumeNode.connect(filterNode)
  filterNode.connect(tributary.audio.context.destination);
  //volumeNode.connect(osc);
  //osc.connect(tributary.audio.context.destination);
  
  song.volumeNode = volumeNode;
  song.osc = osc;
  song.filterNode = filterNode;
  
  
  //song.source.connect(tributary.audio.context.destination);
  song.source.noteOn(0);
  
}

buttons.svg

config.json

{"description":"leap WebAudio API","endpoint":"","display":"svg","public":true,"require":[],"fileconfigs":{"inlet.js":{"default":true,"vim":false,"emacs":false,"fontSize":12},"leap.js":{"default":true,"vim":false,"emacs":false,"fontSize":12},"hud.js":{"default":true,"vim":false,"emacs":false,"fontSize":12},"_.md":{"default":true,"vim":false,"emacs":false,"fontSize":12},"config.json":{"default":true,"vim":false,"emacs":false,"fontSize":12},"style.css":{"default":true,"vim":false,"emacs":false,"fontSize":12},"audio.js":{"default":true,"vim":false,"emacs":false,"fontSize":12},"buttons.svg":{"default":true,"vim":false,"emacs":false,"fontSize":12}},"fullscreen":false,"play":false,"loop":false,"restart":false,"autoinit":true,"pause":true,"loop_type":"period","bv":false,"nclones":15,"clone_opacity":0.4,"duration":3000,"ease":"linear","dt":0.01,"thumbnail":"http://i.imgur.com/4wwSIOp.png"}

inlet.js

//icons from the noun project
//<a href="http://thenounproject.com/noun/play/#icon-No2873" target="_blank">Play</a> designed by <a href="http://thenounproject.com/somerandomdude" target="_blank">P.J. Onori</a> from The Noun Project

var onoffColors = ["#ff0000", "#00ff00"];

var svg = d3.select("svg");

var hub = svg.append("g")
  .classed("hub", true);

hub.append("rect")
  .classed("indicator", true)
  .attr({
    width: 58,
    height: 60,
    x: 21,
    y: 19
  })
  .on("click", tributary.leap.toggle);


var playbutton = d3.select("#play")
.attr({
  transform: "translate(" + [24, 100] + ")scale(" + 0.5 + ")"
});
var pausebutton = d3.select("#pause")
.attr({
  transform: "translate(" + [29, 100] + ")scale(" + 0.5 + ")"
}).style("display", "none")


//console.log(tributary.source);
var playing = false;
hub.append("rect")
  .classed("play", true)
  .attr({
    width: 58,
    height: 60,
    x: 21,
    y: 95,
    "fill-opacity": 0.001,
    stroke: "#000"
    
  })
.on("click", function() {
  if(!playing) {
    //tributary.audio.play(tributary.song, 0)
    tributary.audio.filtered(tributary.song);
    playbutton.style("display", "none")
    pausebutton.style("display", "")
  } else {
    tributary.audio.stop(tributary.song)
    playbutton.style("display", "")
    pausebutton.style("display", "none")
  }
  playing = !playing;
  
});


var format = d3.format(".3g");

var hand1 = makeHand("one")
.attr("transform", "translate(" + [100, 20] + ")");
var hand2 = makeHand("two")
.attr("transform", "translate(" + [311, 20] + ")");

var stats = hub.append("g").classed("stats", true)
.attr("transform", "translate(" + [500, 20] +")");
var maxg = stats.append("text").classed("max", true)
.attr({x: 14, y: 17 }).text("max");
var ming = stats.append("text").classed("min", true)
.attr({x: 15, y: 37}).text("min");


function makeHand(className) {
  var hand = hub.append("g")
   .classed("hand", true)
   .classed(className, true);
  hand.append("rect")
    .classed("bg", true)
  .attr({
    width: 200,
    height: 200
  })
  
  hand.append("text").classed("handid", true)
  .attr({x: 5, y: 12 }).text("hand");
 
  hand.append("text").classed("position", true)
    .attr({x: 5, y: 24 }).text("position");
  hand.append("text").classed("velocity", true)
    .attr({x: 5, y: 36 }).text("velocity");
  
  hand.append("text").classed("normal", true)
    .attr({x: 5, y: 48 }).text("normal");
  
  hand.append("g").classed("pointables", true)
  .attr("transform", "translate(" + [5, 67] + ")");
  
  return hand;
}

function updateHand(hand, pointables) {
  var handId = hand.datum().id;
  hand.select("text.handid").text(function(d) { return "hand: " + d.id });
  
  
  hand.select("text.position").text(function(d) {
    var n = d.palmPosition;
    return "position - " + arr2str(n);
  })
  hand.select("text.velocity").text(function(d) {
    var n = d.palmVelocity;
    return "velocity - " + arr2str(n);
  })
  hand.select("text.normal").text(function(d) {
    var n = d.palmNormal;
    return "normal - " + arr2str(n);
  })
  
  if(!pointables) pointables = [];
  var filtered = pointables.filter(function(d) { return d.handId === handId })
  var pSel = hand.select("g.pointables")
    .selectAll("text.pointable")
    .data(filtered)
    
  
  pSel.enter().append("text")
    .classed("pointable", true);
  
  pSel.exit().remove();
  
  pSel.text(function(d) {
    return d.id + ": " + arr2str(d.tipPosition)
  }).attr({
    dy: function(d,i) { return 14 * i }
  })
  
  
}

var centerY = 293;
var offsetY = 0;

var nbars = 50;
var barh = 395;
var spacing = 2;
var barhh = (barh - spacing * nbars) / nbars;

//scale for getting continuous leap-like values (between 0 and 800)
//from the discrete bars (0 to nbars)
var yscale = d3.scale.linear()
.domain([0, nbars])
.range([526, 65])

var xscale = function(y) {
  var cy = y - centerY + offsetY;
  return cy*cy*cy * 0.000005 + 1
}

var barg = svg.append("g")
.classed("bars", true)
.attr("transform", "translate(" + [312, 256] + ")")


var barSel = barg.selectAll("rect.bar")
.data(d3.range(nbars))

function getWidth(d) {
  var y = yscale(d)
  var w = xscale(y)
  return w;
}

barSel.enter().append("rect").classed("bar", true);
barSel.attr({
  x: function(d,i ){
    var w = getWidth(d);
    if(w < 0) {
      return w;
     } else {
       return 0;
     }
  },
  y: function(d,i) { 
    return i * (barhh + spacing) 
  },
  width: function(d, i) { 
    var w = getWidth(d);
    return Math.abs(w) 
  },
  height: barhh
})

var ginout = svg.append("g");
ginout.append("text").classed("in",true)
.attr("transform", "translate(" +  [400, 400] + ")")
.text("in")
ginout.append("text").classed("out",true)
.attr("transform", "translate(" +  [400, 435] + ")")
.text("out")
ginout.append("text").classed("ind",true)
.attr("transform", "translate(" +  [400, 467] + ")")
.text("index")


update();
function update(d) {
  //show if leap is connected or not
  hub.select("rect.indicator")
  .style("fill", tributary.leap.connected ? onoffColors[1] : onoffColors[0]);
  
  if(!d) return;
  
  //TODO: reuseable chart pattern
  //show numbers for each hand
  if(d.hands && d.hands[0]) {
  hand1.datum(d.hands[0]);
  updateHand(hand1, d.pointables);
  }
   
  if(d.hands && d.hands[1]) {
  hand2.datum(d.hands[1]);
  updateHand(hand2, d.pointables);
  }
  
  var maxy = d3.max(d.hands, function(c) { return c.palmPosition[1] });
  maxy = d3.max([maxy, d3.max(d.pointables, function (c) { return c.tipPosition[1] })]); 
  var miny = d3.min(d.hands, function(d) { return d.palmPosition[1] });
  miny = d3.min([miny, d3.min(d.pointables, function (c) { return c.tipPosition[1] })]); 
  if(maxy)
    maxg.text("max y: " + maxy);
  if(miny)
    ming.text("min y: " + miny);
  
  
  if(d.hands && d.hands[0]) {
    //map the y value of the hand position to the bar graph;
    var y = d.hands[0].palmPosition[1];
    var out = xscale(y);
    var ind = parseInt(yscale.invert(y))
    ginout.select("text.in")
    .text("in: " + format(y))
    ginout.select("text.ind")
    .text("index: " + ind)
    ginout.select("text.out")
    .text("out: " + format(out))
    
    barSel.style({
      "fill": "",
      "stroke": ""
    });
    barSel.filter(function(d) {
      return d === ind;
    }).style({
      fill: "#FF0000",
      stroke: "#ff0000"
    })
    
    if(tributary.song && tributary.song.filterNode) {
      tributary.song.filterNode.frequency.value = y + 300;// + 4000;
      //tributary.song.osc.frequency.value = y + 0;// + 4000;
    }
    
  }
}
              
tributary.leap.events.off("frame");
tributary.leap.events.on("frame", function(d) {
  //console.log(d);
  update(d);
})

tributary.leap.events.off("open");
tributary.leap.events.on("open", function(d) {
  //console.log("open");
  update();
})

tributary.leap.events.off("close");
tributary.leap.events.on("close", function(d) {
  console.log("close");
  update();
})


function arr2str(array3) {
  return "x: " + format(array3[0])
      + " y: " + format(array3[1])
      + " z: " + format(array3[2]);
}

leap.js

tributary.leap = {
  events: _.clone(Backbone.Events)
};

// Support both the WebSocket and MozWebSocket objects
if ((typeof(WebSocket) == 'undefined') &&
    (typeof(MozWebSocket) != 'undefined')) {
  WebSocket = MozWebSocket;
}

// Create the socket with event handlers
tributary.leap.init = function() {

  if(tributary.ws) {
    tributary.ws.close();
  }
  console.log("Starting");
  //Create and open the socket
  tributary.ws = new WebSocket("ws://localhost:6437/");
  var ws = tributary.ws;

  // On successful connection
  ws.onopen = function(event) {
    console.log("Open");
    tributary.leap.connected = true;
    tributary.leap.events.trigger("open");
  };

  // On message received
  ws.onmessage = function(event) {
    tributary.leap.events.trigger("frame", JSON.parse(event.data));
  };

  // On socket close
  ws.onclose = function(event) {
    ws = null;
    tributary.leap.connected = false;
    tributary.leap.events.trigger("close");
  }

  //On socket error
  ws.onerror = function(event) {
    console.error("Received error");
  };
}

var socketState;
if(!tributary.ws || tributary.ws.readyState != 1) {
  tributary.leap.connected = false;
} else {
  tributary.leap.connected = true;
}

tributary.leap.toggle = function() {
  if(tributary.ws.readyState === 1) {
    tributary.ws.close();
  } else {
    tributary.leap.init();
  }
}

tributary.leap.init();

tributary.events.on("restart", function() {
  tributary.leap.init();
})

style.css

.hand .bg {
  fill: #000000;
  fill-opacity: 0.1;
  stroke: #000000;
  stroke-opacity: 0.7
}

.hand text {
  font-size: 10px;
}

.bars {
  stroke: #01056F;
  stroke-width: 1;
  stroke-opacity: 0.5;
  fill: #A3E2F7;
  fill-opacity: 0.5;
}