Rave Theremin

gistfile1.java

import gab.opencv.*;
import processing.video.*;
import org.opencv.core.Mat;
import processing.net.*;
Server myServer; 

Capture video;
OpenCV opencv;
Mat h;
PFont font;

void setup() {
  size(720, 480);
  video = new Capture(this, 720,480);
  opencv = new OpenCV(this, 720, 480);
//  opencv.startBackgroundSubtraction(30, 10, 0.5);
  
  video.start();
  font = createFont("Arial",32,true); // Arial, 16 point, anti-aliasing on
  myServer = new Server(this, 5204); 
};
int last_y=-1;

void draw() {
  opencv.loadImage(video);
  image(video, 0, 0);  
  opencv.useColor(HSB);
  h = opencv.getH();
  opencv.setGray(h);
  opencv.inRange(128,150);
 // opencv.updateBackground();
  opencv.erode();
  opencv.erode();
  opencv.erode();
  opencv.blur(10);
  opencv.dilate();
  
  image(opencv.getSnapshot(),10,10,360,240);

  noFill();
  stroke(255, 0, 0);
  strokeWeight(3);
  for (Contour contour : opencv.findContours()) {
    if(contour.area() > 1000) {
      java.awt.Rectangle r = contour.getBoundingBox();
      int y = (int) r.getY();
      if(last_y==-1) last_y=y;
      if (abs(y-last_y) < 100) {
        last_y=y;
      } else {
        y=last_y;
      }
      String s =  ""+ r.getX() +",  " + (480-y) + "\n";
      myServer.write(s);
      textFont(font, 32);
      fill(255); //background(0);
      text(s, 10, 45);
      contour.draw();
      
    }
  }
}

void captureEvent(Capture m) {
  m.read();
}

gistfile2.rb

# Welcome to Sonic Pi v2.4
require 'socket'

use_bpm 70

client = TCPSocket.new "localhost", 5204
x=400; y=0
in_thread do
  while l=client.gets
    x,y=l.split(",").map(&:to_i)
    puts [x,y]
  end
  client.close
end

define :drum_phrase do |vol_drum = 0.8|
  3.times do
    sample :bd_klub, amp: vol_drum
    sleep 0.5
    sample :drum_heavy_kick, amp: vol_drum
    sample :bd_klub, amp: vol_drum
    sleep 0.5
  end
  sample :bd_klub, amp: vol_drum
  sleep 0.5
  sample :drum_heavy_kick, amp: vol_drum
  sample :bd_klub, amp: vol_drum
  sleep 0.25
  sample :drum_heavy_kick, amp: vol_drum
  sleep 0.25
end

define :bassline_phrase do |myscale|
  with_fx :rlpf, cutoff: 60 do
    use_synth :dsaw
    2.times do    
      bl = play myscale[0], amp: 0.2, detune: 0.7, attack: 0.1, sustain: 2, release: 0.5
      sleep 2
    end
  end
end


define :melody_phrase do |vol=0.3|
  use_synth :saw
  with_fx :lpf do |f| 
    melody = shuffle(chord(:c3, :sus4)+chord(:c3, :major))+[47,0]
    puts [:count, melody.count]
    melody.each do |n|
      play n, amp: vol
      4.times do
        loud = [[0,  y/450.0].max, 1].min
        puts loud
        control f, pre_amp: loud
        wet = [[0, (60 + x/8)].max, 120].min
        puts wet
        control f, cutoff: wet
        sleep 0.125
      end
    end
  end 
end

define :melody2_phrase do |vol=0.6|
  with_fx :lpf do |f| 
    melody = shuffle(chord(:g3, :sus4)+chord(:g3, :major))+[47,0]
    puts [:count, melody.count]
    use_synth :mod_sine
    melody.each do |n|
      play n,  amp: vol
      4.times do
        loud = [[0,  y/450.0].max, 1].min
        puts loud
#        control f, pre_amp: loud
        wet = [[0, (60 + x/8)].max, 120].min
        puts wet
        control f, cutoff: wet
        sleep 0.125
      end
    end
  end 
end



define :bass_section do
  in_thread do 4.times do drum_phrase end end
# A  F#m  D  Bm
  bassline_phrase chord(:a3, :major, num_octaves: 2)
  bassline_phrase chord(note(:f2)+1, :minor, num_octaves: 2)
  bassline_phrase chord(:d2, :major, num_octaves: 2)
  bassline_phrase chord(:b3, :minor, num_octaves: 2)
end

define :buildup_section do |vol_drum=0.8|
  in_thread do 4.times do intro_section end end

  8.times do
    sample :bd_klub, amp: vol_drum
    sleep 0.5
    sample :bd_klub, amp: vol_drum
    sleep 0.5
  end
  16.times do 
    sample :drum_heavy_kick, amp: vol_drum
    sample :bd_klub, amp: vol_drum
    sleep 0.25
  end
  32.times do 
    sample :drum_heavy_kick, amp: vol_drum
    sample :bd_klub, amp: vol_drum
    sleep 0.125
  end
end

define :melody_section do
  in_thread do bass_section end
  4.times do 
    if x<100 
      melody2_phrase 
    else      
      melody_phrase 
    end
  end
end

define :intro_section do
  bassline_phrase [:c2] 
  bassline_phrase [:g2] 
  bassline_phrase [:f2]
  bassline_phrase [:a2]
end

intro_section
live_loop :pay do
  case when y<150
    intro_section
  when y<400 
    melody_section
  else
    buildup_section
    melody_section
  end
end

#melody_section

Before: https://docs.google.com/presentation/d/1vhBXajbb8aElakMhbJkcuwqSvVs4LM9gkioYj20KiEs/edit?usp=sharing

After: https://docs.google.com/presentation/d/1kLldGGutXe-pfNRZSvtdXVhnppSem8fZ-Orasc6xa6Q/edit?usp=sharing

This is the code that the "Rave Theremin" team wrote in the Simply Business 2014 Hackathon. The input half is written with Processing/OpenCV, the output is Sonic Pi, and the two are connected with a TCP socket. It tracks something purple, and then uses the x & y co-ordinates to make decisions on what to play and how to play it.

Sonic Pi is implemented as a Ruby DSL, so the code for reading from the socket is bog-standard Ruby standard library stuff. Note that I haven't seen it written anywhere that Sonic Pi is Ruby underneath, so it may or not be wise to depend on that continuing to work.

You need to import the "OpenCV For Processing" library after starting Processing

I'm not proud of the code quality, though I am at least tolerably pleased with how much we managed to get done in 2 days from a start position of approximately zero knowledge.