Hand gestures trigger music using FoxDot music live coding framework

gestures2music.py

# Original gesture source code:  https://github.com/Sadaival/Hand-Gestures
# Original source code video demonstration: https://youtu.be/v-XcmsYlzjA

from FoxDot import *
import traceback
import cv2
import numpy as np
import math

cap = cv2.VideoCapture(0)


Scale.default = "minor"
Root.default.set("B")
Clock.bpm=80

def note2index(note):
    scale = list(Scale.default)
    return scale[note]
def play2notes(notas,sample="b",dur=1,ritmo=8,oct=0,player=p1):
    rates = []
    for nota in notas:
        nota = note2index(nota)
        rates.append(math.pow(2,(nota+12*oct)/12))
    player >> play(sample,dur=var(ritmo,dur), rate=var(rates,dur),amp=0.5,sample=2)


def intro():
    #d2.solo()
    m2 >> gong(var([p2.pitch],4), dur=1/4, pan=linvar([-1,1],4), amp=linvar([0,2],8)).sometimes('stutter', 4)
    d2 >> play('w' ,dur=PSum([1,6],4), rate=1/4)
    d3 >> play('K', dur=2, sample=3)

def softdrums(hpff=250):
    sust=0.02
    hh >> play('------(-=)--(-=)--(-=)---',dur=1/4,amp=[1,0.3],sus=sust,hpf=hpff,sample=3)
    bd >> play('x  x   xx (x )( x)   x',dur=1/4,amp=0.3,sus=sust,hpf=hpff)
    sn >> play('    o    o  o   ',dur=1/4,amp=0.15,sus=sust,hpf=hpff)

def melody():
    hpff=250
    #p2.reset() >> bell([10,9,7,4,3,2,0],dur=P[3,3,3,3,2,2,16]/4,scale=Scale.lydian,root=-3,oct=4,hpf=hpff)
    p1 >> piano(var([0,2],4), drive=0.3, dur=PSum(7,4),amp=0.7,pan=PWhite(-1,1)).every(4,'stutter',2,dur=1) + var([0,2],8)
    c1 >> glass([0],oct=4)

def heavydrums():
    d2.solo(0)
    play2notes([0],"Q",dur=[4],ritmo=[4],oct=-2,player=q1)

def update_values(values,new_elem):
    values.append(new_elem)
    return values[-10:]

values = ["NOTHING"]
while(1):
        
    try:  #an error comes if it does not find anything in window as it cannot find contour of max area
          #therefore this try error statement
          
        ret, frame = cap.read()
        frame=cv2.flip(frame,1)
        kernel = np.ones((3,3),np.uint8)
        
        #define region of interest
        roi=frame[100:300, 100:300]
        
        
        cv2.rectangle(frame,(100,100),(300,300),(0,255,0),0)    
        hsv = cv2.cvtColor(roi, cv2.COLOR_BGR2HSV)
        
        
    # define range of skin color in HSV
        lower_skin = np.array([0,20,70], dtype=np.uint8)
        upper_skin = np.array([20,255,255], dtype=np.uint8)
        
     #extract skin colur imagw  
        mask = cv2.inRange(hsv, lower_skin, upper_skin)
        
   
        
    #extrapolate the hand to fill dark spots within
        mask = cv2.dilate(mask,kernel,iterations = 4)
        
    #blur the image
        mask = cv2.GaussianBlur(mask,(5,5),100) 
     
    #find contours
        contours,hierarchy= cv2.findContours(mask,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
        #print(contours)
        #print(hierarchy)
   #find contour of max area(hand)
        cnt = max(contours, key = lambda x: cv2.contourArea(x))
        
    #approx the contour a little
        epsilon = 0.0005*cv2.arcLength(cnt,True)
        approx= cv2.approxPolyDP(cnt,epsilon,True)
       
        
    #make convex hull around hand
        hull = cv2.convexHull(cnt)
        
     #define area of hull and area of hand
        areahull = cv2.contourArea(hull)
        areacnt = cv2.contourArea(cnt)
      
    #find the percentage of area not covered by hand in convex hull
        arearatio=((areahull-areacnt)/areacnt)*100
    
     #find the defects in convex hull with respect to hand
        hull = cv2.convexHull(approx, returnPoints=False)
        defects = cv2.convexityDefects(approx, hull)
        
    # l = no. of defects
        l=0
        
    #code for finding no. of defects due to fingers
        for i in range(defects.shape[0]):
            s,e,f,d = defects[i,0]
            start = tuple(approx[s][0])
            end = tuple(approx[e][0])
            far = tuple(approx[f][0])
            pt= (100,180)
            
            
            # find length of all sides of triangle
            a = math.sqrt((end[0] - start[0])**2 + (end[1] - start[1])**2)
            b = math.sqrt((far[0] - start[0])**2 + (far[1] - start[1])**2)
            c = math.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
            s = (a+b+c)/2
            ar = math.sqrt(s*(s-a)*(s-b)*(s-c))
            
            #distance between point and convex hull
            d=(2*ar)/a
            
            # apply cosine rule here
            angle = math.acos((b**2 + c**2 - a**2)/(2*b*c)) * 57
            
        
            # ignore angles > 90 and ignore points very close to convex hull(they generally come due to noise)
            if angle <= 90 and d>30:
                l += 1
                cv2.circle(roi, far, 3, [255,0,0], -1)
            
            #draw lines around hand
            cv2.line(roi,start, end, [0,255,0], 2)
            
            
        l+=1
        
        #print corresponding gestures which are in their ranges
        font = cv2.FONT_HERSHEY_SIMPLEX
        if l==1:
            if areacnt<2000:
                cv2.putText(frame,'Put hand in the box',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA)
            else:
                if arearatio<12:
                    # zero
                    cv2.putText(frame,'',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA)
                    values = update_values(values,"NOTHING")
                elif arearatio<17.5:
                    # Best of luck
                    cv2.putText(frame,'STOP',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA)
                    values = update_values(values,"STOP")
                   
                else:
                    # one
                    cv2.putText(frame,'MELODY',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA)
                    values = update_values(values,"MELODY")

        elif l==2:
            # two
            cv2.putText(frame,'INTRO',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA)
            values = update_values(values,"INTRO")
            
        elif l==3:
         
              if arearatio<27:
                    # 3
                    cv2.putText(frame,'DRUMS',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA)
                    values = update_values(values,"DRUMS")
              else:
                    # ok
                    cv2.putText(frame,'DRUMS',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA)
                    values = update_values(values,"DRUMS")
        elif l==4:
            # 4
            cv2.putText(frame,'SPEED UP',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA)
            values = update_values(values,"SPEED UP")

        elif l==5:
            # 5
            cv2.putText(frame,'MELODY',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA)
            values = update_values(values,"MELODY")
            
        elif l==6:
            #cv2.putText(frame,'reposition',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA)
            pass
        else:
            #cv2.putText(frame,'reposition',(10,50), font, 2, (0,0,255), 3, cv2.LINE_AA)
            pass

        #show the windows
        cv2.imshow('mask',mask)
        cv2.imshow('frame',frame)
    except Exception:
        traceback.print_exc()
        pass
       # break
        
    pose = values[-1]

    if pose == "STOP":
        print(values[:-1].count(pose), pose)
        if values[:-1].count(pose) > 4:
            Clock.clear()
            Clock.bpm = 100
    else:
        if values[:-1].count(pose) > 4:

            if pose == "INTRO":
                intro()
            elif pose == "DRUMS":
                #if random.randint(0,1) == 0:
                softdrums()
            elif pose == "MELODY":
                melody()
            elif pose == "SPEED UP":
                Clock.bpm = int(Clock.bpm*2)
            else:
                pass

    k = cv2.waitKey(5) & 0xFF
    if k == 27:
        break
    
cv2.destroyAllWindows()
cap.release()    

Video demo