Minoru 3D usb webcam image stream to basic MIDI tone array in pygame with OpenCV2 for Raspberry Pi

minoru3d-to-midi-array.py

import pygame.midi
import pygame.camera
import pygame.surfarray as surfarray
import cv2

pygame.init()
pygame.camera.init()
pygame.midi.init()

screen = pygame.display.set_mode((640,480),0)

cam_list = pygame.camera.list_cameras()
#Using Minoru 3D webcam for smoother feeds
webcamL = pygame.camera.Camera(cam_list[0],(32,24))
webcamL.start()
webcamR = pygame.camera.Camera(cam_list[1],(32,24))
webcamR.start()

midiPort = 2 # TODO: using timidity manually (may be settable as environment variable?)
midiOut = pygame.midi.Output(midiPort, 0)

ext = ".bmp"
imagePathR = "rightframecap"+ext
imagePathL = "leftframecap"+ext
diffImagePath = "framediff"+ext
diffArrayHeight = 12 # clarity (24ish) vs generality (5ish)
maxVolume = 127
volume = maxVolume
count = 0

stereo = cv2.StereoBM(cv2.STEREO_BM_NARROW_PRESET, 16, 15)

while True:
    imageL = webcamL.get_image()
    imageR = webcamR.get_image()
    pygame.image.save(imageL, imagePathL)
    pygame.image.save(imageR, imagePathR)
    #Saving images via bmp since it runs faster on Rasperry Pi sd (and they are also handled by another process), but if you're using a disk you'll prob want to keep these in memory only and convert them to a pygame surface from numpy array.
    imgL = cv2.imread(imagePathL,0)
    imgR = cv2.imread(imagePathR,0)

    """
    #can be useful too.
    fgbg = cv2.BackgroundSubtractorMOG()
    fgmask = fgbg.apply(imgL);
    fgmask = fgbg.apply(imgR);
    cv2.imwrite(altDiffImagePath, fgmask)
    altDiffImage = pygame.image.load(altDiffImagePath)
    """

    disparity = stereo.compute(imgR, imgL)
    cv2.imwrite(diffImagePath, disparity)
    diffImage = pygame.image.load(diffImagePath)
    
    # get a slice near the center row
    bar = pygame.transform.scale(diffImage,(13,diffArrayHeight)).convert()
    diffArray = (surfarray.array2d(bar) & 255).transpose()

    screen.blit(pygame.transform.scale(diffImage,(320,240)),(0,240))
    screen.blit(pygame.transform.scale(bar,(320,240)),(320,0))
    screen.blit(pygame.transform.scale(pygame.transform.scale(diffImage,(32,24)).convert(),(320,240)),(0,0))

    # play midi tones at depthmapped volumes
    lookAt = diffArrayHeight / 2
    for x in range(0, 12):
	volume = int(maxVolume * (float(diffArray[lookAt][x]) / float(255)))
	if volume > 0:
		midiOut.note_on(60 + (x*2), volume) # across 2 octaves for range
	else: 
		midiOut.note_off(60 + (x*2), volume)

    pygame.display.update()
    count = count + 1

    # Check for quit events
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            del midiOut
	    pygame.midi.quit()
            webcam.stop()
            webcam2.stop()
            pygame.quit()
	    p.terminate()
            sys.exit()