Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
greenscreen.py: Greenscreen effect without a physical green screen, via OpenCV and Python
#! /usr/bin/env python
'''
greenscreen.py: Greenscreen effect without a physical green screen
This performs background subtraction, and sets the background to "green" for use with "key frame" video editing software
Author: Scott Hawley, https://github.com/drscotthawley
Requirements:
Python, NumPy and OpenCV
I got these via Macports, but Homebrew, etc. work.
Note that on Mac OS X El Capitan, OpenCV > 3.0 causes code to crash after ~30 seconds. I had to install OpenCV 3.0 by hand.
Credits:
Built from Tutorial "Getting Started with Videos"
http://docs.opencv.org/3.0-beta/doc/py_tutorials/py_gui/py_video_display/py_video_display.html
Background-subtraction tutorials were of minimal utility and effectiveness, IMHO. Developed
my own foreground masking method for this
De-noising: http://docs.opencv.org/master/d5/d69/tutorial_py_non_local_means.html#gsc.tab=0
Cropping: http://www.pyimagesearch.com/2015/03/09/capturing-mouse-click-events-with-python-and-opencv/
Time-averaging tutorial (not actually a great effect for this, but I left it in the code):
https://opencvpython.blogspot.com/2012/07/background-extraction-using-running.html
'''
import numpy as np
import cv2
import sys
source = 0 # defaults to local (laptop) camera
if len(sys.argv) > 1:
if ('-h' == sys.argv[1]) or ('--help'==sys.argv[1]):
print "usage: greenscreen.py [-h,--help] [input_file]"
print " If input_file is blank, defaults to live camera capture"
sys.exit()
source = sys.argv[1]
cap = cv2.VideoCapture(source) # Start the video source
ret, ref_img = cap.read() # read background image
if (False == ret):
print "Unable to get any images. Is the camera already in use?"
sys.exit()
# setup for reading & writing movie files
if (source !=0):
fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
video_out = cv2.VideoWriter()
frame_shape = (ref_img.shape[1],ref_img.shape[0])
success = video_out.open('greenscreen_movie_post.mp4v',fourcc, 15.0, frame_shape,True)
# initializing background arrays
green_img = np.zeros(ref_img.shape, np.uint8)
fgmask = np.zeros(ref_img.shape, np.uint8)
green_img[:] = (0, 255, 0)
orig_ref = ref_img
orig_green = green_img
result = ref_img
# this is for automatically setting corners of frame to background
gray_shape = (ref_img.shape[0],ref_img.shape[1])
bg_edges = np.zeros(gray_shape, np.uint8)
# storage format is y,x, color
edge_size_x = bg_edges.shape[1]/8
edge_size_y = bg_edges.shape[0]/4
bg_edges[0:edge_size_y,0:edge_size_x] = 255
bg_edges[0:edge_size_y,-edge_size_x:] = 255
bg_edges[-edge_size_y:,0:edge_size_x] = 255
bg_edges[-edge_size_y:,-edge_size_x:] = 255
avg1 = np.float32(ref_img) # time-averaging array
'''
find_fgmask: Finds the 'foregound mask', i.e. where the foreground objects are
Author: Scott Hawley
It doesn't use any especially clever algorithm (e.g., no BGMOG), just the fruits
of significant trial-and-error on my part, for what seems to work best
Warning: de-noising is slow, and best suited for post-processing only
'''
def find_fgmask(img,ref_img,thresh=13.0,use_denoise=False,h=10.0):
diff1 = cv2.subtract(img,ref_img)
diff2 = cv2.subtract(ref_img,img)
diff = diff1+diff2
sws = int( np.ceil(21*h/10) // 2 * 2 + 1 )
diff[ abs(diff) < thresh] = 0
gray = cv2.cvtColor(diff.astype(np.uint8), cv2.COLOR_BGR2GRAY)
gray[np.abs(gray) < 10] = 0
if (use_denoise):
cv2.fastNlMeansDenoising(gray,gray,h=h,templateWindowSize=5,searchWindowSize=sws)
fgmask = gray.astype(np.uint8)
fgmask[ fgmask >0] = 255
return fgmask
# Cropping stuff
refPt = []
setting_cropping = False
def click_and_crop(event, x, y, flags, param):
# grab references to the global variables
global refPt, cropping
# if the left mouse button was clicked, record the starting
# (x, y) coordinates and indicate that cropping specification is being
# performed
if event == cv2.EVENT_LBUTTONDOWN:
refPt = [(x, y)]
setting_cropping = True
# check to see if the left mouse button was released
elif event == cv2.EVENT_LBUTTONUP:
# record the ending (x, y) coordinates and indicate that
# the cropping operation is finished
refPt.append((x, y))
setting_cropping = False
# draw a rectangle around the region of interest
cv2.rectangle(result, refPt[0], refPt[1], (255, 255, 0), 2)
cv2.imshow("frame", result)
cv2.namedWindow("frame")
cv2.setMouseCallback("frame", click_and_crop) # register mouse events
print "Controls:"
print "D = difference image: take reference image & subtract"
print "C = clear settings: clear reference image, denoising"
print "T = toggle time averaging (probably don't want to use this)"
print "K = toggle crop (Draw box with mouse first, then press K)"
print "N = toggle de-noising (slow)"
print "Up/Down arrows = threshold for foreground: more/less green"
print "Left/Right arrows = size of denoising kernel: less/more noise"
print "Q = quit"
# control parameters
use_diff = True
use_time_avg = False
use_denoise = False
denoise_h = 10.0
use_cropping = False
thresh = 13.0
already_pressed_clear = False
use_edges = False
# Main Loop
while(1):
ret, img = cap.read()
if (False==ret): # end of video capture
print "Saving and terminating"
break
# crop first, for speed
if (use_cropping) and (len(refPt)==2):
cropped_img = img[refPt[0][1]:refPt[1][1], refPt[0][0]:refPt[1][0]]
img = cropped_img
if (img.shape[0] < ref_img.shape[0]) and (img.shape[1] < ref_img.shape[1]):
cropped = ref_img[refPt[0][1]:refPt[1][1], refPt[0][0]:refPt[1][0]]
ref_img = cropped
if (img.shape[0] < green_img.shape[0]) and (img.shape[1] < green_img.shape[1]):
cropped = green_img[refPt[0][1]:refPt[1][1], refPt[0][0]:refPt[1][0]]
green_img = cropped
if (img.shape[0] < avg1.shape[0]) and (img.shape[1] < avg1.shape[1]):
avg1 = np.float32(ref_img)
result = img
if (use_time_avg):
cv2.accumulateWeighted(img,avg1,0.09)
img = cv2.convertScaleAbs(avg1)
if (use_diff):
fgmask = find_fgmask(img,ref_img,thresh=thresh,use_denoise=use_denoise,h=denoise_h)
bgmask = cv2.bitwise_not(fgmask)
if (use_edges):
bgmask = cv2.bitwise_or(bgmask,bg_edges)
fgmask = cv2.bitwise_not(bgmask)
fgimg = cv2.bitwise_and(img,img,mask = fgmask)
bgimg = cv2.bitwise_and(green_img,green_img,mask = bgmask)
sum = cv2.add(fgimg, bgimg)
result = sum
# if there's a cropping rectangle drawn, keep showing the rectangle
if ((2==len(refPt)) and (not use_cropping)):
cv2.rectangle(result, refPt[0], refPt[1], (255, 255, 0), 2)
cv2.imshow('frame',sum)
if (0!=source):
video_out.write(result)
# if there's a cropping rectangle drawn, keep showing the rectangle
if ((2==len(refPt)) and (not use_cropping)):
cv2.rectangle(result, refPt[0], refPt[1], (255, 255, 0), 2)
key = cv2.waitKey(1) & 0xFF
if ord('q') == key:
break
elif ord('d') == key: # take & use difference image
use_diff = True
ref_img = img
elif ord('e') == key: # add extra green to corners
use_edges = not use_edges
elif key == ord('t'): # toggle time-averaging
use_time_avg = not use_time_avg
avg1 = np.float32(img)
elif key == ord('c'): # clear button
print "already_pressed_clear = ",already_pressed_clear
if (already_pressed_clear):
use_diff = use_diff_save
use_time_avg = use_time_avg_save
use_denoise = use_denoise_save
denoise_h = denoise_h_save
thresh = thresh_save
already_pressed_clear = False
else:
use_diff_save = use_diff
use_denoise_save = use_denoise
use_time_avg_save = use_time_avg
denoise_h_save = denoise_h
thresh_save = thresh
already_pressed_clear = True
use_diff = False
use_time_avg = False
use_denoise = False
denoise_h = 10.0
thresh = 13.0
elif key == ord('n'): # toggle denoising (slow)
use_denoise = not use_denoise
if (use_denoise):
denoise_h = 10.0
else:
denoise_h = 0.0
elif (key == ord('k')) and (2==len(refPt)): # cropping
use_cropping = not use_cropping
if (not use_cropping):
ref_img = orig_ref
green_img = orig_green
avg1 = np.float32(ref_img)
elif key == 0: # up arrow
thresh *= 1.1 # increase difference threshold = more green
elif key == 1: # down arrow
thresh /= 1.1
elif key == 2: # left arrow
denoise_h /= 1.1 # increase both denoising weight and kernel
elif key == 3: # right arrow
denoise_h *= 1.1
# Main Loop has finished, shutting down
if source != 0: # close & release any output file
video_out.release()
video_out = None
cap.release() # release the camera
cv2.destroyAllWindows()
@drscotthawley

This comment has been minimized.

Copy link
Owner Author

@drscotthawley drscotthawley commented Jul 1, 2016

Demo video here: https://youtu.be/I7kRFFfTb0Y

You'll want to pay attention to lighting, and to wearing clothes that don't match the background.
Then play with the "threshold" via the up & down arrow keys to get the right amount of green.
Then turn on the de-noising.
(And...maybe this isn't obvious: The camera has to stay in exactly the same position at all times. For moving shots, you will need a real green screen.)

BTW, those arrow key "ord" values are for Mac, but for Linux they'll have different values (80-83 maybe?)

@dheerajmpai

This comment has been minimized.

Copy link

@dheerajmpai dheerajmpai commented Jan 10, 2021

Thanks for the code. There are some recent advancements that you might want to check out. Which is quite nice

https://grail.cs.washington.edu/projects/background-matting/

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment