Skip to content

Instantly share code, notes, and snippets.

@smeschke
Last active June 24, 2024 06:51
Show Gist options
  • Save smeschke/df2c5794287fa7044817dc3e00d61351 to your computer and use it in GitHub Desktop.
Save smeschke/df2c5794287fa7044817dc3e00d61351 to your computer and use it in GitHub Desktop.
Using Keras from the Webcam
import cv2, numpy as np, os
#parameters
working_dir = '/home/stephen/Desktop/keras_demo/'
cap = cv2.VideoCapture(0)
org, font, scale, color, thickness, linetype = (50,50), cv2.FONT_HERSHEY_SIMPLEX, 1.2, (234,12,123), 2, cv2.LINE_AA
#chromakey values
h,s,v,h1,s1,v1 = 16,0,64,123,111,187 #green
h,s,v,h1,s1,v1 = 0,74,53,68,181,157 #skin tone
#amount of data to use
data_size = 1000
#ratio of training data to test data
training_to_test = .75
#amount images are scaled down before being fed to keras
img_size = 100
#image height and width (from the webcam
height, width = 480,640
#returns the region of interest around the largest countour
#the accounts for objects not being centred in the frame
def bbox(img):
try:
bg = np.zeros((1000,1000), np.uint8)
bg[250:250+480, 250:250+640] = img
_, contours, _ = cv2.findContours(img, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
largest_contour = max(contours, key = cv2.contourArea)
rect = cv2.boundingRect(largest_contour)
circ = cv2.minEnclosingCircle(largest_contour)
x,y,w,h = rect
x,y = x+w/2,y+h/2
x,y = x+250, y+250
ddd = 200
return bg[y-ddd:y+ddd, x-ddd:x+ddd]
except: return img
#finds the largest contour in a list of contours
#returns a single contour
def largest_contour(contours):
c = max(contours, key=cv2.contourArea)
return c[0]
#finds the center of a contour
#takes a single contour
#returns (x,y) position of the contour
def contour_center(c):
M = cv2.moments(c)
try: center = int(M['m10']/M['m00']), int(M['m01']/M['m00'])
except: center = 0,0
return center
#takes image and range
#returns parts of image in range
def only_color(img, (h,s,v,h1,s1,v1)):
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
lower, upper = np.array([h,s,v]), np.array([h1,s1,v1])
mask = cv2.inRange(hsv, lower, upper)
kernel = np.ones((15,15), np.uint)
#mask - cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel, iterations=2)
res = cv2.bitwise_and(img, img, mask=mask)
return res, mask
def flatten(dimData, images):
images = np.array(images)
images = images.reshape(len(images), dimData)
images = images.astype('float32')
images /=255
return images
#-------------get train/test data-----------------
images, labels = [],[]
#iterate through tools
tool_name = ''
patterns = []
tool_num = 0
while True:
_, img = cap.read()
cv2.putText(img, 'enter class name (then enter)', org, font, scale, color, thickness, linetype)
cv2.putText(img, 'press esc when finished', (50,100), font, scale, color, thickness, linetype)
cv2.putText(img, tool_name, (50,300), font, 3, (0,0,255), 5, linetype)
cv2.line(img, (330,240), (310,240), (234,123,234), 3)
cv2.line(img, (320,250), (320,230), (234,123,234), 3)
cv2.imshow('img', img)
k = cv2.waitKey(1)
if k>10: tool_name += chr(k)
if k == 27: break
#if tool name has been entered, start collecting the data
current = 0
if k == 13:
while current < data_size:
_, img = cap.read()
img, mask = only_color(img, (h,s,v,h1,s1,v1))
mask = bbox(mask)
images.append(cv2.resize(mask, (img_size, img_size)))
labels.append(tool_num)
current += 1
cv2.line(img, (330,240), (310,240), (234,123,234), 3)
cv2.line(img, (320,250), (320,230), (234,123,234), 3)
cv2.putText(img, 'collecting data', org, font, scale, color, thickness, linetype)
cv2.putText(img, 'data for'+tool_name+':' + str(current), (50,100), font, scale, color, thickness, linetype)
#cv2.imshow('img', img)
cv2.imshow('img', mask)
k = cv2.waitKey(1)
if k == ord('p'): cv2.waitKey(0)
if current == data_size:
patterns.append(tool_name)
tool_name = ''
tool_num += 1
print tool_num
break
#break data into training and test sets
to_train= 0
train_images, test_images, train_labels, test_labels = [],[],[],[]
for image, label in zip(images, labels):
if to_train<3:
train_images.append(image)
train_labels.append(label)
to_train+=1
else:
test_images.append(image)
test_labels.append(label)
to_train = 0
#-----------------keras time --> make the model
from keras.utils import to_categorical
#flatten data
dataDim = np.prod(images[0].shape)
train_data = flatten(dataDim, train_images)
test_data = flatten(dataDim, test_images)
#change labels to categorical
train_labels = np.array(train_labels)
test_labels = np.array(test_labels)
train_labels_one_hot = to_categorical(train_labels)
test_labels_one_hot = to_categorical(test_labels)
#determine the number of classes
classes = np.unique(train_labels)
nClasses = len(classes)
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
model = Sequential()
model.add(Dense(128, activation = 'relu', input_shape = (dataDim,)))
model.add(Dropout(0.5))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(nClasses, activation='softmax'))
model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
history = model.fit(train_data, train_labels_one_hot, batch_size = 256, epochs=5, verbose=1,
validation_data=(test_data, test_labels_one_hot))
#test model
[test_loss, test_acc] = model.evaluate(test_data, test_labels_one_hot)
print("Evaluation result on Test Data : Loss = {}, accuracy = {}".format(test_loss, test_acc))
#save model
#model.save('/home/stephen/Desktop/hand_model.h5')
#---------------display model with webcam
#function to draw prediction on background image
def draw_prediction(bg,prediction, motion):
idxs = [1,2,3,4,5,6,7,8,9]
for i, pattern, idx in zip(prediction, patterns, idxs):
text = pattern + ' '+str(round(i,3))
scale = i*2
if motion: scale = .4
if scale<.95: scale = .95
thickness = 1
if scale>1.5: thickness = 2
if scale>1.95: thickness = 4
scale = scale*.75
org, font, color = (350, idx*70), cv2.FONT_HERSHEY_SIMPLEX, (0,0,0)
cv2.putText(bg, text, org, font, scale, color, thickness, cv2.LINE_AA)
return bg
def draw_bg(prediction):
motion = False
bg = np.zeros((1150,1000,3), np.uint8)
idxs = [1,2,3,4,5,6,7,8,9]
for i, pattern, idx in zip(prediction, patterns, idxs):
text = pattern + ' '+str(round(i,3))
scale = i*2
if motion: scale = .4
if scale<.95: scale = .95
thickness = 1
if scale>1.5: thickness = 2
if scale>1.95: thickness = 4
scale = scale*2
org, font, color = (200, idx*140), cv2.FONT_HERSHEY_SIMPLEX, (12,234,123)
cv2.putText(bg, text, org, font, scale, (255,255,255), 1+thickness, cv2.LINE_AA)
return bg
from keras.models import load_model
#model = load_model('/home/stephen/Desktop/hand_model.h5')
dimData = np.prod([img_size, img_size])
while True:
_, img= cap.read()
_, mask = only_color(img, (h,s,v,h1,s1,v1))
mask = bbox(mask)
mask = cv2.resize(mask, (img_size, img_size))
cv2.imshow('display', mask)
mask = mask.reshape(dimData)
mask = mask.astype('float32')
mask /=255
prediction = model.predict(mask.reshape(1,dimData))[0].tolist()
img = draw_prediction(img, prediction, False)
display = draw_bg(prediction)
cv2.imshow('img', img)
k = cv2.waitKey(10)
if k == 27: break
cap.release()
cv2.destroyAllWindows()
@lh784116211
Copy link

I watched your YouTube video (https://www.youtube.com/watch?v=r9hVypi_6TQ&t=67s) that tracks and identifies multiple objects. The above code does not seem to be mentioned. Can you share it? Thank you very much for sharing.

@smeschke
Copy link
Author

I watched your YouTube video (https://www.youtube.com/watch?v=r9hVypi_6TQ&t=67s) that tracks and identifies multiple objects. The above code does not seem to be mentioned. Can you share it? Thank you very much for sharing.

The complete code for training and making predictions is in this Gist. The model is created om the first part of the Gist and the prediction is done on line 218. This tutorial will explain: https://www.learnopencv.com/image-classification-using-feedforward-neural-network-in-keras/

@lh784116211
Copy link

image
First of all, thank you very much for your answer. In addition, can the above code achieve the effect of the above picture?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment