face_classifier.py

face_classifier.py

"""
# BASH dependencies
apt-get install python-opencv ffmpeg
pip install keras numpy shap matplotlib pillow

rm ./drive/My\ Drive/mlin/training/*/*.jpg
rm ./drive/My\ Drive/mlin/validation/*/*.jpg

"""
# native imports
import json
from random import random

from keras.preprocessing.image import ImageDataGenerator
from keras.applications.vgg16 import preprocess_input
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
import keras.backend as K
import numpy as np
import shap
from PIL import Image
import matplotlib.pyplot as plt

# connect to Google Drive
from google.colab import drive
drive.mount('/content/drive')

# generate video frame training data with CV2 - skip ahead if you already generated frames
import cv2
faceCascade = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_frontalface_default.xml")

def face_only(image):
  gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
  faces = faceCascade.detectMultiScale(
        gray,
        scaleFactor=1.3,
        minNeighbors=3,
        minSize=(50, 50)
  )
  for (x, y, w, h) in faces[:1]:
    if y + int(h/2) - 180 < 0:
      return None
    return image[(y + int(h/2) - 180):(y + int(h/2) + 180), (x + int(w/2) - 180):(x + int(w/2) + 180)]
  return None

# training and validation from original video
vidcap = cv2.VideoCapture("./drive/My Drive/mlin/1224068_C_001.mp4")

success,image = vidcap.read()
count = 0
success = True
while success and count < 300:
  retimg = face_only(image)
  if retimg is not None and random() < 0.6:
    cv2.imwrite("./drive/My Drive/mlin/training/originals/face%d.jpg" % count, retimg)     # save frame as JPEG file
    count += 1
  elif retimg is not None and random() < 0.6:
    cv2.imwrite("./drive/My Drive/mlin/validation/originals/face%d.jpg" % count, retimg)
  success,image = vidcap.read()
  
# training and validation from three fake videos
vidindex = 0
for vid in ["1604866_1224068_C_001.mp4", "1609928_1224068_C_001.mp4", "1916010_1224068_C_001.mp4"]:
  vidindex += 1
  vidcap = cv2.VideoCapture("./drive/My Drive/mlin/" + vid)

  success,image = vidcap.read()
  count = 0
  success = True
  while success and count < 100:
    retimg = face_only(image)
    if retimg is not None and random() < 0.3:
      cv2.imwrite("./drive/My Drive/mlin/training/fakers/face%d_%d.jpg" % (vidindex, count), retimg)     # save frame as JPEG file
      count += 1
    success,image = vidcap.read()

  count = 0
  while success and count < 40:
    retimg = face_only(image)
    if retimg is not None and random() < 0.3:
      cv2.imwrite("./drive/My Drive/mlin/validation/fakers/face%d_%d.jpg" % (vidindex, count), retimg)     # save frame as JPEG file
      count += 1
    success,image = vidcap.read()

# training params
batch_size = 128
epochs = 15
IMG_HEIGHT = 360
IMG_WIDTH = 360

# loading image data
train_image_generator = ImageDataGenerator(rescale=1./255) # Generator for our training data
validation_image_generator = ImageDataGenerator(rescale=1./255) # Generator for our validation data
train_data_gen = train_image_generator.flow_from_directory(batch_size=batch_size,
                                                           directory="./drive/My Drive/mlin/training",
                                                           shuffle=True,
                                                           target_size=(IMG_HEIGHT, IMG_WIDTH),
                                                           class_mode='binary')
val_data_gen = validation_image_generator.flow_from_directory(batch_size=batch_size,
                                                 directory="./drive/My Drive/mlin/validation",
                                                 target_size=(IMG_HEIGHT, IMG_WIDTH),
                                                 class_mode='binary')

if K.image_data_format() == 'channels_first': 
    input_shape = (3, IMG_WIDTH, IMG_HEIGHT) 
else: 
    input_shape = (IMG_WIDTH, IMG_HEIGHT, 3) 

# design the Keras neural network layers
model = Sequential() 
model.add(Conv2D(32, (2, 2), input_shape=input_shape)) 
model.add(Activation('relu')) 
model.add(MaxPooling2D(pool_size=(2, 2))) 
  
model.add(Conv2D(32, (2, 2))) 
model.add(Activation('relu')) 
model.add(MaxPooling2D(pool_size=(2, 2))) 
  
model.add(Conv2D(64, (2, 2))) 
model.add(Activation('relu')) 
model.add(MaxPooling2D(pool_size=(2, 2))) 
  
model.add(Flatten()) 
model.add(Dense(64)) 
model.add(Activation('relu')) 
model.add(Dropout(0.5)) 
model.add(Dense(1)) 
model.add(Activation('sigmoid')) 

model.compile(loss='binary_crossentropy', 
              optimizer='rmsprop', 
              metrics=['accuracy']) 

# training the network
model.fit_generator(train_data_gen, 
    steps_per_epoch = 480 // batch_size, 
    epochs = epochs, validation_data = val_data_gen, 
    validation_steps = 101 // batch_size) 

# prep SHAP on training data
def map2layer(x, layer):
    feed_dict = dict(zip([model.layers[0].input], [preprocess_input(x.copy())]))
    return K.get_session().run(model.layers[layer].input, feed_dict)

e = shap.GradientExplainer(
    (model.layers[7].input, model.layers[-1].output),
    map2layer(train_data_gen[0][0], 8),
    local_smoothing=0 # std dev of smoothing noise
)

# load original video, validation frame
origimg = Image.open('./drive/My Drive/mlin/validation/originals/face100.jpg')
origimgarray = np.array([np.array(origimg)], dtype='float64')
plt.imshow(origimg)

# model's prediction for this image
model.predict(origimgarray)

# SHAP image map for frame
shap_values,indexes = e.shap_values(map2layer(origimgarray, 8), ranked_outputs=2)
shap.image_plot(shap_values, origimgarray, np.array([['a']]))

# load one of the fake video frames from validation
fakeimg = Image.open('./drive/My Drive/mlin/validation/fakers/face2_10.jpg')
fakeimgarray = np.array([np.array(fakeimg)], dtype='float64')
plt.imshow(fakeimg)

# produce a SHAP image map
shap_values,indexes = e.shap_values(map2layer(fakeimgarray, 7), ranked_outputs=3)
shap.image_plot(shap_values, fakeimgarray, np.array([['a']]))

# model's prediction for this image
model.predict(fakeimgarray)