ExtractFaceChip.py

import numpy as np
import cv2
import face_alignment

# Initialize the chip resolution
chipSize = 300
chipCorners = np.float32([[0,0],
                          [chipSize,0],
                          [0,chipSize],
                          [chipSize,chipSize]])

# Initialize the face alignment tracker
fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._3D, flip_input=True, device="cuda")

# Start the webcam capture, exit with 'q'
cap = cv2.VideoCapture(0)
while(not (cv2.waitKey(1) & 0xFF == ord('q'))):
    ret, frame = cap.read()
    if(ret):
        # Run the face alignment tracker on the webcam image
        imagePoints = fa.get_landmarks_from_image(frame)
        if(imagePoints is not None):
            imagePoints = imagePoints[0]

            # Compute the Anchor Landmarks
            # This ensures the eyes and chin will not move within the chip
            rightEyeMean = np.mean(imagePoints[36:42], axis=0)
            leftEyeMean  = np.mean(imagePoints[42:47], axis=0)
            middleEye    = (rightEyeMean + leftEyeMean) * 0.5
            chin         = imagePoints[8]
            #cv2.circle(frame, tuple(rightEyeMean[:2].astype(int)), 30, (255,255,0))
            #cv2.circle(frame, tuple(leftEyeMean [:2].astype(int)), 30, (255,0,255))

            # Compute the chip center and up/side vectors
            mean = ((middleEye * 3) + chin) * 0.25
            centered = imagePoints - mean 
            rightVector = (leftEyeMean - rightEyeMean)
            upVector    = (chin        - middleEye)

            # Divide by the length ratio to ensure a square aspect ratio
            rightVector /= np.linalg.norm(rightVector) / np.linalg.norm(upVector)

            # Compute the corners of the facial chip
            imageCorners = np.float32([(mean + ((-rightVector - upVector)))[:2],
                                       (mean + (( rightVector - upVector)))[:2],
                                       (mean + ((-rightVector + upVector)))[:2],
                                       (mean + (( rightVector + upVector)))[:2]])

            # Compute the Perspective Homography and Extract the chip from the image
            chipMatrix = cv2.getPerspectiveTransform(imageCorners, chipCorners)
            chip = cv2.warpPerspective(frame, chipMatrix, (chipSize, chipSize))

        cv2.imshow('Webcam View', frame)
        cv2.imshow('Chip View', chip)

# When everything is done, release the capture
cap.release()
cv2.destroyAllWindows()

Something's wrong with this code. Here's comparison of my other code that uses dlib vs this code performance on an image

This is because rightVector is not orthogonal to upVector. Also, you have a bug where you scale according to 3D, not 2D. Fixed it here: https://gist.github.com/ibmua/a6c440b3323c0dff851be0e693a07dad