match face position from a video to your webcam in realtime

webcam_face_follow.py

import cv2
import hnswlib
import numpy as np

def set_resolution(cap, width, height):
    cap.set(cv2.CAP_PROP_FRAME_WIDTH, width)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height)

def extract_face_pose(frame, face_detector):
    faces = face_detector.detectMultiScale(frame, scaleFactor=1.1, minNeighbors=5)
    if len(faces) > 0:
        x, y, w, h = faces[0]
        cv2.rectangle(frame, (x, y), (x+w, y+h), (255, 0, 0), 2)  # Draw rectangle
        return np.array([x + w/2, y + h/2])  # Return center of the face
    return None

def build_index_from_video(video_path, index, face_detector):
    cap = cv2.VideoCapture(video_path)
    #set_resolution(cap, 640, 360)
    frame_rate = cap.get(cv2.CAP_PROP_FPS)
    frame_count = 0

    while cap.isOpened():
        ret, frame = cap.read()
        if not ret:
            break

        if frame_count % int(frame_rate) == 0:
            pose = extract_face_pose(frame, face_detector)
            if pose is not None:
                index.add_items(np.array([pose]), np.array([frame_count]))
        
        frame_count += 1

    cap.release()

def process_webcam_and_find_similar_faces(video_path, index, face_detector):
    cap = cv2.VideoCapture(0)
    #set_resolution(cap, 640, 360)

    video_cap = cv2.VideoCapture(video_path)
    #set_resolution(video_cap, 640, 360)

    while cap.isOpened():
        ret, frame = cap.read()
        if not ret:
            break

        pose = extract_face_pose(frame, face_detector)
        print(f"current pose: {pose}")
        if pose is not None:
            labels, distances = index.knn_query(np.array([pose]), k=1)
            matching_frame_number = labels[0][0]
            
            video_cap.set(cv2.CAP_PROP_POS_FRAMES, matching_frame_number)
            ret, matching_frame = video_cap.read()
            if ret:
                print(f"nearest neighbor: {labels} dist: {distances}")  
                extract_face_pose(matching_frame, face_detector)  # Draw bounding box
                cv2.imshow('Matching Frame', matching_frame)

        cv2.imshow('Webcam', frame)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    cap.release()
    cv2.destroyAllWindows()

# Initialize the k-NN index
index = hnswlib.Index(space='l2', dim=2)  # Dimension matches face pose data (x, y)
index.init_index(max_elements=10000, ef_construction=200, M=16)

# Load the face detector
face_detector = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')

# Build the index from the video
video_path = 'test.mov'
build_index_from_video(video_path, index, face_detector)

# Process webcam and find similar faces
process_webcam_and_find_similar_faces(video_path, index, face_detector)