handy_mediapipe_script.py

from __future__ import annotations
import cv2
import mediapipe as mp
from dataclasses import dataclass, field
from glm import vec3
import json

mp_drawing = mp.solutions.drawing_utils  # type: ignore
mp_drawing_styles = mp.solutions.drawing_styles  # type: ignore
mp_hands = mp.solutions.hands  # type: ignore


class JSONEncoder(json.JSONEncoder):
    def default(self, o):
        if isinstance(o, vec3):
            return o.to_list()
        elif isinstance(o, Hand):
            return {
                "id": o.id,
                "wrist": o.wrist,
                "points": o.points,
                "fingers": o.fingers
            }
        elif isinstance(o, Finger):
            return {
                "id": o.id,
                "points": o.points
            }
        return json.JSONEncoder.default(self, o)


@dataclass
class Finger:
    id: int
    points: list[vec3] = field(default_factory=list)


@dataclass
class Hand:
    id: int
    wrist: vec3 = field(default_factory=vec3)
    points: list[vec3] = field(default_factory=list)
    fingers: list[Finger] = field(default_factory=list)


def construct_hand(idx, hand_landmarks):
    hand = Hand(
        id=idx,
        wrist=vec3(
            x=hand_landmarks.landmark[0].x,
            y=hand_landmarks.landmark[0].y,
            z=hand_landmarks.landmark[0].z
        ),
        points=[
            vec3(x=landmark.x, y=landmark.y, z=landmark.z)
            for landmark in hand_landmarks.landmark
        ]
    )
    hand.fingers = [
        Finger(
            id=i,
            points=[hand.points[4*i + j] for j in range(4)]
        ) for i in range(5)
    ]
    return hand


# For webcam input:
cap = cv2.VideoCapture(1)
with mp_hands.Hands(
        model_complexity=0,
        min_detection_confidence=0.5,
        min_tracking_confidence=0.5) as hands:
    while cap.isOpened():
        success, image = cap.read()
        if not success:
            print("Ignoring empty camera frame.")
            # If loading a video, use 'break' instead of 'continue'.
            continue

        # To improve performance, optionally mark the image as not writeable to
        # pass by reference.
        image.flags.writeable = False
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        results = hands.process(image)

        # Draw the hand annotations on the image.
        image.flags.writeable = True
        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
        if results.multi_hand_landmarks:
            for hand_landmarks in results.multi_hand_landmarks:
                mp_drawing.draw_landmarks(
                    image,
                    hand_landmarks,
                    mp_hands.HAND_CONNECTIONS,
                    mp_drawing_styles.get_default_hand_landmarks_style(),
                    mp_drawing_styles.get_default_hand_connections_style())

            hand_data = [
                construct_hand(i, hand_landmarks)
                for i, hand_landmarks in enumerate(results.multi_hand_landmarks)
            ]
            print(json.dumps(hand_data, indent=2, cls=JSONEncoder))

        # Flip the image horizontally for a selfie-view display.
        cv2.imshow('MediaPipe Hands', cv2.flip(image, 1))
        if cv2.waitKey(5) & 0xFF == 27:
            break
cap.release()