MediaPipe Holistic Tracking for Android

BUILD

# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

licenses(["notice"])

package(default_visibility = ["//visibility:private"])

cc_binary(
    name = "libmediapipe_jni.so",
    linkshared = 1,
    linkstatic = 1,
    deps = [
        "//mediapipe/graphs/holistic_tracking:holistic_tracking_gpu_deps",
        "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
    ],
)

cc_library(
    name = "mediapipe_jni_lib",
    srcs = [":libmediapipe_jni.so"],
    alwayslink = 1,
)

android_binary(
    name = "holistictrackinggpu",
    srcs = glob(["*.java"]),
    assets = [
        "//mediapipe/graphs/holistic_tracking:holistic_tracking_gpu.binarypb",
        "//mediapipe/modules/face_detection:face_detection_front.tflite",
        "//mediapipe/modules/face_landmark:face_landmark.tflite",
        "//mediapipe/modules/hand_landmark:hand_landmark.tflite",
        "//mediapipe/modules/hand_landmark:handedness.txt",
        "//mediapipe/modules/holistic_landmark:hand_recrop.tflite",
        "//mediapipe/modules/pose_detection:pose_detection.tflite",
        "//mediapipe/modules/pose_landmark:pose_landmark_full.tflite",
    ],
    assets_dir = "",
    manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml",
    manifest_values = {
        "applicationId": "com.google.mediapipe.apps.holistictrackinggpu",
        "appName": "Holistic Tracking",
        "mainActivity": "com.google.mediapipe.apps.basic.MainActivity",
        "cameraFacingFront": "False",
        "binaryGraphName": "holistic_tracking_gpu.binarypb",
        "inputVideoStreamName": "input_video",
        "outputVideoStreamName": "output_video",
        "flipFramesVertically": "True",
        "converterNumBuffers": "3",
    },
    multidex = "native",
    deps = [
        ":mediapipe_jni_lib",
        "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib",
        "//mediapipe/framework/formats:landmark_java_proto_lite",
        "//mediapipe/java/com/google/mediapipe/framework:android_framework",
    ],
)

holistic_landmark_gpu.pbtxt

# Predicts pose + left/right hand + face landmarks.
#
# It is required that:
# - "face_detection_front.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_front.tflite"
#
# - "face_landmark.tflite" is available at
# "mediapipe/modules/face_landmark/face_landmark.tflite"
#
# - "hand_landmark.tflite" is available at
# "mediapipe/modules/hand_landmark/hand_landmark.tflite"
#
# - "hand_recrop.tflite" is available at
# "mediapipe/modules/holistic_landmark/hand_recrop.tflite"
#
# - "handedness.txt" is available at
# "mediapipe/modules/hand_landmark/handedness.txt"
#
# - "pose_detection.tflite" is available at
# "mediapipe/modules/pose_detection/pose_detection.tflite"
#
# - "pose_landmark_lite.tflite" or "pose_landmark_full.tflite" or
# "pose_landmark_heavy.tflite" is available at
# "mediapipe/modules/pose_landmark/pose_landmark_lite.tflite" or
# "mediapipe/modules/pose_landmark/pose_landmark_full.tflite" or
# "mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite"
# path respectively during execution, depending on the specification in the
# MODEL_COMPLEXITY input side packet.
#
# EXAMPLE:
#   node {
#     calculator: "HolisticLandmarkGpu"
#     input_stream: "IMAGE:input_video"
#     input_side_packet: "MODEL_COMPLEXITY:model_complexity"
#     input_side_packet: SMOOTH_LANDMARKS:smooth_landmarks
#     output_stream: "POSE_LANDMARKS:pose_landmarks"
#     output_stream: "FACE_LANDMARKS:face_landmarks"
#     output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
#     output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
#   }
#
# NOTE: if a pose/hand/face output is not present in the image, for this
# particular timestamp there will not be an output packet in the corresponding
# output stream below. However, the MediaPipe framework will internally inform
# the downstream calculators of the absence of this packet so that they don't
# wait for it unnecessarily.

type: "HolisticLandmarkGpu"

# GPU image. (GpuBuffer)
input_stream: "IMAGE:image"

# Complexity of the pose landmark model: 0, 1 or 2. Landmark accuracy as well as
# inference latency generally go up with the model complexity. If unspecified,
# functions as set to 1. (int)
input_side_packet: "MODEL_COMPLEXITY:model_complexity"

# Whether to filter landmarks across different input images to reduce jitter.
# If unspecified, functions as set to true. (bool)
input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"

# Pose landmarks. (NormalizedLandmarkList)
# 33 pose landmarks.
output_stream: "POSE_LANDMARKS:pose_landmarks"
# 21 left hand landmarks. (NormalizedLandmarkList)
output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
# 21 right hand landmarks. (NormalizedLandmarkList)
output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
# Face detection
# output_stream: "FACE_LANDMARKS:face_landmarks"

# Debug outputs
output_stream: "POSE_ROI:pose_landmarks_roi"
output_stream: "POSE_DETECTION:pose_detection"

# Predicts pose landmarks.
node {
  calculator: "PoseLandmarkGpu"
  input_stream: "IMAGE:image"
  input_side_packet: "MODEL_COMPLEXITY:model_complexity"
  input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks"
  output_stream: "LANDMARKS:pose_landmarks"
  output_stream: "ROI_FROM_LANDMARKS:pose_landmarks_roi"
  output_stream: "DETECTION:pose_detection"
}

# Predicts left and right hand landmarks based on the initial pose landmarks.
node {
  calculator: "HandLandmarksLeftAndRightGpu"
  input_stream: "IMAGE:image"
  input_stream: "POSE_LANDMARKS:pose_landmarks"
  output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
  output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
}

# Extracts face-related pose landmarks.
#node {
#  calculator: "SplitNormalizedLandmarkListCalculator"
#  input_stream: "pose_landmarks"
#  output_stream: "face_landmarks_from_pose"
#  options: {
#    [mediapipe.SplitVectorCalculatorOptions.ext] {
#      ranges: { begin: 0 end: 11 }
#    }
#  }
#}

# Predicts face landmarks based on the initial pose landmarks.
#node {
#  calculator: "FaceLandmarksFromPoseGpu"
#  input_stream: "IMAGE:image"
#  input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
#  output_stream: "FACE_LANDMARKS:face_landmarks"
#}

holistic_tracking_gpu.pbtxt

# Tracks and renders pose + hands + face landmarks.

# GPU buffer. (GpuBuffer)
input_stream: "input_video"

# GPU image with rendered results. (GpuBuffer)
output_stream: "output_video"
output_stream: "pose_landmarks"
output_stream: "left_hand_landmarks"
output_stream: "right_hand_landmarks"

# Throttles the images flowing downstream for flow control. It passes through
# the very first incoming image unaltered, and waits for downstream nodes
# (calculators and subgraphs) in the graph to finish their tasks before it
# passes through another image. All images that come in while waiting are
# dropped, limiting the number of in-flight images in most part of the graph to
# 1. This prevents the downstream nodes from queuing up incoming images and data
# excessively, which leads to increased latency and memory usage, unwanted in
# real-time mobile applications. It also eliminates unnecessarily computation,
# e.g., the output produced by a node may get dropped downstream if the
# subsequent nodes are still busy processing previous inputs.
node {
  calculator: "FlowLimiterCalculator"
  input_stream: "input_video"
  input_stream: "FINISHED:output_video"
  input_stream_info: {
    tag_index: "FINISHED"
    back_edge: true
  }
  output_stream: "throttled_input_video"
  node_options: {
    [type.googleapis.com/mediapipe.FlowLimiterCalculatorOptions] {
      max_in_flight: 1
      max_in_queue: 1
      # Timeout is disabled (set to 0) as first frame processing can take more
      # than 1 second.
      in_flight_timeout: 0
    }
  }
}

node {
  calculator: "HolisticLandmarkGpu"
  input_stream: "IMAGE:throttled_input_video"
  output_stream: "POSE_LANDMARKS:pose_landmarks"
  output_stream: "POSE_ROI:pose_roi"
  output_stream: "POSE_DETECTION:pose_detection"
  output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
  output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
}

# Gets image size.
node {
  calculator: "ImagePropertiesCalculator"
  input_stream: "IMAGE_GPU:throttled_input_video"
  output_stream: "SIZE:image_size"
}

# Converts pose, hands and face landmarks to a render data vector.
node {
  calculator: "HolisticTrackingToRenderData"
  input_stream: "IMAGE_SIZE:image_size"
  input_stream: "POSE_LANDMARKS:pose_landmarks"
  input_stream: "POSE_ROI:pose_roi"
  input_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
  input_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
  output_stream: "RENDER_DATA_VECTOR:render_data_vector"
}

# Draws annotations and overlays them on top of the input images.
node {
  calculator: "AnnotationOverlayCalculator"
  input_stream: "IMAGE_GPU:throttled_input_video"
  input_stream: "VECTOR:render_data_vector"
  output_stream: "IMAGE_GPU:output_video"
}

holistic_tracking_to_render_data.pbtxt

# Converts pose + hands + face landmarks to a render data vector.

type: "HolisticTrackingToRenderData"

# Image size. (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# Pose landmarks. (NormalizedLandmarkList)
input_stream: "POSE_LANDMARKS:landmarks"
# Region of interest calculated based on pose landmarks. (NormalizedRect)
input_stream: "POSE_ROI:roi"
# Left hand landmarks. (NormalizedLandmarkList)
input_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
# Right hand landmarks. (NormalizedLandmarkList)
input_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"

# Render data vector. (std::vector<RenderData>)
output_stream: "RENDER_DATA_VECTOR:render_data_vector"

# --------------------------------------------------------------------------- #
# ------------------ Calculates scale for render objects -------------------- #
# --------------------------------------------------------------------------- #

# Calculates rendering scale based on the pose bounding box.
node {
  calculator: "RectToRenderScaleCalculator"
  input_stream: "NORM_RECT:roi"
  input_stream: "IMAGE_SIZE:image_size"
  output_stream: "RENDER_SCALE:render_scale"
  node_options: {
    [type.googleapis.com/mediapipe.RectToRenderScaleCalculatorOptions] {
      multiplier: 0.0008
    }
  }
}

# --------------------------------------------------------------------------- #
# --------------- Combines pose and hands into pose skeleton ---------------- #
# --------------------------------------------------------------------------- #

# Gets pose landmarks before wrists.
node {
  calculator: "SplitNormalizedLandmarkListCalculator"
  input_stream: "landmarks"
  output_stream: "landmarks_before_wrist"
  node_options: {
    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
      ranges: { begin: 11 end: 15 }
    }
  }
}

# Gets pose left wrist landmark.
node {
  calculator: "SplitNormalizedLandmarkListCalculator"
  input_stream: "landmarks"
  output_stream: "landmarks_left_wrist"
  node_options: {
    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
      ranges: { begin: 15 end: 16 }
    }
  }
}

# Gets pose right wrist landmark.
node {
  calculator: "SplitNormalizedLandmarkListCalculator"
  input_stream: "landmarks"
  output_stream: "landmarks_right_wrist"
  node_options: {
    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
      ranges: { begin: 16 end: 17 }
    }
  }
}

# Gets pose landmarks after wrists.
node {
  calculator: "SplitNormalizedLandmarkListCalculator"
  input_stream: "landmarks"
  output_stream: "landmarks_after_wrist"
  node_options: {
    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
      ranges: { begin: 23 end: 33 }
    }
  }
}

# Gets left hand wrist landmark.
node {
  calculator: "HandWristForPose"
  input_stream: "HAND_LANDMARKS:left_hand_landmarks"
  output_stream: "WRIST_LANDMARK:left_hand_wrist_landmark"
}

# Gets left hand wrist landmark or keep pose wrist landmark if hand was not
# predicted.
node {
  calculator: "MergeCalculator"
  input_stream: "left_hand_wrist_landmark"
  input_stream: "landmarks_left_wrist"
  output_stream: "merged_left_hand_wrist_landmark"
}

# Gets right hand wrist landmark.
node {
  calculator: "HandWristForPose"
  input_stream: "HAND_LANDMARKS:right_hand_landmarks"
  output_stream: "WRIST_LANDMARK:right_hand_wrist_landmark"
}

# Gets right hand wrist landmark or keep pose wrist landmark if hand was not
# predicted.
node {
  calculator: "MergeCalculator"
  input_stream: "right_hand_wrist_landmark"
  input_stream: "landmarks_right_wrist"
  output_stream: "merged_right_hand_wrist_landmark"
}

# Combines pose landmarks all together.
node {
  calculator: "ConcatenateNormalizedLandmarkListCalculator"
  input_stream: "landmarks_before_wrist"
  input_stream: "merged_left_hand_wrist_landmark"
  input_stream: "merged_right_hand_wrist_landmark"
  input_stream: "landmarks_after_wrist"
  output_stream: "landmarks_merged"
  node_options: {
    [type.googleapis.com/mediapipe.ConcatenateVectorCalculatorOptions] {
      only_emit_if_all_present: true
    }
  }
}

# Takes left pose landmarks.
node {
  calculator: "SplitNormalizedLandmarkListCalculator"
  input_stream: "landmarks_merged"
  output_stream: "landmarks_left_side"
  node_options: {
    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
      ranges: { begin: 0 end: 1 }
      ranges: { begin: 2 end: 3 }
      ranges: { begin: 4 end: 5 }
      ranges: { begin: 6 end: 7 }
      ranges: { begin: 8 end: 9 }
      ranges: { begin: 10 end: 11 }
      ranges: { begin: 12 end: 13 }
      ranges: { begin: 14 end: 15 }
      combine_outputs: true
    }
  }
}

# Takes right pose landmarks.
node {
  calculator: "SplitNormalizedLandmarkListCalculator"
  input_stream: "landmarks_merged"
  output_stream: "landmarks_right_side"
  node_options: {
    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
      ranges: { begin: 1 end: 2 }
      ranges: { begin: 3 end: 4 }
      ranges: { begin: 5 end: 6 }
      ranges: { begin: 7 end: 8 }
      ranges: { begin: 9 end: 10 }
      ranges: { begin: 11 end: 12 }
      ranges: { begin: 13 end: 14 }
      ranges: { begin: 15 end: 16 }
      combine_outputs: true
    }
  }
}

# --------------------------------------------------------------------------- #
# ---------------------------------- Pose ----------------------------------- #
# --------------------------------------------------------------------------- #

# Converts pose connections to white lines.
node {
  calculator: "LandmarksToRenderDataCalculator"
  input_stream: "NORM_LANDMARKS:landmarks_merged"
  input_stream: "RENDER_SCALE:render_scale"
  output_stream: "RENDER_DATA:landmarks_render_data"
  node_options: {
    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
      landmark_connections: 0
      landmark_connections: 1
      landmark_connections: 0
      landmark_connections: 2
      landmark_connections: 2
      landmark_connections: 4
      landmark_connections: 1
      landmark_connections: 3
      landmark_connections: 3
      landmark_connections: 5
      landmark_connections: 0
      landmark_connections: 6
      landmark_connections: 1
      landmark_connections: 7
      landmark_connections: 6
      landmark_connections: 7
      landmark_connections: 6
      landmark_connections: 8
      landmark_connections: 7
      landmark_connections: 9
      landmark_connections: 8
      landmark_connections: 10
      landmark_connections: 9
      landmark_connections: 11
      landmark_connections: 10
      landmark_connections: 12
      landmark_connections: 11
      landmark_connections: 13
      landmark_connections: 12
      landmark_connections: 14
      landmark_connections: 13
      landmark_connections: 15
      landmark_connections: 10
      landmark_connections: 14
      landmark_connections: 11
      landmark_connections: 15

      landmark_color { r: 255 g: 255 b: 255 }
      connection_color { r: 255 g: 255 b: 255 }
      thickness: 3.0
      visualize_landmark_depth: false
      utilize_visibility: true
      visibility_threshold: 0.1
    }
  }
}

# Converts pose joints to big white circles.
node {
  calculator: "LandmarksToRenderDataCalculator"
  input_stream: "NORM_LANDMARKS:landmarks_merged"
  input_stream: "RENDER_SCALE:render_scale"
  output_stream: "RENDER_DATA:landmarks_background_joints_render_data"
  node_options: {
    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
      landmark_color { r: 255 g: 255 b: 255 }
      connection_color { r: 255 g: 255 b: 255 }
      thickness: 5.0
      visualize_landmark_depth: false
      utilize_visibility: true
      visibility_threshold: 0.5
    }
  }
}

# Converts pose left side joints to orange circles (inside white ones).
node {
  calculator: "LandmarksToRenderDataCalculator"
  input_stream: "NORM_LANDMARKS:landmarks_left_side"
  input_stream: "RENDER_SCALE:render_scale"
  output_stream: "RENDER_DATA:landmarks_left_joints_render_data"
  node_options: {
    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
      landmark_color { r: 255 g: 138 b: 0 }
      connection_color { r: 255 g: 138 b: 0 }
      thickness: 3.0
      visualize_landmark_depth: false
      utilize_visibility: true
      visibility_threshold: 0.5
    }
  }
}

# Converts pose right side joints to cyan circles (inside white ones).
node {
  calculator: "LandmarksToRenderDataCalculator"
  input_stream: "NORM_LANDMARKS:landmarks_right_side"
  input_stream: "RENDER_SCALE:render_scale"
  output_stream: "RENDER_DATA:landmarks_right_joints_render_data"
  node_options: {
    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
      landmark_color { r: 0 g: 217 b: 231 }
      connection_color { r: 0 g: 217 b: 231 }
      thickness: 3.0
      visualize_landmark_depth: false
      utilize_visibility: true
      visibility_threshold: 0.5
    }
  }
}

# --------------------------------------------------------------------------- #
# ------------------------------- Left hand --------------------------------- #
# --------------------------------------------------------------------------- #

# Converts left hand connections to white lines.
node {
  calculator: "LandmarksToRenderDataCalculator"
  input_stream: "NORM_LANDMARKS:left_hand_landmarks"
  input_stream: "RENDER_SCALE:render_scale"
  output_stream: "RENDER_DATA:left_hand_landmarks_connections_rd"
  node_options: {
    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
      landmark_connections: 0
      landmark_connections: 1
      landmark_connections: 1
      landmark_connections: 2
      landmark_connections: 2
      landmark_connections: 3
      landmark_connections: 3
      landmark_connections: 4
      landmark_connections: 0
      landmark_connections: 5
      landmark_connections: 5
      landmark_connections: 6
      landmark_connections: 6
      landmark_connections: 7
      landmark_connections: 7
      landmark_connections: 8
      landmark_connections: 5
      landmark_connections: 9
      landmark_connections: 9
      landmark_connections: 10
      landmark_connections: 10
      landmark_connections: 11
      landmark_connections: 11
      landmark_connections: 12
      landmark_connections: 9
      landmark_connections: 13
      landmark_connections: 13
      landmark_connections: 14
      landmark_connections: 14
      landmark_connections: 15
      landmark_connections: 15
      landmark_connections: 16
      landmark_connections: 13
      landmark_connections: 17
      landmark_connections: 0
      landmark_connections: 17
      landmark_connections: 17
      landmark_connections: 18
      landmark_connections: 18
      landmark_connections: 19
      landmark_connections: 19
      landmark_connections: 20
      landmark_color { r: 255 g: 255 b: 255 }
      connection_color { r: 255 g: 255 b: 255 }
      thickness: 4.0
      visualize_landmark_depth: false
    }
  }
}

# Converts left hand color joints.
node {
  calculator: "LandmarksToRenderDataCalculator"
  input_stream: "NORM_LANDMARKS:left_hand_landmarks"
  input_stream: "RENDER_SCALE:render_scale"
  output_stream: "RENDER_DATA:left_hand_landmarks_joints_rd"
  node_options: {
    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
      landmark_color { r: 255 g: 138 b: 0 }
      connection_color { r: 255 g: 138 b: 0 }
      thickness: 3.0
      visualize_landmark_depth: false
    }
  }
}

# --------------------------------------------------------------------------- #
# -------------------------------- Right hand ------------------------------- #
# --------------------------------------------------------------------------- #

# Converts right hand connections to white lines.
node {
  calculator: "LandmarksToRenderDataCalculator"
  input_stream: "NORM_LANDMARKS:right_hand_landmarks"
  input_stream: "RENDER_SCALE:render_scale"
  output_stream: "RENDER_DATA:right_hand_landmarks_connections_rd"
  node_options: {
    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
      landmark_connections: 0
      landmark_connections: 1
      landmark_connections: 1
      landmark_connections: 2
      landmark_connections: 2
      landmark_connections: 3
      landmark_connections: 3
      landmark_connections: 4
      landmark_connections: 0
      landmark_connections: 5
      landmark_connections: 5
      landmark_connections: 6
      landmark_connections: 6
      landmark_connections: 7
      landmark_connections: 7
      landmark_connections: 8
      landmark_connections: 5
      landmark_connections: 9
      landmark_connections: 9
      landmark_connections: 10
      landmark_connections: 10
      landmark_connections: 11
      landmark_connections: 11
      landmark_connections: 12
      landmark_connections: 9
      landmark_connections: 13
      landmark_connections: 13
      landmark_connections: 14
      landmark_connections: 14
      landmark_connections: 15
      landmark_connections: 15
      landmark_connections: 16
      landmark_connections: 13
      landmark_connections: 17
      landmark_connections: 0
      landmark_connections: 17
      landmark_connections: 17
      landmark_connections: 18
      landmark_connections: 18
      landmark_connections: 19
      landmark_connections: 19
      landmark_connections: 20
      landmark_color { r: 255 g: 255 b: 255 }
      connection_color { r: 255 g: 255 b: 255 }
      thickness: 4.0
      visualize_landmark_depth: false
    }
  }
}

# Converts right hand color joints.
node {
  calculator: "LandmarksToRenderDataCalculator"
  input_stream: "NORM_LANDMARKS:right_hand_landmarks"
  input_stream: "RENDER_SCALE:render_scale"
  output_stream: "RENDER_DATA:right_hand_landmarks_joints_rd"
  node_options: {
    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
      landmark_color { r: 0 g: 217 b: 231 }
      connection_color { r: 0 g: 217 b: 231 }
      thickness: 3.0
      visualize_landmark_depth: false
    }
  }
}

# Concatenates all render data.
node {
  calculator: "ConcatenateRenderDataVectorCalculator"
  input_stream: "landmarks_render_data"
  input_stream: "landmarks_background_joints_render_data"
  input_stream: "landmarks_left_joints_render_data"
  input_stream: "landmarks_right_joints_render_data"

  # Left hand.
  input_stream: "left_hand_landmarks_connections_rd"
  input_stream: "left_hand_landmarks_joints_rd"

  # Right hand.
  input_stream: "right_hand_landmarks_connections_rd"
  input_stream: "right_hand_landmarks_joints_rd"

  output_stream: "render_data_vector"
}

MainActivity.java

// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.mediapipe.apps.basic;

import android.content.pm.ApplicationInfo;
import android.content.pm.PackageManager;
import android.content.pm.PackageManager.NameNotFoundException;
import android.graphics.SurfaceTexture;
import android.os.Bundle;
import androidx.appcompat.app.AppCompatActivity;
import android.util.Log;
import android.util.Size;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import android.view.View;
import android.view.ViewGroup;

import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList;
import com.google.mediapipe.components.CameraHelper;
import com.google.mediapipe.components.CameraXPreviewHelper;
import com.google.mediapipe.components.ExternalTextureConverter;
import com.google.mediapipe.components.FrameProcessor;
import com.google.mediapipe.components.PermissionHelper;
import com.google.mediapipe.framework.AndroidAssetUtil;
import com.google.mediapipe.glutil.EglManager;

/** Main activity of MediaPipe basic app. */
public class MainActivity extends AppCompatActivity {
  private static final String TAG = "MainActivity";

  // Flips the camera-preview frames vertically by default, before sending them into FrameProcessor
  // to be processed in a MediaPipe graph, and flips the processed frames back when they are
  // displayed. This maybe needed because OpenGL represents images assuming the image origin is at
  // the bottom-left corner, whereas MediaPipe in general assumes the image origin is at the
  // top-left corner.
  // NOTE: use "flipFramesVertically" in manifest metadata to override this behavior.
  private static final boolean FLIP_FRAMES_VERTICALLY = true;

  // Number of output frames allocated in ExternalTextureConverter.
  // NOTE: use "converterNumBuffers" in manifest metadata to override number of buffers. For
  // example, when there is a FlowLimiterCalculator in the graph, number of buffers should be at
  // least `max_in_flight + max_in_queue + 1` (where max_in_flight and max_in_queue are used in
  // FlowLimiterCalculator options). That's because we need buffers for all the frames that are in
  // flight/queue plus one for the next frame from the camera.
  private static final int NUM_BUFFERS = 2;

  static {
    // Load all native libraries needed by the app.
    System.loadLibrary("mediapipe_jni");
    try {
      System.loadLibrary("opencv_java3");
    } catch (java.lang.UnsatisfiedLinkError e) {
      // Some example apps (e.g. template matching) require OpenCV 4.
      System.loadLibrary("opencv_java4");
    }
  }

  // Sends camera-preview frames into a MediaPipe graph for processing, and displays the processed
  // frames onto a {@link Surface}.
  protected FrameProcessor processor;
  // Handles camera access via the {@link CameraX} Jetpack support library.
  protected CameraXPreviewHelper cameraHelper;

  // {@link SurfaceTexture} where the camera-preview frames can be accessed.
  private SurfaceTexture previewFrameTexture;
  // {@link SurfaceView} that displays the camera-preview frames processed by a MediaPipe graph.
  private SurfaceView previewDisplayView;

  // Creates and manages an {@link EGLContext}.
  private EglManager eglManager;
  // Converts the GL_TEXTURE_EXTERNAL_OES texture from Android camera into a regular texture to be
  // consumed by {@link FrameProcessor} and the underlying MediaPipe graph.
  private ExternalTextureConverter converter;

  // ApplicationInfo for retrieving metadata defined in the manifest.
  private ApplicationInfo applicationInfo;

  @Override
  protected void onCreate(Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);
    setContentView(getContentViewLayoutResId());

    try {
      applicationInfo =
          getPackageManager().getApplicationInfo(getPackageName(), PackageManager.GET_META_DATA);
    } catch (NameNotFoundException e) {
      Log.e(TAG, "Cannot find application info: " + e);
    }

    previewDisplayView = new SurfaceView(this);
    setupPreviewDisplayView();

    // Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
    // binary graphs.
    AndroidAssetUtil.initializeNativeAssetManager(this);
    eglManager = new EglManager(null);
    processor =
        new FrameProcessor(
            this,
            eglManager.getNativeContext(),
            applicationInfo.metaData.getString("binaryGraphName"),
            applicationInfo.metaData.getString("inputVideoStreamName"),
            applicationInfo.metaData.getString("outputVideoStreamName"));
    processor
        .getVideoSurfaceOutput()
        .setFlipY(
            applicationInfo.metaData.getBoolean("flipFramesVertically", FLIP_FRAMES_VERTICALLY));
    processor
        .addPacketCallback("pose_landmarks", (packet) -> {
            List<NormalizedLandmarkList> poseLandmarks =
                            PacketGetter.getProtoVector(packet, NormalizedLandmarkList.parser());
            // TODO: process the landmarks

        });

    PermissionHelper.checkAndRequestCameraPermissions(this);
  }

  // Used to obtain the content view for this application. If you are extending this class, and
  // have a custom layout, override this method and return the custom layout.
  protected int getContentViewLayoutResId() {
    return R.layout.activity_main;
  }

  @Override
  protected void onResume() {
    super.onResume();
    converter =
        new ExternalTextureConverter(
            eglManager.getContext(),
            applicationInfo.metaData.getInt("converterNumBuffers", NUM_BUFFERS));
    converter.setFlipY(
        applicationInfo.metaData.getBoolean("flipFramesVertically", FLIP_FRAMES_VERTICALLY));
    converter.setConsumer(processor);
    if (PermissionHelper.cameraPermissionsGranted(this)) {
      startCamera();
    }
  }

  @Override
  protected void onPause() {
    super.onPause();
    converter.close();

    // Hide preview display until we re-open the camera again.
    previewDisplayView.setVisibility(View.GONE);
  }

  @Override
  public void onRequestPermissionsResult(
      int requestCode, String[] permissions, int[] grantResults) {
    super.onRequestPermissionsResult(requestCode, permissions, grantResults);
    PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults);
  }

  protected void onCameraStarted(SurfaceTexture surfaceTexture) {
    previewFrameTexture = surfaceTexture;
    // Make the display view visible to start showing the preview. This triggers the
    // SurfaceHolder.Callback added to (the holder of) previewDisplayView.
    previewDisplayView.setVisibility(View.VISIBLE);
  }

  protected Size cameraTargetResolution() {
    return null; // No preference and let the camera (helper) decide.
  }

  public void startCamera() {
    cameraHelper = new CameraXPreviewHelper();
    cameraHelper.setOnCameraStartedListener(
        surfaceTexture -> {
          onCameraStarted(surfaceTexture);
        });
    CameraHelper.CameraFacing cameraFacing =
        applicationInfo.metaData.getBoolean("cameraFacingFront", false)
            ? CameraHelper.CameraFacing.FRONT
            : CameraHelper.CameraFacing.BACK;
    cameraHelper.startCamera(
        this, cameraFacing, /*unusedSurfaceTexture=*/ null, cameraTargetResolution());
  }

  protected Size computeViewSize(int width, int height) {
    return new Size(width, height);
  }

  protected void onPreviewDisplaySurfaceChanged(
      SurfaceHolder holder, int format, int width, int height) {
    // (Re-)Compute the ideal size of the camera-preview display (the area that the
    // camera-preview frames get rendered onto, potentially with scaling and rotation)
    // based on the size of the SurfaceView that contains the display.
    Size viewSize = computeViewSize(width, height);
    Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
    boolean isCameraRotated = cameraHelper.isCameraRotated();

    // Connect the converter to the camera-preview frames as its input (via
    // previewFrameTexture), and configure the output width and height as the computed
    // display size.
    converter.setSurfaceTextureAndAttachToGLContext(
        previewFrameTexture,
        isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
        isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
  }

  private void setupPreviewDisplayView() {
    previewDisplayView.setVisibility(View.GONE);
    ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
    viewGroup.addView(previewDisplayView);

    previewDisplayView
        .getHolder()
        .addCallback(
            new SurfaceHolder.Callback() {
              @Override
              public void surfaceCreated(SurfaceHolder holder) {
                processor.getVideoSurfaceOutput().setSurface(holder.getSurface());
              }

              @Override
              public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
                onPreviewDisplaySurfaceChanged(holder, format, width, height);
              }

              @Override
              public void surfaceDestroyed(SurfaceHolder holder) {
                processor.getVideoSurfaceOutput().setSurface(null);
              }
            });
  }
}

Hello, I have a question, can these files be tested in android studio? You won't have the interface file, the .xml, by chance.