MediaPipe + OpenCVで指先から炎エフェクトを出す

multi_hand_tracking_mad_burnish.cc

// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// An example of sending OpenCV webcam frames into a MediaPipe graph.

#include <cstdlib>
#include <memory>
#include <vector>
#include <cmath>
#include <stddef.h>
#include <stdlib.h>

#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_video_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"

constexpr char kWindowName[] = "MediaPipe";
constexpr char kCalculatorGraphConfigFile[] =
    "mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt";
// Input and output streams.
constexpr char kInputStream[] = "input_video";
constexpr char kOutputStream[] = "output_video";
constexpr char kMultiHandLandmarksOutputStream[] = "multi_hand_landmarks";

DEFINE_string(input_video_path, "",
              "Full path of video to load. "
              "If not provided, attempt to use a webcam.");
DEFINE_string(output_video_path, "",
              "Full path of where to save result (.mp4 only). "
              "If not provided, show result in a window.");

cv::Rect rect; 
int counter = 0;

// Gamma Correction 
// 全体的に画像を暗くして炎を目立たせたい時に使う
// void gamma(cv::Mat &srcImg){  
  
//     double gamma = 1.1;
//     uchar  lut[256];
//     for (int i = 0; i < 256; i++) {
//       lut[i] = pow(i / 255.0, gamma) * 255.0;
//     }
//     cv::LUT(srcImg, cv::Mat(1, 256, CV_8UC1, lut), srcImg);
// }

// Read fire image
cv::Mat getCurrentFireImage(int cnt_offset)
{
    counter++;
    int img_num = (counter + cnt_offset*2) % 50;
    std::string file_name = "image_file_name" + std::to_string(img_num) + ".png";
    return cv::imread(file_name, cv::IMREAD_UNCHANGED);
}

// Add fire image
void lightFire(cv::Mat &srcImg, mediapipe::NormalizedLandmark finger, int cnt_offset)
{
    cv::Mat img_rgb, img_aaa, img_1ma;
	  std::vector<cv::Mat>planes_rgba, planes_rgb, planes_aaa, planes_1ma;
    std::vector<cv::Point2f> tgtPt;
    int maxVal = pow(2, 8*srcImg.elemSize1())-1;

#if DEBUG
    LOG(INFO) << "lightFire [width]: " << srcImg.cols;
    LOG(INFO) << "lightFire [height]: " << srcImg.rows;
    LOG(INFO) << "lightFire [x]: " << finger.x() * srcImg.cols;
    LOG(INFO) << "lightFire [y]: " << finger.y() * srcImg.rows;
#endif

    cv::Mat overlay = getCurrentFireImage(cnt_offset);
    resize(overlay, overlay, cv::Size(), 0.4, 0.4);

    try
    {
        // check NULL or 0
        CV_Assert((finger.x() >= 0) && (finger.y() >= 0));
        CV_Assert((srcImg.rows > 0) && (srcImg.cols > 0));
        CV_Assert(((finger.x()*srcImg.cols) + overlay.cols <= srcImg.cols));
        CV_Assert(((finger.y()*srcImg.rows) + overlay.rows <= srcImg.rows));

        //Set coordinates for foreground image
        //参考: https://qiita.com/locutus1990/items/f0e2e62a7c4935dffe7c
        int ltx = (finger.x()*srcImg.cols);
        int lty = (finger.y()*srcImg.rows);
        int ww  = overlay.cols;
        int hh  = overlay.rows;

        // 炎画像のを表示する座標（調整してください）
        tgtPt.push_back(cv::Point2f(ltx-ww+ww/2   , lty-hh+hh/5)); 
        tgtPt.push_back(cv::Point2f(ltx+ww/2, lty-hh+hh/5));
        tgtPt.push_back(cv::Point2f(ltx+ww/2, lty+hh/5));
        tgtPt.push_back(cv::Point2f(ltx-ww+ww/2   , lty+hh/5));

        //Create T-matrix
        std::vector<cv::Point2f>srcPt;
        srcPt.push_back( cv::Point2f(0, 0) );
        srcPt.push_back( cv::Point2f(overlay.cols-1, 0) );
        srcPt.push_back( cv::Point2f(overlay.cols-1, overlay.rows-1) );
        srcPt.push_back( cv::Point2f(0, overlay.rows-1) );
        cv::Mat mat = cv::getPerspectiveTransform(srcPt, tgtPt);

        cv::Mat alpha0(srcImg.rows, srcImg.cols, overlay.type());
        alpha0 = cv::Scalar::all(0);
        cv::warpPerspective(overlay, alpha0, mat, alpha0.size(), cv::INTER_CUBIC, cv::BORDER_TRANSPARENT);

        cv::split(alpha0, planes_rgba);

        //RGBA画像をRGBに変換   
        planes_rgb.push_back(planes_rgba[0]);
        planes_rgb.push_back(planes_rgba[1]);
        planes_rgb.push_back(planes_rgba[2]);
        merge(planes_rgb, img_rgb);

        //Extract RGBA values  
        planes_aaa.push_back(planes_rgba[3]);
        planes_aaa.push_back(planes_rgba[3]);
        planes_aaa.push_back(planes_rgba[3]);
        merge(planes_aaa, img_aaa);

        //alpha channel for background image  
        planes_1ma.push_back(maxVal-planes_rgba[3]);
        planes_1ma.push_back(maxVal-planes_rgba[3]);
        planes_1ma.push_back(maxVal-planes_rgba[3]);
        merge(planes_1ma, img_1ma);

	      srcImg = img_rgb.mul(img_aaa, 1.0/(double)maxVal) + srcImg.mul(img_1ma, 1.0/(double)maxVal);  
        //gamma(srcImg);     
    }
    catch(const std::exception& e)
    {
        std::cerr << " LightFireException : " << e.what() << '\n';
    }
}

void createCandleFire(cv::Mat &base, cv::Rect rect, std::vector<mediapipe::NormalizedLandmarkList> landmarks){
    int loopIndex[] = {4, 8, 12, 16, 20}; // Thumb, Index, Middle, Ring, Little
    int handIndex = 0;

    for (const auto& single_hand_landmarks: landmarks) {
        handIndex++;
        for (int i = 0; i < sizeof loopIndex/sizeof loopIndex[0]; i++){
            const mediapipe::NormalizedLandmark finger = single_hand_landmarks.landmark(loopIndex[i]);

            LOG(INFO) << "lightFire : index: " << handIndex << "[row-x]: " << finger.x(); //　ここで0
            LOG(INFO) << "lightFire : index: " << handIndex << "[row-y]: " << finger.y();

            if (finger.x() != NULL && finger.y() != NULL)
            {
                if (finger.x() > 0 && finger.y() > 0)
                {
                    lightFire(base, finger, loopIndex[i]);
                    LOG(INFO) << "Light a Fire! : " << loopIndex[i];
                }
            }
        }
    }
}

::mediapipe::Status RunMPPGraph(
    std::unique_ptr<::mediapipe::CalculatorGraph> graph) {

  LOG(INFO) << "Initialize the camera or load the video.";
  cv::VideoCapture capture;
  const bool load_video = !FLAGS_input_video_path.empty();
  if (load_video) {
    capture.open(FLAGS_input_video_path);
  } else {
    capture.open(0);
  }
  RET_CHECK(capture.isOpened());

  cv::VideoWriter writer;
  const bool save_video = !FLAGS_output_video_path.empty();
  if (!save_video) {
    cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
#if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2)
    capture.set(cv::CAP_PROP_FRAME_WIDTH, 640);
    capture.set(cv::CAP_PROP_FRAME_HEIGHT, 480);
    capture.set(cv::CAP_PROP_FPS, 30);
#endif
  }

  LOG(INFO) << "Start running the calculator graph.";
  ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller poller,
                   graph->AddOutputStreamPoller(kOutputStream));
  ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller multi_hand_landmarks_poller,
                   graph->AddOutputStreamPoller(kMultiHandLandmarksOutputStream));
  MP_RETURN_IF_ERROR(graph->StartRun({}));

  LOG(INFO) << "Start grabbing and processing frames.";
  bool grab_frames = true;
  while (grab_frames) {
    // Capture opencv camera or video frame.
    cv::Mat camera_frame_raw;
    capture >> camera_frame_raw;
    if (camera_frame_raw.empty()) break;  // End of video.
    cv::Mat camera_frame;
    cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB);
    if (!load_video) {
      cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1);
    }

    // Wrap Mat into an ImageFrame.
    auto input_frame = absl::make_unique<::mediapipe::ImageFrame>(
        ::mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows,
        ::mediapipe::ImageFrame::kDefaultAlignmentBoundary);
    cv::Mat input_frame_mat = ::mediapipe::formats::MatView(input_frame.get());
    camera_frame.copyTo(input_frame_mat);

    // Send image packet into the graph.
    size_t frame_timestamp_us =
        (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6;
    MP_RETURN_IF_ERROR(graph->AddPacketToInputStream(
        kInputStream, ::mediapipe::Adopt(input_frame.release())
                          .At(::mediapipe::Timestamp(frame_timestamp_us))));

    // Get the graph result packet, or stop if that fails.
    ::mediapipe::Packet packet;
    if (!poller.Next(&packet)) break;
    auto& output_frame = packet.Get<::mediapipe::ImageFrame>();

    // Get the packet containing multi_hand_landmarks.
    ::mediapipe::Packet multi_hand_landmarks_packet;
    if (!multi_hand_landmarks_poller.Next(&multi_hand_landmarks_packet)) break;
    std::vector<mediapipe::NormalizedLandmarkList> multi_hand_landmarks =
        multi_hand_landmarks_packet.Get<
            std::vector<::mediapipe::NormalizedLandmarkList>>();

#if DEBUG
    LOG(INFO) << "#Multi Hand landmarks: " << multi_hand_landmarks.size();
    int hand_id = 0;
    for (const auto& single_hand_landmarks: multi_hand_landmarks) {
      ++hand_id;
      LOG(INFO) << "Hand [" << hand_id << "]:";
      for (int i = 0; i < single_hand_landmarks.landmark_size(); ++i) {
        const auto& landmark = single_hand_landmarks.landmark(i);
        LOG(INFO) << "\tLandmark [" << i << "]: ("
                  << landmark.x() << ", "
                  << landmark.y() << ", "
                  << landmark.z() << ")";
      }
    }
#endif

    // Convert back to opencv for display or saving.
    cv::Mat output_frame_mat = ::mediapipe::formats::MatView(&output_frame); // Get frame from MediaPipe

    // Get Window Image 
    rect = cv::getWindowImageRect(kWindowName);

    // Light On Fire 
    createCandleFire(output_frame_mat, rect, multi_hand_landmarks);
    LOG(INFO) << "multi_hand_landmarks.size() == 1";
    

    cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR);
    if (save_video) {
      if (!writer.isOpened()) {
        LOG(INFO) << "Prepare video writer.";
        writer.open(FLAGS_output_video_path,
                    ::mediapipe::fourcc('a', 'v', 'c', '1'),  // .mp4
                    capture.get(cv::CAP_PROP_FPS), output_frame_mat.size());
        RET_CHECK(writer.isOpened());
      }
      writer.write(output_frame_mat);
    } else {
      cv::imshow(kWindowName, output_frame_mat);
      // Press any key to exit.
      const int pressed_key = cv::waitKey(5);
      if (pressed_key >= 0 && pressed_key != 255) grab_frames = false;
    }
  }

  LOG(INFO) << "Shutting down.";
  if (writer.isOpened()) writer.release();
  MP_RETURN_IF_ERROR(graph->CloseInputStream(kInputStream));
  return graph->WaitUntilDone();
}

::mediapipe::Status InitializeAndRunMPPGraph() {

  std::string calculator_graph_config_contents;
  MP_RETURN_IF_ERROR(::mediapipe::file::GetContents(
      kCalculatorGraphConfigFile, &calculator_graph_config_contents));
  LOG(INFO) << "Get calculator graph config contents: "
            << calculator_graph_config_contents;
  mediapipe::CalculatorGraphConfig config =
      mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
          calculator_graph_config_contents);

  LOG(INFO) << "Initialize the calculator graph.";
  std::unique_ptr<::mediapipe::CalculatorGraph> graph =
      absl::make_unique<::mediapipe::CalculatorGraph>();
  MP_RETURN_IF_ERROR(graph->Initialize(config));

  return RunMPPGraph(std::move(graph));
}

int main(int argc, char** argv) {
  google::InitGoogleLogging(argv[0]);
  gflags::ParseCommandLineFlags(&argc, &argv, true);
  ::mediapipe::Status run_status = InitializeAndRunMPPGraph();
  if (!run_status.ok()) {
    LOG(ERROR) << "Failed to run the graph: " << run_status.message();
    return EXIT_FAILURE;
  } else {
    LOG(INFO) << "Success!";
  }
  return EXIT_SUCCESS;
}