Created
August 24, 2019 21:34
-
-
Save sandipan1/b724405cfab7eee09df4ba4027c55662 to your computer and use it in GitHub Desktop.
adding absl::make_unique
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// imports | |
#include "mediapipe/framework/calculator_framework.h" | |
#include "mediapipe/framework/port/status.h" | |
#include "mediapipe/framework/formats/image_frame.h" | |
#include "mediapipe/framework/port/ret_check.h" | |
#include <vector> | |
/* | |
node { | |
calculator:"ColorSliderCalculator" | |
input_stream: "Red" | |
input_stream: "Green" | |
input_stream: "Blue" | |
output_stream:"RGB_OUT:out_array" | |
} | |
*/ | |
//calculator ColoSlider | |
namespace mediapipe { | |
class ColorSliderCalculator : public CalculatorBase { | |
public: | |
ColorSliderCalculator() = default; | |
~ColorSliderCalculator() override = default; | |
//ColorSliderCalculator() : initialized_(false){} | |
static ::mediapipe::Status GetContract(CalculatorContract* cc); | |
::mediapipe::Status Open(CalculatorContext* cc) override; | |
::mediapipe::Status Process(CalculatorContext* cc) override; | |
void make_array(int r,int g,int b,std::array<int,3>* out); | |
}; | |
REGISTER_CALCULATOR(ColorSliderCalculator); | |
//static | |
::mediapipe::Status ColorSliderCalculator::GetContract (CalculatorContract *cc){ | |
cc->Inputs().Index(0).Set<int>(); | |
cc->Inputs().Index(1).Set<int>(); | |
cc->Inputs().Index(2).Set<int>(); | |
if (cc->Outputs().HasTag("RGB_OUT")){ | |
cc->Outputs().Tag("RGB_OUT").Set<std::int<int,3>>(); | |
} | |
return ::mediapipe::OkStatus(); | |
} | |
::mediapipe::Status ColorSliderCalculator::Open(CalculatorContext* cc) { | |
cc->SetOffset(TimestampDiff(0)); | |
return ::mediapipe::OkStatus(); | |
} | |
::mediapipe::Status ColorSliderCalculator::Process(CalculatorContext* cc) { | |
if (cc->Inputs().NumEntries() == 0) { | |
return tool::StatusStop(); | |
} | |
int red_buffer = cc->Inputs().Index(0).Value().Get<int>(); | |
int green_buffer = cc->Inputs().Index(1).Value().Get<int>(); | |
int blue_buffer = cc->Inputs().Index(2).Value().Get<int>(); | |
auto out = absl::make_unique<std::array<int,3>>(); | |
make_array(red_buffer,green_buffer,blue_buffer, out.get()); | |
// auto rgb = absl::make_unique<std::array<int, 3>>(); | |
const auto out = {red_buffer, green_buffer,blue_buffer}; | |
cc->Outputs().Tag("RGB_OUT").Add(&out, cc->InputTimestamp()); | |
return ::mediapipe::OkStatus(); | |
} | |
void make_array(int r,int g,int b,std::array<int>* out){ | |
(*out)[0] = r; | |
(*out)[1] =g; | |
(*out)[2] =b; | |
} | |
} //end namespace | |
// REGISTER_CALCULATOR(::mediapipe::ColorSliderCalculator); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
08-25 03:01:49.284 2672 2672 I Timeline: Timeline: Activity_launch_request time:14160307 intent:Intent { act=android.intent.action.MAIN cat=[android.intent.category.LAUNCHER] flg=0x10200000 cmp=com.google.mediapipe.apps.hairsegmentationgpu/.MainActivity bnds=[238,1084][439,1416] } | |
08-25 03:01:49.453 20996 20996 E AndroidRuntime: Process: com.google.mediapipe.apps.hairsegmentationgpu, PID: 20996 | |
08-25 03:01:49.453 20996 20996 E AndroidRuntime: java.lang.UnsatisfiedLinkError: dlopen failed: cannot locate symbol "_ZN9mediapipe21ColorSliderCalculator10make_arrayEiiiPNSt6__ndk15arrayIiLm3EEE" referenced by "/data/app/com.google.mediapipe.apps.hairsegmentationgpu-1/lib/arm64/libmediapipe_jni.so"... | |
08-25 03:01:49.453 20996 20996 E AndroidRuntime: at com.google.mediapipe.apps.hairsegmentationgpu.MainActivity.<clinit>(MainActivity.java:60) | |
08-25 03:01:49.457 2582 2749 I WtProcessController: Receive am_crash event for pid: 20996 pkg = com.google.mediapipe.apps.hairsegmentationgpu | |
08-25 03:01:49.458 1555 2563 W ActivityManager: Force finishing activity com.google.mediapipe.apps.hairsegmentationgpu/.MainActivity | |
08-25 03:01:49.983 1555 1570 W ActivityManager: Activity pause timeout for ActivityRecord{49a2e31 u0 com.google.mediapipe.apps.hairsegmentationgpu/.MainActivity t49754 f} | |
08-25 03:02:00.385 1555 1570 W ActivityManager: Activity destroy timeout for ActivityRecord{49a2e31 u0 com.google.mediapipe.apps.hairsegmentationgpu/.MainActivity t49754 f} | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Copyright 2019 The MediaPipe Authors. | |
// | |
// Licensed under the Apache License, Version 2.0 (the "License"); | |
// you may not use this file except in compliance with the License. | |
// You may obtain a copy of the License at | |
// | |
// http://www.apache.org/licenses/LICENSE-2.0 | |
// | |
// Unless required by applicable law or agreed to in writing, software | |
// distributed under the License is distributed on an "AS IS" BASIS, | |
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
// See the License for the specific language governing permissions and | |
// limitations under the License. | |
#include <vector> | |
#include "mediapipe/calculators/image/recolor_calculator.pb.h" | |
#include "mediapipe/framework/calculator_framework.h" | |
#include "mediapipe/framework/formats/image_frame.h" | |
#include "mediapipe/framework/port/ret_check.h" | |
#include "mediapipe/framework/port/status.h" | |
#include "mediapipe/util/color.pb.h" | |
#if defined(__ANDROID__) | |
#include "mediapipe/gpu/gl_calculator_helper.h" | |
#include "mediapipe/gpu/gl_simple_shaders.h" | |
#include "mediapipe/gpu/gpu_buffer.h" | |
#include "mediapipe/gpu/shader_util.h" | |
#endif // __ANDROID__ | |
namespace { | |
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; | |
} // namespace | |
namespace mediapipe { | |
// A calculator to recolor a masked area of an image to a specified color. | |
// | |
// A mask image is used to specify where to overlay a user defined color. | |
// The luminance of the input image is used to adjust the blending weight, | |
// to help preserve image textures. | |
// | |
// TODO implement cpu support. | |
// | |
// Inputs: | |
// One of the following IMAGE tags: | |
// IMAGE: An ImageFrame input image, RGB or RGBA. | |
// IMAGE_GPU: A GpuBuffer input image, RGBA. | |
// One of the following MASK tags: | |
// MASK: An ImageFrame input mask, Gray, RGB or RGBA. | |
// MASK_GPU: A GpuBuffer input mask, RGBA. | |
// RGB_ARRAY : rgb array that contain r g b values | |
// Output: | |
// One of the following IMAGE tags: | |
// IMAGE: An ImageFrame output image. | |
// IMAGE_GPU: A GpuBuffer output image. | |
// | |
// Options: | |
// color_rgb (required): A map of RGB values [0-255]. | |
// mask_channel (optional): Which channel of mask image is used [RED or ALPHA] | |
// | |
// Usage example: | |
// node { | |
// calculator: "RecolorCalculator" | |
// input_stream: "IMAGE_GPU:input_image" | |
// input_stream: "MASK_GPU:input_mask" | |
// input_stream: "RGB_ARRAY" | |
// output_stream: "IMAGE_GPU:output_image" | |
// node_options: { | |
// [mediapipe.RecolorCalculatorOptions] { | |
// color { r: 0 g: 0 b: 255 } | |
// mask_channel: RED | |
// } | |
// } | |
// } | |
// | |
class RecolorCalculator : public CalculatorBase { | |
public: | |
RecolorCalculator() = default; | |
~RecolorCalculator() override = default; | |
static ::mediapipe::Status GetContract(CalculatorContract* cc); | |
::mediapipe::Status Open(CalculatorContext* cc) override; | |
::mediapipe::Status Process(CalculatorContext* cc) override; | |
::mediapipe::Status Close(CalculatorContext* cc) override; | |
private: | |
::mediapipe::Status LoadOptions(CalculatorContext* cc); | |
::mediapipe::Status InitGpu(CalculatorContext* cc); | |
::mediapipe::Status RenderGpu(CalculatorContext* cc); | |
::mediapipe::Status RenderCpu(CalculatorContext* cc); | |
void GlRender(); | |
bool initialized_ = false; | |
std::vector<float> color_; | |
std::vector<float> my_color; | |
mediapipe::RecolorCalculatorOptions::MaskChannel mask_channel_; | |
bool use_gpu_ = false; | |
#if defined(__ANDROID__) | |
mediapipe::GlCalculatorHelper gpu_helper_; | |
GLuint program_ = 0; | |
#endif // __ANDROID__ | |
}; | |
REGISTER_CALCULATOR(RecolorCalculator); | |
// static | |
::mediapipe::Status RecolorCalculator::GetContract(CalculatorContract* cc) { | |
RET_CHECK(!cc->Inputs().GetTags().empty()); | |
RET_CHECK(!cc->Outputs().GetTags().empty()); | |
#if defined(__ANDROID__) | |
if (cc->Inputs().HasTag("IMAGE_GPU")) { | |
cc->Inputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>(); | |
} | |
#endif // __ANDROID__ | |
if (cc->Inputs().HasTag("IMAGE")) { | |
cc->Inputs().Tag("IMAGE").Set<ImageFrame>(); | |
} | |
#if defined(__ANDROID__) | |
if (cc->Inputs().HasTag("MASK_GPU")) { | |
cc->Inputs().Tag("MASK_GPU").Set<mediapipe::GpuBuffer>(); | |
} | |
#endif // __ANDROID__ | |
if (cc->Inputs().HasTag("MASK")) { | |
cc->Inputs().Tag("MASK").Set<ImageFrame>(); | |
} | |
if (cc->Inputs().HasTag("RGB_ARRAY")) { | |
cc->Inputs().Tag("RGB_ARRAY").Set<std::array<int,3>>(); | |
} | |
#if defined(__ANDROID__) | |
if (cc->Outputs().HasTag("IMAGE_GPU")) { | |
cc->Outputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>(); | |
} | |
#endif // __ANDROID__ | |
if (cc->Outputs().HasTag("IMAGE")) { | |
cc->Outputs().Tag("IMAGE").Set<ImageFrame>(); | |
} | |
#if defined(__ANDROID__) | |
RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); | |
#endif // __ANDROID__ | |
return ::mediapipe::OkStatus(); | |
} | |
::mediapipe::Status RecolorCalculator::Open(CalculatorContext* cc) { | |
if (cc->Inputs().HasTag("IMAGE_GPU")) { | |
use_gpu_ = true; | |
#if defined(__ANDROID__) | |
RETURN_IF_ERROR(gpu_helper_.Open(cc)); | |
#endif // __ANDROID__ | |
} | |
RETURN_IF_ERROR(LoadOptions(cc)); | |
return ::mediapipe::OkStatus(); | |
} | |
::mediapipe::Status RecolorCalculator::Process(CalculatorContext* cc) { | |
if (use_gpu_) { | |
#if defined(__ANDROID__) | |
RETURN_IF_ERROR( | |
gpu_helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status { | |
if (!initialized_) { | |
RETURN_IF_ERROR(InitGpu(cc)); | |
initialized_ = true; | |
} | |
RETURN_IF_ERROR(RenderGpu(cc)); | |
return ::mediapipe::OkStatus(); | |
})); | |
#endif // __ANDROID__ | |
} else { | |
RETURN_IF_ERROR(RenderCpu(cc)); | |
} | |
return ::mediapipe::OkStatus(); | |
} | |
::mediapipe::Status RecolorCalculator::Close(CalculatorContext* cc) { | |
#if defined(__ANDROID__) | |
gpu_helper_.RunInGlContext([this] { | |
if (program_) glDeleteProgram(program_); | |
program_ = 0; | |
}); | |
#endif // __ANDROID__ | |
return ::mediapipe::OkStatus(); | |
} | |
::mediapipe::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) { | |
return ::mediapipe::UnimplementedError("CPU support is not implemented yet."); | |
} | |
::mediapipe::Status RecolorCalculator::RenderGpu(CalculatorContext* cc) { | |
if (cc->Inputs().Tag("MASK_GPU").IsEmpty()) { | |
return ::mediapipe::OkStatus(); | |
} | |
#if defined(__ANDROID__) | |
// Get inputs and setup output. | |
const Packet& input_packet = cc->Inputs().Tag("IMAGE_GPU").Value(); | |
const Packet& mask_packet = cc->Inputs().Tag("MASK_GPU").Value(); | |
const Packet& rgb_packet = cc->Inputs().Tag("RGB_ARRAY").Value(); | |
const auto& input_buffer = input_packet.Get<mediapipe::GpuBuffer>(); | |
const auto& mask_buffer = mask_packet.Get<mediapipe::GpuBuffer>(); | |
const auto& rgb_buffer = rgb_packet.Get<std::array<int,3>>(); | |
my_color.push_back(rgb_buffer[0] / 255.0); | |
my_color.push_back(rgb_buffer[1] / 255.0); | |
my_color.push_back(rgb_buffer[2] / 255.0); | |
auto img_tex = gpu_helper_.CreateSourceTexture(input_buffer); | |
auto mask_tex = gpu_helper_.CreateSourceTexture(mask_buffer); | |
auto dst_tex = gpu_helper_.CreateDestinationTexture(img_tex.width(), img_tex.height()); | |
// Run recolor shader on GPU. | |
{ | |
gpu_helper_.BindFramebuffer(dst_tex); // GL_TEXTURE0 | |
glActiveTexture(GL_TEXTURE1); | |
glBindTexture(img_tex.target(), img_tex.name()); | |
glActiveTexture(GL_TEXTURE2); | |
glBindTexture(mask_tex.target(), mask_tex.name()); | |
GlRender(); | |
glActiveTexture(GL_TEXTURE2); | |
glBindTexture(GL_TEXTURE_2D, 0); | |
glActiveTexture(GL_TEXTURE1); | |
glBindTexture(GL_TEXTURE_2D, 0); | |
glFlush(); | |
} | |
// Send result image in GPU packet. | |
auto output = dst_tex.GetFrame<mediapipe::GpuBuffer>(); | |
cc->Outputs().Tag("IMAGE_GPU").Add(output.release(), cc->InputTimestamp()); | |
// Cleanup | |
img_tex.Release(); | |
mask_tex.Release(); | |
dst_tex.Release(); | |
#endif // __ANDROID__ | |
return ::mediapipe::OkStatus(); | |
} | |
void RecolorCalculator::GlRender() { | |
#if defined(__ANDROID__) | |
static const GLfloat square_vertices[] = { | |
-1.0f, -1.0f, // bottom left | |
1.0f, -1.0f, // bottom right | |
-1.0f, 1.0f, // top left | |
1.0f, 1.0f, // top right | |
}; | |
static const GLfloat texture_vertices[] = { | |
0.0f, 0.0f, // bottom left | |
1.0f, 0.0f, // bottom right | |
0.0f, 1.0f, // top left | |
1.0f, 1.0f, // top right | |
}; | |
// program | |
glUseProgram(program_); | |
glUniform3f(glGetUniformLocation(program_, "recolor"), my_color[0], my_color[1], my_color[2]); | |
// vertex storage | |
GLuint vbo[2]; | |
glGenBuffers(2, vbo); | |
GLuint vao; | |
glGenVertexArrays(1, &vao); | |
glBindVertexArray(vao); | |
// vbo 0 | |
glBindBuffer(GL_ARRAY_BUFFER, vbo[0]); | |
glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), square_vertices, | |
GL_STATIC_DRAW); | |
glEnableVertexAttribArray(ATTRIB_VERTEX); | |
glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr); | |
// vbo 1 | |
glBindBuffer(GL_ARRAY_BUFFER, vbo[1]); | |
glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), texture_vertices, | |
GL_STATIC_DRAW); | |
glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION); | |
glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr); | |
// draw | |
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); | |
// cleanup | |
glDisableVertexAttribArray(ATTRIB_VERTEX); | |
glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION); | |
glBindBuffer(GL_ARRAY_BUFFER, 0); | |
glBindVertexArray(0); | |
glDeleteVertexArrays(1, &vao); | |
glDeleteBuffers(2, vbo); | |
#endif // __ANDROID__ | |
} | |
::mediapipe::Status RecolorCalculator::LoadOptions(CalculatorContext* cc) { | |
const auto& options = cc->Options<mediapipe::RecolorCalculatorOptions>(); | |
mask_channel_ = options.mask_channel(); | |
if (!options.has_color()) RET_CHECK_FAIL() << "Missing color option."; | |
color_.push_back(options.color().r() / 255.0); | |
color_.push_back(options.color().g() / 255.0); | |
color_.push_back(options.color().b() / 255.0); | |
return ::mediapipe::OkStatus(); | |
} | |
::mediapipe::Status RecolorCalculator::InitGpu(CalculatorContext* cc) { | |
#if defined(__ANDROID__) | |
const GLint attr_location[NUM_ATTRIBUTES] = { | |
ATTRIB_VERTEX, | |
ATTRIB_TEXTURE_POSITION, | |
}; | |
const GLchar* attr_name[NUM_ATTRIBUTES] = { | |
"position", | |
"texture_coordinate", | |
}; | |
std::string mask_component; | |
switch (mask_channel_) { | |
case mediapipe::RecolorCalculatorOptions_MaskChannel_UNKNOWN: | |
case mediapipe::RecolorCalculatorOptions_MaskChannel_RED: | |
mask_component = "r"; | |
break; | |
case mediapipe::RecolorCalculatorOptions_MaskChannel_ALPHA: | |
mask_component = "a"; | |
break; | |
} | |
// A shader to blend a color onto an image where the mask > 0. | |
// The blending is based on the input image luminosity. | |
const std::string frag_src = R"( | |
#if __VERSION__ < 130 | |
#define in varying | |
#endif // __VERSION__ < 130 | |
#ifdef GL_ES | |
#define fragColor gl_FragColor | |
precision highp float; | |
#else | |
#define lowp | |
#define mediump | |
#define highp | |
#define texture2D texture | |
out vec4 fragColor; | |
#endif // defined(GL_ES) | |
#define MASK_COMPONENT )" + mask_component + | |
R"( | |
in vec2 sample_coordinate; | |
uniform sampler2D frame; | |
uniform sampler2D mask; | |
uniform vec3 recolor; | |
void main() { | |
vec4 weight = texture2D(mask, sample_coordinate); | |
vec4 color1 = texture2D(frame, sample_coordinate); | |
vec4 color2 = vec4(recolor, 1.0); | |
float luminance = dot(color1.rgb, vec3(0.299, 0.587, 0.114)); | |
float mix_value = weight.MASK_COMPONENT * luminance; | |
fragColor = mix(color1, color2, mix_value); | |
} | |
)"; | |
// shader program and params | |
mediapipe::GlhCreateProgram(mediapipe::kBasicVertexShader, frag_src.c_str(), | |
NUM_ATTRIBUTES, &attr_name[0], attr_location, | |
&program_); | |
RET_CHECK(program_) << "Problem initializing the program."; | |
glUseProgram(program_); | |
glUniform1i(glGetUniformLocation(program_, "frame"), 1); | |
glUniform1i(glGetUniformLocation(program_, "mask"), 2); | |
glUniform3f(glGetUniformLocation(program_, "recolor"), color_[0], color_[1], | |
color_[2]); | |
#endif // __ANDROID__ | |
return ::mediapipe::OkStatus(); | |
} | |
} // namespace mediapipe |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment