Skip to content

Instantly share code, notes, and snippets.

@sandipan1
Created August 24, 2019 21:34
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save sandipan1/b724405cfab7eee09df4ba4027c55662 to your computer and use it in GitHub Desktop.
Save sandipan1/b724405cfab7eee09df4ba4027c55662 to your computer and use it in GitHub Desktop.
adding absl::make_unique
// imports
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/port/ret_check.h"
#include <vector>
/*
node {
calculator:"ColorSliderCalculator"
input_stream: "Red"
input_stream: "Green"
input_stream: "Blue"
output_stream:"RGB_OUT:out_array"
}
*/
//calculator ColoSlider
namespace mediapipe {
class ColorSliderCalculator : public CalculatorBase {
public:
ColorSliderCalculator() = default;
~ColorSliderCalculator() override = default;
//ColorSliderCalculator() : initialized_(false){}
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
void make_array(int r,int g,int b,std::array<int,3>* out);
};
REGISTER_CALCULATOR(ColorSliderCalculator);
//static
::mediapipe::Status ColorSliderCalculator::GetContract (CalculatorContract *cc){
cc->Inputs().Index(0).Set<int>();
cc->Inputs().Index(1).Set<int>();
cc->Inputs().Index(2).Set<int>();
if (cc->Outputs().HasTag("RGB_OUT")){
cc->Outputs().Tag("RGB_OUT").Set<std::int<int,3>>();
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status ColorSliderCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
}
::mediapipe::Status ColorSliderCalculator::Process(CalculatorContext* cc) {
if (cc->Inputs().NumEntries() == 0) {
return tool::StatusStop();
}
int red_buffer = cc->Inputs().Index(0).Value().Get<int>();
int green_buffer = cc->Inputs().Index(1).Value().Get<int>();
int blue_buffer = cc->Inputs().Index(2).Value().Get<int>();
auto out = absl::make_unique<std::array<int,3>>();
make_array(red_buffer,green_buffer,blue_buffer, out.get());
// auto rgb = absl::make_unique<std::array<int, 3>>();
const auto out = {red_buffer, green_buffer,blue_buffer};
cc->Outputs().Tag("RGB_OUT").Add(&out, cc->InputTimestamp());
return ::mediapipe::OkStatus();
}
void make_array(int r,int g,int b,std::array<int>* out){
(*out)[0] = r;
(*out)[1] =g;
(*out)[2] =b;
}
} //end namespace
// REGISTER_CALCULATOR(::mediapipe::ColorSliderCalculator);
08-25 03:01:49.284 2672 2672 I Timeline: Timeline: Activity_launch_request time:14160307 intent:Intent { act=android.intent.action.MAIN cat=[android.intent.category.LAUNCHER] flg=0x10200000 cmp=com.google.mediapipe.apps.hairsegmentationgpu/.MainActivity bnds=[238,1084][439,1416] }
08-25 03:01:49.453 20996 20996 E AndroidRuntime: Process: com.google.mediapipe.apps.hairsegmentationgpu, PID: 20996
08-25 03:01:49.453 20996 20996 E AndroidRuntime: java.lang.UnsatisfiedLinkError: dlopen failed: cannot locate symbol "_ZN9mediapipe21ColorSliderCalculator10make_arrayEiiiPNSt6__ndk15arrayIiLm3EEE" referenced by "/data/app/com.google.mediapipe.apps.hairsegmentationgpu-1/lib/arm64/libmediapipe_jni.so"...
08-25 03:01:49.453 20996 20996 E AndroidRuntime: at com.google.mediapipe.apps.hairsegmentationgpu.MainActivity.<clinit>(MainActivity.java:60)
08-25 03:01:49.457 2582 2749 I WtProcessController: Receive am_crash event for pid: 20996 pkg = com.google.mediapipe.apps.hairsegmentationgpu
08-25 03:01:49.458 1555 2563 W ActivityManager: Force finishing activity com.google.mediapipe.apps.hairsegmentationgpu/.MainActivity
08-25 03:01:49.983 1555 1570 W ActivityManager: Activity pause timeout for ActivityRecord{49a2e31 u0 com.google.mediapipe.apps.hairsegmentationgpu/.MainActivity t49754 f}
08-25 03:02:00.385 1555 1570 W ActivityManager: Activity destroy timeout for ActivityRecord{49a2e31 u0 com.google.mediapipe.apps.hairsegmentationgpu/.MainActivity t49754 f}
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <vector>
#include "mediapipe/calculators/image/recolor_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/util/color.pb.h"
#if defined(__ANDROID__)
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gl_simple_shaders.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/shader_util.h"
#endif // __ANDROID__
namespace {
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
} // namespace
namespace mediapipe {
// A calculator to recolor a masked area of an image to a specified color.
//
// A mask image is used to specify where to overlay a user defined color.
// The luminance of the input image is used to adjust the blending weight,
// to help preserve image textures.
//
// TODO implement cpu support.
//
// Inputs:
// One of the following IMAGE tags:
// IMAGE: An ImageFrame input image, RGB or RGBA.
// IMAGE_GPU: A GpuBuffer input image, RGBA.
// One of the following MASK tags:
// MASK: An ImageFrame input mask, Gray, RGB or RGBA.
// MASK_GPU: A GpuBuffer input mask, RGBA.
// RGB_ARRAY : rgb array that contain r g b values
// Output:
// One of the following IMAGE tags:
// IMAGE: An ImageFrame output image.
// IMAGE_GPU: A GpuBuffer output image.
//
// Options:
// color_rgb (required): A map of RGB values [0-255].
// mask_channel (optional): Which channel of mask image is used [RED or ALPHA]
//
// Usage example:
// node {
// calculator: "RecolorCalculator"
// input_stream: "IMAGE_GPU:input_image"
// input_stream: "MASK_GPU:input_mask"
// input_stream: "RGB_ARRAY"
// output_stream: "IMAGE_GPU:output_image"
// node_options: {
// [mediapipe.RecolorCalculatorOptions] {
// color { r: 0 g: 0 b: 255 }
// mask_channel: RED
// }
// }
// }
//
class RecolorCalculator : public CalculatorBase {
public:
RecolorCalculator() = default;
~RecolorCalculator() override = default;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
private:
::mediapipe::Status LoadOptions(CalculatorContext* cc);
::mediapipe::Status InitGpu(CalculatorContext* cc);
::mediapipe::Status RenderGpu(CalculatorContext* cc);
::mediapipe::Status RenderCpu(CalculatorContext* cc);
void GlRender();
bool initialized_ = false;
std::vector<float> color_;
std::vector<float> my_color;
mediapipe::RecolorCalculatorOptions::MaskChannel mask_channel_;
bool use_gpu_ = false;
#if defined(__ANDROID__)
mediapipe::GlCalculatorHelper gpu_helper_;
GLuint program_ = 0;
#endif // __ANDROID__
};
REGISTER_CALCULATOR(RecolorCalculator);
// static
::mediapipe::Status RecolorCalculator::GetContract(CalculatorContract* cc) {
RET_CHECK(!cc->Inputs().GetTags().empty());
RET_CHECK(!cc->Outputs().GetTags().empty());
#if defined(__ANDROID__)
if (cc->Inputs().HasTag("IMAGE_GPU")) {
cc->Inputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>();
}
#endif // __ANDROID__
if (cc->Inputs().HasTag("IMAGE")) {
cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
}
#if defined(__ANDROID__)
if (cc->Inputs().HasTag("MASK_GPU")) {
cc->Inputs().Tag("MASK_GPU").Set<mediapipe::GpuBuffer>();
}
#endif // __ANDROID__
if (cc->Inputs().HasTag("MASK")) {
cc->Inputs().Tag("MASK").Set<ImageFrame>();
}
if (cc->Inputs().HasTag("RGB_ARRAY")) {
cc->Inputs().Tag("RGB_ARRAY").Set<std::array<int,3>>();
}
#if defined(__ANDROID__)
if (cc->Outputs().HasTag("IMAGE_GPU")) {
cc->Outputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>();
}
#endif // __ANDROID__
if (cc->Outputs().HasTag("IMAGE")) {
cc->Outputs().Tag("IMAGE").Set<ImageFrame>();
}
#if defined(__ANDROID__)
RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
#endif // __ANDROID__
return ::mediapipe::OkStatus();
}
::mediapipe::Status RecolorCalculator::Open(CalculatorContext* cc) {
if (cc->Inputs().HasTag("IMAGE_GPU")) {
use_gpu_ = true;
#if defined(__ANDROID__)
RETURN_IF_ERROR(gpu_helper_.Open(cc));
#endif // __ANDROID__
}
RETURN_IF_ERROR(LoadOptions(cc));
return ::mediapipe::OkStatus();
}
::mediapipe::Status RecolorCalculator::Process(CalculatorContext* cc) {
if (use_gpu_) {
#if defined(__ANDROID__)
RETURN_IF_ERROR(
gpu_helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
if (!initialized_) {
RETURN_IF_ERROR(InitGpu(cc));
initialized_ = true;
}
RETURN_IF_ERROR(RenderGpu(cc));
return ::mediapipe::OkStatus();
}));
#endif // __ANDROID__
} else {
RETURN_IF_ERROR(RenderCpu(cc));
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status RecolorCalculator::Close(CalculatorContext* cc) {
#if defined(__ANDROID__)
gpu_helper_.RunInGlContext([this] {
if (program_) glDeleteProgram(program_);
program_ = 0;
});
#endif // __ANDROID__
return ::mediapipe::OkStatus();
}
::mediapipe::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) {
return ::mediapipe::UnimplementedError("CPU support is not implemented yet.");
}
::mediapipe::Status RecolorCalculator::RenderGpu(CalculatorContext* cc) {
if (cc->Inputs().Tag("MASK_GPU").IsEmpty()) {
return ::mediapipe::OkStatus();
}
#if defined(__ANDROID__)
// Get inputs and setup output.
const Packet& input_packet = cc->Inputs().Tag("IMAGE_GPU").Value();
const Packet& mask_packet = cc->Inputs().Tag("MASK_GPU").Value();
const Packet& rgb_packet = cc->Inputs().Tag("RGB_ARRAY").Value();
const auto& input_buffer = input_packet.Get<mediapipe::GpuBuffer>();
const auto& mask_buffer = mask_packet.Get<mediapipe::GpuBuffer>();
const auto& rgb_buffer = rgb_packet.Get<std::array<int,3>>();
my_color.push_back(rgb_buffer[0] / 255.0);
my_color.push_back(rgb_buffer[1] / 255.0);
my_color.push_back(rgb_buffer[2] / 255.0);
auto img_tex = gpu_helper_.CreateSourceTexture(input_buffer);
auto mask_tex = gpu_helper_.CreateSourceTexture(mask_buffer);
auto dst_tex = gpu_helper_.CreateDestinationTexture(img_tex.width(), img_tex.height());
// Run recolor shader on GPU.
{
gpu_helper_.BindFramebuffer(dst_tex); // GL_TEXTURE0
glActiveTexture(GL_TEXTURE1);
glBindTexture(img_tex.target(), img_tex.name());
glActiveTexture(GL_TEXTURE2);
glBindTexture(mask_tex.target(), mask_tex.name());
GlRender();
glActiveTexture(GL_TEXTURE2);
glBindTexture(GL_TEXTURE_2D, 0);
glActiveTexture(GL_TEXTURE1);
glBindTexture(GL_TEXTURE_2D, 0);
glFlush();
}
// Send result image in GPU packet.
auto output = dst_tex.GetFrame<mediapipe::GpuBuffer>();
cc->Outputs().Tag("IMAGE_GPU").Add(output.release(), cc->InputTimestamp());
// Cleanup
img_tex.Release();
mask_tex.Release();
dst_tex.Release();
#endif // __ANDROID__
return ::mediapipe::OkStatus();
}
void RecolorCalculator::GlRender() {
#if defined(__ANDROID__)
static const GLfloat square_vertices[] = {
-1.0f, -1.0f, // bottom left
1.0f, -1.0f, // bottom right
-1.0f, 1.0f, // top left
1.0f, 1.0f, // top right
};
static const GLfloat texture_vertices[] = {
0.0f, 0.0f, // bottom left
1.0f, 0.0f, // bottom right
0.0f, 1.0f, // top left
1.0f, 1.0f, // top right
};
// program
glUseProgram(program_);
glUniform3f(glGetUniformLocation(program_, "recolor"), my_color[0], my_color[1], my_color[2]);
// vertex storage
GLuint vbo[2];
glGenBuffers(2, vbo);
GLuint vao;
glGenVertexArrays(1, &vao);
glBindVertexArray(vao);
// vbo 0
glBindBuffer(GL_ARRAY_BUFFER, vbo[0]);
glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), square_vertices,
GL_STATIC_DRAW);
glEnableVertexAttribArray(ATTRIB_VERTEX);
glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr);
// vbo 1
glBindBuffer(GL_ARRAY_BUFFER, vbo[1]);
glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), texture_vertices,
GL_STATIC_DRAW);
glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr);
// draw
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
// cleanup
glDisableVertexAttribArray(ATTRIB_VERTEX);
glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
glBindBuffer(GL_ARRAY_BUFFER, 0);
glBindVertexArray(0);
glDeleteVertexArrays(1, &vao);
glDeleteBuffers(2, vbo);
#endif // __ANDROID__
}
::mediapipe::Status RecolorCalculator::LoadOptions(CalculatorContext* cc) {
const auto& options = cc->Options<mediapipe::RecolorCalculatorOptions>();
mask_channel_ = options.mask_channel();
if (!options.has_color()) RET_CHECK_FAIL() << "Missing color option.";
color_.push_back(options.color().r() / 255.0);
color_.push_back(options.color().g() / 255.0);
color_.push_back(options.color().b() / 255.0);
return ::mediapipe::OkStatus();
}
::mediapipe::Status RecolorCalculator::InitGpu(CalculatorContext* cc) {
#if defined(__ANDROID__)
const GLint attr_location[NUM_ATTRIBUTES] = {
ATTRIB_VERTEX,
ATTRIB_TEXTURE_POSITION,
};
const GLchar* attr_name[NUM_ATTRIBUTES] = {
"position",
"texture_coordinate",
};
std::string mask_component;
switch (mask_channel_) {
case mediapipe::RecolorCalculatorOptions_MaskChannel_UNKNOWN:
case mediapipe::RecolorCalculatorOptions_MaskChannel_RED:
mask_component = "r";
break;
case mediapipe::RecolorCalculatorOptions_MaskChannel_ALPHA:
mask_component = "a";
break;
}
// A shader to blend a color onto an image where the mask > 0.
// The blending is based on the input image luminosity.
const std::string frag_src = R"(
#if __VERSION__ < 130
#define in varying
#endif // __VERSION__ < 130
#ifdef GL_ES
#define fragColor gl_FragColor
precision highp float;
#else
#define lowp
#define mediump
#define highp
#define texture2D texture
out vec4 fragColor;
#endif // defined(GL_ES)
#define MASK_COMPONENT )" + mask_component +
R"(
in vec2 sample_coordinate;
uniform sampler2D frame;
uniform sampler2D mask;
uniform vec3 recolor;
void main() {
vec4 weight = texture2D(mask, sample_coordinate);
vec4 color1 = texture2D(frame, sample_coordinate);
vec4 color2 = vec4(recolor, 1.0);
float luminance = dot(color1.rgb, vec3(0.299, 0.587, 0.114));
float mix_value = weight.MASK_COMPONENT * luminance;
fragColor = mix(color1, color2, mix_value);
}
)";
// shader program and params
mediapipe::GlhCreateProgram(mediapipe::kBasicVertexShader, frag_src.c_str(),
NUM_ATTRIBUTES, &attr_name[0], attr_location,
&program_);
RET_CHECK(program_) << "Problem initializing the program.";
glUseProgram(program_);
glUniform1i(glGetUniformLocation(program_, "frame"), 1);
glUniform1i(glGetUniformLocation(program_, "mask"), 2);
glUniform3f(glGetUniformLocation(program_, "recolor"), color_[0], color_[1],
color_[2]);
#endif // __ANDROID__
return ::mediapipe::OkStatus();
}
} // namespace mediapipe
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment