Skip to content

Instantly share code, notes, and snippets.

@bsenftner
Created February 8, 2018 15:09
Show Gist options
  • Save bsenftner/c7f983fda99ff463ba34e6e4bd69ebd1 to your computer and use it in GitHub Desktop.
Save bsenftner/c7f983fda99ff463ba34e6e4bd69ebd1 to your computer and use it in GitHub Desktop.
A C++ class, an implementation of a general use AVFilterGraph for video playback within FFMPEG's libav libraries
//
// This file contains two code fragments:
// 1) A C++ class, an implementation of a general use AVFilterGraph for video playback. This class demonstrates a "best guess" of
// how an AVFilterGraph should be constructed, yet some WMV and AVI sourced video frames do not convert from the codec native format
// (yuv420p?) to the requested RGBA.
//
// 2) A C++ code fragment demonstrating how the above CE_LIBAV_FrameFilter class is used, and the error correcting logic catching
// when YUV to RGB conversion is incorrect, and a work-around for the pixel format conversion bug
//
// It appears a barebones AVFilterGraph is required to receive stable playback when working with a wide variety of formats & devices.
// Before implementation and use of the CE_LIBAV_FrameFilter, partial and corrupt frames were causing lots of problems. A bare bones
// AVFilterGraph appears to fix 99% of our bad-frame-related playback issues.
//
#pragma once
#ifndef _CE_LIBAV_VIDEO_FRAMEFILTERING_H_
#define _CE_LIBAV_VIDEO_FRAMEFILTERING_H_
extern "C" {
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libavdevice/avdevice.h"
#include "libavutil/imgutils.h"
#include "libswscale/swscale.h"
#include "libavutil/error.h"
// syncing logic with ffplay, these are used:
#include "libavfilter/avfiltergraph.h"
#include "libavfilter/buffersink.h"
#include "libavfilter/buffersrc.h"
#include "libavutil/avstring.h"
}
#pragma comment(lib, "avformat.lib")
// -------------------------------------------------------------------------------------------------------------------------------
// Usage: whatever the libav based video source, use an instance of this to filter out corrupt/partial frames and convert to RGBA
// Please read the header blocks for Init() and FilterFrame() for the easy and required usage of this frame filtering class.
// -------------------------------------------------------------------------------------------------------------------------------
class CE_LIBAV_FrameFilter
{
public:
CE_LIBAV_FrameFilter()
{
mp_buffersrc_ctx = NULL;
mp_buffersink_ctx = NULL;
mp_filter_graph = NULL;
// these "last" members are used to track the last active w, h & pixel format:
m_last_width = 0;
m_last_height = 0;
m_last_format = AV_PIX_FMT_NONE;
}
~CE_LIBAV_FrameFilter()
{
if (mp_filter_graph)
{
// I believe all the memory allocated to
// mp_buffersrc_ctx & mp_buffersink_ctx
// is also deallocated by this:
avfilter_graph_free( &mp_filter_graph );
}
};
// ------------------------------------------------------------------------------------------------
// Ignore this; it is called automatically for you by FilterFrame(), below:
// ------------------------------------------------------------------------------------------------
int Init( AVFormatContext* p_format_context, AVStream* p_video_stream, AVFrame* p_video_frame )
{
// we'll be recreating the filter graph, so if one exists, delete it:
if (mp_filter_graph)
{
// I believe all the memory allocated to
// mp_buffersrc_ctx & mp_buffersink_ctx
// is also deallocated by this:
avfilter_graph_free( &mp_filter_graph );
}
mp_filter_graph = avfilter_graph_alloc();
if (!mp_filter_graph)
{
av_log(NULL, AV_LOG_ERROR, "CE_LIBAV_FrameFilter::Init out of memory\n");
return AVERROR(ENOMEM);
}
AVFilterInOut* p_outputs = NULL;
AVFilterInOut* p_inputs = NULL;
// this controls the type of scale interpolation:
mp_filter_graph->scale_sws_opts = av_strdup("flags=bicubic");
char graph_filter_text[1024]; // the "source", the "description", the filter itself
AVCodecParameters* p_codecpars = p_video_stream->codecpar;
AVRational& time_base = p_video_stream->time_base;
AVRational fr = av_guess_frame_rate( p_format_context, p_video_stream, NULL);
// buffer video source; the decoded frames from the decoder will be filtered thru here:
snprintf( graph_filter_text, sizeof(graph_filter_text),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
p_video_frame->width, p_video_frame->height, p_video_frame->format,
time_base.num, time_base.den,
p_codecpars->sample_aspect_ratio.num, FFMAX(p_codecpars->sample_aspect_ratio.den, 1) );
//
if (fr.num && fr.den)
{
av_strlcatf(graph_filter_text, sizeof(graph_filter_text), ":frame_rate=%d/%d", fr.num, fr.den);
}
//
int ret = avfilter_graph_create_filter( &mp_buffersrc_ctx, avfilter_get_by_name("buffer"), "in", graph_filter_text, NULL, mp_filter_graph );
if (ret < 0)
{
av_log(NULL, AV_LOG_ERROR, "CE_LIBAV_FrameFilter::Init Cannot create buffer source\n");
goto end;
}
// buffer video sink; this terminates the filter chain:
ret = avfilter_graph_create_filter( &mp_buffersink_ctx, avfilter_get_by_name("buffersink"), "out", NULL, NULL, mp_filter_graph );
if (ret < 0)
{
av_log(NULL, AV_LOG_ERROR, "CE_LIBAV_FrameFilter::Init Cannot create buffer sink\n");
goto end;
}
// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
// It appears with some wmv and avi files, converting to RGBA produces incorrect linesize[0] (bytes per image row) (2 extra pixels)
//
static const enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_RGBA, AV_PIX_FMT_NONE };
//
ret = av_opt_set_int_list( mp_buffersink_ctx, "pix_fmts", pix_fmts, AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
if (ret < 0)
{
av_log(NULL, AV_LOG_ERROR, "CE_LIBAV_FrameFilter::Init Cannot set output pixel format\n");
goto end;
}
AVFilterContext* last_filter = mp_buffersink_ctx;
// this can be set to a text string defining an unlimited series of video filters, chained together,
// see https://ffmpeg.org/ffmpeg-filters.html
const char* optional_post_processing_filters = NULL; // facility disabled for bug tracking
if (optional_post_processing_filters)
{
p_outputs = avfilter_inout_alloc();
p_inputs = avfilter_inout_alloc();
if (!p_outputs || !p_inputs)
{
av_log(NULL, AV_LOG_ERROR, "CE_LIBAV_FrameFilter::Init out of memory 2\n");
ret = AVERROR(ENOMEM);
goto end;
}
// Set the endpoints for the filter graph. The filter_graph will be linked to the graph described by filters_descr.
//
// The buffer source output must be connected to the input pad of the first filter described by filters_descr;
// since the first filter input label is not specified, it is set to "in" by default.
//
p_outputs->name = av_strdup("in");
p_outputs->filter_ctx = mp_buffersrc_ctx;
p_outputs->pad_idx = 0;
p_outputs->next = NULL;
//
// The buffer sink input must be connected to the output pad of the last filter described by filters_descr;
// since the last filter output label is not specified, it is set to "out" by default.
//
p_inputs->name = av_strdup("out");
p_inputs->filter_ctx = mp_buffersink_ctx;
p_inputs->pad_idx = 0;
p_inputs->next = NULL;
if ((ret = avfilter_graph_parse_ptr( mp_filter_graph, optional_post_processing_filters, &p_inputs, &p_outputs, NULL)) < 0)
{
av_log(NULL, AV_LOG_ERROR, "CE_LIBAV_FrameFilter::Init avfilter_graph_parse_ptr failed\n");
goto end;
}
}
else // this is the normal (usual) path of logic: (no special filtering)
{
ret = avfilter_link( mp_buffersrc_ctx, 0, mp_buffersink_ctx, 0 );
if (ret < 0)
{
av_log(NULL, AV_LOG_ERROR, "CE_LIBAV_FrameFilter::Init avfilter_link failed\n");
goto end;
}
}
// "Reorder the filters to ensure that inputs of the custom filters are merged first"
ce_uint nb_filters = mp_filter_graph->nb_filters;
for (ce_uint i = 0; i < mp_filter_graph->nb_filters - nb_filters; i++)
{
FFSWAP(AVFilterContext*, mp_filter_graph->filters[i], mp_filter_graph->filters[i + nb_filters]);
}
// And this is construction of the filter that fixes corrupt video frames:
if ((ret = avfilter_graph_config( mp_filter_graph, NULL)) < 0)
goto end;
end:
avfilter_inout_free( &p_inputs );
avfilter_inout_free( &p_outputs );
return ret;
}
// ------------------------------------------------------------------------------------------------------------
// As each fresh AVFrame is received from avcodec_receive_frame(), this filtering logic will correct for frame
// corruption, as well as convert whatever pixel format the codec gives to the RGBA we want for our needs.
// ------------------------------------------------------------------------------------------------------------
int FilterFrame( AVFormatContext* p_format_context, AVStream* p_video_stream, AVFrame* p_video_frame )
{
int ret = 0;
// check if a new filter graph is needed:
if (m_last_width != p_video_frame->width || // Note: ffplay also has tests for vfilter_idx & pkt_serial
m_last_height != p_video_frame->height || // but so far I've not figured out what those are...
m_last_format != p_video_frame->format)
{
ret = Init( p_format_context, p_video_stream, p_video_frame );
if (ret >= 0)
{
// success
m_last_width = p_video_frame->width;
m_last_height = p_video_frame->height;
m_last_format = p_video_frame->format;
}
}
if (ret >= 0)
ret = av_buffersrc_add_frame( mp_buffersrc_ctx, p_video_frame );
// examples, such as ffplay.c, have this in a "while (ret >= 0)" loop; I had
// a more complex architecture at first allowing multiple frames emitted from
// such a while loop, but profiling showed no such multiple frames ever occur:
if (ret >= 0)
{
ret = av_buffersink_get_frame_flags( mp_buffersink_ctx, p_video_frame, 0 );
if (ret < 0)
{
if (ret == AVERROR_EOF) // note: has never happened
av_log(NULL, AV_LOG_INFO, "CE_LIBAV_FrameFilter::FilterFrame av_buffersink_get_frame_flags returned AVERROR_EOF\n");
ret = 0;
}
}
return ret;
}
AVFilterContext* mp_buffersrc_ctx;
AVFilterContext* mp_buffersink_ctx;
AVFilterGraph* mp_filter_graph;
int m_last_width, m_last_height, m_last_format;
};
// The logic which owns and controls a CE_LIBAV_FrameFilter has a circular buffer of AVFrames. A separate thread
// fills the circular buffer with AVFrames from use of avcodec_receive_frame(), insuring not to overwrite undisplayed
// AVFrames.
// (Note that an alternative version using avcodec_decode_video2() sometimes crashes with our troublesome videos.)
// When displaying a frame, first the frame is passed through the above CE_LIBAV_FrameFilter::FrameFilter(), and then
// this logic is used to catch cases where the RGBA conversion is wrong, handing back image rows two pixels too long:
void CE_LIBAV_FrameDispatch::DeliverFrameToClient( bool first_frame, AVFrame* src_frame, ce_uint display_index )
{
CE_Image& im = mp_parent->m_im; // this is an internal image format with lots of computer vision methods
// check for an empty frame:
if (src_frame->format == AV_PIX_FMT_NONE)
{
m_frame_count++; // ? happens sometimes...
return;
}
// always apply frame filtering:
int ret = mp_frame_filter->FilterFrame( mp_parent->mp_parent->mp_format_context, mp_parent->mp_parent->mp_video_stream, src_frame );
if (ret < 0)
return;
// continue if the frame is being delivered to the client:
if ((first_frame) || ((display_index % (unsigned int)m_frame_interval) == 0))
{
int true_bytes_per_row = src_frame->width * 4; // each RGBA is 4 bytes
// check for incorrect pixel format conversion:
if (src_frame->linesize[0] != true_bytes_per_row)
{
av_log( NULL, AV_LOG_ERROR, "after frame filter, linesize[0] should be %d but is %d!\n",
true_bytes_per_row, src_frame->linesize[0] );
if (src_frame->linesize[0] > true_bytes_per_row)
{
for (int i = 0; i < src_frame->height; i++)
{
// copy 1 row of RGBA pixels into our image storage:
std::memcpy( &im.m_data[i*src_frame->width],
&src_frame->data[0][i*src_frame->linesize[0]],
true_bytes_per_row );
}
}
else
{
return; // boo!
}
}
else
{
// copy RGBA pixels into our image storage:
std::size_t pixels_size = (std::size_t)src_frame->height * (std::size_t)src_frame->width * sizeof(CE_Colour);
std::memcpy( im.m_data, src_frame->data[0], pixels_size );
}
if (first_frame)
{
if (mp_process_first_frame)
(mp_process_first_frame)(mp_process_first_frame_object, im, display_index );
}
if (mp_process_frame)
(mp_process_frame)(mp_process_frame_object, im, display_index );
}
m_frame_count++;
}
#endif
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment