A C++ class, an implementation of a general use AVFilterGraph for video playback within FFMPEG's libav libraries

gistfile1.txt

// 
// This file contains two code fragments:
// 1) A C++ class, an implementation of a general use AVFilterGraph for video playback. This class demonstrates a "best guess" of 
// how an AVFilterGraph should be constructed, yet some WMV and AVI sourced video frames do not convert from the codec native format 
// (yuv420p?) to the requested RGBA. 
//
// 2) A C++ code fragment demonstrating how the above CE_LIBAV_FrameFilter class is used, and the error correcting logic catching
// when YUV to RGB conversion is incorrect, and a work-around for the pixel format conversion bug
//
// It appears a barebones AVFilterGraph is required to receive stable playback when working with a wide variety of formats & devices.
// Before implementation and use of the CE_LIBAV_FrameFilter, partial and corrupt frames were causing lots of problems. A bare bones
// AVFilterGraph appears to fix 99% of our bad-frame-related playback issues. 
//


#pragma once
#ifndef _CE_LIBAV_VIDEO_FRAMEFILTERING_H_
#define _CE_LIBAV_VIDEO_FRAMEFILTERING_H_

extern "C" {
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libavdevice/avdevice.h"
#include "libavutil/imgutils.h"
#include "libswscale/swscale.h"
#include "libavutil/error.h"
// syncing logic with ffplay, these are used:
#include "libavfilter/avfiltergraph.h"
#include "libavfilter/buffersink.h"
#include "libavfilter/buffersrc.h"
#include "libavutil/avstring.h"
}
#pragma comment(lib, "avformat.lib")

// -------------------------------------------------------------------------------------------------------------------------------
// Usage: whatever the libav based video source, use an instance of this to filter out corrupt/partial frames and convert to RGBA
// Please read the header blocks for Init() and FilterFrame() for the easy and required usage of this frame filtering class. 
// -------------------------------------------------------------------------------------------------------------------------------
class CE_LIBAV_FrameFilter
{
public:
		CE_LIBAV_FrameFilter()
		{
			mp_buffersrc_ctx  = NULL;
			mp_buffersink_ctx = NULL;
			mp_filter_graph   = NULL;

			// these "last" members are used to track the last active w, h & pixel format:
			m_last_width  = 0;		
			m_last_height = 0;
			m_last_format = AV_PIX_FMT_NONE;
		}
		~CE_LIBAV_FrameFilter() 
		{			
			if (mp_filter_graph)
			{
				// I believe all the memory allocated to 
				// mp_buffersrc_ctx & mp_buffersink_ctx
				// is also deallocated by this:	
				avfilter_graph_free( &mp_filter_graph );
			}
		};

		// ------------------------------------------------------------------------------------------------
		// Ignore this; it is called automatically for you by FilterFrame(), below:
		// ------------------------------------------------------------------------------------------------
		int Init( AVFormatContext* p_format_context, AVStream* p_video_stream, AVFrame* p_video_frame )
		{
			// we'll be recreating the filter graph, so if one exists, delete it: 
			if (mp_filter_graph)
			{
				// I believe all the memory allocated to 
				// mp_buffersrc_ctx & mp_buffersink_ctx
				// is also deallocated by this:	
				avfilter_graph_free( &mp_filter_graph );
			}
			mp_filter_graph = avfilter_graph_alloc();
			if (!mp_filter_graph)
			{
				av_log(NULL, AV_LOG_ERROR, "CE_LIBAV_FrameFilter::Init out of memory\n");
        return AVERROR(ENOMEM);
			}

			AVFilterInOut* p_outputs = NULL; 
			AVFilterInOut* p_inputs  = NULL;

			// this controls the type of scale interpolation:
			mp_filter_graph->scale_sws_opts = av_strdup("flags=bicubic"); 

			char graph_filter_text[1024]; // the "source", the "description", the filter itself

			AVCodecParameters* p_codecpars = p_video_stream->codecpar; 
			AVRational&     time_base      = p_video_stream->time_base;
			AVRational      fr             = av_guess_frame_rate( p_format_context, p_video_stream, NULL);

			// buffer video source; the decoded frames from the decoder will be filtered thru here:
			snprintf( graph_filter_text, sizeof(graph_filter_text),
                "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
                p_video_frame->width, p_video_frame->height, p_video_frame->format,
                time_base.num, time_base.den,
                p_codecpars->sample_aspect_ratio.num, FFMAX(p_codecpars->sample_aspect_ratio.den, 1) );
			//
			if (fr.num && fr.den)
			{
				av_strlcatf(graph_filter_text, sizeof(graph_filter_text), ":frame_rate=%d/%d", fr.num, fr.den);
			}
			//
			int ret = avfilter_graph_create_filter( &mp_buffersrc_ctx, avfilter_get_by_name("buffer"), "in", graph_filter_text, NULL, mp_filter_graph );
			if (ret < 0) 
			{
        av_log(NULL, AV_LOG_ERROR, "CE_LIBAV_FrameFilter::Init Cannot create buffer source\n");
        goto end;
			}	

			// buffer video sink; this terminates the filter chain:
			ret = avfilter_graph_create_filter( &mp_buffersink_ctx, avfilter_get_by_name("buffersink"), "out", NULL, NULL, mp_filter_graph );
			if (ret < 0) 
			{
        av_log(NULL, AV_LOG_ERROR, "CE_LIBAV_FrameFilter::Init Cannot create buffer sink\n");
        goto end;
			}	

			// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
			// It appears with some wmv and avi files, converting to RGBA produces incorrect linesize[0] (bytes per image row) (2 extra pixels)
			//
			static const enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_RGBA, AV_PIX_FMT_NONE };
			//
			ret = av_opt_set_int_list( mp_buffersink_ctx, "pix_fmts", pix_fmts, AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
			if (ret < 0) 
			{
        av_log(NULL, AV_LOG_ERROR, "CE_LIBAV_FrameFilter::Init Cannot set output pixel format\n");
        goto end;
			}	

			AVFilterContext* last_filter = mp_buffersink_ctx;

			// this can be set to a text string defining an unlimited series of video filters, chained together, 
			// see https://ffmpeg.org/ffmpeg-filters.html
			const char* optional_post_processing_filters = NULL; // facility disabled for bug tracking

			if (optional_post_processing_filters)
			{
				p_outputs	= avfilter_inout_alloc();
        p_inputs	= avfilter_inout_alloc();
				if (!p_outputs || !p_inputs)
				{
					av_log(NULL, AV_LOG_ERROR, "CE_LIBAV_FrameFilter::Init out of memory 2\n");
					ret = AVERROR(ENOMEM);
					goto end;
				}
				// Set the endpoints for the filter graph. The filter_graph will be linked to the graph described by filters_descr.
				//
				// The buffer source output must be connected to the input pad of the first filter described by filters_descr; 
				// since the first filter input label is not specified, it is set to "in" by default.
				//
				p_outputs->name       = av_strdup("in");
				p_outputs->filter_ctx = mp_buffersrc_ctx;
				p_outputs->pad_idx    = 0;
				p_outputs->next       = NULL;
				//
				// The buffer sink input must be connected to the output pad of the last filter described by filters_descr; 
				// since the last filter output label is not specified, it is set to "out" by default.
				//
				p_inputs->name       = av_strdup("out");
				p_inputs->filter_ctx = mp_buffersink_ctx;
				p_inputs->pad_idx    = 0;
				p_inputs->next       = NULL;

				if ((ret = avfilter_graph_parse_ptr( mp_filter_graph, optional_post_processing_filters, &p_inputs, &p_outputs, NULL)) < 0)
				{
					av_log(NULL, AV_LOG_ERROR, "CE_LIBAV_FrameFilter::Init avfilter_graph_parse_ptr failed\n");
					goto end;
				}
			}
			else // this is the normal (usual) path of logic: (no special filtering)
			{
				ret = avfilter_link( mp_buffersrc_ctx, 0, mp_buffersink_ctx, 0 );
				if (ret < 0)
				{
					av_log(NULL, AV_LOG_ERROR, "CE_LIBAV_FrameFilter::Init avfilter_link failed\n");
					goto end;
				}
			}

			// "Reorder the filters to ensure that inputs of the custom filters are merged first"
			ce_uint nb_filters = mp_filter_graph->nb_filters;
			for (ce_uint i = 0; i < mp_filter_graph->nb_filters - nb_filters; i++)
			{
				FFSWAP(AVFilterContext*, mp_filter_graph->filters[i], mp_filter_graph->filters[i + nb_filters]);
			}

			// And this is construction of the filter that fixes corrupt video frames:
			if ((ret = avfilter_graph_config( mp_filter_graph, NULL)) < 0)
         goto end;

			end:
			avfilter_inout_free( &p_inputs  );
			avfilter_inout_free( &p_outputs );

			return ret;
		}

		// ------------------------------------------------------------------------------------------------------------
		// As each fresh AVFrame is received from avcodec_receive_frame(), this filtering logic will correct for frame
		// corruption, as well as convert whatever pixel format the codec gives to the RGBA we want for our needs.
		// ------------------------------------------------------------------------------------------------------------
		int FilterFrame( AVFormatContext* p_format_context, AVStream* p_video_stream, AVFrame* p_video_frame )
		{
			int ret = 0;

			// check if a new filter graph is needed: 
			if (m_last_width  != p_video_frame->width  ||  // Note: ffplay also has tests for vfilter_idx & pkt_serial
				  m_last_height != p_video_frame->height ||  // but so far I've not figured out what those are...
				  m_last_format != p_video_frame->format)
			{
				ret = Init( p_format_context, p_video_stream, p_video_frame ); 
				if (ret >= 0)
				{
					// success
					m_last_width  = p_video_frame->width;
					m_last_height = p_video_frame->height;
					m_last_format = p_video_frame->format;
				}
			}

			if (ret >= 0)
				ret = av_buffersrc_add_frame( mp_buffersrc_ctx, p_video_frame );
			
			// examples, such as ffplay.c, have this in a "while (ret >= 0)" loop; I had
			// a more complex architecture at first allowing multiple frames emitted from
			// such a while loop, but profiling showed no such multiple frames ever occur: 
			if (ret >= 0)
			{
				ret = av_buffersink_get_frame_flags( mp_buffersink_ctx, p_video_frame, 0 );
				if (ret < 0)
				{
					if (ret == AVERROR_EOF) // note: has never happened
						 av_log(NULL, AV_LOG_INFO, "CE_LIBAV_FrameFilter::FilterFrame av_buffersink_get_frame_flags returned AVERROR_EOF\n");
					ret = 0;
				}
			}

			return ret;
		}

		AVFilterContext*   mp_buffersrc_ctx;
		AVFilterContext*   mp_buffersink_ctx;
		AVFilterGraph*     mp_filter_graph;
		int                m_last_width, m_last_height, m_last_format;
};



// The logic which owns and controls a CE_LIBAV_FrameFilter has a circular buffer of AVFrames. A separate thread
// fills the circular buffer with AVFrames from use of avcodec_receive_frame(), insuring not to overwrite undisplayed
// AVFrames. 
// (Note that an alternative version using avcodec_decode_video2() sometimes crashes with our troublesome videos.)
// When displaying a frame, first the frame is passed through the above CE_LIBAV_FrameFilter::FrameFilter(), and then
// this logic is used to catch cases where the RGBA conversion is wrong, handing back image rows two pixels too long:

void CE_LIBAV_FrameDispatch::DeliverFrameToClient( bool first_frame, AVFrame* src_frame, ce_uint display_index )
{
	CE_Image& im = mp_parent->m_im; // this is an internal image format with lots of computer vision methods

	// check for an empty frame:
	if (src_frame->format == AV_PIX_FMT_NONE)
	{
		m_frame_count++; // ? happens sometimes...
		return;
	}

	// always apply frame filtering:
	int ret = mp_frame_filter->FilterFrame( mp_parent->mp_parent->mp_format_context, mp_parent->mp_parent->mp_video_stream, src_frame );
	if (ret < 0)
		return;

	// continue if the frame is being delivered to the client:
	if ((first_frame) || ((display_index % (unsigned int)m_frame_interval) == 0))
	{		
				int true_bytes_per_row = src_frame->width * 4; // each RGBA is 4 bytes

				// check for incorrect pixel format conversion: 
				if (src_frame->linesize[0] != true_bytes_per_row)
				{
					av_log( NULL, AV_LOG_ERROR, "after frame filter, linesize[0] should be %d but is %d!\n",
							    true_bytes_per_row, src_frame->linesize[0] );

					if (src_frame->linesize[0] > true_bytes_per_row)
					{
						for (int i = 0; i < src_frame->height; i++)
						{
							// copy 1 row of RGBA pixels into our image storage:
							std::memcpy( &im.m_data[i*src_frame->width], 
								           &src_frame->data[0][i*src_frame->linesize[0]], 
													 true_bytes_per_row );
						}
					}
					else
					{
						return; // boo!
					}
				}
				else
				{
					// copy RGBA pixels into our image storage:
					std::size_t pixels_size = (std::size_t)src_frame->height * (std::size_t)src_frame->width * sizeof(CE_Colour);
					std::memcpy( im.m_data, src_frame->data[0], pixels_size );
				}
		
				if (first_frame)
				{ 
					if (mp_process_first_frame)
							(mp_process_first_frame)(mp_process_first_frame_object, im, display_index );
				}
		
				if (mp_process_frame) 
						(mp_process_frame)(mp_process_frame_object, im, display_index ); 
	}

	m_frame_count++;
}



#endif