FFmpeg example to extract a frame from a video in Rust Lang

ffmpeg.rs

/// This 

unsafe {
        ffmpeg::av_register_all();

        // This portion of code was written by taking as resource: http://dranger.com/ffmpeg/tutorial01.html
        // This article is outdated, and some APIs got deprecated, here I used the non deprecated version.
        //
        // The idea of FFmpeg is to
        // 1. open the file
        // 2. Find the codec stream
        // 3. Read all the packets
        // 4. Decodify the packet data using the codec
        // 5. Clean it out.
        //
        // This is the main concept to get video information using FFmpeg.

        // 1. Open the video and take the codec to read the frame
        // Open the video
        let mut video_input_context: *mut ffmpeg::AVFormatContext = std::ptr::null_mut();
        if ffmpeg::avformat_open_input(
            &mut video_input_context,
            CString::new("resources/test_video_3.mp4").unwrap().as_ptr(),
            std::ptr::null_mut(),
            std::ptr::null_mut(),
        ) != 0
        {
            panic!("Can't open the Video");
        }
        // Get the stream information
        if ffmpeg::avformat_find_stream_info(video_input_context, std::ptr::null_mut()) < 0 {
            panic!("Can't read video stream information");
        }

        // Find the video stream
        let (video_stream_codec_ctx, video_stream_id) = {
            let mut video_stream_id = -1isize;
            for i in 0isize..(*video_input_context).nb_streams as isize {
                if (*(*(*(*video_input_context).streams.offset(i))).codec).codec_type
                    == ffmpeg::AVMediaType::AVMEDIA_TYPE_VIDEO
                {
                    video_stream_id = i;
                    break;
                }
            }

            if video_stream_id < 0 {
                panic!("Was not possible find the video stream on this video.")
            }

            (
                (*(*(*video_input_context).streams.offset(video_stream_id))).codec,
                video_stream_id,
            )
        };

        // Find the actual codec
        let video_codec = ffmpeg::avcodec_find_decoder((*video_stream_codec_ctx).codec_id);
        if video_codec == std::ptr::null_mut() {
            panic!("The codec of this video is not available.");
        }

        // Duplicate the codec context to avoid use the stream codec, so we can reuse the input context
        // without concerning about closing its codecs (since these got never opened)
        let mut video_codec_ctx = {
            let video_codec_ctx = ffmpeg::avcodec_alloc_context3(video_codec);
            let mut params = ffmpeg::avcodec_parameters_alloc();
            ffmpeg::avcodec_parameters_from_context(params, video_stream_codec_ctx);
            ffmpeg::avcodec_parameters_to_context(video_codec_ctx, params);
            ffmpeg::avcodec_parameters_free(&mut params);
            video_codec_ctx
        };

        // Open the new decoder codec context
        if ffmpeg::avcodec_open2(video_codec_ctx, video_codec, std::ptr::null_mut()) < 0 {
            panic!("Can't open the codec.");
        }

        // 2. Creates the video frames where to put video data into

        // Note:
        // The video frame has its own format, that depends on the codification, so we need to convert it
        // to something meaningful to the GPU.
        // So here I'm allocating two video frames:
        // - `video_frame` the video frame (any format)
        // - `rgb_video_frame` the converted video frame (RGB24 layout = {R:8 G:8 B:8})
        let mut video_frame = ffmpeg::av_frame_alloc();
        let mut rgb_video_frame = ffmpeg::av_frame_alloc();

        // The `rgb_video_frame` is a wrapper for the frame data buffer and it needs to have a buffer
        // since the API `sws_scale` require it.
        // To assign the buffer is possible to use the API `avpicture_fill`.
        //
        // Why don't we assign to the `video_frame` a buffer?
        // The reason is that the decoding function allocates or assign a memory buffer:
        // https://www.ffmpeg.org/doxygen/3.4/structAVCodecContext.html#a17d91e3ddcf7cc8fb235ba4f15777282
        //
        // (**1)
        // The standard way of creating a buffer would be this one:
        // ```
        // let buffer = ffmpeg::av_malloc(buffer_bytes_size * std::mem::size_of::<u8>()) as *const u8;
        // ```
        // By assigning this buffer to the `rgb_video_frame`, once the `rgb_video_frame` is filled,
        // it is necessary to copy its data inside a rust safe container like a `Vec<u8>`.
        // Note:
        // (Free the buffer using `ffmpeg::av_free(buffer as *mut _);` when you don't need it anymore).
        //
        // To avoid this extra copy, instead to follow the "standard way", I'm passing directly the
        // `Vec<u8>` pointer. Just keep in mind that altering the vector, in any way, while its pointer
        // is assigned as buffer of the `rgb_video_frame` will summon a Dragon.
        let buffer_bytes_size = ffmpeg::avpicture_get_size(
            ffmpeg::AVPixelFormat::AV_PIX_FMT_RGB24,
            (*video_codec_ctx).width,
            (*video_codec_ctx).height,
        ) as usize;
        video_texture_data = Vec::with_capacity(buffer_bytes_size);
        video_texture_data.set_len(buffer_bytes_size);
        if ffmpeg::avpicture_fill(
            rgb_video_frame as *mut ffmpeg::AVPicture, // Safe cast (Check FFmpeg docs)
            video_texture_data.as_mut_ptr(),
            ffmpeg::AVPixelFormat::AV_PIX_FMT_RGB24,
            (*video_codec_ctx).width,
            (*video_codec_ctx).height,
        ) <= 0
        {
            panic!("Was not possible to fill the picture with the buffer.");
        }

        // 3. Read the data from the video

        // Creates the converter software context
        let sws_context = ffmpeg::sws_getContext(
            (*video_codec_ctx).width,                // Source
            (*video_codec_ctx).height,               // Source
            (*video_codec_ctx).pix_fmt,              // Source
            (*video_codec_ctx).width,                // Destination
            (*video_codec_ctx).height,               // Destination
            ffmpeg::AVPixelFormat::AV_PIX_FMT_RGB24, // Destination
            ffmpeg::SWS_BILINEAR,
            std::ptr::null_mut(),
            std::ptr::null_mut(),
            std::ptr::null_mut(),
        );

        let mut frames = 0;
        dbg!((*video_codec_ctx).refcounted_frames);
        // A packet is a wrapper of some useful information, related to the video playback,
        // like the video frames, the audio, the subtitles, etc...
        // With `av_read_frame` it is possible to retrieve a series of packets, that decoded allow to
        // get the video information.
        // Doc: https://ffmpeg.org/doxygen/2.8/structAVPacket.html#details
        let mut packet: ffmpeg::AVPacket = std::mem::zeroed();
        while ffmpeg::av_read_frame(video_input_context, &mut packet) >= 0 {
            if packet.stream_index == video_stream_id as i32 {
                // Set the packet to decode
                if ffmpeg::avcodec_send_packet(video_codec_ctx, &packet) < 0 {
                    panic!("Can't send this packet to the decoder")
                }

                // Decode all the frames inside that packet
                while ffmpeg::avcodec_receive_frame(video_codec_ctx, video_frame) >= 0 {
                    // Convert the frame to RGB24
                    ffmpeg::sws_scale(
                        sws_context,
                        (*video_frame).data.as_ptr() as *const *const _,
                        (*video_frame).linesize.as_ptr() as *mut _,
                        0,
                        (*video_codec_ctx).height as libc::c_int,
                        (*rgb_video_frame).data.as_ptr() as *const *const _,
                        (*rgb_video_frame).linesize.as_ptr() as *mut _,
                    );

                    // Do nothing.
                    //
                    // Note:
                    // As mentioned before, (**1), I'm not using an allocated buffer, as the standard
                    // way would suggest. Rather, I'm assigning the vector pointer as buffer of
                    // `rgb_video_frame` so the API `sws_scale` operate directly on the vector memory.
                    //
                    // If I would follow the standard way, at this point I should copy the content
                    // of the allocated buffer here:
                    // ```
                    // video_texture_data = Vec::with_capacity(buffer_bytes_size);
                    // video_texture_data.set_len(buffer_bytes_size);
                    // std::ptr::copy_nonoverlapping((*rgb_video_frame).data[0], video_texture_data.as_mut_ptr(), buffer_bytes_size);
                    // ```

                    frames += 1;
                    // Just take 1 frame
                    break;
                }

                if frames > 5 {
                    break; // TODO just extract the first frame! (Test)
                }
            }
            // Always free the read packet
            ffmpeg::av_packet_unref(&mut packet);
        }

        texture_size = Vector2::new(
            (*video_codec_ctx).width as f32,
            (*video_codec_ctx).height as f32,
        );

        // 4. Release all the allocated memory and close codecs and the input
        ffmpeg::sws_freeContext(sws_context);
        ffmpeg::av_free(rgb_video_frame as *mut std::ffi::c_void);
        ffmpeg::av_free(video_frame as *mut std::ffi::c_void);
        ffmpeg::avcodec_close(video_codec_ctx);
        ffmpeg::avcodec_free_context(&mut video_codec_ctx);
        ffmpeg::avformat_close_input(&mut video_input_context);
    }