Godot GDNative FFMPEG Streaming

frame_decoder.c

#include <gdnative_api_struct.gen.h>
#include <libavcodec/avcodec.h>
#include <libavutil/avutil.h>
#include <libavformat/avformat.h>
#include <libavutil/imgutils.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

const godot_gdnative_core_api_struct *api = NULL;
const godot_gdnative_ext_nativescript_api_struct *nativescript_api = NULL;

void *sensor_decoder_constructor(godot_object *p_instance, void *p_method_data);
void sensor_decoder_destructor(godot_object *p_instance, void *p_method_data, void *p_user_data);
godot_variant sensor_decoder_get_data(godot_object *p_instance, void *p_method_data,
        void *p_user_data, int p_num_args, godot_variant **p_args);

void godot_log(const wchar_t * message){
    godot_string s;
    api->godot_string_new_with_wide_string(&s, message, wcslen(message));
    api->godot_print(&s);
    api->godot_string_destroy(&s);
}

#define LOG(format, ...) do {\
    const wchar_t buf[512];\
    if(swprintf((wchar_t *)buf, sizeof(buf), format, ##__VA_ARGS__) <= 0){\
        godot_log(L"Error printing to log");\
    } else {\
        godot_log(buf);\
    }\
} while(0);

void GDN_EXPORT godot_gdnative_init(godot_gdnative_init_options *p_options) {
    api = p_options->api_struct;

    // Now find our extensions.
    for (int i = 0; i < api->num_extensions; i++) {
        switch (api->extensions[i]->type) {
            case GDNATIVE_EXT_NATIVESCRIPT: {
                nativescript_api = (godot_gdnative_ext_nativescript_api_struct *)api->extensions[i];
            }; break;
            default: break;
        }
    }
}


void GDN_EXPORT godot_gdnative_terminate(godot_gdnative_terminate_options *p_options) {
    api = NULL;
    nativescript_api = NULL;
}

void GDN_EXPORT godot_nativescript_init(void *p_handle) {
    godot_instance_create_func create = { NULL, NULL, NULL };
    create.create_func = &sensor_decoder_constructor;

    godot_instance_destroy_func destroy = { NULL, NULL, NULL };
    destroy.destroy_func = &sensor_decoder_destructor;

    nativescript_api->godot_nativescript_register_class(p_handle, "SENSORDECODER", "Reference",
            create, destroy);

    godot_instance_method get_data = { NULL, NULL, NULL };
    get_data.method = &sensor_decoder_get_data;

    godot_method_attributes attributes = { GODOT_METHOD_RPC_MODE_DISABLED };

    nativescript_api->godot_nativescript_register_method(p_handle, "SENSORDECODER", "get_data",
            attributes, get_data);
}

typedef struct user_data_struct {
    pthread_t decoder_thread;
    pthread_rwlock_t frame_lock;
    uint8_t * frame_data;
    size_t frame_size;
    size_t buf_size;
    godot_pool_byte_array frame;
    int stop_decoder;
} user_data_struct;

void * decoder_thread(void * decoder_args) {
    user_data_struct  *user_data    = (user_data_struct *) decoder_args;
    AVFormatContext   *pFormatCtx   = NULL;
    AVCodecContext    *pCodecCtx    = NULL;
    AVCodec           *pCodec       = NULL;
    AVCodecParameters *pCodecParams = NULL;
    AVFrame           *pFrame       = NULL;
    AVDictionary      *optionsDict  = NULL;
    AVPacket          packet;
    int               videoStream;
    int               frameFinished;
    int               stop;



    if(avformat_open_input(&pFormatCtx, "http://192.168.4.63:5000/stream/depth", NULL, NULL) !=0) {
        LOG(L"Couldn't open file");
        return NULL; // Couldn't open file
    }

    if(avformat_find_stream_info(pFormatCtx, NULL)<0) {
        LOG(L"Couldn't find stream information");
        return NULL;
    }

    // Find the first video stream
    videoStream = -1;
    for(int i=0; i<pFormatCtx->nb_streams; i++) {
        if(pFormatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
            videoStream = i;
            pCodecParams = pFormatCtx->streams[i]->codecpar;
            break;
        }
    }
    if(videoStream == -1) {
        LOG(L"Couldn't find video stream");
        return NULL; // Didn't find a video stream
    }



   // Find the decoder for the video stream
   pCodec = avcodec_find_decoder(pCodecParams->codec_id);

   if(pCodec==NULL) {
       LOG(L"Unsupported codec!");
       return NULL;
    }

   // Get a pointer to the codec context for the video stream
   pCodecCtx = avcodec_alloc_context3(pCodec);
    if(pCodecCtx == NULL){
        LOG(L"Couldn't create codec context");
        return NULL;
    }

   if(avcodec_open2(pCodecCtx, pCodec, &optionsDict) < 0) {
       LOG(L"Couldn't open codec");
       return NULL;
   }


   pFrame = av_frame_alloc();// Allocate video frame
   stop = 0;
   // Read frames and save first five frames to disk
   while(stop == 0 && av_read_frame(pFormatCtx, &packet) >= 0) {
     if(pthread_rwlock_rdlock(&user_data->frame_lock) != 0){
         continue;
     }
     stop = user_data->stop_decoder;
     if(pthread_rwlock_unlock(&user_data->frame_lock) != 0){
         LOG(L"Could not unlock locked thread, bailing");
         break;
     }
     if(stop == 1){
         LOG(L"Exiting decoder thread");
         break;
     }
     // Is this a packet from the video stream?
     if(packet.stream_index == videoStream) {
       // Decode video frame
       avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &packet);

       // Did we get a video frame?
       if(frameFinished)
       {
           if(pthread_rwlock_wrlock(&user_data->frame_lock) != 0){
               LOG(L"Could not lock to frame write, dropping frame");
               continue;
           }
           int copied = av_image_copy_to_buffer(user_data->frame_data, user_data->buf_size, pFrame->data, pFrame->linesize, pFrame->format, pFrame->width, pFrame->height, 1);
           if(copied < 0){
               LOG(L"Could not copy frame contents, dropping frame");
               memset(user_data->frame_data, 0, user_data->frame_size);
               continue;
           } else {
               user_data->frame_size = copied;
           }
           if(pthread_rwlock_unlock(&user_data->frame_lock) != 0){
               LOG(L"Could not unlock frame write after locked, bailing");
               break;
           }
       }
     }
     // Free the packet that was allocated by av_read_frame
     av_free_packet(&packet);
   }
   av_free(pFrame);// Free the video frame
   avcodec_close(pCodecCtx);// Close the codec
   avformat_close_input(&pFormatCtx);// Close the video file
    LOG(L"Decode finished");

    return NULL;
}

void *sensor_decoder_constructor(godot_object *p_instance, void *p_method_data) {
    user_data_struct * user_data = api->godot_alloc(sizeof(user_data_struct));
    user_data->buf_size = 1048576;
    user_data->frame_data = (uint8_t *) malloc(sizeof(uint8_t) * user_data->buf_size);
    user_data->frame_size = 0;
    api->godot_pool_byte_array_new(&user_data->frame);
    user_data->stop_decoder = 0;
    if(pthread_rwlock_init(&user_data->frame_lock, NULL) != 0){
        LOG(L"Lock failed to initialize");
    }
    if(pthread_create(&user_data->decoder_thread, NULL, decoder_thread, (void *)user_data) != 0){
        LOG(L"Thread start failed");
    } else {
        LOG(L"Thread start");
    }
    return user_data;
}

void sensor_decoder_destructor(godot_object *p_instance, void *p_method_data, void *p_user_data) {
    user_data_struct * user_data = (user_data_struct *) p_user_data;
    if(pthread_rwlock_wrlock(&user_data->frame_lock) != 0) {
        LOG(L"Could not acquire lock");
    } else {
        user_data->stop_decoder = 1;
        if(pthread_rwlock_unlock(&user_data->frame_lock) != 0){
            LOG(L"Could not unlock frame lock");
        }
    }
    if(pthread_join(user_data->decoder_thread, NULL) != 0){
        LOG(L"Could not safely exit decoder thread");
    }
    if(pthread_rwlock_destroy(&user_data->frame_lock) != 0){
        LOG(L"Could not destroy frame lock");
    }
    free(user_data->frame_data);
    api->godot_pool_byte_array_destroy(&user_data->frame);
    api->godot_free(p_user_data);
    LOG(L"Destroying");
}

godot_variant sensor_decoder_get_data(godot_object *p_instance, void *p_method_data,
        void *p_user_data, int p_num_args, godot_variant **p_args) {
    user_data_struct * user_data = (user_data_struct *) p_user_data;
    godot_variant ret;
    if(pthread_rwlock_rdlock(&user_data->frame_lock) != 0){
        LOG(L"Could not lock to frame read");
        // TODO
    }
    godot_int size = api->godot_pool_byte_array_size(&user_data->frame);
    if(size != user_data->frame_size){
        api->godot_pool_byte_array_resize(&user_data->frame, user_data->frame_size);
    }
    godot_pool_byte_array_write_access * write_access = api->godot_pool_byte_array_write(&user_data->frame);
    uint8_t * ptr = api->godot_pool_byte_array_write_access_ptr(write_access);
    memcpy(ptr, user_data->frame_data, user_data->frame_size);
    api->godot_pool_byte_array_write_access_destroy(write_access);
    if(pthread_rwlock_unlock(&user_data->frame_lock) != 0){
        LOG(L"Could not unlock frame read after locked, bailing");
        // TODO
    }
    api->godot_variant_new_pool_byte_array(&ret, &user_data->frame);

    return ret;
}

Note that this sets up a separate decoding thread and just copies in frames on demand from the calling Godot process - not really essential to the whole video decoding thing, so ignore all the locks and threading bits if that isn't part of what you need.

Also it occurs to me that I used LOG in a separate thread here believing that is safe, but I would not be shocked if it's totally not safe. So, if you copy this, probably remove the LOG statements in the decoder_thread, or at least verify that Godot can handle logs coming in from separate threads in a safe way.

Hi. It takes a single picture? I was looking for something like that. But I need also the preview in real time.

It spawns a decoder thread that reads MJPEG frames from a video source at this line: https://gist.github.com/netshade/867cef0c749ebb5624d9e0a0d1ff59f6#file-frame_decoder-c-L99

Then the method sensor_decoder_get_data gets called in the actual Godot project, and copies out whatever the decoder thread has most recently copied from that frame source. So the decoder thread in this case is running as fast as you care to let it run, and then your project / object is responsible for displaying the most recent decoder thread copy as you need it.

I'm working on something similar for Godot 4 using a newer version of ffmpeg and h264 frames in an mpegts container. I can receive the frames fine. I essentially get to the point where I have the h264 frame decoded to RGB24. I'm curious what you do with the data you're loading into the Godot pool byte array. How do you actually display it in the engine? any tips?

My apologies because my memory of what I did here is hazy, but I used this in concert with a texture object and set the textures data to the pool byte array returned by the get data method. Sorry for poor memory here, when next I’m able to actually look at the project files I’ll give you a more exact answer.

No worries. Thanks for the reply. I figued it was something simple, like just load the data as a bitmap and update a texture. I can do that no problem, but either my ffmpeg code has an error, or I don't understand some core concept about my specific use-case (i.e. h.264 decoding). With the new ffmpeg API, there's no longer avcodec_decode_video2, but in the same spot in my code I use sws_scale to convert the h264 frame to RGB24 and for debugging I write the frame out as a bitmap and it comes out all janky looking.

This is what things looked like on the Godot side of things, and I realize I sort of omitted some important details here:

func _process(delta):
	var byte_array = data.get_data() // Where "data" is an initialized version of the plugin
	if byte_array.size() > 0:
		var img = Image.new()
		img.create_from_data(640, 480, false, Image.FORMAT_L8, byte_array)
		var texture = ImageTexture.new()
		texture.create_from_image(img)
		$ImageButton.texture_normal = texture

So, my depth camera was capturing in 640x480, and sort of everything was predicated on th at path. Similarly, the image format was only L8, since the camera I was using ( an Intel True Depth camera ) was just outputting 8 bit depth values and nothing else. So I had sort of everything wired together on my particular setup and it all worked in concert, but it was by no means very flexible or durable to changes.

Thanks! this helps a lot. It confirms what I was thinking is possible at least. Just gotta get it working for my use-case.

@vix597 Are you still working on that? I'm trying to do the same thing, but in Rust. Maybe I can help somehow

Hey, I actually came up with a solid solution that works for what I need. I wrote a plugin in C++ using the new GDExtension API for Godot 4.x. My class extends from TextureRect and uses the FFmpeg libavcodec APIs to open a stream via a provided URL (e.g. tcp://somehost:port) and it decodes it on a thread, gets the raw RGB and writes it to the texture property of the texture rect on each decoded frame. If I can open source any part of it I plan to. Just have to double check.

@vix597 Hey, any update on your GDExtension plugin? The only other plugin on Github related to RTSP is this one that serves RTSP instead of being a client.

I'm working on figuring out approvals to open source the plugin I wrote. I have no idea how long it could take, but we pivoted away from ffmpeg so I'm hoping it will be easy since we don't really use the plugin anymore for what we were building.

Seconded, would love to see the example of your plugin @vix597

Agreed, hope you will publish it soon!

FWIW this repo has a semi working player - it pretty broken for live streams but it does play something: https://github.com/Elly2018/elly_videoplayer
I'm looking at seeing if I can troubleshoot it - the latency and the packet loss is terrible at the moment.

I modified one of the forks of the FFMPEG integration and got RTSP streaming working here: https://github.com/IvanWoehr/godot-ffmpeg-rtsp2

Just a basic checkin, and I've only tested RTSP streaming H264. PLease excuse any mess, I just wanted to checkin since I was so happy it's working.:)