Godot GDNative FFMPEG Streaming

frame_decoder.c

#include <gdnative_api_struct.gen.h>
#include <libavcodec/avcodec.h>
#include <libavutil/avutil.h>
#include <libavformat/avformat.h>
#include <libavutil/imgutils.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

const godot_gdnative_core_api_struct *api = NULL;
const godot_gdnative_ext_nativescript_api_struct *nativescript_api = NULL;

void *sensor_decoder_constructor(godot_object *p_instance, void *p_method_data);
void sensor_decoder_destructor(godot_object *p_instance, void *p_method_data, void *p_user_data);
godot_variant sensor_decoder_get_data(godot_object *p_instance, void *p_method_data,
        void *p_user_data, int p_num_args, godot_variant **p_args);

void godot_log(const wchar_t * message){
    godot_string s;
    api->godot_string_new_with_wide_string(&s, message, wcslen(message));
    api->godot_print(&s);
    api->godot_string_destroy(&s);
}

#define LOG(format, ...) do {\
    const wchar_t buf[512];\
    if(swprintf((wchar_t *)buf, sizeof(buf), format, ##__VA_ARGS__) <= 0){\
        godot_log(L"Error printing to log");\
    } else {\
        godot_log(buf);\
    }\
} while(0);

void GDN_EXPORT godot_gdnative_init(godot_gdnative_init_options *p_options) {
    api = p_options->api_struct;

    // Now find our extensions.
    for (int i = 0; i < api->num_extensions; i++) {
        switch (api->extensions[i]->type) {
            case GDNATIVE_EXT_NATIVESCRIPT: {
                nativescript_api = (godot_gdnative_ext_nativescript_api_struct *)api->extensions[i];
            }; break;
            default: break;
        }
    }
}


void GDN_EXPORT godot_gdnative_terminate(godot_gdnative_terminate_options *p_options) {
    api = NULL;
    nativescript_api = NULL;
}

void GDN_EXPORT godot_nativescript_init(void *p_handle) {
    godot_instance_create_func create = { NULL, NULL, NULL };
    create.create_func = &sensor_decoder_constructor;

    godot_instance_destroy_func destroy = { NULL, NULL, NULL };
    destroy.destroy_func = &sensor_decoder_destructor;

    nativescript_api->godot_nativescript_register_class(p_handle, "SENSORDECODER", "Reference",
            create, destroy);

    godot_instance_method get_data = { NULL, NULL, NULL };
    get_data.method = &sensor_decoder_get_data;

    godot_method_attributes attributes = { GODOT_METHOD_RPC_MODE_DISABLED };

    nativescript_api->godot_nativescript_register_method(p_handle, "SENSORDECODER", "get_data",
            attributes, get_data);
}

typedef struct user_data_struct {
    pthread_t decoder_thread;
    pthread_rwlock_t frame_lock;
    uint8_t * frame_data;
    size_t frame_size;
    size_t buf_size;
    godot_pool_byte_array frame;
    int stop_decoder;
} user_data_struct;

void * decoder_thread(void * decoder_args) {
    user_data_struct  *user_data    = (user_data_struct *) decoder_args;
    AVFormatContext   *pFormatCtx   = NULL;
    AVCodecContext    *pCodecCtx    = NULL;
    AVCodec           *pCodec       = NULL;
    AVCodecParameters *pCodecParams = NULL;
    AVFrame           *pFrame       = NULL;
    AVDictionary      *optionsDict  = NULL;
    AVPacket          packet;
    int               videoStream;
    int               frameFinished;
    int               stop;



    if(avformat_open_input(&pFormatCtx, "http://192.168.4.63:5000/stream/depth", NULL, NULL) !=0) {
        LOG(L"Couldn't open file");
        return NULL; // Couldn't open file
    }

    if(avformat_find_stream_info(pFormatCtx, NULL)<0) {
        LOG(L"Couldn't find stream information");
        return NULL;
    }

    // Find the first video stream
    videoStream = -1;
    for(int i=0; i<pFormatCtx->nb_streams; i++) {
        if(pFormatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
            videoStream = i;
            pCodecParams = pFormatCtx->streams[i]->codecpar;
            break;
        }
    }
    if(videoStream == -1) {
        LOG(L"Couldn't find video stream");
        return NULL; // Didn't find a video stream
    }



   // Find the decoder for the video stream
   pCodec = avcodec_find_decoder(pCodecParams->codec_id);

   if(pCodec==NULL) {
       LOG(L"Unsupported codec!");
       return NULL;
    }

   // Get a pointer to the codec context for the video stream
   pCodecCtx = avcodec_alloc_context3(pCodec);
    if(pCodecCtx == NULL){
        LOG(L"Couldn't create codec context");
        return NULL;
    }

   if(avcodec_open2(pCodecCtx, pCodec, &optionsDict) < 0) {
       LOG(L"Couldn't open codec");
       return NULL;
   }


   pFrame = av_frame_alloc();// Allocate video frame
   stop = 0;
   // Read frames and save first five frames to disk
   while(stop == 0 && av_read_frame(pFormatCtx, &packet) >= 0) {
     if(pthread_rwlock_rdlock(&user_data->frame_lock) != 0){
         continue;
     }
     stop = user_data->stop_decoder;
     if(pthread_rwlock_unlock(&user_data->frame_lock) != 0){
         LOG(L"Could not unlock locked thread, bailing");
         break;
     }
     if(stop == 1){
         LOG(L"Exiting decoder thread");
         break;
     }
     // Is this a packet from the video stream?
     if(packet.stream_index == videoStream) {
       // Decode video frame
       avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &packet);

       // Did we get a video frame?
       if(frameFinished)
       {
           if(pthread_rwlock_wrlock(&user_data->frame_lock) != 0){
               LOG(L"Could not lock to frame write, dropping frame");
               continue;
           }
           int copied = av_image_copy_to_buffer(user_data->frame_data, user_data->buf_size, pFrame->data, pFrame->linesize, pFrame->format, pFrame->width, pFrame->height, 1);
           if(copied < 0){
               LOG(L"Could not copy frame contents, dropping frame");
               memset(user_data->frame_data, 0, user_data->frame_size);
               continue;
           } else {
               user_data->frame_size = copied;
           }
           if(pthread_rwlock_unlock(&user_data->frame_lock) != 0){
               LOG(L"Could not unlock frame write after locked, bailing");
               break;
           }
       }
     }
     // Free the packet that was allocated by av_read_frame
     av_free_packet(&packet);
   }
   av_free(pFrame);// Free the video frame
   avcodec_close(pCodecCtx);// Close the codec
   avformat_close_input(&pFormatCtx);// Close the video file
    LOG(L"Decode finished");

    return NULL;
}

void *sensor_decoder_constructor(godot_object *p_instance, void *p_method_data) {
    user_data_struct * user_data = api->godot_alloc(sizeof(user_data_struct));
    user_data->buf_size = 1048576;
    user_data->frame_data = (uint8_t *) malloc(sizeof(uint8_t) * user_data->buf_size);
    user_data->frame_size = 0;
    api->godot_pool_byte_array_new(&user_data->frame);
    user_data->stop_decoder = 0;
    if(pthread_rwlock_init(&user_data->frame_lock, NULL) != 0){
        LOG(L"Lock failed to initialize");
    }
    if(pthread_create(&user_data->decoder_thread, NULL, decoder_thread, (void *)user_data) != 0){
        LOG(L"Thread start failed");
    } else {
        LOG(L"Thread start");
    }
    return user_data;
}

void sensor_decoder_destructor(godot_object *p_instance, void *p_method_data, void *p_user_data) {
    user_data_struct * user_data = (user_data_struct *) p_user_data;
    if(pthread_rwlock_wrlock(&user_data->frame_lock) != 0) {
        LOG(L"Could not acquire lock");
    } else {
        user_data->stop_decoder = 1;
        if(pthread_rwlock_unlock(&user_data->frame_lock) != 0){
            LOG(L"Could not unlock frame lock");
        }
    }
    if(pthread_join(user_data->decoder_thread, NULL) != 0){
        LOG(L"Could not safely exit decoder thread");
    }
    if(pthread_rwlock_destroy(&user_data->frame_lock) != 0){
        LOG(L"Could not destroy frame lock");
    }
    free(user_data->frame_data);
    api->godot_pool_byte_array_destroy(&user_data->frame);
    api->godot_free(p_user_data);
    LOG(L"Destroying");
}

godot_variant sensor_decoder_get_data(godot_object *p_instance, void *p_method_data,
        void *p_user_data, int p_num_args, godot_variant **p_args) {
    user_data_struct * user_data = (user_data_struct *) p_user_data;
    godot_variant ret;
    if(pthread_rwlock_rdlock(&user_data->frame_lock) != 0){
        LOG(L"Could not lock to frame read");
        // TODO
    }
    godot_int size = api->godot_pool_byte_array_size(&user_data->frame);
    if(size != user_data->frame_size){
        api->godot_pool_byte_array_resize(&user_data->frame, user_data->frame_size);
    }
    godot_pool_byte_array_write_access * write_access = api->godot_pool_byte_array_write(&user_data->frame);
    uint8_t * ptr = api->godot_pool_byte_array_write_access_ptr(write_access);
    memcpy(ptr, user_data->frame_data, user_data->frame_size);
    api->godot_pool_byte_array_write_access_destroy(write_access);
    if(pthread_rwlock_unlock(&user_data->frame_lock) != 0){
        LOG(L"Could not unlock frame read after locked, bailing");
        // TODO
    }
    api->godot_variant_new_pool_byte_array(&ret, &user_data->frame);

    return ret;
}

This is what things looked like on the Godot side of things, and I realize I sort of omitted some important details here:

func _process(delta):
	var byte_array = data.get_data() // Where "data" is an initialized version of the plugin
	if byte_array.size() > 0:
		var img = Image.new()
		img.create_from_data(640, 480, false, Image.FORMAT_L8, byte_array)
		var texture = ImageTexture.new()
		texture.create_from_image(img)
		$ImageButton.texture_normal = texture

So, my depth camera was capturing in 640x480, and sort of everything was predicated on th at path. Similarly, the image format was only L8, since the camera I was using ( an Intel True Depth camera ) was just outputting 8 bit depth values and nothing else. So I had sort of everything wired together on my particular setup and it all worked in concert, but it was by no means very flexible or durable to changes.

Thanks! this helps a lot. It confirms what I was thinking is possible at least. Just gotta get it working for my use-case.

@vix597 Are you still working on that? I'm trying to do the same thing, but in Rust. Maybe I can help somehow

Hey, I actually came up with a solid solution that works for what I need. I wrote a plugin in C++ using the new GDExtension API for Godot 4.x. My class extends from TextureRect and uses the FFmpeg libavcodec APIs to open a stream via a provided URL (e.g. tcp://somehost:port) and it decodes it on a thread, gets the raw RGB and writes it to the texture property of the texture rect on each decoded frame. If I can open source any part of it I plan to. Just have to double check.

@vix597 Hey, any update on your GDExtension plugin? The only other plugin on Github related to RTSP is this one that serves RTSP instead of being a client.

I'm working on figuring out approvals to open source the plugin I wrote. I have no idea how long it could take, but we pivoted away from ffmpeg so I'm hoping it will be easy since we don't really use the plugin anymore for what we were building.

Seconded, would love to see the example of your plugin @vix597

Agreed, hope you will publish it soon!

FWIW this repo has a semi working player - it pretty broken for live streams but it does play something: https://github.com/Elly2018/elly_videoplayer
I'm looking at seeing if I can troubleshoot it - the latency and the packet loss is terrible at the moment.

I modified one of the forks of the FFMPEG integration and got RTSP streaming working here: https://github.com/IvanWoehr/godot-ffmpeg-rtsp2

Just a basic checkin, and I've only tested RTSP streaming H264. PLease excuse any mess, I just wanted to checkin since I was so happy it's working.:)