Godot GDNative FFMPEG Streaming

frame_decoder.c

#include <gdnative_api_struct.gen.h>
#include <libavcodec/avcodec.h>
#include <libavutil/avutil.h>
#include <libavformat/avformat.h>
#include <libavutil/imgutils.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

const godot_gdnative_core_api_struct *api = NULL;
const godot_gdnative_ext_nativescript_api_struct *nativescript_api = NULL;

void *sensor_decoder_constructor(godot_object *p_instance, void *p_method_data);
void sensor_decoder_destructor(godot_object *p_instance, void *p_method_data, void *p_user_data);
godot_variant sensor_decoder_get_data(godot_object *p_instance, void *p_method_data,
        void *p_user_data, int p_num_args, godot_variant **p_args);

void godot_log(const wchar_t * message){
    godot_string s;
    api->godot_string_new_with_wide_string(&s, message, wcslen(message));
    api->godot_print(&s);
    api->godot_string_destroy(&s);
}

#define LOG(format, ...) do {\
    const wchar_t buf[512];\
    if(swprintf((wchar_t *)buf, sizeof(buf), format, ##__VA_ARGS__) <= 0){\
        godot_log(L"Error printing to log");\
    } else {\
        godot_log(buf);\
    }\
} while(0);

void GDN_EXPORT godot_gdnative_init(godot_gdnative_init_options *p_options) {
    api = p_options->api_struct;

    // Now find our extensions.
    for (int i = 0; i < api->num_extensions; i++) {
        switch (api->extensions[i]->type) {
            case GDNATIVE_EXT_NATIVESCRIPT: {
                nativescript_api = (godot_gdnative_ext_nativescript_api_struct *)api->extensions[i];
            }; break;
            default: break;
        }
    }
}


void GDN_EXPORT godot_gdnative_terminate(godot_gdnative_terminate_options *p_options) {
    api = NULL;
    nativescript_api = NULL;
}

void GDN_EXPORT godot_nativescript_init(void *p_handle) {
    godot_instance_create_func create = { NULL, NULL, NULL };
    create.create_func = &sensor_decoder_constructor;

    godot_instance_destroy_func destroy = { NULL, NULL, NULL };
    destroy.destroy_func = &sensor_decoder_destructor;

    nativescript_api->godot_nativescript_register_class(p_handle, "SENSORDECODER", "Reference",
            create, destroy);

    godot_instance_method get_data = { NULL, NULL, NULL };
    get_data.method = &sensor_decoder_get_data;

    godot_method_attributes attributes = { GODOT_METHOD_RPC_MODE_DISABLED };

    nativescript_api->godot_nativescript_register_method(p_handle, "SENSORDECODER", "get_data",
            attributes, get_data);
}

typedef struct user_data_struct {
    pthread_t decoder_thread;
    pthread_rwlock_t frame_lock;
    uint8_t * frame_data;
    size_t frame_size;
    size_t buf_size;
    godot_pool_byte_array frame;
    int stop_decoder;
} user_data_struct;

void * decoder_thread(void * decoder_args) {
    user_data_struct  *user_data    = (user_data_struct *) decoder_args;
    AVFormatContext   *pFormatCtx   = NULL;
    AVCodecContext    *pCodecCtx    = NULL;
    AVCodec           *pCodec       = NULL;
    AVCodecParameters *pCodecParams = NULL;
    AVFrame           *pFrame       = NULL;
    AVDictionary      *optionsDict  = NULL;
    AVPacket          packet;
    int               videoStream;
    int               frameFinished;
    int               stop;



    if(avformat_open_input(&pFormatCtx, "http://192.168.4.63:5000/stream/depth", NULL, NULL) !=0) {
        LOG(L"Couldn't open file");
        return NULL; // Couldn't open file
    }

    if(avformat_find_stream_info(pFormatCtx, NULL)<0) {
        LOG(L"Couldn't find stream information");
        return NULL;
    }

    // Find the first video stream
    videoStream = -1;
    for(int i=0; i<pFormatCtx->nb_streams; i++) {
        if(pFormatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
            videoStream = i;
            pCodecParams = pFormatCtx->streams[i]->codecpar;
            break;
        }
    }
    if(videoStream == -1) {
        LOG(L"Couldn't find video stream");
        return NULL; // Didn't find a video stream
    }



   // Find the decoder for the video stream
   pCodec = avcodec_find_decoder(pCodecParams->codec_id);

   if(pCodec==NULL) {
       LOG(L"Unsupported codec!");
       return NULL;
    }

   // Get a pointer to the codec context for the video stream
   pCodecCtx = avcodec_alloc_context3(pCodec);
    if(pCodecCtx == NULL){
        LOG(L"Couldn't create codec context");
        return NULL;
    }

   if(avcodec_open2(pCodecCtx, pCodec, &optionsDict) < 0) {
       LOG(L"Couldn't open codec");
       return NULL;
   }


   pFrame = av_frame_alloc();// Allocate video frame
   stop = 0;
   // Read frames and save first five frames to disk
   while(stop == 0 && av_read_frame(pFormatCtx, &packet) >= 0) {
     if(pthread_rwlock_rdlock(&user_data->frame_lock) != 0){
         continue;
     }
     stop = user_data->stop_decoder;
     if(pthread_rwlock_unlock(&user_data->frame_lock) != 0){
         LOG(L"Could not unlock locked thread, bailing");
         break;
     }
     if(stop == 1){
         LOG(L"Exiting decoder thread");
         break;
     }
     // Is this a packet from the video stream?
     if(packet.stream_index == videoStream) {
       // Decode video frame
       avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &packet);

       // Did we get a video frame?
       if(frameFinished)
       {
           if(pthread_rwlock_wrlock(&user_data->frame_lock) != 0){
               LOG(L"Could not lock to frame write, dropping frame");
               continue;
           }
           int copied = av_image_copy_to_buffer(user_data->frame_data, user_data->buf_size, pFrame->data, pFrame->linesize, pFrame->format, pFrame->width, pFrame->height, 1);
           if(copied < 0){
               LOG(L"Could not copy frame contents, dropping frame");
               memset(user_data->frame_data, 0, user_data->frame_size);
               continue;
           } else {
               user_data->frame_size = copied;
           }
           if(pthread_rwlock_unlock(&user_data->frame_lock) != 0){
               LOG(L"Could not unlock frame write after locked, bailing");
               break;
           }
       }
     }
     // Free the packet that was allocated by av_read_frame
     av_free_packet(&packet);
   }
   av_free(pFrame);// Free the video frame
   avcodec_close(pCodecCtx);// Close the codec
   avformat_close_input(&pFormatCtx);// Close the video file
    LOG(L"Decode finished");

    return NULL;
}

void *sensor_decoder_constructor(godot_object *p_instance, void *p_method_data) {
    user_data_struct * user_data = api->godot_alloc(sizeof(user_data_struct));
    user_data->buf_size = 1048576;
    user_data->frame_data = (uint8_t *) malloc(sizeof(uint8_t) * user_data->buf_size);
    user_data->frame_size = 0;
    api->godot_pool_byte_array_new(&user_data->frame);
    user_data->stop_decoder = 0;
    if(pthread_rwlock_init(&user_data->frame_lock, NULL) != 0){
        LOG(L"Lock failed to initialize");
    }
    if(pthread_create(&user_data->decoder_thread, NULL, decoder_thread, (void *)user_data) != 0){
        LOG(L"Thread start failed");
    } else {
        LOG(L"Thread start");
    }
    return user_data;
}

void sensor_decoder_destructor(godot_object *p_instance, void *p_method_data, void *p_user_data) {
    user_data_struct * user_data = (user_data_struct *) p_user_data;
    if(pthread_rwlock_wrlock(&user_data->frame_lock) != 0) {
        LOG(L"Could not acquire lock");
    } else {
        user_data->stop_decoder = 1;
        if(pthread_rwlock_unlock(&user_data->frame_lock) != 0){
            LOG(L"Could not unlock frame lock");
        }
    }
    if(pthread_join(user_data->decoder_thread, NULL) != 0){
        LOG(L"Could not safely exit decoder thread");
    }
    if(pthread_rwlock_destroy(&user_data->frame_lock) != 0){
        LOG(L"Could not destroy frame lock");
    }
    free(user_data->frame_data);
    api->godot_pool_byte_array_destroy(&user_data->frame);
    api->godot_free(p_user_data);
    LOG(L"Destroying");
}

godot_variant sensor_decoder_get_data(godot_object *p_instance, void *p_method_data,
        void *p_user_data, int p_num_args, godot_variant **p_args) {
    user_data_struct * user_data = (user_data_struct *) p_user_data;
    godot_variant ret;
    if(pthread_rwlock_rdlock(&user_data->frame_lock) != 0){
        LOG(L"Could not lock to frame read");
        // TODO
    }
    godot_int size = api->godot_pool_byte_array_size(&user_data->frame);
    if(size != user_data->frame_size){
        api->godot_pool_byte_array_resize(&user_data->frame, user_data->frame_size);
    }
    godot_pool_byte_array_write_access * write_access = api->godot_pool_byte_array_write(&user_data->frame);
    uint8_t * ptr = api->godot_pool_byte_array_write_access_ptr(write_access);
    memcpy(ptr, user_data->frame_data, user_data->frame_size);
    api->godot_pool_byte_array_write_access_destroy(write_access);
    if(pthread_rwlock_unlock(&user_data->frame_lock) != 0){
        LOG(L"Could not unlock frame read after locked, bailing");
        // TODO
    }
    api->godot_variant_new_pool_byte_array(&ret, &user_data->frame);

    return ret;
}

I'm working on figuring out approvals to open source the plugin I wrote. I have no idea how long it could take, but we pivoted away from ffmpeg so I'm hoping it will be easy since we don't really use the plugin anymore for what we were building.

Seconded, would love to see the example of your plugin @vix597

Agreed, hope you will publish it soon!

FWIW this repo has a semi working player - it pretty broken for live streams but it does play something: https://github.com/Elly2018/elly_videoplayer
I'm looking at seeing if I can troubleshoot it - the latency and the packet loss is terrible at the moment.

I modified one of the forks of the FFMPEG integration and got RTSP streaming working here: https://github.com/IvanWoehr/godot-ffmpeg-rtsp2

Just a basic checkin, and I've only tested RTSP streaming H264. PLease excuse any mess, I just wanted to checkin since I was so happy it's working.:)