reflow-arekinath-nvidia-v2.patch

From 0e97680a5492a5a834a9b747c77787228922826c Mon Sep 17 00:00:00 2001
From: Tony Cheneau <tony.cheneau@amnesiak.org>
Date: Wed, 29 Apr 2020 23:00:21 +0200
Subject: [PATCH] arekinath-nvidia-reflow-for-nixos

---
 src/Makefile.am       |  2 +-
 src/debug.c           | 31 ++++++++++++++++++++++---------
 src/debug.h           |  2 +-
 src/object_heap.c     |  9 +++++++--
 src/vdpau_buffer.c    | 31 ++++++++++++++++++++++++++++---
 src/vdpau_buffer.h    |  2 ++
 src/vdpau_decode.c    | 39 ++++++++++++++++++++++++++++++++++-----
 src/vdpau_driver.c    | 11 ++++++++---
 src/vdpau_driver.h    |  4 +++-
 src/vdpau_mixer.c     |  1 +
 src/vdpau_video.c     |  1 +
 src/vdpau_video.h     |  1 +
 src/vdpau_video_x11.c |  9 +++++++--
 13 files changed, 116 insertions(+), 27 deletions(-)

diff --git a/src/Makefile.am b/src/Makefile.am
index cec36c1..0dc2dad 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -75,7 +75,7 @@ EXTRA_DIST = \
 install-data-hook:
 	cd $(DESTDIR)$(LIBVA_DRIVERS_PATH) ;			\
 	for drv in $(DRIVERS); do				\
-	    ln -s vdpau_drv_video.so $${drv}_drv_video.so;	\
+	    ln -sf vdpau_drv_video.so $${drv}_drv_video.so;	\
 	done
 
 # Extra clean files so that maintainer-clean removes *everything*
diff --git a/src/debug.c b/src/debug.c
index 870d956..2c98f8d 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -22,6 +22,7 @@
 #include "debug.h"
 #include "utils.h"
 #include <stdarg.h>
+#include <unistd.h>
 
 static void do_vfprintf(FILE *fp, const char *msg, va_list args)
 {
@@ -43,9 +44,9 @@ void vdpau_error_message(const char *msg, ...)
 {
     va_list args;
 
-    do_fprintf(stderr, "%s: error: ", PACKAGE_NAME);
+    do_fprintf(stdout, "%s: error: ", PACKAGE_NAME);
     va_start(args, msg);
-    do_vfprintf(stderr, msg, args);
+    do_vfprintf(stdout, msg, args);
     va_end(args);
 }
 
@@ -69,17 +70,28 @@ static int debug_enabled(void)
     return g_debug_enabled;
 }
 
+static FILE *debug_file(void)
+{
+    static FILE *g_debug_file = NULL;
+    if (g_debug_file == NULL) {
+        g_debug_file = fopen("/tmp/libva-vdpau-debug.log", "a");
+    }
+    return g_debug_file;
+}
+
 void debug_message(const char *msg, ...)
 {
     va_list args;
+    FILE *f = debug_file();
 
     if (!debug_enabled())
         return;
 
-    do_fprintf(stdout, "%s: ", PACKAGE_NAME);
+    do_fprintf(f, "%s(%d): ", PACKAGE_NAME, getpid());
     va_start(args, msg);
-    do_vfprintf(stdout, msg, args);
+    do_vfprintf(f, msg, args);
     va_end(args);
+    fflush(f);
 }
 
 static int g_trace_is_new_line  = 1;
@@ -113,25 +125,26 @@ void trace_indent(int inc)
 void trace_print(const char *format, ...)
 {
     va_list args;
+    FILE *f = debug_file();
 
     if (g_trace_is_new_line) {
         int i, j, n;
-        printf("%s: ", PACKAGE_NAME);
+        fprintf(f, "%s: ", PACKAGE_NAME);
         n = trace_indent_width();
         for (i = 0; i < g_trace_indent; i++) {
             for (j = 0; j < n / 4; j++)
-                printf("    ");
+                fprintf(f, "    ");
             for (j = 0; j < n % 4; j++)
-                printf(" ");
+                fprintf(f, " ");
         }
     }
 
     va_start(args, format);
-    vfprintf(stdout, format, args);
+    vfprintf(f, format, args);
     va_end(args);
 
     g_trace_is_new_line = (strchr(format, '\n') != NULL);
 
     if (g_trace_is_new_line)
-        fflush(stdout);
+        fflush(f);
 }
diff --git a/src/debug.h b/src/debug.h
index f10b2ed..0c30688 100644
--- a/src/debug.h
+++ b/src/debug.h
@@ -30,7 +30,7 @@ void vdpau_information_message(const char *msg, ...)
 void debug_message(const char *msg, ...)
     attribute_hidden;
 
-#if DEBUG && USE_DEBUG
+#if (DEBUG && USE_DEBUG)
 # define D(x) x
 # define bug debug_message
 #else
diff --git a/src/object_heap.c b/src/object_heap.c
index 96b8cb6..a21d8bc 100644
--- a/src/object_heap.c
+++ b/src/object_heap.c
@@ -99,7 +99,6 @@ object_heap_expand(object_heap_p heap)
 int
 object_heap_init(object_heap_p heap, int object_size, int id_offset)
 {
-    pthread_mutex_init(&heap->mutex, NULL);
     heap->object_size = object_size;
     heap->id_offset = id_offset & OBJECT_HEAP_OFFSET_MASK;
     heap->heap_size = 0;
@@ -107,7 +106,13 @@ object_heap_init(object_heap_p heap, int object_size, int id_offset)
     heap->next_free = LAST_FREE;
     heap->num_buckets = 0;
     heap->bucket = NULL;
-    return object_heap_expand(heap);
+    if (object_heap_expand(heap) == 0) {
+        pthread_mutex_init(&heap->mutex, NULL);
+        return 0;
+    } else {
+        free(heap->bucket);
+        return -1;
+    }
 }
 
 /*
diff --git a/src/vdpau_buffer.c b/src/vdpau_buffer.c
index f7883df..989cb2f 100644
--- a/src/vdpau_buffer.c
+++ b/src/vdpau_buffer.c
@@ -41,6 +41,8 @@ destroy_dead_va_buffers(
     if (obj_context->dead_buffers_count < 1)
         return;
 
+    return;
+
     ASSERT(obj_context->dead_buffers);
     for (i = 0; i < obj_context->dead_buffers_count; i++) {
         obj_buffer = VDPAU_BUFFER(obj_context->dead_buffers[i]);
@@ -79,6 +81,7 @@ create_va_buffer(
     obj_buffer->buffer_data      = malloc(obj_buffer->buffer_size);
     obj_buffer->mtime            = 0;
     obj_buffer->delayed_destroy  = 0;
+    obj_buffer->next_pre_slice_params = NULL;
 
     if (!obj_buffer->buffer_data) {
         destroy_va_buffer(driver_data, obj_buffer);
@@ -96,6 +99,7 @@ destroy_va_buffer(
 {
     if (!obj_buffer)
         return;
+    D(bug("destroying va buffer %x\n", obj_buffer->base.id));
 
     if (obj_buffer->buffer_data) {
         free(obj_buffer->buffer_data);
@@ -142,8 +146,8 @@ vdpau_CreateBuffer(
 {
     VDPAU_DRIVER_DATA_INIT;
 
-    if (buf_id)
-        *buf_id = VA_INVALID_BUFFER;
+    /*if (buf_id)
+        *buf_id = VA_INVALID_BUFFER;*/
 
     /* Validate type */
     switch (type) {
@@ -165,9 +169,25 @@ vdpau_CreateBuffer(
     if (!obj_buffer)
         return VA_STATUS_ERROR_ALLOCATION_FAILED;
 
+    switch (type) {
+    case VAPictureParameterBufferType:
+        obj_buffer->typestr = "VAPictureParameterBufferType"; break;
+    case VAIQMatrixBufferType:
+        obj_buffer->typestr = "VAIQMatrixBufferType"; break;
+    case VASliceParameterBufferType:
+        obj_buffer->typestr = "VASliceParameterBufferType"; break;
+    case VASliceDataBufferType:
+        obj_buffer->typestr = "VASliceDataBufferType"; break;
+    case VABitPlaneBufferType:
+        obj_buffer->typestr = "VABitPlaneBufferType"; break;
+    case VAImageBufferType:
+        obj_buffer->typestr = "VAImageBufferType"; break;
+    }
+
     if (data)
         memcpy(obj_buffer->buffer_data, data, obj_buffer->buffer_size);
 
+    D(bug("allocated buffer id %x (%s)\n", obj_buffer->base.id, obj_buffer->typestr));
     if (buf_id)
         *buf_id = obj_buffer->base.id;
 
@@ -185,6 +205,7 @@ vdpau_DestroyBuffer(
 
     object_buffer_p obj_buffer = VDPAU_BUFFER(buffer_id);
 
+    D(bug("destroy req for buffer id %x\n", buffer_id));
     if (obj_buffer && !obj_buffer->delayed_destroy)
         destroy_va_buffer(driver_data, obj_buffer);
 
@@ -223,8 +244,12 @@ vdpau_MapBuffer(
     VDPAU_DRIVER_DATA_INIT;
 
     object_buffer_p obj_buffer = VDPAU_BUFFER(buf_id);
-    if (!obj_buffer)
+    if (!obj_buffer) {
+        D(bug("tried to map invalid buffer id %x\n", buf_id));
         return VA_STATUS_ERROR_INVALID_BUFFER;
+    }
+
+    D(bug("map buffer id %x\n", buf_id));
 
     if (pbuf)
         *pbuf = obj_buffer->buffer_data;
diff --git a/src/vdpau_buffer.h b/src/vdpau_buffer.h
index 20581e5..0ce9483 100644
--- a/src/vdpau_buffer.h
+++ b/src/vdpau_buffer.h
@@ -28,6 +28,8 @@ struct object_buffer {
     struct object_base  base;
     VAContextID         va_context;
     VABufferType        type;
+    const char *        typestr;
+    struct object_buffer *next_pre_slice_params;
     void               *buffer_data;
     unsigned int        buffer_size;
     unsigned int        max_num_elements;
diff --git a/src/vdpau_decode.c b/src/vdpau_decode.c
index 20bd897..5d3bd36 100644
--- a/src/vdpau_decode.c
+++ b/src/vdpau_decode.c
@@ -463,6 +463,13 @@ translate_VASliceDataBuffer(
 )
 {
     if (obj_context->vdp_codec == VDP_CODEC_H264) {
+        /* Queue up any slicedata that arrive before sliceparams */
+        if (obj_context->last_slice_params_count == 0) {
+            obj_buffer->next_pre_slice_params = obj_context->last_slice_data;
+            obj_context->last_slice_data = obj_buffer;
+            return 1;
+        }
+        D(bug("start h264 data buffer (slices = %d)\n", obj_context->last_slice_params_count));
         /* Check we have the start code */
         /* XXX: check for other codecs too? */
         /* XXX: this assumes we get SliceParams before SliceData */
@@ -472,6 +479,7 @@ translate_VASliceDataBuffer(
         for (i = 0; i < obj_context->last_slice_params_count; i++) {
             VASliceParameterBufferH264 * const slice_param = &slice_params[i];
             uint8_t *buf = (uint8_t *)obj_buffer->buffer_data + slice_param->slice_data_offset;
+            D(bug("h264 slice at +0x%x (%u long) (#%d)\n", slice_param->slice_data_offset, slice_param->slice_data_size, i));
             if (memcmp(buf, start_code_prefix, sizeof(start_code_prefix)) != 0) {
                 if (append_VdpBitstreamBuffer(obj_context,
                                               start_code_prefix,
@@ -483,6 +491,7 @@ translate_VASliceDataBuffer(
                                           slice_param->slice_data_size) < 0)
                 return 0;
         }
+        D(bug("h264 data buffer done\n"));
         return 1;
     }
 
@@ -902,6 +911,19 @@ translate_VASliceParameterBufferH264(
     pic_info->num_ref_idx_l1_active_minus1 = slice_param->num_ref_idx_l1_active_minus1;
     obj_context->last_slice_params         = obj_buffer->buffer_data;
     obj_context->last_slice_params_count   = obj_buffer->num_elements;
+
+    if (obj_context->last_slice_data) {
+        object_buffer_p sdata_buffer = obj_context->last_slice_data, sdata_next = NULL;
+        while (sdata_buffer != NULL) {
+            if (!translate_VASliceDataBuffer(driver_data, obj_context, sdata_buffer))
+                return 0;
+            sdata_next = sdata_buffer->next_pre_slice_params;
+            sdata_buffer->next_pre_slice_params = NULL;
+            sdata_buffer = sdata_next;
+        }
+        obj_context->last_slice_data = NULL;
+    }
+
     return 1;
 }
 
@@ -1152,6 +1174,7 @@ vdpau_BeginPicture(
 
     obj_surface->va_surface_status           = VASurfaceRendering;
     obj_context->last_pic_param              = NULL;
+    obj_context->last_slice_data             = NULL;
     obj_context->last_slice_params           = NULL;
     obj_context->last_slice_params_count     = 0;
     obj_context->current_render_target       = obj_surface->base.id;
@@ -1209,27 +1232,29 @@ vdpau_RenderPicture(
     /* Translate buffers */
     for (i = 0; i < num_buffers; i++) {
         object_buffer_p obj_buffer = VDPAU_BUFFER(buffers[i]);
+        D(bug("try translate buffer %x (type = %d/%s)\n", buffers[i], obj_buffer->type, obj_buffer->typestr));
         if (!translate_buffer(driver_data, obj_context, obj_buffer))
             return VA_STATUS_ERROR_UNSUPPORTED_BUFFERTYPE;
+        D(bug("translated buffer %x\n", buffers[i]));
         /* Release any buffer that is not VASliceDataBuffer */
         /* VASliceParameterBuffer is also needed to check for start_codes */
         switch (obj_buffer->type) {
         case VASliceParameterBufferType:
         case VASliceDataBufferType:
-            schedule_destroy_va_buffer(driver_data, obj_buffer);
+            //schedule_destroy_va_buffer(driver_data, obj_buffer);
             break;
         case VAPictureParameterBufferType:
             /* Preserve VAPictureParameterBufferMPEG4 */
             if (obj_context->vdp_codec == VDP_CODEC_MPEG4) {
-                schedule_destroy_va_buffer(driver_data, obj_buffer);
+                //schedule_destroy_va_buffer(driver_data, obj_buffer);
                 break;
             }
             /* fall-through */
         default:
-            destroy_va_buffer(driver_data, obj_buffer);
+            //destroy_va_buffer(driver_data, obj_buffer);
             break;
         }
-        buffers[i] = VA_INVALID_BUFFER;
+        //buffers[i] = VA_INVALID_BUFFER;
     }
 
     return VA_STATUS_SUCCESS;
@@ -1277,6 +1302,8 @@ vdpau_EndPicture(
             dump_VdpBitstreamBuffer(&obj_context->vdp_bitstream_buffers[i]);
     }
 
+    D(bug("rendering to surface %x\n", obj_context->current_render_target));
+
     VAStatus va_status;
     VdpStatus vdp_status;
     vdp_status = ensure_decoder_with_max_refs(
@@ -1284,6 +1311,7 @@ vdpau_EndPicture(
         obj_context,
         get_num_ref_frames(obj_context)
     );
+    D(bug("vdp_status after ensure = %d\n", vdp_status));
     if (vdp_status == VDP_STATUS_OK)
         vdp_status = vdpau_decoder_render(
             driver_data,
@@ -1294,9 +1322,10 @@ vdpau_EndPicture(
             obj_context->vdp_bitstream_buffers
         );
     va_status = vdpau_get_VAStatus(vdp_status);
+    D(bug("vdp_status after render = %d\n", vdp_status));
 
     /* XXX: assume we are done with rendering right away */
-    obj_context->current_render_target = VA_INVALID_SURFACE;
+    //obj_context->current_render_target = VA_INVALID_SURFACE;
 
     /* Release pending buffers */
     destroy_dead_va_buffers(driver_data, obj_context);
diff --git a/src/vdpau_driver.c b/src/vdpau_driver.c
index 6fc35c6..929ee7f 100644
--- a/src/vdpau_driver.c
+++ b/src/vdpau_driver.c
@@ -174,7 +174,7 @@ vdpau_common_Terminate(vdpau_driver_data_t *driver_data)
     }
     vdpau_gate_exit(driver_data);
 
-    if (driver_data->vdp_dpy) {
+    if ((!driver_data->x_fallback) && driver_data->vdp_dpy) {
         XCloseDisplay(driver_data->vdp_dpy);
         driver_data->vdp_dpy = NULL;
     }
@@ -187,8 +187,13 @@ vdpau_common_Initialize(vdpau_driver_data_t *driver_data)
     /* Create a dedicated X11 display for VDPAU purposes */
     const char * const x11_dpy_name = XDisplayString(driver_data->x11_dpy);
     driver_data->vdp_dpy = XOpenDisplay(x11_dpy_name);
-    if (!driver_data->vdp_dpy)
-        return VA_STATUS_ERROR_UNKNOWN;
+    /* Fallback to existing X11 display */
+    driver_data->x_fallback = false;
+    if (!driver_data->vdp_dpy) {
+        driver_data->x_fallback = true;
+        driver_data->vdp_dpy = driver_data->x11_dpy;
+        printf("Failed to create dedicated X11 display!\n");
+    }
 
     VdpStatus vdp_status;
     driver_data->vdp_device = VDP_INVALID_HANDLE;
diff --git a/src/vdpau_driver.h b/src/vdpau_driver.h
index 54d37fb..9a81513 100644
--- a/src/vdpau_driver.h
+++ b/src/vdpau_driver.h
@@ -23,6 +23,7 @@
 
 #include <linux/videodev2.h>
 
+#include <stdbool.h>
 #include <va/va_backend.h>
 #include "vaapi_compat.h"
 #include "vdpau_gate.h"
@@ -65,7 +66,7 @@
 #define VDPAU_MAX_DISPLAY_ATTRIBUTES    6
 #define VDPAU_MAX_OUTPUT_SURFACES       2
 #define VDPAU_STR_DRIVER_VENDOR         "Splitted-Desktop Systems"
-#define VDPAU_STR_DRIVER_NAME           "VDPAU backend for VA-API"
+#define VDPAU_STR_DRIVER_NAME           "VDPAU backend for VA-API (arekinath)"
 
 /* Check we have MPEG-4 support in VDPAU and the necessary VAAPI extensions */
 #define USE_VDPAU_MPEG4                                         \
@@ -102,6 +103,7 @@ struct vdpau_driver_data {
     uint64_t                    va_display_attrs_mtime[VDPAU_MAX_DISPLAY_ATTRIBUTES];
     unsigned int                va_display_attrs_count;
     char                        va_vendor[256];
+    bool                        x_fallback;
 };
 
 typedef struct object_config   *object_config_p;
diff --git a/src/vdpau_mixer.c b/src/vdpau_mixer.c
index 0fcec9c..76ddd23 100644
--- a/src/vdpau_mixer.c
+++ b/src/vdpau_mixer.c
@@ -21,6 +21,7 @@
 #include "sysdeps.h"
 #include "vdpau_mixer.h"
 #include "vdpau_video.h"
+#include "debug.h"
 #include <math.h>
 
 #define VDPAU_MAX_VIDEO_MIXER_PARAMS    4
diff --git a/src/vdpau_video.c b/src/vdpau_video.c
index c940949..10453ec 100644
--- a/src/vdpau_video.c
+++ b/src/vdpau_video.c
@@ -462,6 +462,7 @@ vdpau_CreateSurfaces(
         }
 
         int va_surface = object_heap_allocate(&driver_data->surface_heap);
+        D(bug("created %dx%d surface at id %x, vdp handle %x\n", width, height, va_surface, vdp_surface));
         object_surface_p obj_surface = VDPAU_SURFACE(va_surface);
         if (!obj_surface) {
             va_status = VA_STATUS_ERROR_ALLOCATION_FAILED;
diff --git a/src/vdpau_video.h b/src/vdpau_video.h
index fc3fd65..452749d 100644
--- a/src/vdpau_video.h
+++ b/src/vdpau_video.h
@@ -60,6 +60,7 @@ struct object_context {
     void                        *last_pic_param;
     void                        *last_slice_params;
     unsigned int                 last_slice_params_count;
+    struct object_buffer        *last_slice_data;
     VdpCodec                     vdp_codec;
     VdpDecoderProfile            vdp_profile;
     VdpDecoder                   vdp_decoder;
diff --git a/src/vdpau_video_x11.c b/src/vdpau_video_x11.c
index 8171549..6c0bf70 100644
--- a/src/vdpau_video_x11.c
+++ b/src/vdpau_video_x11.c
@@ -401,7 +401,7 @@ render_surface(
     src_rect.y0 = source_rect->y;
     src_rect.x1 = source_rect->x + source_rect->width;
     src_rect.y1 = source_rect->y + source_rect->height;
-    ensure_bounds(&src_rect, obj_surface->width, obj_surface->height);   
+    ensure_bounds(&src_rect, obj_surface->width, obj_surface->height);
 
     VdpRect dst_rect;
     dst_rect.x0 = target_rect->x;
@@ -785,6 +785,9 @@ vdpau_PutSurface(
 {
     VDPAU_DRIVER_DATA_INIT;
 
+    VAStatus ret;
+
+    D(bug("putsurface(%x, (%d,%d+%dx%d=>%d,%d+%dx%d))\n", surface, srcx, srcy, srcw, srch, destx, desty, destw, desth));
     vdpau_set_display_type(driver_data, VA_DISPLAY_X11);
 
     /* XXX: no clip rects supported */
@@ -805,5 +808,7 @@ vdpau_PutSurface(
     dst_rect.y      = desty;
     dst_rect.width  = destw;
     dst_rect.height = desth;
-    return put_surface(driver_data, surface, xid, w, h, &src_rect, &dst_rect, flags);
+    ret = put_surface(driver_data, surface, xid, w, h, &src_rect, &dst_rect, flags);
+
+    return ret;
 }
-- 
2.23.1