Proof of concept: Create Pixmap via DRI3 from shared memory via udmabuf

main.c

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/syscall.h>
#include <sys/mman.h>
#include <linux/udmabuf.h>

#include <xcb/xcb.h>
#include <xcb/dri3.h>
#include <xcb/present.h>

struct App
{
  int                size;
  int                devFd, memFd, dmaFd;
  void             * mem;

  xcb_connection_t * xcb;
  xcb_screen_t     * screen;
  xcb_window_t       window;
  int                fd;
  xcb_pixmap_t       pixmap;
};

struct App app = { 0 };

static int get_udmabuf(const int size)
{
  int ret;

  app.size  = size;
  app.devFd = open("/dev/udmabuf", O_RDWR);
  if (app.devFd < 0)
  {
    perror("open /dev/udmabuf failed");
    return app.devFd;
  }

  app.memFd = memfd_create("udmabuf-test", MFD_ALLOW_SEALING);
  if (app.memFd < 0)
  {
    perror("memfd_create failed");
    return app.memFd;
  }

  app.mem = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, app.memFd, 0);
  if (!app.mem)
  {
    perror("mmap failed");
    return -1;
  }

  ret = fcntl(app.memFd, F_ADD_SEALS, F_SEAL_SHRINK);
  if (ret < 0)
  {
    perror("fcntl");
    return ret;
  }

  ret = ftruncate(app.memFd, size);
  if (ret == -1)
  {
    perror("ftruncate failed");
    return ret;
  }

  struct udmabuf_create create = {
    .memfd  = app.memFd,
    .offset = 0,
    .size   = size
  };

  app.dmaFd = ioctl(app.devFd, UDMABUF_CREATE, &create);
  if (app.dmaFd < 0)
  {
    perror("UDMABUF_CREATE failed");
    return -1;
  }

  return 0;
}

static void close_udmabuf()
{
  if (app.mem        ) munmap(app.mem, app.size);
  if (app.dmaFd >= 0 ) close(app.dmaFd);
  if (app.memFd  >= 0) close(app.memFd );
  if (app.devFd  >= 0) close(app.devFd );
}

static int dri3_open()
{
  xcb_dri3_open_cookie_t c =
    xcb_dri3_open(app.xcb, app.window, 0);
  xcb_dri3_open_reply_t * r =
    xcb_dri3_open_reply(app.xcb, c, NULL);
  if (!r)
  {
    printf("failed to open dri3\n");
    return -1;
  }

  if (r->nfd != 1)
  {
    free(r);
    printf("nfd != 1\n");
    return -1;
  }

  int fd = xcb_dri3_open_reply_fds(app.xcb, r)[0];
  if (fd < 0)
  {
    free(r);
    return fd;
  }

  fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
  free(r);
  return fd;
}

int main()
{
  const uint32_t width  = 200;
  const uint32_t height = 200;
  const uint32_t bpp    = 24;
  const uint32_t pitch  = 32;
  const int size        = (width * height * (pitch << 3) + (getpagesize() - 1)) & ~(getpagesize() - 1);

  if (!get_udmabuf(size) == 0)
    return -1;

  app.xcb = xcb_connect(NULL, NULL);
  app.screen     = xcb_setup_roots_iterator(xcb_get_setup(app.xcb)).data;

  const uint32_t values[2] = {
    app.screen->white_pixel,
    XCB_EVENT_MASK_EXPOSURE | XCB_EVENT_MASK_KEY_PRESS
  };

  app.window = xcb_generate_id(app.xcb);
  xcb_create_window(
    app.xcb,
    XCB_COPY_FROM_PARENT,
    app.window,
    app.screen->root,
    0, 0, 240, 240,
    0,
    XCB_WINDOW_CLASS_INPUT_OUTPUT,
    app.screen->root_visual,
    XCB_CW_BACK_PIXEL | XCB_CW_EVENT_MASK,
    values
  );

  xcb_map_window(app.xcb, app.window);
  xcb_flush(app.xcb);

  app.fd = dri3_open();
  if (app.fd < 0)
  {
    close_udmabuf();
    return app.fd;
  }

  app.pixmap = xcb_generate_id(app.xcb);
  {
    xcb_void_cookie_t c =
      xcb_dri3_pixmap_from_buffer_checked(
        app.xcb,
        app.pixmap,
        app.window,
        width * height * (pitch >> 3),
        width,
        height,
        width * (bpp >> 3),
        24,
        32,
        app.dmaFd
      );

    xcb_generic_error_t *error;
    if ((error = xcb_request_check(app.xcb, c)))
    {
      printf("dri3_pixmap_from_buffer failure: code=%d value=%d\n",
          error->error_code, error->resource_id);
      free(error);
      goto exit;
    }
  }

  int id = 0;
  while(1)
  {
    xcb_generic_event_t * e;
    if ((e = xcb_poll_for_event(app.xcb)) != NULL)
    {
      switch(e->response_type)
      {
        case XCB_KEY_PRESS:
        {
          xcb_key_release_event_t *ev = (xcb_key_release_event_t *)e;
          if (ev->detail == 9)
          {
            free(e);
            goto exit;
          }
        }

        case XCB_EXPOSE:
          break;
      }

      free(e);
    }

    memset(app.mem, id, app.size);

    xcb_present_pixmap(
      app.xcb,
      app.window,
      app.pixmap,
      ++id,
      0,  // valid
      0,  // dirty
      20, // x
      20, // y
      0,
      0,
      0,
      XCB_PRESENT_OPTION_COPY,
      0, 0, 0, 0,
      NULL
    );

    xcb_flush(app.xcb);
  }

exit:
  close(app.fd);
  xcb_disconnect(app.xcb);
  close_udmabuf();
  return 0;
}

Makefile

all:
	gcc -D_GNU_SOURCE main.c -Wall -Werror -g -Og -o test -lX11-xcb -lX11 -lxcb -lxcb-dri3 -lxcb-present

Epilepsy Warning: Puts a rapidly fading square on screen.