Trying to break nvidia's glGetTexImage

main.c

#define _GNU_SOURCE
#include <assert.h>
#include <fcntl.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <unistd.h>
#include <wayland-client.h>
#include <wayland-client-protocol.h>
#include <linux/input-event-codes.h>

#include "xdg-shell-client-protocol.h"

#include <EGL/egl.h>
#include <EGL/eglext.h>
#include <GLES2/gl2.h>
#include <GL/gl.h>

#include <gbm.h>
#include <drm/drm_fourcc.h>

static const char* eglStrError(EGLint err)
{
    switch (err){
        case EGL_SUCCESS:           return "EGL_SUCCESS";
        case EGL_NOT_INITIALIZED:   return "EGL_NOT_INITIALIZED";
        case EGL_BAD_ACCESS:        return "EGL_BAD_ACCESS";
        case EGL_BAD_ALLOC:         return "EGL_BAD_ALLOC";
        case EGL_BAD_ATTRIBUTE:     return "EGL_BAD_ATTRIBUTE";
        case EGL_BAD_CONFIG:        return "EGL_BAD_CONFIG";
        case EGL_BAD_CONTEXT:       return "EGL_BAD_CONTEXT";
        case EGL_BAD_CURRENT_SURFACE: return "EGL_BAD_CURRENT_SURFACE";
        case EGL_BAD_DISPLAY:       return "EGL_BAD_DISPLAY";
        case EGL_BAD_MATCH:         return "EGL_BAD_MATCH";
        case EGL_BAD_NATIVE_PIXMAP: return "EGL_BAD_NATIVE_PIXMAP";
        case EGL_BAD_NATIVE_WINDOW: return "EGL_BAD_NATIVE_WINDOW";
        case EGL_BAD_PARAMETER:     return "EGL_BAD_PARAMETER";
        case EGL_BAD_SURFACE:       return "EGL_BAD_SURFACE";
        case EGL_CONTEXT_LOST:      return "EGL_CONTEXT_LOST";
        default: return "UNKNOWN";
    }
}

static const char* glStrErr(GLenum err) {
    switch (err) {
        case GL_NO_ERROR: return "GL_NO_ERROR";
        case GL_INVALID_ENUM: return "GL_INVALID_ENUM";
        case GL_INVALID_VALUE: return "GL_INVALID_VALUE";
        case GL_INVALID_OPERATION: return "GL_INVALID_OPERATION";
        case GL_INVALID_FRAMEBUFFER_OPERATION: return "GL_INVALID_FRAMEBUFFER_OPERATION";
        case GL_OUT_OF_MEMORY: return "GL_OUT_OF_MEMORY";
        case GL_STACK_UNDERFLOW: return "GL_STACK_UNDERFLOW";
        case GL_STACK_OVERFLOW: return "GL_STACK_OVERFLOW";
        default: return "UNKNOWN";
    }
}

static bool
weston_check_egl_extension(const char *extensions, const char *extension)
{
    size_t extlen = strlen(extension);
    const char *end = extensions + strlen(extensions);

    while (extensions < end) {
        size_t n = 0;

        /* Skip whitespaces, if any */
        if (*extensions == ' ') {
            extensions++;
            continue;
        }

        n = strcspn(extensions, " ");

        /* Compare strings */
        if (n == extlen && strncmp(extension, extensions, n) == 0)
            return true; /* Found */

        extensions += n;
    }

    /* Not found */
    return false;
}

#define MAX_BUFFER_PLANES 4



static bool running = true;

static struct wl_shm *shm = NULL;
static struct wl_compositor *compositor = NULL;
static struct wl_subcompositor *subcompositor = NULL;
static struct xdg_wm_base *xdg_wm_base = NULL;

static struct wl_surface *surface = NULL;
static struct xdg_toplevel *xdg_toplevel = NULL;


static void noop() {
    // This space intentionally left blank
}

static void xdg_surface_handle_configure(void *data,
        struct xdg_surface *xdg_surface, uint32_t serial) {
    xdg_surface_ack_configure(xdg_surface, serial);
    wl_surface_commit(surface);
}

static const struct xdg_surface_listener xdg_surface_listener = {
    .configure = xdg_surface_handle_configure,
};

static void xdg_toplevel_handle_close(void *data,
        struct xdg_toplevel *xdg_toplevel) {
    running = false;
}

static const struct xdg_toplevel_listener xdg_toplevel_listener = {
    .configure = noop,
    .close = xdg_toplevel_handle_close,
};

static void handle_global(void *data, struct wl_registry *registry,
        uint32_t name, const char *interface, uint32_t version) {
    if (strcmp(interface, wl_shm_interface.name) == 0) {
        shm = wl_registry_bind(registry, name, &wl_shm_interface, 1);
    } else if (strcmp(interface, wl_compositor_interface.name) == 0) {
        compositor = wl_registry_bind(registry, name, &wl_compositor_interface, 1);
    } else if (strcmp(interface, xdg_wm_base_interface.name) == 0) {
        xdg_wm_base = wl_registry_bind(registry, name, &xdg_wm_base_interface, 1);
    } else if (strcmp(interface, wl_subcompositor_interface.name) == 0) {
        subcompositor = wl_registry_bind(registry, name, &wl_subcompositor_interface, 1);
    }
}

static const struct wl_registry_listener registry_listener = {
    .global = handle_global,
};

static struct gbm_device *gbm_dev;

static void render_gl_to(void *out, int width, int height) {
    PFNEGLQUERYDMABUFFORMATSEXTPROC eglQueryDmaBufFormatsEXT = (void *)eglGetProcAddress("eglQueryDmaBufFormatsEXT");
    PFNEGLCREATEIMAGEKHRPROC eglCreateImageKHR = (void *) eglGetProcAddress("eglCreateImageKHR");
    PFNGLEGLIMAGETARGETTEXTURE2DOESPROC glEGLImageTargetTexture2DOES = (void *) eglGetProcAddress("glEGLImageTargetTexture2DOES");

    uint32_t format = DRM_FORMAT_ARGB8888;

    EGLint formats[60] = {0};
    EGLint num_formats;
    eglQueryDmaBufFormatsEXT(eglGetCurrentDisplay(), 50, &formats[0], &num_formats);
    for (int i = 0; i < num_formats; i++) {
        if ((uint32_t)formats[i] == format) {
            printf("Format supported\n");
        }
    }

    struct gbm_bo *bo = gbm_bo_create(gbm_dev, width, height, format, GBM_BO_USE_RENDERING);
    assert(bo != NULL);
    int planes = gbm_bo_get_plane_count(bo);

    printf("Planes %d\n", planes);

    const char *extensions = eglQueryString(eglGetCurrentDisplay(), EGL_EXTENSIONS);
    bool has_dmabuf_import_modifiers = weston_check_egl_extension(extensions, "EGL_EXT_image_dma_buf_import_modifiers");
    printf("Has import modifiers %d\n", has_dmabuf_import_modifiers);

    int dmabuf_fds[MAX_BUFFER_PLANES];
    uint32_t strides[MAX_BUFFER_PLANES];
    uint32_t offsets[MAX_BUFFER_PLANES];
    for (int i = 0; i < planes; i++) {
        dmabuf_fds[i] = gbm_bo_get_fd_for_plane(bo, i);
        strides[i] = gbm_bo_get_stride_for_plane(bo, i);
        offsets[i] = gbm_bo_get_offset(bo, i);
    }
    uint64_t mod = gbm_bo_get_modifier(bo);
    printf("Mod %"PRIx64"\n", mod);

    static const int general_attribs = 3;
    static const int plane_attribs = 5;
    static const int entries_per_attrib = 2;
    EGLint attribs[(general_attribs + plane_attribs * MAX_BUFFER_PLANES) *
            entries_per_attrib + 1];
    unsigned int atti = 0;

    attribs[atti++] = EGL_WIDTH;
    attribs[atti++] = width;
    attribs[atti++] = EGL_HEIGHT;
    attribs[atti++] = height;
    attribs[atti++] = EGL_LINUX_DRM_FOURCC_EXT;
    attribs[atti++] = format;

#define ADD_PLANE_ATTRIBS(plane_idx) { \
    attribs[atti++] = EGL_DMA_BUF_PLANE ## plane_idx ## _FD_EXT; \
    attribs[atti++] = dmabuf_fds[plane_idx]; \
    attribs[atti++] = EGL_DMA_BUF_PLANE ## plane_idx ## _OFFSET_EXT; \
    attribs[atti++] = (int) offsets[plane_idx]; \
    attribs[atti++] = EGL_DMA_BUF_PLANE ## plane_idx ## _PITCH_EXT; \
    attribs[atti++] = (int) strides[plane_idx]; \
    if (has_dmabuf_import_modifiers) { \
        attribs[atti++] = EGL_DMA_BUF_PLANE ## plane_idx ## _MODIFIER_LO_EXT; \
        attribs[atti++] = mod & 0xFFFFFFFF; \
        attribs[atti++] = EGL_DMA_BUF_PLANE ## plane_idx ## _MODIFIER_HI_EXT; \
        attribs[atti++] = mod >> 32; \
    } \
    }

    if (planes > 0)
        ADD_PLANE_ATTRIBS(0);
    if (planes > 1)
        ADD_PLANE_ATTRIBS(1);
    if (planes > 2)
        ADD_PLANE_ATTRIBS(2);
    if (planes > 3)
        ADD_PLANE_ATTRIBS(3);

#undef ADD_PLANE_ATTRIBS

    attribs[atti] = EGL_NONE;
    assert(atti < (sizeof(attribs) / sizeof(attribs[0])));

    EGLImage dmabuf_image = eglCreateImageKHR(eglGetCurrentDisplay(), EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, NULL, attribs);
    printf("Create image %s\n", eglStrError(eglGetError()));
    assert(dmabuf_image != EGL_NO_IMAGE);

    GLuint dmabuf_texture;
    glGenTextures(1, &dmabuf_texture);
    glBindTexture(GL_TEXTURE_2D, dmabuf_texture);
    glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, dmabuf_image);

    GLuint dmabuf_fb;
    glGenFramebuffers(1, &dmabuf_fb);
    glBindFramebuffer(GL_FRAMEBUFFER, dmabuf_fb);
    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, dmabuf_texture, 0);
    glClearColor(0, 255, 0, 255);
    glClear(GL_COLOR_BUFFER_BIT);

#if 0
    GLuint fbo;
    glGenFramebuffers(1, &fbo);
    glBindFramebuffer(GL_FRAMEBUFFER, fbo);
    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, dmabuf_texture, 0);
    glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, out);
#else
    glBindTexture(GL_TEXTURE_2D, dmabuf_texture);
    glGetTexImage(GL_TEXTURE_2D, 0, GL_RGBA, GL_UNSIGNED_BYTE, out);
#endif
    printf("Read back: %s\n", glStrErr(glGetError()));
}

static struct wl_buffer *create_buffer(int width, int height) {
    int stride = width * 4;
    int size = stride * height;

    int fd = memfd_create("buffer", 0);
    ftruncate(fd, size);
    void *shm_data = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);

    render_gl_to(shm_data, width, height);

    struct wl_shm_pool *pool = wl_shm_create_pool(shm, fd, size);
    struct wl_buffer *buffer = wl_shm_pool_create_buffer(pool, 0, width, height,
        stride, WL_SHM_FORMAT_ARGB8888);
    wl_shm_pool_destroy(pool);

    return buffer;
}

void egl_init(int width, int height) {
    PFNEGLGETPLATFORMDISPLAYEXTPROC eglGetPlatformDisplayEXT = (void *) eglGetProcAddress("eglGetPlatformDisplayEXT");
    EGLDisplay egl_dpy = eglGetPlatformDisplayEXT(EGL_PLATFORM_GBM_KHR, gbm_dev, NULL);
    eglInitialize(egl_dpy, NULL, NULL);
    printf("eglInitialize: %s\n", eglStrError(eglGetError()));

    EGLConfig config;
    int num_config;
    EGLint dpy_attrs[] = {
        EGL_RED_SIZE, 8,
        EGL_GREEN_SIZE, 8,
        EGL_BLUE_SIZE, 8,
        EGL_ALPHA_SIZE, 8,
        EGL_NONE };
    eglChooseConfig(egl_dpy, dpy_attrs, &config, 1, &num_config);
    printf("eglChooseConfig: %s\n", eglStrError(eglGetError()));

    eglBindAPI(EGL_OPENGL_API);
    printf("eglBindAPI: %s\n", eglStrError(eglGetError()));

    EGLint context_attrs[] = {
        EGL_CONTEXT_MAJOR_VERSION, 3,
        EGL_CONTEXT_MAJOR_VERSION, 1,
        EGL_NONE };
    EGLContext ctx = eglCreateContext(egl_dpy, config,  EGL_NO_CONTEXT, context_attrs);
    printf("eglCreateContext: %s\n", eglStrError(eglGetError()));

    eglMakeCurrent(egl_dpy, EGL_NO_SURFACE, EGL_NO_SURFACE, ctx);
    printf("eglMakeCurrent: %s\n", eglStrError(eglGetError()));
}

int main(int argc, char *argv[]) {
    gbm_dev = gbm_create_device(open("/dev/dri/renderD128", O_RDWR));
    egl_init(512, 512);

    struct wl_display *display = wl_display_connect(NULL);

    struct wl_registry *registry = wl_display_get_registry(display);
    wl_registry_add_listener(registry, &registry_listener, NULL);
    wl_display_roundtrip(display);

    struct wl_buffer *buffer = create_buffer(512, 512);
    surface = wl_compositor_create_surface(compositor);
    struct xdg_surface *xdg_surface =
        xdg_wm_base_get_xdg_surface(xdg_wm_base, surface);
    xdg_toplevel = xdg_surface_get_toplevel(xdg_surface);

    xdg_surface_add_listener(xdg_surface, &xdg_surface_listener, NULL);
    xdg_toplevel_add_listener(xdg_toplevel, &xdg_toplevel_listener, NULL);

    wl_surface_commit(surface);
    wl_display_roundtrip(display);

    wl_surface_attach(surface, buffer, 0, 0);
    wl_surface_commit(surface);

    while (wl_display_dispatch(display) != -1 && running) {
        // This space intentionally left blank
    }

    return EXIT_SUCCESS;
}

Makefile

CFLAGS ?= -std=c11 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-error=unused-function -Wno-error=unused-variable -Wno-error=unused-but-set-variable -g
PKG_CONFIG ?= pkg-config

# Host deps
WAYLAND_FLAGS = $(shell $(PKG_CONFIG) wayland-client --cflags --libs)
WAYLAND_PROTOCOLS_DIR = $(shell $(PKG_CONFIG) wayland-protocols --variable=pkgdatadir)

# Build deps
WAYLAND_SCANNER = $(shell pkg-config --variable=wayland_scanner wayland-scanner)

XDG_SHELL_PROTOCOL = $(WAYLAND_PROTOCOLS_DIR)/stable/xdg-shell/xdg-shell.xml

HEADERS=xdg-shell-client-protocol.h
SOURCES=main.c xdg-shell-protocol.c

all: hello-wayland

hello-wayland: $(HEADERS) $(SOURCES)
	$(CC) $(CFLAGS) -o $@ $(SOURCES) -lrt -lGL -lEGL -lgbm $(WAYLAND_FLAGS)

xdg-shell-client-protocol.h:
	$(WAYLAND_SCANNER) client-header $(XDG_SHELL_PROTOCOL) xdg-shell-client-protocol.h

xdg-shell-protocol.c:
	$(WAYLAND_SCANNER) private-code $(XDG_SHELL_PROTOCOL) xdg-shell-protocol.c

.PHONY: clean
clean:
	$(RM) hello-wayland cat.h xdg-shell-protocol.c xdg-shell-client-protocol.h