Quick Hack to test out 32bit nvfbc without having to patch the driver or rebuild the affected app

LD_PRELOAD_nvidia-patch.c

#define _GNU_SOURCE

// compile:
//  gcc -m64 -shared -I../Capture_Linux_v7.1.9/NvFBC/inc -fPIC dl_prog3.c -o nvfbc_preload.64.so -ldl
//  gcc -m64  -shared -I../Capture_Linux_v7.1.9/NvFBC/inc -fPIC dl_prog3.c -o nvfbc_preload.32.so -ldl
// Run: 
// LD_PRELOAD="$PWD/nvfbc_preload.32.so $PWD/nvfbc_preload.64.so" DISPLAY=:0 ../Capture_Linux_v7.1.9/NvFBC/samples/NvFBCToGLEnc/NvFBCToGLEnc -f 10


#include "NvFBC.h"
#include <string.h>
#include <stdio.h>
#include <dlfcn.h>

#define LIB_NVFBC_NAME "libnvidia-fbc.so.1"
PNVFBCCREATEHANDLE real_dnvFBCCreateHandle = NULL;;

NVFBCSTATUS NVFBCAPI _nvFBCCreateHandle(NVFBC_SESSION_HANDLE *pSessionHandle, NVFBC_CREATE_HANDLE_PARAMS *pParams)
{
    printf("fake _nvFBCCreateHandle\n");
    // https://github.com/keylase/nvidia-patch/blob/8b0d2238c1391e7fadfc3df2aaf8d4dd398ed18d/win/nvfbcwrp/nvfbcwrp_main.cpp#L23
    int magic[] = {0xAEF57AC5, 0x401D1A39, 0x1B856BBE, 0x9ED0CEBA};
    pParams->privateData = &magic;
    pParams->privateDataSize = sizeof(magic);
    return real_dnvFBCCreateHandle(pSessionHandle, pParams);
}

NVFBCSTATUS NVFBCAPI _nvFBCCreateInstance(NVFBC_API_FUNCTION_LIST *pFunctionList)
{
    PNVFBCCREATEINSTANCE nvFBCCreateInstance = dlsym(dlopen(LIB_NVFBC_NAME, RTLD_NOW), "RealNvFBCCreateInstance");
    printf("fake _nvFBCCreateInstance\n");
    NVFBCSTATUS status = nvFBCCreateInstance(pFunctionList);
   if (real_dnvFBCCreateHandle == NULL)
        real_dnvFBCCreateHandle = pFunctionList->nvFBCCreateHandle;
    pFunctionList->nvFBCCreateHandle = _nvFBCCreateHandle;

    return status;
}

// https://stackoverflow.com/questions/15599026/how-can-i-intercept-dlsym-calls-using-ld-preload
extern void *_dl_sym(void *, const char *, void *);
extern void *dlsym(void *handle, const char *name)
{
    static void *(*real_dlsym)(void *, const char *) = NULL;
    if (real_dlsym == NULL)
        real_dlsym = _dl_sym(RTLD_NEXT, "dlsym", dlsym);
    /* my target binary is even asking for dlsym() via dlsym()... */
    if (!strcmp(name, "dlsym"))
    {
        printf("dlsym() hooked %s\n", name);
        return (void *)dlsym;
    }

    if (!strcmp(name, "RealNvFBCCreateInstance"))
    {
        return real_dlsym(handle, "NvFBCCreateInstance");
    }
    if (!strcmp(name, "NvFBCCreateInstance"))
    {
        printf("dlsym() hooked %s\n", name);
        return _nvFBCCreateInstance;
    }

    return real_dlsym(handle, name);
}