directx 11 luajit ffi obslua obs studio https://obsproject.com/forum/threads/tips-and-tricks-for-lua-scripts.132256/page-4#post-644707

demo_feature.lua

obs = obslua
S = obs
ffi = require "ffi"

function try_load_library(alias, name)
  if ffi.os == "OSX" then name = name .. ".0.dylib" end
  ok, _G[alias] = pcall(ffi.load, name)
  if not ok then 
    print(("WARNING:%s:Has failed to load, %s is nil"):format(name, alias))
  end
end

try_load_library("obsffi", "obs")

ffi.cdef[[
void *gs_get_device_obj(void);

struct d3ddeviceVTBL {
  void *QueryInterface;
  void *AddRef;
  void *Release;
  void *CreateBuffer;
  void *CreateTexture1D;
  void *CreateTexture2D;
  void *CreateTexture3D;
  void *CreateShaderResourceView;
  void *CreateUnorderedAccessView;
  void *CreateRenderTargetView;
  void *CreateDepthStencilView;
  void *CreateInputLayout;
  void *CreateVertexShader;
  void *CreateGeometryShader;
  void *CreateGeometryShaderWithStreamOutput;
  void *CreatePixelShader;
  void *CreateHullShader;
  void *CreateDomainShader;
  void *CreateComputeShader;
  void *CreateClassLinkage;
  void *CreateBlendState;
  void *CreateDepthStencilState;
  void *CreateRasterizerState;
  void *CreateSamplerState;
  void *CreateQuery;
  void *CreatePredicate;
  void *CreateCounter;
  void *CreateDeferredContext;
  void *OpenSharedResource;
  void *CheckFormatSupport;
  void *CheckMultisampleQualityLevels;
  void *CheckCounterInfo;
  void *CheckCounter;
  void *CheckFeatureSupport;
  void *GetPrivateData;
  void *SetPrivateData;
  void *SetPrivateDataInterface;
  void *GetFeatureLevel;
  void *GetCreationFlags;
  void *GetDeviceRemovedReason;
  void *GetImmediateContext;
  void *SetExceptionMode;
  void *GetExceptionMode;
};
struct d3ddevice {
  struct d3ddeviceVTBL** lpVtbl;
};

struct d3ddevicecontextVTBL {
  void *QueryInterface;
  void *Addref;
  void *Release;
  void *GetDevice;
  void *GetPrivateData;
  void *SetPrivateData;
  void *SetPrivateDataInterface;
  void *VSSetConstantBuffers;
  void *PSSetShaderResources;
  void *PSSetShader;
  void *SetSamplers;
  void *SetShader;
  void *DrawIndexed;
  void *Draw;
  void *Map;
  void *Unmap;
  void *PSSetConstantBuffer;
  void *IASetInputLayout;
  void *IASetVertexBuffers;
  void *IASetIndexBuffer;
  void *DrawIndexedInstanced;
  void *DrawInstanced;
  void *GSSetConstantBuffers;
  void *GSSetShader;
  void *IASetPrimitiveTopology;
  void *VSSetShaderResources;
  void *VSSetSamplers;
  void *Begin;
  void *End;
  void *GetData;
  void *GSSetPredication;
  void *GSSetShaderResources;
  void *GSSetSamplers;
  void *OMSetRenderTargets;
  void *OMSetRenderTargetsAndUnorderedAccessViews;
  void *OMSetBlendState;
  void *OMSetDepthStencilState;
  void *SOSetTargets;
  void *DrawAuto;
  void *DrawIndexedInstancedIndirect;
  void *DrawInstancedIndirect;
  void *Dispatch;
  void *DispatchIndirect;
  void *RSSetState;
  void *RSSetViewports;
  void *RSSetScissorRects;
  void *CopySubresourceRegion;
  void *CopyResource;
  void *UpdateSubresource;
  void *CopyStructureCount;
  void *ClearRenderTargetView;
  void *ClearUnorderedAccessViewUint;
  void *ClearUnorderedAccessViewFloat;
  void *ClearDepthStencilView;
  void *GenerateMips;
  void *SetResourceMinLOD;
  void *GetResourceMinLOD;
  void *ResolveSubresource;
  void *ExecuteCommandList;
  void *HSSetShaderResources;
  void *HSSetShader;
  void *HSSetSamplers;
  void *HSSetConstantBuffers;
  void *DSSetShaderResources;
  void *DSSetShader;
  void *DSSetSamplers;
  void *DSSetConstantBuffers;
  void *DSSetShaderResources;
  void *CSSetUnorderedAccessViews;
  void *CSSetShader;
  void *CSSetSamplers;
  void *CSSetConstantBuffers;
  void *VSGetConstantBuffers;
  void *PSGetShaderResources;
  void *PSGetShader;
  void *PSGetSamplers;
  void *VSGetShader;
  void *PSGetConstantBuffers;
  void *IAGetInputLayout;
  void *IAGetVertexBuffers;
  void *IAGetIndexBuffer;
  void *GSGetConstantBuffers;
  void *GSGetShader;
  void *IAGetPrimitiveTopology;
  void *VSGetShaderResources;
  void *VSGetSamplers;
  void *GetPredication;
  void *GSGetShaderResources;
  void *GSGetSamplers;
  void *OMGetRenderTargets;
  void *OMGetRenderTargetsAndUnorderedAccessViews;
  void *OMGetBlendState;
  void *OMGetDepthStencilState;
  void *SOGetTargets;
  void *RSGetState;
  void *RSGetViewports;
  void *RSGetScissorRects;
  void *HSGetShaderResources;
  void *HSGetShader;
  void *HSGetSamplers;
  void *HSGetConstantBuffers;
  void *DSGetShaderResources;
  void *DSGetShader;
  void *DSGetSamplers;
  void *DSGetConstantBuffers;
  void *CSGetShaderResources;
  void *CSGetUnorderedAccessViews;
  void *CSGetShader;
  void *CSGetSamplers;
  void *CSGetConstantBuffers;
  void *ClearState;
  void *Flush;
  void *GetType;
  void *GetContextFlags;
  void *FinishCommandList;
};

struct d3ddevicecontext {
  struct d3ddevicecontextVTBL** lpVtbl;
};

]]

time = 0.0
function script_tick(seconds)
  time = time + seconds
end

local source_def = {}
source_def.id = "lua_vertex_check"
source_def.output_flags = bit.bor(S.OBS_SOURCE_VIDEO, S.OBS_SOURCE_CUSTOM_DRAW)

source_def.get_name = function()
  return "vertex texture fetch check"
end

source_def.create = function(source, settings)
  local data = {}
  S.obs_enter_graphics()
  data.effect = S.gs_effect_create(EFFECT, "my_effect_test_check", nil)
  data.params = {}
  data.texture_a = S.gs_texrender_create(S.GS_RGBA, S.GS_ZS_NONE)  -- random texture
  local color = S.vec4()
  local color1 = S.vec4()
  local color2 = S.vec4()
  local color3 = S.vec4()
  local effect_solid = S.obs_get_base_effect(S.OBS_EFFECT_SOLID)
  local color_param = S.gs_effect_get_param_by_name(effect_solid, "color");
  local color_param1 = S.gs_effect_get_param_by_name(effect_solid, "randomvals1");
  local color_param2 = S.gs_effect_get_param_by_name(effect_solid, "randomvals2");
  local color_param3 = S.gs_effect_get_param_by_name(effect_solid, "randomvals3");
  if S.gs_texrender_begin(data.texture_a, 2560, 1440) then
    S.vec4_from_rgba(color, 0xff0f008a);
    S.vec4_from_rgba(color1, 0xff00ff8a);
    S.vec4_from_rgba(color2, 0xffff0909);
    S.vec4_from_rgba(color3, 0xff0f002a);
    S.gs_effect_set_vec4(color_param, color);
    S.gs_effect_set_vec4(color_param1, color1);
    S.gs_effect_set_vec4(color_param2, color2);
    S.gs_effect_set_vec4(color_param3, color3);
    while S.gs_effect_loop(effect_solid, "Random") do
      S.gs_draw_sprite(nil, 0, 2560, 1440)
    end
    S.gs_texrender_end(data.texture_a)
  end

  data.params.tar_tex = S.gs_effect_get_param_by_name(data.effect, "target_tex")
  data.params.itime = S.gs_effect_get_param_by_name(data.effect, "itime")
  data.tex1 = S.gs_texrender_get_texture(data.texture_a)

  data.device = obsffi.gs_get_device_obj()

  data.pDevice = ffi.cast("struct d3ddevice*", data.device)
  data.GetImmediateContext = ffi.cast("long(__stdcall*)(void*, void**)", data.pDevice.lpVtbl[40])
  data._arg1 = ffi.new("unsigned long[1]")
  data.pContext = ffi.cast("void**", data._arg1)
  data.GetImmediateContext(data.pDevice, data.pContext)
  data.pContext2 = ffi.cast("struct d3ddevicecontext*", data.pContext[0])
  data.Release_pContext = ffi.cast("unsigned long(__stdcall*)(void*)", data.pContext2.lpVtbl[2])
  data.Release_pDevice = ffi.cast("unsigned long(__stdcall*)(void*)", data.pDevice.lpVtbl[2])
  data.VSSetShaderResources = ffi.cast("long(__stdcall*)(void*, unsigned int, unsigned int, void**)", data.pContext2.lpVtbl[25])
  data.PSGetShaderResources = ffi.cast("long(__stdcall*)(void*, unsigned int, unsigned int, void**)", data.pContext2.lpVtbl[73])
  data.VSSetSamplers = ffi.cast("long(__stdcall*)(void*, unsigned int, unsigned int, void**)", data.pContext2.lpVtbl[26])
  data._arg2 = ffi.new("unsigned long[1]")
  data.pRes = ffi.cast("void**", data._arg2)


  S.obs_leave_graphics()
  -- Calls the destroy function if the effect was not compiled properly
  if data.effect == nil then
    S.blog(S.LOG_ERROR, "Effect compilation failed")
    source_def.destroy(data)
    return nil
  end
  return data
end

source_def.destroy = function(data)
  S.obs_enter_graphics()
  data.Release_pContext(data.pContext2)
  --data.Release_pDevice(data.pDevice)
  S.gs_texrender_destroy(data.texture_a)
  S.obs_leave_graphics()
end

source_def.video_render = function(data, effect)
  S.gs_effect_set_float(data.params.itime, time+0.0)
  S.gs_effect_set_texture(data.params.tar_tex, data.tex1)
  while S.gs_effect_loop(data.effect, "Custom1") do
    S.gs_load_texture(data.tex1, 0) -- sets pixel shader resource view
    data.PSGetShaderResources(data.pContext2, 0, 1, data.pRes)
    data.VSSetShaderResources(data.pContext2, 0, 1, data.pRes)
    S.gs_draw_sprite(nil, 0, 2560, 1440)
  end
end

source_def.get_width = function(data)
  return 2560
end

source_def.get_height = function(data)
  return 1440
end

EFFECT = [[


uniform float4x4 ViewProj;
uniform texture2d target_tex;
uniform float itime;
sampler_state def_sampler {
  Filter   = Linear;
  AddressU = Clamp;
  AddressV = Clamp;
    MinLOD = 0;
    MaxLOD = 0;
};
struct VertIn {
  float4 pos : POSITION;
  float2 uv  : TEXCOORD0;
};
struct VertOut {
  float4 pos : POSITION;
  float2 uv  : TEXCOORD0;
  float4 color: COLOR0;
  float test1 : TEXCOORD1;
  float3 test2 : TEXCOORD2;
};
VertOut VSShader(VertIn vert_in)
{
  VertOut vert_out;
  float coord = abs(50*sin(itime*0.2));
  float4 c1 = target_tex.Load(int3(coord,  coord*2, 0)); 
  //float4 c2 = target_tex.SampleLevel(def_sampler, float4(coord, coord*2, 0, 0), 0);
  vert_out.test1 = c1.b;
  vert_out.test2 = c1.rgb;
  vert_out.pos = mul(float4(vert_in.pos.xyz, 1.0), ViewProj);
  vert_out.uv = vert_in.uv;
  return vert_out;
}
float4 PSShader(VertOut vert_in) : TARGET
{
  float4 not_used = target_tex.Sample(def_sampler, vert_in.uv);
  float4 debug_col = float4(0.0, 0.0, 0.0, 1.0); // full black
  //debug_col.r = vert_in.test1;
  debug_col.rgb = vert_in.test2;
  return debug_col;
  //return target_tex.Sample(def_sampler, vert_in.uv); // random pattern no zeros
}
technique Custom1 {
  pass {
    vertex_shader = VSShader(vert_in);
    pixel_shader  = PSShader(vert_in);
  }
}
]]

S.obs_register_source(source_def)