Tip for implementing VLC custom callback with DirectX

pseudo_code.cpp

// VLC prepares to render a video frame.
static void* lock(void* data, void** p_pixels) {
  return NULL; // Picture identifier, not needed here.
}

// VLC just rendered a video frame.
static void unlock(void* data, void* id, void* const* p_pixels) {
}

// VLC wants to display a video frame.
static void display(void* data, void* id) {
}


// 1. set callback functions to vlc.
libvlc_video_set_callbacks(mp, lock, unlock, display, userData);
libvlc_media_add_option(m, ":avcodec-hw=d3d11va"); // enable hw-decode for DirectX 11
// libvlc_media_add_option(m, ":avcodec-hw=dxva2"); // enable hw-decode for DirectX 9
// libvlc_media_add_option(m, ":avcodec-hw="); // enable hw-decode for auto

// 2. get pitch from texture
D3D11_MAPPED_SUBRESOURCE resource;
UINT subresource = D3D11CalcSubresource(0, 0, 0);
HRESULT hr = DeviceContext->Map(texture, subresource, D3D11_MAP_WRITE_DISCARD, 0, &resource);
DeviceContext->Unmap(texture, subresource);


// 3. set video setup callback functions to vlc
unsigned cb_setup_video_format (void **opaque, char *chroma,
                                unsigned *width, unsigned *height,
                                unsigned *pitches,
                                unsigned *lines) {
  // ...
  chroma = "YV12";
  pitches[0] = resource.RowPitch; // row length in byte.
  lines[0] = videoHeight; // row count
  
  // ...
}

void cb_video_cleanup (void *opaque) {
  // ...
}

libvlc_video_set_format_callbacks(mp, cb_setup_video_format, cb_video_cleanup or nullptr);