Fix for return value -12780 when calling VTCompressionSessionEncodeFrame(). You need to pass the `nbytes` into the `CVPixelBufferCreateWithPlanarBytes()` function otherwise you'll get this return value.

Encoding.cpp

#include "Encoding.h"

/* ------------------------------------------------------------------- */ 

static void output_callback(void* outputCallbackRefCon,
                            void* sourceFrameRefCon,
                            OSStatus status,
                            VTEncodeInfoFlags infoFlags,
                            CMSampleBufferRef sampleBuffer);
                            
static void release_callback(void *releaseRefCon,     
                             const void *dataPtr, 
                             size_t dataSize,
                             size_t numberOfPlanes,
                             const void *planeAddresses[]);



/* ------------------------------------------------------------------- */ 

Encoding::Encoding() 
  :width(640)
  ,height(480)
  ,num_frames(0)
  ,nbytes(0)
  ,session(NULL)
{
}

Encoding::~Encoding() {
  printf("+ todo: cleanup encoding.\n");
}

int Encoding::init() {

  CFNumberRef numref = NULL;

  printf("+ width: %d\n", width);
  printf("+ height: %d\n", height);

  if (NULL != session) {
    printf("+ error: already initialized.\n");
    return -1;
  }
  
  CFMutableDictionaryRef image_attribs = NULL;
  CFMutableDictionaryRef encoder_specs = NULL;

  OSStatus status = VTCompressionSessionCreate(NULL,
                                               width,
                                               height,
                                               kCMVideoCodecType_H264,
                                               encoder_specs,
                                               image_attribs,
                                               NULL,
                                               output_callback,
                                               this,
                                               &session);

  if (noErr != status) {
    printf("+ error: VTCompressionSessionCreate was not `noErr`: %d\n", (int)status);
    return -2;
  }

  /* BITRATE */
  {
    int bitrate = 700;
    numref = CFNumberCreate(NULL, kCFNumberSInt32Type, &bitrate);
    if (NULL == numref) {
      printf("+ error: Failed to create a CFNumber.\n");
      return -5;
    }

    status = VTSessionSetProperty(session, 
                                  kVTCompressionPropertyKey_AverageBitRate,
                                  numref);
    CFRelease(numref);
    numref = NULL;

    if (noErr != status) {
      printf("+ error: Cannot set: kVTCompressionPropertyKey_AverageBitRate.\n");
      return -6;
    }
  }

  /* PROFILE */
  status = VTSessionSetProperty(session,
                                kVTCompressionPropertyKey_ProfileLevel,
                                kVTProfileLevel_H264_Baseline_AutoLevel);
  if (noErr != status) {
    printf("+ error: Cannot set kVTCompressionPropertyKey_ProfileLevel.\n");
    return -4;
  }
  
  /* REALTIME */
  status = VTSessionSetProperty(session, 
                                kVTCompressionPropertyKey_RealTime,
                                kCFBooleanTrue);
  if (noErr != status) {
    printf("+ error: Cannot set kVTCompressionPropertyKey_RealTime.\n");
    return -3;
  }

  /* KEYFRAME INTERVAL */
  {
    int interval = 50;
    numref = CFNumberCreate(NULL, kCFNumberSInt32Type, &interval);
    if (NULL == numref) {
      printf("+ error: Failed to create a CFNumber.\n");
      return -7;
    }

    status = VTSessionSetProperty(session,
                                 kVTCompressionPropertyKey_MaxKeyFrameInterval,
                                 numref);

    CFRelease(numref);
    numref = NULL;

    if (noErr != status) {
      printf("+ error: Failed to set: kVTCompressionPropertyKey_MaxKeyFrameInterval.\n");
      return -8;
    }
  }
  
                
  return 0;
}

int Encoding::encodeBiPlanarBuffer(void* planes[2], 
                                   size_t strides[2], 
                                   size_t widths[2],
                                   size_t heights[2])
{

  if (NULL == session)          { printf("+ error: the session member is NULL.\n");            return -1;  }
  if (NULL == planes)           { printf("+ error: the planes param is NULL.\n");              return -2;  }
  if (NULL == planes[0])        { printf("+ error: planes[0] is NULL.\n");                     return -3;  }
  if (NULL == planes[1])        { printf("+ error: planes[1] is NULL.\n");                     return -4;  }
  if (width != strides[0])      { printf("+ error: we expect strides[0] to be %d\n", width);   return -5;  } 
  if (width != strides[1])      { printf("+ error: we expect strides[1] to be %d\n", width);   return -6;  } 
  if (height != heights[0])     { printf("+ error: we expect height[0] to be %d\n", height);   return -7;  } 
  if ((height/2) != heights[1]) { printf("+ error: we expect height[1] to be %d\n", height/2); return -8;  } 
  if (width != widths[0])       { printf("+ error: we expect widths[0] to be %d\n", width);    return -9;  } 
  if ((width/2) != widths[1])   { printf("+ error: we expect widths[1] to be %d\n", width/1);  return -10; } 

  nbytes = strides[0] * heights[0] + strides[1] * heights[1];

  CVPixelBufferRef pixbuf = NULL;

  CVReturn r = CVPixelBufferCreateWithPlanarBytes(kCFAllocatorDefault,
                                                  width,
                                                  height,
                                                  kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,
                                                  &info,
                                                  nbytes,
                                                  2,
                                                  planes,
                                                  widths,
                                                  heights,
                                                  strides,
                                                  release_callback,
                                                  this,
                                                  NULL,
                                                  &pixbuf);
  
  if (kCVReturnSuccess != r) {
    printf("+ error: CVPixelBufferCreateWithPlanarBytes failed.\n");
    return -11;
  }

  if (NULL == pixbuf) {
    printf("+ error: the created pixbuf is NULL.\n");
    return -12;
  }

  CMTime presentation_timestamp = CMTimeMake(num_frames, 1000.0);
  CMTime duration = CMTimeMake(1, 25);

  OSStatus status = VTCompressionSessionEncodeFrame(session,
                                                    (CVImageBufferRef)pixbuf,
                                                    presentation_timestamp,
                                                    duration,
                                                    NULL,
                                                    (void*)planes,
                                                    NULL);

  if (noErr != status) {
    printf("+ error: VTCompressionSessionEncodeFrame result is not noErr but: %d\n", (int)status);
    return -13;
  }

  num_frames++;

  return 0;
}



/* ------------------------------------------------------------------- */ 

static void output_callback(void* outputCallbackRefCon,
                            void* sourceFrameRefCon,
                            OSStatus status,
                            VTEncodeInfoFlags infoFlags,
                            CMSampleBufferRef sampleBuffer)
{
  printf("+ output_callback.status: %d", (int)status);
}


static void release_callback(void *releaseRefCon,     
                             const void *dataPtr, 
                             size_t dataSize,
                             size_t numberOfPlanes,
                             const void *planeAddresses[])
{
  printf("+ release_callback\n");
}

Encoding.h

#ifndef ENCODING_H
#define ENCODING_H

#include <stdio.h>
#include <VideoToolbox/VideoToolbox.h>
#include <VideoToolbox/VTVideoEncoderList.h>
#include <CoreMedia/CMTime.h>

class Encoding {
 public:
  Encoding();
  ~Encoding();
  int init();                                          /* returns 0 on success, otherwise < 0 */
  int encodeBiPlanarBuffer(void* planes[2], 
                           size_t strides[2], 
                           size_t widths[2],
                           size_t heights[2]);
 public:
  int width;
  int height;
  int nbytes;
  int num_frames;
  VTCompressionSessionRef session;
  CVPlanarPixelBufferInfo_YCbCrPlanar info;
};

#endif