How to get some camera frames with AVCaptureDevice APIs

mac-camera-playground.swift

import AVFoundation
import Cocoa

var greeting = "Hello, playground"

var camera_type = AVCaptureDevice.DeviceType.builtInWideAngleCamera
var unknown_type = AVCaptureDevice.DeviceType.externalUnknown

var discovery = AVCaptureDevice.DiscoverySession(
    deviceTypes: [camera_type, unknown_type],
    mediaType: AVMediaType.video,
    position: AVCaptureDevice.Position.unspecified)


print("\nDevice discovery")
for device in discovery.devices {
    print(device)
}

var CoreVideoPixelFormats: [OSType:String] = [
    kCVPixelFormatType_128RGBAFloat: "128RGBAFloat",
    kCVPixelFormatType_14Bayer_BGGR: "BGGR",
    kCVPixelFormatType_14Bayer_GBRG: "GBRG",
    kCVPixelFormatType_14Bayer_GRBG: "GRBG",
    kCVPixelFormatType_14Bayer_RGGB: "RGGB",
    kCVPixelFormatType_16BE555: "16BE555",
    kCVPixelFormatType_16BE565: "16BE565",
    kCVPixelFormatType_16Gray: "16Gray",
    kCVPixelFormatType_16LE5551: "16LE5551",
    kCVPixelFormatType_16LE555: "16LE555",
    kCVPixelFormatType_16LE565: "16LE565",
    kCVPixelFormatType_16VersatileBayer: "16VersatileBayer",
    kCVPixelFormatType_1IndexedGray_WhiteIsZero: "WhiteIsZero",
    kCVPixelFormatType_1Monochrome: "1Monochrome",
    kCVPixelFormatType_24BGR: "24BGR",
    kCVPixelFormatType_24RGB: "24RGB",
    kCVPixelFormatType_2Indexed: "2Indexed",
    kCVPixelFormatType_2IndexedGray_WhiteIsZero: "WhiteIsZero",
    kCVPixelFormatType_30RGB: "30RGB",
    kCVPixelFormatType_30RGBLEPackedWideGamut: "30RGBLEPackedWideGamut",
    kCVPixelFormatType_32ABGR: "32ABGR",
    kCVPixelFormatType_32ARGB: "32ARGB",
    kCVPixelFormatType_32AlphaGray: "32AlphaGray",
    kCVPixelFormatType_32BGRA: "32BGRA",
    kCVPixelFormatType_32RGBA: "32RGBA",
    kCVPixelFormatType_40ARGBLEWideGamut: "40ARGBLEWideGamut",
    kCVPixelFormatType_40ARGBLEWideGamutPremultiplied: "40ARGBLEWideGamutPremultiplied",
    kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: "420YpCbCr10BiPlanarFullRange",
    kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: "420YpCbCr10BiPlanarVideoRange",
    kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: "420YpCbCr8BiPlanarFullRange",
    kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: "420YpCbCr8BiPlanarVideoRange",
    kCVPixelFormatType_420YpCbCr8Planar: "420YpCbCr8Planar",
    kCVPixelFormatType_420YpCbCr8PlanarFullRange: "420YpCbCr8PlanarFullRange",
    kCVPixelFormatType_420YpCbCr8VideoRange_8A_TriPlanar: "TriPlanar",
    kCVPixelFormatType_422YpCbCr10: "422YpCbCr10",
    kCVPixelFormatType_422YpCbCr10BiPlanarFullRange: "422YpCbCr10BiPlanarFullRange",
    kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange: "422YpCbCr10BiPlanarVideoRange",
    kCVPixelFormatType_422YpCbCr16: "422YpCbCr16",
    kCVPixelFormatType_422YpCbCr16BiPlanarVideoRange: "422YpCbCr16BiPlanarVideoRange",
    kCVPixelFormatType_422YpCbCr8: "422YpCbCr8",
    kCVPixelFormatType_422YpCbCr8BiPlanarFullRange: "422YpCbCr8BiPlanarFullRange",
    kCVPixelFormatType_422YpCbCr8BiPlanarVideoRange: "422YpCbCr8BiPlanarVideoRange",
    kCVPixelFormatType_422YpCbCr8FullRange: "422YpCbCr8FullRange",
    kCVPixelFormatType_422YpCbCr8_yuvs: "yuvs",
    kCVPixelFormatType_422YpCbCr_4A_8BiPlanar: "8BiPlanar",
    kCVPixelFormatType_4444AYpCbCr16: "4444AYpCbCr16",
    kCVPixelFormatType_4444AYpCbCr8: "4444AYpCbCr8",
    kCVPixelFormatType_4444YpCbCrA8: "4444YpCbCrA8",
    kCVPixelFormatType_4444YpCbCrA8R: "4444YpCbCrA8R",
    kCVPixelFormatType_444YpCbCr10: "444YpCbCr10",
    kCVPixelFormatType_444YpCbCr10BiPlanarFullRange: "444YpCbCr10BiPlanarFullRange",
    kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange: "444YpCbCr10BiPlanarVideoRange",
    kCVPixelFormatType_444YpCbCr16BiPlanarVideoRange: "444YpCbCr16BiPlanarVideoRange",
    kCVPixelFormatType_444YpCbCr16VideoRange_16A_TriPlanar: "TriPlanar",
    kCVPixelFormatType_444YpCbCr8: "444YpCbCr8",
    kCVPixelFormatType_444YpCbCr8BiPlanarFullRange: "444YpCbCr8BiPlanarFullRange",
    kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange: "444YpCbCr8BiPlanarVideoRange",
    kCVPixelFormatType_48RGB: "48RGB",
    kCVPixelFormatType_4Indexed: "4Indexed",
    kCVPixelFormatType_4IndexedGray_WhiteIsZero: "WhiteIsZero",
    kCVPixelFormatType_64ARGB: "64ARGB",
    kCVPixelFormatType_64RGBAHalf: "64RGBAHalf",
    kCVPixelFormatType_64RGBALE: "64RGBALE",
    kCVPixelFormatType_64RGBA_DownscaledProResRAW: "DownscaledProResRAW",
    kCVPixelFormatType_8Indexed: "8Indexed",
    kCVPixelFormatType_8IndexedGray_WhiteIsZero: "WhiteIsZero",
    kCVPixelFormatType_ARGB2101010LEPacked: "ARGB2101010LEPacked",
    kCVPixelFormatType_DepthFloat16: "DepthFloat16",
    kCVPixelFormatType_DepthFloat32: "DepthFloat32",
    kCVPixelFormatType_DisparityFloat16: "DisparityFloat16",
    kCVPixelFormatType_DisparityFloat32: "DisparityFloat32",
    kCVPixelFormatType_Lossless_32BGRA: "32BGRA",
    kCVPixelFormatType_Lossless_420YpCbCr10PackedBiPlanarVideoRange: "420YpCbCr10PackedBiPlanarVideoRange",
    kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarFullRange: "420YpCbCr8BiPlanarFullRange",
    kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarVideoRange: "420YpCbCr8BiPlanarVideoRange",
    kCVPixelFormatType_Lossless_422YpCbCr10PackedBiPlanarVideoRange: "422YpCbCr10PackedBiPlanarVideoRange",
    kCVPixelFormatType_Lossy_32BGRA: "32BGRA",
    kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange: "420YpCbCr10PackedBiPlanarVideoRange",
    kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange: "420YpCbCr8BiPlanarFullRange",
    kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange: "420YpCbCr8BiPlanarVideoRange",
    kCVPixelFormatType_Lossy_422YpCbCr10PackedBiPlanarVideoRange: "422YpCbCr10PackedBiPlanarVideoRange",
    kCVPixelFormatType_OneComponent10: "OneComponent10",
    kCVPixelFormatType_OneComponent12: "OneComponent12",
    kCVPixelFormatType_OneComponent16: "OneComponent16",
    kCVPixelFormatType_OneComponent16Half: "OneComponent16Half",
    kCVPixelFormatType_OneComponent32Float: "OneComponent32Float",
    kCVPixelFormatType_OneComponent8: "OneComponent8",
    kCVPixelFormatType_TwoComponent16: "TwoComponent16",
    kCVPixelFormatType_TwoComponent16Half: "TwoComponent16Half",
    kCVPixelFormatType_TwoComponent32Float: "TwoComponent32Float",
    kCVPixelFormatType_TwoComponent8: "TwoComponent8",

]

func main() {
    var device = discovery.devices.count == 2 ? discovery.devices[1] : AVCaptureDevice.default(for: AVMediaType.video)
    if device == nil {
        return
    }
    var camera = device!
    print("default: \(camera.localizedName)")
    for format in camera.formats {
        print(format.formatDescription.dimensions)
        print(format.formatDescription.extensions)
        print()
    }
    
    print(camera.activeFormat.formatDescription.dimensions)
    
    var cam_input = try! AVCaptureDeviceInput(device: camera)
    var capture = AVCaptureSession();
    if !capture.canAddInput(cam_input) { return }
    capture.addInput(cam_input)
    var cam_output = AVCaptureVideoDataOutput()
    
    print("Pixel formats:")
    for format in cam_output.availableVideoPixelFormatTypes {
        print(CoreVideoPixelFormats[format] ?? "unknown pixel format")
    }
    
    print("Video settings:")
    print(cam_output.videoSettings ?? "nil")
    
    // see AVVideoSettings.h and CVPixelBuffer.h for available keys for compressed and uncompressed output
    cam_output.videoSettings = [:]
    cam_output.alwaysDiscardsLateVideoFrames = true
    
    var frame_printer = FramePrinter()
    cam_output.setSampleBufferDelegate(frame_printer, queue: frame_printer.queue)
    
    capture.addOutput(cam_output)
    
    capture.startRunning()
    
    sleep(2)
    
    capture.stopRunning()
    
    print("stopRunning")
}

class FramePrinter: NSObject {
    var queue = DispatchQueue(label:"frame-printer")
}

extension FramePrinter: AVCaptureVideoDataOutputSampleBufferDelegate {
    func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
        var display_size = CVImageBufferGetDisplaySize(sampleBuffer.imageBuffer!)
        print("frame \(display_size)")
    }
}

main()