Sampel Renderer

H265Error.swift

import Foundation
import AVFoundation
import VideoToolbox
import remoteplay

public enum H265Error : Error, CustomStringConvertible {
    case invalidNALUType
    case cmBlockBufferCreateWithMemoryBlock(OSStatus)
    case cmBlockBufferAppendBufferReference(OSStatus)
    case cmSampleBufferCreateReady(OSStatus)
    case cmVideoFormatDescriptionCreateFromH265ParameterSets(OSStatus)

    public var description : String {
        switch self {
        case .invalidNALUType: return "H265Error.InvalidNALUType"
        case let .cmBlockBufferCreateWithMemoryBlock(status): return "H265Error.CMBlockBufferCreateWithMemoryBlock(\(status))"
        case let .cmBlockBufferAppendBufferReference(status): return "H265Error.CMBlockBufferAppendBufferReference(\(status))"
        case let .cmSampleBufferCreateReady(status): return "H265Error.CMSampleBufferCreateReady(\(status))"
        case let .cmVideoFormatDescriptionCreateFromH265ParameterSets(status): return "H265Error.CMVideoFormatDescriptionCreateFromH265ParameterSets(\(status))"
        }
    }
}

open class H265Decoder: Decoder {
    fileprivate var vps : H265NALU?
    fileprivate var sps : H265NALU?
    fileprivate var pps : H265NALU?
    fileprivate var formatDescription : CMVideoFormatDescription!
    
    deinit {
        invalidateVideo()
    }

    open func initVideoSession(vpsData: Data, spsData: Data, ppsData: Data) throws {
        let vpsNalu = H265NALU(vpsData, naluTypeOffset: 0)
        let spsNalu = H265NALU(spsData, naluTypeOffset: 0)
        let ppsNalu = H265NALU(ppsData, naluTypeOffset: 0)
        if vpsNalu.type == .vps && spsNalu.type == .sps && ppsNalu.type == .pps {
            if vps == nil || sps == nil || pps == nil || vps!.equals(vpsNalu) || sps!.equals(spsNalu) || pps!.equals(ppsNalu) {
                invalidateVideo()
                vps = vpsNalu
                sps = spsNalu
                pps = ppsNalu
                do {
                    try initVideoSession()
                } catch {
                    vps = nil
                    sps = nil
                    pps = nil
                    throw error
                }
            }
        } else {
            throw H265Error.invalidNALUType
        }
    }

    func decode(_ videoFrameInfo: VideoFrameInfo) throws -> CMSampleBuffer {
        let nalu = H265NALU(videoFrameInfo, 4)
        if nalu.type == .undefined {
            throw H265Error.invalidNALUType
        }

        //if nalu.type != .idrNLp && nalu.type != .trailR {
        //    throw H265Error.invalidNALUType
        //}
        return try nalu.sampleBuffer(formatDescription)
    }

    fileprivate func invalidateVideo() {
        formatDescription = nil
        vps = nil
        sps = nil
        pps = nil
    }
    
    fileprivate func initVideoSession() throws {
        formatDescription = nil
        var _formatDescription : CMFormatDescription?
        let vpsVideoFrame = vps!.videoFrameInfo
        let spsVideoFrame = sps!.videoFrameInfo
        let ppsVideoFrame = pps!.videoFrameInfo
        
        let vpsUnsafePointer = vpsVideoFrame.rawPointer.bindMemory(to: UInt8.self, capacity: vpsVideoFrame.size)
        let spsUnsafePointer = spsVideoFrame.rawPointer.bindMemory(to: UInt8.self, capacity: spsVideoFrame.size)
        let ppsUnsafePointer = ppsVideoFrame.rawPointer.bindMemory(to: UInt8.self, capacity: ppsVideoFrame.size)
        let parameterSetPointers : [UnsafePointer<UInt8>] = [ UnsafePointer(vpsUnsafePointer), UnsafePointer(spsUnsafePointer), UnsafePointer(ppsUnsafePointer) ]
        
        let parameterSetSizes : [Int] = [ vpsVideoFrame.size, spsVideoFrame.size, ppsVideoFrame.size ]
        let status = CMVideoFormatDescriptionCreateFromHEVCParameterSets(allocator: kCFAllocatorDefault, parameterSetCount: 3, parameterSetPointers: parameterSetPointers, parameterSetSizes: parameterSetSizes, nalUnitHeaderLength: 4, extensions: nil, formatDescriptionOut: &_formatDescription)
        if status != noErr {
            throw H265Error.cmVideoFormatDescriptionCreateFromH265ParameterSets(status)
        }
        formatDescription = _formatDescription!
    }
}

H265NALU.swift

import CoreMedia
import remoteplay

public enum H265NALUType : UInt8, CustomStringConvertible {
    case trailN = 0
    case trailR = 1
    case tsaN = 2
    case tsaR = 3
    case stsaN = 4
    case stsaR = 5
    case radlN = 6
    case radlR = 7
    case raslN = 8
    case raslR = 9
    case blaWLp = 16
    case blaWRadl = 17
    case blaNLp = 18
    case idrWRadl = 19
    case idrNLp = 20
    case craNut = 21
    case vps = 32
    case sps = 33
    case pps = 34
    case aud = 35
    case eosNut = 36
    case eobNut = 37
    case fdNut = 38
    case seiPrefix = 39
    case seiSuffix = 40

    case undefined = 99
    
    public var description : String {
        switch self {
        case .aud: return "aud"
        case .blaNLp: return "blaNLp"
        case .blaWLp: return "blaWLp"
        case .blaWRadl: return "blaWRadl"
        case .craNut: return "craNut"
        case .eobNut: return "eobNut"
        case .eosNut: return "eosNut"
        case .fdNut: return "fdNut"
        case .idrNLp: return "idrNLp"
        case .idrWRadl: return "idrWRadl"
        case .pps: return "pps"
        case .radlN: return "radlN"
        case .radlR: return "radlR"
        case .raslN: return "raslN"
        case .raslR: return "raslR"
        case .seiPrefix: return "seiPrefix"
        case .seiSuffix: return "seiSuffix"
        case .sps: return "sps"
        case .stsaN: return "stsaN"
        case .stsaR: return "stsaR"
        case .trailN: return "trailN"
        case .trailR: return "trailR"
        case .tsaN: return "tsaN"
        case .tsaR: return "tsaR"
        case .vps: return "vps"
        default: return "Undefined"
        }
    }
}

open class H265NALU {
    var videoFrameInfo : VideoFrameInfo
    var type : H265NALUType
    
    private static let kfcSampleArracgmentKeyDisplayImmediately = unsafeBitCast(kCMSampleAttachmentKey_DisplayImmediately, to: UnsafeRawPointer.self)
    private static let kCMSampleAttachmentKeyIsDependedOnByOthers = unsafeBitCast(kCMSampleAttachmentKey_IsDependedOnByOthers, to: UnsafeRawPointer.self)
    private static let kCMSampleAttachmentKeyDependsOnOthers = unsafeBitCast(kCMSampleAttachmentKey_DependsOnOthers, to: UnsafeRawPointer.self)
    private static let kCMSampleAttachmentKeyNotSync = unsafeBitCast(kCMSampleAttachmentKey_NotSync, to: UnsafeRawPointer.self)
    private static let kCFBooleanTrueValue = unsafeBitCast(kCFBooleanTrue, to: UnsafeRawPointer.self)
    private static let kCFBooleanFalseValue = unsafeBitCast(kCFBooleanFalse, to: UnsafeRawPointer.self)

    public init(_ videoFrameInfo: VideoFrameInfo, _ naluTypeOffset: Int) {
        var type : H265NALUType?
        self.videoFrameInfo = videoFrameInfo
        if videoFrameInfo.size > naluTypeOffset {
            type = H265NALUType(rawValue: (videoFrameInfo.frameType & 0x7E) >> 1) // type
        }
        self.type = type ?? .undefined
    }

    public convenience init(_ bytes: Data, naluTypeOffset: Int) {
        self.init(VideoFrameInfo(rawPointer: UnsafeMutableRawPointer(mutating: (bytes as NSData).bytes), size: bytes.count, frameType: bytes[naluTypeOffset]), naluTypeOffset)
    }

    open var naluTypeName : String {
        type.description
    }

    open func equals(_ nalu: H265NALU) -> Bool {
        let size = videoFrameInfo.size
        if nalu.videoFrameInfo.size != size {
            return false
        }
        return memcmp(nalu.videoFrameInfo.rawPointer, videoFrameInfo.rawPointer, size) == 0
    }
    
    open func sampleBuffer(_ formatDescription : CMVideoFormatDescription) throws -> CMSampleBuffer {
        var sampleBuffer : CMSampleBuffer?
        
        let status = CMSampleBufferCreate(
            allocator: kCFAllocatorDefault,
            dataBuffer: try blockBuffer(),
            dataReady: true,
            makeDataReadyCallback: nil,
            refcon: nil,
            formatDescription: formatDescription,
            sampleCount: 1,
            sampleTimingEntryCount: 0,
            sampleTimingArray: nil,
            sampleSizeEntryCount: 0,
            sampleSizeArray: nil,
            sampleBufferOut: &sampleBuffer)
        if status != noErr {
            throw H265Error.cmSampleBufferCreateReady(status)
        }
        
        var attachments: CFArray? = nil
        if let sampleBuffer = sampleBuffer {
            attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: true)
        }
        let dict = unsafeBitCast(CFArrayGetValueAtIndex(attachments, 0), to: CFMutableDictionary.self)

        CFDictionarySetValue(dict, H265NALU.kfcSampleArracgmentKeyDisplayImmediately, H265NALU.kCFBooleanTrueValue)
        if type == H265NALUType.idrNLp || type == H265NALUType.idrWRadl {
            CFDictionarySetValue(dict, H265NALU.kCMSampleAttachmentKeyNotSync, H265NALU.kCFBooleanFalseValue)
        } else {
            CFDictionarySetValue(dict, H265NALU.kCMSampleAttachmentKeyNotSync, H265NALU.kCFBooleanTrueValue)
        }
        // Tried different things here but nothing seems to work
      
        //CFDictionarySetValue(dict, H265NALU.kCMSampleAttachmentKeyIsDependedOnByOthers, H265NALU.kCFBooleanFalseValue)
        //CFDictionarySetValue(dict, H265NALU.kCMSampleAttachmentKeyIsDependedOnByOthers, H265NALU.kCFBooleanFalseValue)
        //CFDictionarySetValue(dict, H265NALU.kCMSampleAttachmentKeyNotSync, H265NALU.kCFBooleanFalseValue)
        //CFDictionarySetValue(dict, H265NALU.kCMSampleAttachmentKeyDependsOnOthers, H265NALU.kCFBooleanTrueValue)
        
        if type.rawValue != 1 && type.rawValue != 20 { // ToDo check
            print("!!! type \(String(type.rawValue))")
        }
        
        if type == H265NALUType.idrNLp || type == H265NALUType.idrWRadl {
            print("idr frame received")
        }
        
        //CFDictionarySetValue(dict, H265NALU.kCMSampleAttachmentKeyNotSync, H265NALU.kCFBooleanTrueValue)
        //CFDictionarySetValue(dict, H265NALU.kCMSampleAttachmentKeyDependsOnOthers, H265NALU.kCFBooleanTrueValue)
        
        /*if type == H265NALUType.idrNLp || type == H265NALUType.idrWRadl {
            print("idr frame received")
            // I-frame
            //CFDictionarySetValue(dict, H265NALU.kCMSampleAttachmentKeyNotSync, H265NALU.kCFBooleanFalseValue)
            CFDictionarySetValue(dict, H265NALU.kCMSampleAttachmentKeyDependsOnOthers, H265NALU.kCFBooleanFalseValue)
            CFDictionarySetValue(dict, H265NALU.kCMSampleAttachmentKeyIsDependedOnByOthers, H265NALU.kCFBooleanFalseValue)
        } else {
            // P-frame
            //CFDictionarySetValue(dict, H265NALU.kCMSampleAttachmentKeyNotSync, H265NALU.kCFBooleanTrueValue)
            CFDictionarySetValue(dict, H265NALU.kCMSampleAttachmentKeyDependsOnOthers, H265NALU.kCFBooleanTrueValue)
            CFDictionarySetValue(dict, H265NALU.kCMSampleAttachmentKeyIsDependedOnByOthers, H265NALU.kCFBooleanTrueValue)
        }*/
        
        return sampleBuffer!
    }
    
    private func blockBuffer() throws -> CMBlockBuffer {
        var bufferData : CMBlockBuffer?
        
        let status = CMBlockBufferCreateWithMemoryBlock(allocator: nil, memoryBlock: videoFrameInfo.rawPointer, blockLength: videoFrameInfo.size, blockAllocator: kCFAllocatorDefault, customBlockSource: nil, offsetToData: 0, dataLength: videoFrameInfo.size, flags: 0, blockBufferOut: &bufferData)
        if status != noErr || bufferData == nil {
            throw H265Error.cmBlockBufferCreateWithMemoryBlock(status)
        }
        
        return bufferData!
    }
}

VideoDecoderRenderer.swift

import Foundation
import AVFoundation
import SwiftUI
import remoteplay
import logger

class VideoDecoderRenderer<T: StreamingView> {
    private let logger = Logger.logger
    
    private let fps: UInt32
    private let decoderType: DecoderType
    private let aspectRatio: AspectRatio
    private weak var idrRequestHandler: IdrFrameRequestHandler?
    private weak var videoEventHandler: VideoDecoderEventListener?
    private var resolutionPayload: ResolutionPayload?
    private var displayLayer: AVSampleBufferDisplayLayer?

    private var isDecoderDisposed: Bool = false
    
    private var isDecoderConfigured: Bool = false
    
    private var decoder: Decoder?
    
    private var missedIdrFrameCounter = 0
    
    private let maxFramesUntilIdrFrame: UInt32
    
    private var notfiedDecoderErrorCount = 0

    init(fps: UInt32,
         decoderType: DecoderType,
         aspectRatio: AspectRatio,
         idrRequestHandler: IdrFrameRequestHandler,
         videoEventHandler: VideoDecoderEventListener){
        self.fps = fps
        self.decoderType = decoderType
        self.aspectRatio = aspectRatio
        self.idrRequestHandler = idrRequestHandler
        self.videoEventHandler = videoEventHandler
        self.maxFramesUntilIdrFrame = self.fps * 3
    }
    
    deinit {
        // no logging as this could lead to a crash in logger
        disposeDecoder()
    }
    
    func initializeDisplayLayer(streamView: T) {
        let oldLayer = displayLayer

        let bounds = streamView.getSize()
        displayLayer = AVSampleBufferDisplayLayer()
        displayLayer?.backgroundColor = UIColor.black.cgColor

        displayLayer?.bounds = bounds
        displayLayer?.position = CGPoint(x: bounds.midX , y: bounds.midY)
        displayLayer?.videoGravity = getVideoGravity()

        if let oldLayer = oldLayer {
            // Switch out the old display layer with the new one
            if let displayLayer = displayLayer {
                streamView.getDecoderView().replaceCoderView(coderView: displayLayer, oldCoderView: oldLayer)
            }
        } else {
            if let displayLayer = displayLayer {
                streamView.getDecoderView().addCoderView(coderView: displayLayer)
            }
        }
    }
    
    func reinitializeDisplayLayerBounds(streamView: T) {
        if let displayLayer = displayLayer {
            let bounds = streamView.getSize()
            displayLayer.bounds = bounds

            displayLayer.position = CGPoint(x: bounds.midX , y: bounds.midY)
            displayLayer.videoGravity = getVideoGravity()
        }
    }
    
    func setResolutionPayload(resolutionPayload: ResolutionPayload) {
        self.resolutionPayload = resolutionPayload
    }

    func parseFrame(videoFrameInfo: VideoFrameInfo) {
        interpretNalu(videoFrameInfo: videoFrameInfo)
    }

    func disposeDecoder() {
        decoder = nil
        isDecoderDisposed = true
        isDecoderConfigured = false
        displayLayer?.bounds = CGRect()
        idrRequestHandler = nil
        videoEventHandler = nil
    }

    /*+++++++++++++++++++*/
    /*+ private methods +*/
    /*+++++++++++++++++++*/

    private func isKeyFrame(videoFrameInfo: VideoFrameInfo) -> Bool {
        if isDecoderDisposed {
            return false
        }
        
        if decoderType == DecoderType.h264 {
            return videoFrameInfo.frameType == 101
        } else {
            return videoFrameInfo.frameType == 40
        }
    }
    
    private func interpretNalu(videoFrameInfo: VideoFrameInfo) {
        if isDecoderConfigured {
            do {
                let sampleBuffer = try decoder!.decode(videoFrameInfo)
                submitFrameBuffer(sampleBuffer: sampleBuffer)
            }  catch let error {
                print("Decoder error \(error)")
            }
        } else if isKeyFrame(videoFrameInfo: videoFrameInfo) {
            let buffers = resolutionPayload!.videoHeader.split(separator: Data.init([0x00, 0x00, 0x00, 0x01]))
            
            if decoderType == .h264 {
                if buffers.count < 2 {
                    return // no valid header sps buffer and pps buffer missing
                }
                let spsBuffer = Data(buffers[0])
                let ppsBuffer = Data(buffers[1])
                do {
                    let h264Decoder = H264Decoder()
                    decoder = h264Decoder
                    do {
                        try h264Decoder.initVideoSession(spsData: spsBuffer, ppsData: ppsBuffer)
                        isDecoderConfigured = true
                    }  catch {
                        return
                    }
                    let sampleBuffer = try decoder!.decode(videoFrameInfo)
                    submitFrameBuffer(sampleBuffer: sampleBuffer)
                    
                    DispatchQueue.main.async {
                        self.videoEventHandler?.onFirstIdrFrame()
                    }
                } catch let error{
                    if notfiedDecoderErrorCount < 5 {
                        notfiedDecoderErrorCount += 1
                        DispatchQueue.main.async {
                            self.videoEventHandler?.onDecoderError(error: error)
                        }
                    }
                }
            } else {
                if buffers.count < 3 {
                    return // no valid header vps, sps buffer and pps buffer missing
                }
                let vpsBuffer = Data(buffers[0])
                let spsBuffer = Data(buffers[1])
                let ppsBuffer = Data(buffers[2])
                
                do {
                    let h265Decoder = H265Decoder()
                    decoder = h265Decoder
                    do {
                        try h265Decoder.initVideoSession(vpsData: vpsBuffer, spsData: spsBuffer, ppsData: ppsBuffer)
                        isDecoderConfigured = true
                    }  catch {
                        return
                    }
                    let sampleBuffer = try decoder!.decode(videoFrameInfo)
                    submitFrameBuffer(sampleBuffer: sampleBuffer)
                    
                    DispatchQueue.main.async {
                        self.videoEventHandler?.onFirstIdrFrame()
                    }
                } catch let error {
                    if notfiedDecoderErrorCount < 5 {
                        notfiedDecoderErrorCount += 1
                        DispatchQueue.main.async {
                            self.videoEventHandler?.onDecoderError(error: error)
                        }
                    }
                }
            }
        } else {
            videoFrameInfo.rawPointer.deallocate()
            if !isDecoderDisposed {
                missedIdrFrameCounter = missedIdrFrameCounter + 1
                if missedIdrFrameCounter >= maxFramesUntilIdrFrame {
                    missedIdrFrameCounter = 0
                    idrRequestHandler?.requestIdrFrame()
                }
            }
        }
    }

    private func getVideoGravity() -> AVLayerVideoGravity {
        var result: AVLayerVideoGravity = .resizeAspect
        switch aspectRatio {
        case .keepAspectRatio:
            result = .resizeAspect
        case .stretched:
            result = .resize
        case .zoomed:
            result = .resizeAspectFill
        }
        return result
    }
    
    /*+++++++++++++++++++*/
    /*+ decoder methods +*/
    /*+++++++++++++++++++*/

    private func submitFrameBuffer(sampleBuffer: CMSampleBuffer) {
        guard let displayLayer = displayLayer else {
            return
        }

        // Enqueue video samples
        //DispatchQueue.main.async(execute: { [self] in // not needed to do this on main thread it seems
        // Enqueue the next frame
        if displayLayer.status == .failed {
            // https://stackoverflow.com/questions/43687169/ios-ignoring-enqueuesamplebuffer-because-status-is-failed
            print("!!!status failed")
            displayLayer.flush()
        }
        
        if !displayLayer.isReadyForMoreMediaData {
            print("!!!not ready")
        }
        
        displayLayer.enqueue(sampleBuffer)
        //})
    }
}

VideoFrameInfo.swift

import Foundation

public struct VideoFrameInfo {

    public let rawPointer: UnsafeMutableRawPointer

    public let size: Int

    public let frameType: UInt8

    public init(rawPointer: UnsafeMutableRawPointer, size: Int, frameType: UInt8) {
        self.rawPointer = rawPointer
        self.size = size
        self.frameType = frameType
    }
}