Possible GetClosestScaleForResolution Bug

ProcessClipMetal.mm

/* -LICENSE-START-
 ** Copyright (c) 2018 Blackmagic Design
 **
 ** Permission is hereby granted, free of charge, to any person or organization
 ** obtaining a copy of the software and accompanying documentation covered by
 ** this license (the "Software") to use, reproduce, display, distribute,
 ** execute, and transmit the Software, and to prepare derivative works of the
 ** Software, and to permit third-parties to whom the Software is furnished to
 ** do so, all subject to the following:
 **
 ** The copyright notices in the Software and this entire statement, including
 ** the above license grant, this restriction and the following disclaimer,
 ** must be included in all copies of the Software, in whole or in part, and
 ** all derivative works of the Software, unless such copies or derivative
 ** works are solely in the form of machine-executable object code generated by
 ** a source language processor.
 **
 ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 ** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
 ** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
 ** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
 ** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 ** DEALINGS IN THE SOFTWARE.
 ** -LICENSE-END-
 */

#include "BlackmagicRawAPI.h"

#include <algorithm>	// std::min
#include <atomic>		// std::atomic
#include <cassert>		// assert
#include <chrono>		// std::chrono
#include <cstdint>		// uint32_t
#include <iostream>		// std::cout, std::cerr
#include <thread>		// std::thread

#include <ImageIO/ImageIO.h>
#include <CoreServices/CoreServices.h>

#import <Metal/Metal.h>

#ifdef DEBUG
	#define VERIFY(condition) assert(SUCCEEDED(condition))
#else
	#define VERIFY(condition) ((void)(condition))
#endif

static const BlackmagicRawResourceFormat	s_resourceFormat		= blackmagicRawResourceFormatRGBAU8;

static const int							s_maxJobsInFlight		= 3;
static std::atomic<int>						s_jobsInFlight			= {0};

static IBlackmagicRawPipelineDevice*		s_metalDevice			= nullptr;

struct UserData
{
	unsigned long long              frameIndex;
    BlackmagicRawResolutionScale    decodeQuality;
};

static void OutputImage(CFStringRef outputFileName, uint32_t width, uint32_t height, uint32_t sizeBytes, void* imageData)
{
	bool success = false;
	const char* outputFileNameAsCString = CFStringGetCStringPtr(outputFileName, kCFStringEncodingMacRoman);

	CFURLRef file = CFURLCreateWithFileSystemPath(kCFAllocatorDefault, outputFileName, kCFURLPOSIXPathStyle, false);
	if (file != nullptr)
	{
		const uint32_t bitsPerComponent	= 8;
		const uint32_t bitsPerPixel		= 32;
		const uint32_t bytesPerRow		= (bitsPerPixel * width) / 8U;

		CGColorSpaceRef space			= CGColorSpaceCreateWithName(kCGColorSpaceSRGB);
		CGBitmapInfo bitmapInfo			= kCGImageAlphaNoneSkipLast | kCGImageByteOrderDefault;
		CGDataProviderRef provider		= CGDataProviderCreateWithData(nullptr, imageData, sizeBytes, nullptr);
		const CGFloat* decode			= nullptr;
		bool shouldInterpolate			= false;
		CGColorRenderingIntent intent	= kCGRenderingIntentDefault;

		CGImageRef imageRef = CGImageCreate(width, height, bitsPerComponent, bitsPerPixel, bytesPerRow, space, bitmapInfo, provider, decode, shouldInterpolate, intent);
		if (imageRef != nullptr)
		{
			CGImageDestinationRef destination = CGImageDestinationCreateWithURL(file, kUTTypePNG, 1, nullptr);
			if (destination)
			{
				CGImageDestinationAddImage(destination, imageRef, nil);
				CGImageDestinationFinalize(destination);

				CFRelease(destination);

				std::cout << "Created " << outputFileNameAsCString << std::endl;
				success = true;
			}

			CGImageRelease(imageRef);
		}

		CGDataProviderRelease(provider);
		CGColorSpaceRelease(space);

		CFRelease(file);
	}

	if (! success)
		std::cerr << "Failed to create " << outputFileNameAsCString << "!" << std::endl;
}

class CameraCodecCallback : public IBlackmagicRawCallback
{
public:
	explicit CameraCodecCallback() = default;
	virtual ~CameraCodecCallback() = default;

	virtual void ReadComplete(IBlackmagicRawJob* readJob, HRESULT result, IBlackmagicRawFrame* frame)
	{
		UserData* userData = nullptr;
		VERIFY(readJob->GetUserData((void**)&userData));

		IBlackmagicRawJob* decodeAndProcessJob = nullptr;

		if (result == S_OK)
			VERIFY(frame->SetResourceFormat(s_resourceFormat));

        if (result == S_OK) {
            BlackmagicRawResolutionScale decodeQuality = userData->decodeQuality;
            VERIFY(frame->SetResolutionScale(decodeQuality));
        }
        
		if (result == S_OK)
			result = frame->CreateJobDecodeAndProcessFrame(nullptr, nullptr, &decodeAndProcessJob);

		if (result == S_OK)
			VERIFY(decodeAndProcessJob->SetUserData(userData));

		if (result == S_OK)
			result = decodeAndProcessJob->Submit();

		if (result != S_OK)
		{
			if (decodeAndProcessJob)
				decodeAndProcessJob->Release();

			delete userData;
		}

		readJob->Release();
	}

	virtual void ProcessComplete(IBlackmagicRawJob* job, HRESULT result, IBlackmagicRawProcessedImage* processedImage)
	{
		UserData* userData = nullptr;
		VERIFY(job->GetUserData((void**)&userData));

		unsigned long long frameIndex = userData->frameIndex;
		std::cout << "Processed frame index: " << frameIndex << " ... " << std::flush;
		delete userData;

		job->Release();
		--s_jobsInFlight;

		// Query the device for the native Metal commandQueue
		BlackmagicRawPipeline pipeline;
		void* context;
		void* commandQueue;
		VERIFY(s_metalDevice->GetPipeline(&pipeline, &context, &commandQueue) == S_OK);

		// "Toll free" bridge cast the void* representation of the MTLCommandQueue to the usable protocol
		id<MTLCommandQueue> metalCommandQueue = (__bridge id<MTLCommandQueue>)commandQueue;
		assert(metalCommandQueue != nil);

		uint32_t processedImageSizeBytes = 0;
		uint32_t processedImageWidthPixels = 0;
		uint32_t processedImageHeightPixels = 0;
		void* processedResourceOnGPU = nullptr;
		VERIFY(processedImage->GetWidth(&processedImageWidthPixels) == S_OK);
		VERIFY(processedImage->GetHeight(&processedImageHeightPixels) == S_OK);
		VERIFY(processedImage->GetResourceSizeBytes(&processedImageSizeBytes) == S_OK);
		VERIFY(processedImage->GetResource(&processedResourceOnGPU) == S_OK);

		// Create a managed "staging" buffer to receive the contents of the processed GPU buffer so that we may access it with the CPU (to save to disk)
		id<MTLBuffer> stagingMetalBuffer = [metalCommandQueue.device newBufferWithLength:processedImageSizeBytes options:MTLResourceStorageModeManaged];
		assert(stagingMetalBuffer != nil);

		id<MTLCommandBuffer> commandBuffer = [metalCommandQueue commandBuffer];
		assert(commandBuffer != nil);

		id<MTLBlitCommandEncoder> blitEncoder = [commandBuffer blitCommandEncoder];
		assert(blitEncoder != nil);

		// Copy the processed image from a private GPU buffer into a managed (CPU accessible) buffer
		id<MTLBuffer> processedMetalBuffer = (__bridge id<MTLBuffer>)processedResourceOnGPU;
		[blitEncoder copyFromBuffer:processedMetalBuffer sourceOffset:0 toBuffer:stagingMetalBuffer destinationOffset:0 size:processedMetalBuffer.length];
		[blitEncoder synchronizeResource:stagingMetalBuffer];
		[blitEncoder endEncoding];

		[commandBuffer commit];
		[commandBuffer waitUntilCompleted];

        NSString *desktopPath = [NSHomeDirectory() stringByAppendingString:@"/Desktop/AAAoutputRGBAU8_frame%llu.png"];
        CFStringRef desktopPathRef = (__bridge CFStringRef)desktopPath;
        
		CFStringRef outputFileName = CFStringCreateWithFormat(NULL, NULL, desktopPathRef, frameIndex);
		OutputImage(outputFileName, processedImageWidthPixels, processedImageHeightPixels, processedImageSizeBytes, stagingMetalBuffer.contents);
		CFRelease(outputFileName);

		[stagingMetalBuffer release];
	}

	virtual void DecodeComplete(IBlackmagicRawJob*, HRESULT) {}
	virtual void TrimProgress(IBlackmagicRawJob*, float) {}
	virtual void TrimComplete(IBlackmagicRawJob*, HRESULT) {}
	virtual void SidecarMetadataParseWarning(IBlackmagicRawClip*, CFStringRef, uint32_t, CFStringRef) {}
	virtual void SidecarMetadataParseError(IBlackmagicRawClip*, CFStringRef, uint32_t, CFStringRef) {}
	virtual void PreparePipelineComplete(void*, HRESULT) {}

	virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
	{
		return E_NOTIMPL;
	}

	virtual ULONG STDMETHODCALLTYPE AddRef(void)
	{
		return 0;
	}

	virtual ULONG STDMETHODCALLTYPE Release(void)
	{
		return 0;
	}
};

HRESULT ProcessClip(IBlackmagicRawClip* clip)
{
	HRESULT result;

	unsigned long long frameCount = 0;
	unsigned long long frameIndex = 0;

	result = clip->GetFrameCount(&frameCount);

	// Extract at most 8 frames so as not to fill the disk
	frameCount = std::min(frameCount, 8ULL);

	while (frameIndex < frameCount)
	{
		if (s_jobsInFlight >= s_maxJobsInFlight)
		{
			std::this_thread::sleep_for(std::chrono::microseconds(100));
			continue;
		}

		IBlackmagicRawJob* jobRead = nullptr;
		if (result == S_OK)
			result = clip->CreateJobReadFrame(frameIndex, &jobRead);

		UserData* userData = nullptr;
		if (result == S_OK)
		{
			userData = new UserData();
			userData->frameIndex = frameIndex;
                        
            IBlackmagicRawClipResolutions *clipResolutions;
            clip->QueryInterface(IID_IBlackmagicRawClipResolutions, (void**)&clipResolutions);
            
            uint32_t decodeQuality;
            
            //
            // NOTE: Feeding `GetClosestScaleForResolution` with 1920x1080 and `requestUpsideDown` as `true` gives upside down image,
            //       however, feeding it 112x63 and `requestUpsideDown` as `true` gives a right-side-up image?!?
            //
            uint32_t destinationWidth   = 112; //1920;
            uint32_t destinationHeight  = 63;  //1080;
            bool requestUpsideDown      = true;

            clipResolutions->GetClosestScaleForResolution(destinationWidth, destinationHeight, requestUpsideDown, &decodeQuality);
            
            userData->decodeQuality = decodeQuality;
            
			VERIFY(jobRead->SetUserData(userData));
		}

		if (result == S_OK)
			result = jobRead->Submit();

		if (result != S_OK)
		{
			if (userData != nullptr)
				delete userData;

			if (jobRead != nullptr)
				jobRead->Release();

			break;
		}

		++s_jobsInFlight;

		frameIndex++;
	}

	return result;
}

int main(int argc, const char* argv[])
{
	if (argc > 2)
	{
		std::cerr << "Usage: " << argv[0] << " clipName.braw" << std::endl;
		return 1;
	}

	CFStringRef clipName;
	bool clipNameProvided = argc == 2;
	if (clipNameProvided)
	{
		clipName = CFStringCreateWithCString(NULL, argv[1], kCFStringEncodingUTF8);
	}
	else
	{
		clipName = CFSTR("/Applications/Blackmagic RAW/Blackmagic RAW SDK/Media/sample.braw");
	}

	HRESULT result = S_OK;

	IBlackmagicRawFactory* factory = nullptr;
	IBlackmagicRaw* codec = nullptr;
	IBlackmagicRawClip* clip = nullptr;
	IBlackmagicRawConfiguration* config = nullptr;

	CameraCodecCallback callback;

	do
	{
		factory = CreateBlackmagicRawFactoryInstanceFromPath(CFSTR("/Applications/Blackmagic RAW/Blackmagic RAW SDK/Mac/Libraries"));
		if (factory == nullptr)
		{
			std::cerr << "Failed to create IBlackmagicRawFactory!" << std::endl;
			break;
		}

		result = factory->CreateCodec(&codec);
		if (result != S_OK)
		{
			std::cerr << "Failed to create IBlackmagicRaw!" << std::endl;
			break;
		}

		IBlackmagicRawPipelineDeviceIterator* pipelineDeviceIterator = nullptr;
		result = factory->CreatePipelineDeviceIterator(blackmagicRawPipelineMetal, blackmagicRawInteropOpenGL, &pipelineDeviceIterator);
		if (result != S_OK)
		{
			std::cerr << "Failed to create device iterator for blackmagicRawPipelineMetal with blackmagicRawInteropOpenGL!" << std::endl;
			break;
		}

		// Create the (first) device
		result = pipelineDeviceIterator->CreateDevice(&s_metalDevice);
		if (result != S_OK)
		{
			std::cerr << "Failed to create the first metal device as indicated by pipelineDeviceIterator!" << std::endl;
			break;
		}

		// Release the device iterator as it is no longer required
		pipelineDeviceIterator->Release();

		result = codec->QueryInterface(IID_IBlackmagicRawConfiguration, (void**)&config);
		if (result != S_OK)
		{
			std::cerr << "Failed to get IID_IBlackmagicRawConfiguration!" << std::endl;
			break;
		}

		result = config->SetFromDevice(s_metalDevice);
		if (result != S_OK)
		{
			std::cerr << "Failed to set the config from the metal device!" << std::endl;
			break;
		}

		result = codec->OpenClip(clipName, &clip);
		if (result != S_OK)
		{
			std::cerr << "Failed to open IBlackmagicRawClip!" << std::endl;
			break;
		}

		result = codec->SetCallback(&callback);
		if (result != S_OK)
		{
			std::cerr << "Failed to set IBlackmagicRawCallback!" << std::endl;
			break;
		}

		result = ProcessClip(clip);
		codec->FlushJobs();

	} while(0);

	if (config != nullptr)
		config->Release();

	if (clip != nullptr)
		clip->Release();

	if (codec != nullptr)
		codec->Release();

	if (s_metalDevice != nullptr)
		s_metalDevice->Release();

	if (factory != nullptr)
		factory->Release();

	CFRelease(clipName);

	return result;
}