-
-
Save gregcotten/2583ba117de2a6c79969b3a964239bc8 to your computer and use it in GitHub Desktop.
Performs a test to verify that 8, 10, 12, and 16-bit code values can be converted to normalized 32-bit floats, rounded to only 5 decimal places (!!!), and reconverted losslessly back to the original integer code value.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import numpy as np | |
def float32PrecisionTest(bitdepth, floatingPointDecimalPlaces): | |
maximumCodeValue = pow(2, bitdepth) - 1 | |
maximumCodeValueAsFloat = np.float32(maximumCodeValue) | |
failedCodeValues = [] | |
for codeValue in range(0, pow(2, bitdepth)): | |
normalizedFloatValue = np.round(np.float32(codeValue) / maximumCodeValueAsFloat, floatingPointDecimalPlaces) | |
rescaledCodeValue = np.uint16(np.round(normalizedFloatValue * maximumCodeValueAsFloat)) | |
if codeValue != rescaledCodeValue: | |
failedCodeValues.append(codeValue) | |
return failedCodeValues | |
floatingPointDecimalPlaces = 5 | |
for bitdepth in [8, 10, 12, 16]: | |
result = float32PrecisionTest(bitdepth, floatingPointDecimalPlaces) | |
if len(result) is 0: | |
print(str(bitdepth) + "-bit test passed.") | |
else: | |
print(str(bitdepth) + "-bit test failed for " + str(len(result)) + " code values") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Output of running this: