Created
August 27, 2023 07:14
-
-
Save halcwb/eefda640dc887aed1f9607074e14587e to your computer and use it in GitHub Desktop.
Makes a floating point number "readable"
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/// <summary> | |
/// Fixes the precision of a double number. | |
/// </summary> | |
/// <param name="dblNum">The double number to fix the precision of.</param> | |
/// <param name="intPrec">The desired precision.</param> | |
/// <returns>The double number with the fixed precision.</returns> | |
let fixPrecision (dblNum: double) (intPrec: int) : double = | |
let strNum = dblNum.ToString("F99", CultureInfo.InvariantCulture) | |
let colNum = List.ofArray (strNum.Split('.')) | |
let strN, strD = | |
match colNum with | |
| [n] -> (n, "") | |
| [n; d] -> (n, d) | |
| _ -> failwith "Unexpected number format" | |
let mutable intCount = 0 | |
let mutable blnFirst = true | |
for ch in strD do | |
if blnFirst && ch = '0' then | |
intCount <- intCount + 1 | |
else | |
blnFirst <- false | |
let intP = | |
if strN = "0" then intPrec else intPrec - strN.Length | |
let intP = | |
if intP < 0 then 0 else intP | |
let intP = | |
if String.IsNullOrEmpty(strD) || dblNum >= 1.0 then intP | |
else intCount + intP | |
let dblFix = Math.Round(dblNum * Math.Pow(10.0, float intP), 0) | |
dblFix / Math.Pow(10.0, float intP) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment