bio printf: Avoid using rounding errors in range check

[openssl.git] / crypto / bio / b_print.c

diff --git a/crypto/bio/b_print.c b/crypto/bio/b_print.c
index 0d6fafcc2d411da4158526b024173bab58f0b981..6b995f82334be0db7474dce562004a07eacb1647 100644 (file)
--- a/crypto/bio/b_print.c
+++ b/crypto/bio/b_print.c
@@ -635,7 +635,13 @@ fmtfp(char **sbuffer,
             fvalue = tmpvalue;
     }
     ufvalue = abs_val(fvalue);
-    if (ufvalue > ULONG_MAX) {
+    /*
+     * By subtracting 65535 (2^16-1) we cancel the low order 15 bits
+     * of ULONG_MAX to avoid using imprecise floating point values.
+     * The second condition is necessary to catch NaN values.
+     */
+    if (ufvalue >= (double)(ULONG_MAX - 65535) + 65536.0
+            || !(ufvalue == ufvalue) /* NaN */) {
         /* Number too big */
         return 0;
     }