Fix BN_gcd errors for some curves
authorBilly Brumley <bbrumley@gmail.com>
Wed, 20 Jan 2016 11:18:21 +0000 (13:18 +0200)
committerRich Salz <rsalz@openssl.org>
Thu, 4 Feb 2016 13:02:48 +0000 (08:02 -0500)
Those even order that do not play nicely with Montgomery arithmetic

Signed-off-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Emilia Käsper <emilia@openssl.org>
crypto/ec/ec_lib.c

index abb15a5..a34113c 100644 (file)
@@ -327,13 +327,18 @@ int EC_GROUP_set_generator(EC_GROUP *group, const EC_POINT *generator,
     } else
         BN_zero(group->cofactor);
 
     /*
-     * We ignore the return value because some groups have an order with
+     * Some groups have an order with
      * factors of two, which makes the Montgomery setup fail.
      * |group->mont_data| will be NULL in this case.
      */
-    ec_precompute_mont_data(group);
+    if (BN_is_odd(group->order)) {
+        return ec_precompute_mont_data(group);
+    }
 
+    BN_MONT_CTX_free(group->mont_data);
+    group->mont_data = NULL;
     return 1;
 }