RT4320/GH705: Fix PEM parsing bug.

[openssl.git] / crypto / bn / bn_gf2m.c

diff --git a/crypto/bn/bn_gf2m.c b/crypto/bn/bn_gf2m.c
index 7c0d4afc7198b6f0076a2d3e10af899e9e297109..84837a0f579ea2f5dfa154ef79f83ddc90dae426 100644 (file)
--- a/crypto/bn/bn_gf2m.c
+++ b/crypto/bn/bn_gf2m.c
@@ -1,4 +1,3 @@
-/* crypto/bn/bn_gf2m.c */
 /* ====================================================================
  * Copyright 2002 Sun Microsystems, Inc. ALL RIGHTS RESERVED.
  *
@@ -92,7 +91,7 @@
 #include <assert.h>
 #include <limits.h>
 #include <stdio.h>
-#include "cryptlib.h"
+#include "internal/cryptlib.h"
 #include "bn_lcl.h"
 
 #ifndef OPENSSL_NO_EC2M
@@ -450,8 +449,7 @@ int BN_GF2m_mod_arr(BIGNUM *r, const BIGNUM *a, const int p[])
             d0 = p[k] % BN_BITS2;
             d1 = BN_BITS2 - d0;
             z[n] ^= (zz << d0);
-            tmp_ulong = zz >> d1;
-            if (d0 && tmp_ulong)
+            if (d0 && (tmp_ulong = zz >> d1))
                 z[n + 1] ^= tmp_ulong;
         }
 
@@ -473,8 +471,8 @@ int BN_GF2m_mod(BIGNUM *r, const BIGNUM *a, const BIGNUM *p)
     int arr[6];
     bn_check_top(a);
     bn_check_top(p);
-    ret = BN_GF2m_poly2arr(p, arr, sizeof(arr) / sizeof(arr[0]));
-    if (!ret || ret > (int)(sizeof(arr) / sizeof(arr[0]))) {
+    ret = BN_GF2m_poly2arr(p, arr, OSSL_NELEM(arr));
+    if (!ret || ret > (int)OSSL_NELEM(arr)) {
         BNerr(BN_F_BN_GF2M_MOD, BN_R_INVALID_LENGTH);
         return 0;
     }
@@ -551,7 +549,7 @@ int BN_GF2m_mod_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b,
     bn_check_top(a);
     bn_check_top(b);
     bn_check_top(p);
-    if ((arr = OPENSSL_malloc(sizeof(int) * max)) == NULL)
+    if ((arr = OPENSSL_malloc(sizeof(*arr) * max)) == NULL)
         goto err;
     ret = BN_GF2m_poly2arr(p, arr, max);
     if (!ret || ret > max) {
@@ -575,7 +573,7 @@ int BN_GF2m_mod_sqr_arr(BIGNUM *r, const BIGNUM *a, const int p[],
     bn_check_top(a);
     BN_CTX_start(ctx);
     if ((s = BN_CTX_get(ctx)) == NULL)
-        return 0;
+        goto err;
     if (!bn_wexpand(s, 2 * a->top))
         goto err;
 
@@ -609,7 +607,7 @@ int BN_GF2m_mod_sqr(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
 
     bn_check_top(a);
     bn_check_top(p);
-    if ((arr = OPENSSL_malloc(sizeof(int) * max)) == NULL)
+    if ((arr = OPENSSL_malloc(sizeof(*arr) * max)) == NULL)
         goto err;
     ret = BN_GF2m_poly2arr(p, arr, max);
     if (!ret || ret > max) {
@@ -692,23 +690,27 @@ int BN_GF2m_mod_inv(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
     }
 # else
     {
-        int i, ubits = BN_num_bits(u), vbits = BN_num_bits(v), /* v is copy
-                                                                * of p */
-            top = p->top;
+        int i;
+        int ubits = BN_num_bits(u);
+        int vbits = BN_num_bits(v); /* v is copy of p */
+        int top = p->top;
         BN_ULONG *udp, *bdp, *vdp, *cdp;
 
-        bn_wexpand(u, top);
+        if (!bn_wexpand(u, top))
+            goto err;
         udp = u->d;
         for (i = u->top; i < top; i++)
             udp[i] = 0;
         u->top = top;
-        bn_wexpand(b, top);
+        if (!bn_wexpand(b, top))
+          goto err;
         bdp = b->d;
         bdp[0] = 1;
         for (i = 1; i < top; i++)
             bdp[i] = 0;
         b->top = top;
-        bn_wexpand(c, top);
+        if (!bn_wexpand(c, top))
+          goto err;
         cdp = c->d;
         for (i = 0; i < top; i++)
             cdp[i] = 0;
@@ -738,8 +740,12 @@ int BN_GF2m_mod_inv(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
                 ubits--;
             }
 
-            if (ubits <= BN_BITS2 && udp[0] == 1)
-                break;
+            if (ubits <= BN_BITS2) {
+                if (udp[0] == 0) /* poly was reducible */
+                    goto err;
+                if (udp[0] == 1)
+                    break;
+            }
 
             if (ubits < vbits) {
                 i = ubits;
@@ -1025,7 +1031,7 @@ int BN_GF2m_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *b,
     bn_check_top(a);
     bn_check_top(b);
     bn_check_top(p);
-    if ((arr = OPENSSL_malloc(sizeof(int) * max)) == NULL)
+    if ((arr = OPENSSL_malloc(sizeof(*arr) * max)) == NULL)
         goto err;
     ret = BN_GF2m_poly2arr(p, arr, max);
     if (!ret || ret > max) {
@@ -1084,7 +1090,7 @@ int BN_GF2m_mod_sqrt(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
     int *arr = NULL;
     bn_check_top(a);
     bn_check_top(p);
-    if ((arr = OPENSSL_malloc(sizeof(int) * max)) == NULL)
+    if ((arr = OPENSSL_malloc(sizeof(*arr) * max)) == NULL)
         goto err;
     ret = BN_GF2m_poly2arr(p, arr, max);
     if (!ret || ret > max) {
@@ -1214,7 +1220,7 @@ int BN_GF2m_mod_solve_quad(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
     int *arr = NULL;
     bn_check_top(a);
     bn_check_top(p);
-    if ((arr = OPENSSL_malloc(sizeof(int) * max)) == NULL)
+    if ((arr = OPENSSL_malloc(sizeof(*arr) * max)) == NULL)
         goto err;
     ret = BN_GF2m_poly2arr(p, arr, max);
     if (!ret || ret > max) {