Fix BN_kronecker so that it works correctly if 'a' is negative

[openssl.git] / crypto / bn / bn_kron.c

diff --git a/crypto/bn/bn_kron.c b/crypto/bn/bn_kron.c
index aba48dda4a22a2f96ba8878468522abaa3df9d6c..0dd8a194cbe3cb11ed939e734a60475d3274e950 100644 (file)
--- a/crypto/bn/bn_kron.c
+++ b/crypto/bn/bn_kron.c
@@ -65,7 +65,7 @@
 int BN_kronecker(const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)
        {
        int i;
 int BN_kronecker(const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)
        {
        int i;

-       int ret;
+       int ret = -2; /* avoid 'uninitialized' warning */

        int err = 0;
        BIGNUM *A, *B, *tmp;
        /* In 'tab', only odd-indexed entries are relevant:
        int err = 0;
        BIGNUM *A, *B, *tmp;
        /* In 'tab', only odd-indexed entries are relevant:


@@ -146,7 +146,7 @@ int BN_kronecker(const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)
 
                if (BN_is_zero(A))
                        {
 
                if (BN_is_zero(A))
                        {

-                       ret = BN_is_one(B);
+                       ret = BN_is_one(B) ? ret : 0;

                        goto end;
                        }
 
                        goto end;
                        }
 


@@ -165,7 +165,7 @@ int BN_kronecker(const BIGNUM *a, const BIGNUM *b, BN_CTX *ctx)
        
                /* Cohen's step 4: */
                /* multiply 'ret' by  $(-1)^{(A-1)(B-1)/4}$ */
        
                /* Cohen's step 4: */
                /* multiply 'ret' by  $(-1)^{(A-1)(B-1)/4}$ */

-               if (BN_lsw(A) & BN_lsw(B) & 2)
+               if ((A->neg ? ~BN_lsw(A) : BN_lsw(A)) & BN_lsw(B) & 2)

                        ret = -ret;
                
                /* (A, B) := (B mod |A|, |A|) */
                        ret = -ret;
                
                /* (A, B) := (B mod |A|, |A|) */