RT3192: spurious error in DSA verify

[openssl.git] / crypto / dsa / dsa_ossl.c

diff --git a/crypto/dsa/dsa_ossl.c b/crypto/dsa/dsa_ossl.c
index fd757082f955f3768b08e44a8ba371067123650d..846e16275b9a9829f692a01a0f71feac7b96cecf 100644 (file)
--- a/crypto/dsa/dsa_ossl.c
+++ b/crypto/dsa/dsa_ossl.c
@@ -58,6 +58,8 @@
 
 /* Original version from Steven Schoch <schoch@sheba.arc.nasa.gov> */
 
+#define OPENSSL_FIPSAPI
+
 #include <stdio.h>
 #include "cryptlib.h"
 #include <openssl/bn.h>
@@ -70,7 +72,10 @@
 #endif
 
 static DSA_SIG *dsa_do_sign(const unsigned char *dgst, int dlen, DSA *dsa);
-static int dsa_sign_setup(DSA *dsa, BN_CTX *ctx_in, BIGNUM **kinvp, BIGNUM **rp);
+static int dsa_sign_setup_no_digest(DSA *dsa, BN_CTX *ctx_in, BIGNUM **kinvp, BIGNUM **rp);
+static int dsa_sign_setup(DSA *dsa, BN_CTX *ctx_in,
+                                     BIGNUM **kinvp, BIGNUM **rp,
+                                     const unsigned char *dgst, int dlen);
 static int dsa_do_verify(const unsigned char *dgst, int dgst_len, DSA_SIG *sig,
                         DSA *dsa);
 static int dsa_init(DSA *dsa);
@@ -79,7 +84,7 @@ static int dsa_finish(DSA *dsa);
 static DSA_METHOD openssl_dsa_meth = {
 "OpenSSL DSA method",
 dsa_do_sign,
-dsa_sign_setup,
+dsa_sign_setup_no_digest,
 dsa_do_verify,
 NULL, /* dsa_mod_exp, */
 NULL, /* dsa_bn_mod_exp, */
@@ -148,11 +153,14 @@ static DSA_SIG *dsa_do_sign(const unsigned char *dgst, int dlen, DSA *dsa)
            return NULL;
            }
 
-       if (FIPS_mode() && (BN_num_bits(dsa->p) < OPENSSL_DSA_FIPS_MIN_MODULUS_BITS))
+       if (FIPS_module_mode() && !(dsa->flags & DSA_FLAG_NON_FIPS_ALLOW) 
+               && (BN_num_bits(dsa->p) < OPENSSL_DSA_FIPS_MIN_MODULUS_BITS))
                {
                DSAerr(DSA_F_DSA_DO_SIGN, DSA_R_KEY_SIZE_TOO_SMALL);
                return NULL;
                }
+       if (!fips_check_dsa_prng(dsa, 0, 0))
+               goto err;
 #endif
 
        BN_init(&m);
@@ -171,7 +179,8 @@ static DSA_SIG *dsa_do_sign(const unsigned char *dgst, int dlen, DSA *dsa)
 redo:
        if ((dsa->kinv == NULL) || (dsa->r == NULL))
                {
-               if (!DSA_sign_setup(dsa,ctx,&kinv,&r)) goto err;
+               if (!dsa_sign_setup(dsa,ctx,&kinv,&r,dgst,dlen))
+                       goto err;
                }
        else
                {
@@ -230,7 +239,14 @@ err:
        return(ret);
        }
 
-static int dsa_sign_setup(DSA *dsa, BN_CTX *ctx_in, BIGNUM **kinvp, BIGNUM **rp)
+static int dsa_sign_setup_no_digest(DSA *dsa, BN_CTX *ctx_in,
+                         BIGNUM **kinvp, BIGNUM **rp) {
+       return dsa_sign_setup(dsa, ctx_in, kinvp, rp, NULL, 0);
+}
+
+static int dsa_sign_setup(DSA *dsa, BN_CTX *ctx_in,
+                                     BIGNUM **kinvp, BIGNUM **rp,
+                                     const unsigned char *dgst, int dlen)
        {
        BN_CTX *ctx;
        BIGNUM k,kq,*K,*kinv=NULL,*r=NULL;
@@ -256,8 +272,22 @@ static int dsa_sign_setup(DSA *dsa, BN_CTX *ctx_in, BIGNUM **kinvp, BIGNUM **rp)
 
        /* Get random k */
        do
-               if (!BN_rand_range(&k, dsa->q)) goto err;
-       while (BN_is_zero(&k));
+               {
+#ifndef OPENSSL_NO_SHA512
+               if (dgst != NULL)
+                       {
+                       /* We calculate k from SHA512(private_key + H(message)
+                        * + random). This protects the private key from a weak
+                        * PRNG. */
+                       if (!BN_generate_dsa_nonce(&k, dsa->q, dsa->priv_key, dgst,
+                                                  dlen, ctx))
+                               goto err;
+                       }
+               else
+#endif
+                       if (!BN_rand_range(&k, dsa->q)) goto err;
+               } while (BN_is_zero(&k));
+
        if ((dsa->flags & DSA_FLAG_NO_EXP_CONSTTIME) == 0)
                {
                BN_set_flags(&k, BN_FLG_CONSTTIME);
@@ -349,7 +379,8 @@ static int dsa_do_verify(const unsigned char *dgst, int dgst_len, DSA_SIG *sig,
            return -1;
            }
 
-       if (FIPS_mode() && (BN_num_bits(dsa->p) < OPENSSL_DSA_FIPS_MIN_MODULUS_BITS))
+       if (FIPS_module_mode() && !(dsa->flags & DSA_FLAG_NON_FIPS_ALLOW) 
+               && (BN_num_bits(dsa->p) < OPENSSL_DSA_FIPS_MIN_MODULUS_BITS))
                {
                DSAerr(DSA_F_DSA_DO_VERIFY, DSA_R_KEY_SIZE_TOO_SMALL);
                return -1;
@@ -418,9 +449,7 @@ static int dsa_do_verify(const unsigned char *dgst, int dgst_len, DSA_SIG *sig,
        ret=(BN_ucmp(&u1, sig->r) == 0);
 
        err:
-       /* XXX: surely this is wrong - if ret is 0, it just didn't verify;
-          there is no error in BN. Test should be ret == -1 (Ben) */
-       if (ret != 1) DSAerr(DSA_F_DSA_DO_VERIFY,ERR_R_BN_LIB);
+       if (ret < 0) DSAerr(DSA_F_DSA_DO_VERIFY,ERR_R_BN_LIB);
        if (ctx != NULL) BN_CTX_free(ctx);
        BN_free(&u1);
        BN_free(&u2);