set FIPS permitted flag before initalising digest

[openssl.git] / ssl / s3_enc.c

diff --git a/ssl/s3_enc.c b/ssl/s3_enc.c
index a7943aba81e140d36f68e1a5b5fb4e7fa8ca37df..0ddfe192bc60748e41720c528a93de2622c9bed2 100644 (file)
--- a/ssl/s3_enc.c
+++ b/ssl/s3_enc.c
@@ -170,6 +170,7 @@ static int ssl3_generate_key_block(SSL *s, unsigned char *km, int num)
 #endif
        k=0;
        EVP_MD_CTX_init(&m5);
+       EVP_MD_CTX_set_flags(&m5, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
        EVP_MD_CTX_init(&s1);
        for (i=0; (int)i<num; i+=MD5_DIGEST_LENGTH)
                {
@@ -214,7 +215,7 @@ static int ssl3_generate_key_block(SSL *s, unsigned char *km, int num)
 
 int ssl3_change_cipher_state(SSL *s, int which)
        {
-       unsigned char *p,*key_block,*mac_secret;
+       unsigned char *p,*mac_secret;
        unsigned char exp_key[EVP_MAX_KEY_LENGTH];
        unsigned char exp_iv[EVP_MAX_IV_LENGTH];
        unsigned char *ms,*key,*iv,*er1,*er2;
@@ -231,13 +232,14 @@ int ssl3_change_cipher_state(SSL *s, int which)
        is_exp=SSL_C_IS_EXPORT(s->s3->tmp.new_cipher);
        c=s->s3->tmp.new_sym_enc;
        m=s->s3->tmp.new_hash;
+       /* m == NULL will lead to a crash later */
+       OPENSSL_assert(m);
 #ifndef OPENSSL_NO_COMP
        if (s->s3->tmp.new_compression == NULL)
                comp=NULL;
        else
                comp=s->s3->tmp.new_compression->method;
 #endif
-       key_block=s->s3->tmp.key_block;
 
        if (which & SSL3_CC_READ)
                {
@@ -313,6 +315,8 @@ int ssl3_change_cipher_state(SSL *s, int which)
 
        p=s->s3->tmp.key_block;
        i=EVP_MD_size(m);
+       if (i < 0)
+               goto err2;
        cl=EVP_CIPHER_key_length(c);
        j=is_exp ? (cl < SSL_C_EXPORT_KEYLENGTH(s->s3->tmp.new_cipher) ?
                 cl : SSL_C_EXPORT_KEYLENGTH(s->s3->tmp.new_cipher)) : cl;
@@ -407,7 +411,11 @@ int ssl3_setup_key_block(SSL *s)
        s->s3->tmp.new_compression=comp;
 #endif
 
-       num=EVP_CIPHER_key_length(c)+EVP_MD_size(hash)+EVP_CIPHER_iv_length(c);
+       num=EVP_MD_size(hash);
+       if (num < 0)
+               return 0;
+
+       num=EVP_CIPHER_key_length(c)+num+EVP_CIPHER_iv_length(c);
        num*=2;
 
        ssl3_cleanup_key_block(s);
@@ -561,12 +569,12 @@ void ssl3_free_digest_list(SSL *s)
        OPENSSL_free(s->s3->handshake_dgst);
        s->s3->handshake_dgst=NULL;
        }       
-               
+
 
 
 void ssl3_finish_mac(SSL *s, const unsigned char *buf, int len)
        {
-       if (s->s3->handshake_buffer) 
+       if (s->s3->handshake_buffer && !(s->s3->flags & TLS1_FLAGS_KEEP_HANDSHAKE)) 
                {
                BIO_write (s->s3->handshake_buffer,(void *)buf,len);
                } 
@@ -580,37 +588,57 @@ void ssl3_finish_mac(SSL *s, const unsigned char *buf, int len)
                        }
                }       
        }
-void ssl3_digest_cached_records(SSL *s)
+
+int ssl3_digest_cached_records(SSL *s)
        {
-               int i;
-               long mask;
-               const EVP_MD *md;
-               long hdatalen;
-               void *hdata;
-               /* Allocate handshake_dgst array */
-               ssl3_free_digest_list(s);
-               s->s3->handshake_dgst = OPENSSL_malloc(SSL_MAX_DIGEST * sizeof(EVP_MD_CTX *));
-               memset(s->s3->handshake_dgst,0,SSL_MAX_DIGEST *sizeof(EVP_MD_CTX *));
-               hdatalen = BIO_get_mem_data(s->s3->handshake_buffer,&hdata);
-               /* Loop through bitso of algorithm2 field and create MD_CTX-es */
-               for (i=0;ssl_get_handshake_digest(i,&mask,&md); i++) 
+       int i;
+       long mask;
+       const EVP_MD *md;
+       long hdatalen;
+       void *hdata;
+
+       /* Allocate handshake_dgst array */
+       ssl3_free_digest_list(s);
+       s->s3->handshake_dgst = OPENSSL_malloc(SSL_MAX_DIGEST * sizeof(EVP_MD_CTX *));
+       memset(s->s3->handshake_dgst,0,SSL_MAX_DIGEST *sizeof(EVP_MD_CTX *));
+       hdatalen = BIO_get_mem_data(s->s3->handshake_buffer,&hdata);
+       if (hdatalen <= 0)
+               {
+               SSLerr(SSL_F_SSL3_DIGEST_CACHED_RECORDS, SSL_R_BAD_HANDSHAKE_LENGTH);
+               return 0;
+               }
+
+       /* Loop through bitso of algorithm2 field and create MD_CTX-es */
+       for (i=0;ssl_get_handshake_digest(i,&mask,&md); i++) 
+               {
+               if ((mask & ssl_get_algorithm2(s)) && md) 
                        {
-                               if ((mask & s->s3->tmp.new_cipher->algorithm2) && md) 
+                       s->s3->handshake_dgst[i]=EVP_MD_CTX_create();
+#ifdef OPENSSL_FIPS
+                       if (EVP_MD_nid(md) == NID_md5)
                                {
-                                       s->s3->handshake_dgst[i]=EVP_MD_CTX_create();
-                                       EVP_DigestInit_ex(s->s3->handshake_dgst[i],md,NULL);
-                                       EVP_DigestUpdate(s->s3->handshake_dgst[i],hdata,hdatalen);
-                               } 
-                               else 
-                               {       
-                                       s->s3->handshake_dgst[i]=NULL;
+                               EVP_MD_CTX_set_flags(s->s3->handshake_dgst[i],
+                                               EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
                                }
+#endif
+                       EVP_DigestInit_ex(s->s3->handshake_dgst[i],md,NULL);
+                       EVP_DigestUpdate(s->s3->handshake_dgst[i],hdata,hdatalen);
+                       } 
+               else 
+                       {       
+                       s->s3->handshake_dgst[i]=NULL;
                        }
+               }
+       if (!(s->s3->flags & TLS1_FLAGS_KEEP_HANDSHAKE))
+               {
                /* Free handshake_buffer BIO */
                BIO_free(s->s3->handshake_buffer);
                s->s3->handshake_buffer = NULL;
+               }
 
+       return 1;
        }
+
 int ssl3_cert_verify_mac(SSL *s, int md_nid, unsigned char *p)
        {
        return(ssl3_handshake_mac(s,md_nid,NULL,0,p));
@@ -632,10 +660,12 @@ static int ssl3_handshake_mac(SSL *s, int md_nid,
        unsigned int i;
        unsigned char md_buf[EVP_MAX_MD_SIZE];
        EVP_MD_CTX ctx,*d=NULL;
+
        if (s->s3->handshake_buffer) 
-               ssl3_digest_cached_records(s);
+               if (!ssl3_digest_cached_records(s))
+                       return 0;
 
-       /* Search for djgest of specified type  in the handshake_dgst
+       /* Search for digest of specified type in the handshake_dgst
         * array*/
        for (i=0;i<SSL_MAX_DIGEST;i++) 
                {
@@ -650,8 +680,12 @@ static int ssl3_handshake_mac(SSL *s, int md_nid,
                return 0;
        }       
        EVP_MD_CTX_init(&ctx);
+       EVP_MD_CTX_set_flags(&ctx, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
        EVP_MD_CTX_copy_ex(&ctx,d);
        n=EVP_MD_CTX_size(&ctx);
+       if (n < 0)
+               return 0;
+
        npad=(48/n)*n;
        if (sender != NULL)
                EVP_DigestUpdate(&ctx,sender,len);
@@ -672,7 +706,7 @@ static int ssl3_handshake_mac(SSL *s, int md_nid,
        return((int)ret);
        }
 
-int ssl3_mac(SSL *ssl, unsigned char *md, int send)
+int n_ssl3_mac(SSL *ssl, unsigned char *md, int send)
        {
        SSL3_RECORD *rec;
        unsigned char *mac_sec,*seq;
@@ -681,6 +715,7 @@ int ssl3_mac(SSL *ssl, unsigned char *md, int send)
        unsigned char *p,rec_char;
        unsigned int md_size;
        int npad;
+       int t;
 
        if (send)
                {
@@ -697,7 +732,10 @@ int ssl3_mac(SSL *ssl, unsigned char *md, int send)
                hash=ssl->read_hash;
                }
 
-       md_size=EVP_MD_CTX_size(hash);
+       t=EVP_MD_CTX_size(hash);
+       if (t < 0)
+               return -1;
+       md_size=t;
        npad=(48/md_size)*md_size;
 
        /* Chop the digest off the end :-) */