PR: 2120

[openssl.git] / ssl / t1_enc.c

diff --git a/ssl/t1_enc.c b/ssl/t1_enc.c
index f19d68c08ab9e62cfe7b59588e3b92ae851dcf4a..d9cb059d0c21685d8b0e9c7c2cba816fe4b70350 100644 (file)
--- a/ssl/t1_enc.c
+++ b/ssl/t1_enc.c
@@ -137,7 +137,9 @@
 
 #include <stdio.h>
 #include "ssl_locl.h"
+#ifndef OPENSSL_NO_COMP
 #include <openssl/comp.h>
+#endif
 #include <openssl/evp.h>
 #include <openssl/hmac.h>
 #include <openssl/md5.h>
@@ -163,6 +165,7 @@ static void tls1_P_hash(const EVP_MD *md, const unsigned char *sec,
        unsigned int A1_len;
        
        chunk=EVP_MD_size(md);
+       OPENSSL_assert(chunk >= 0);
 
        HMAC_CTX_init(&ctx);
        HMAC_CTX_init(&ctx_tmp);
@@ -311,16 +314,15 @@ int tls1_change_cipher_state(SSL *s, int which)
        printf("\talg= %ld/%ld, comp= %p\n",
               s->s3->tmp.new_cipher->algorithm_mkey,
               s->s3->tmp.new_cipher->algorithm_auth,
-              (void *)comp);
-       printf("\tevp_cipher == %p ==? &d_cbc_ede_cipher3\n", (void *)c);
-       printf("\tevp_cipher: nid, blksz= %d, %ld, keylen=%ld, ivlen=%ld\n",
-                c->nid,(unsigned long)c->block_size,
-               (unsigned long)c->key_len,(unsigned long)c->iv_len);
+              comp);
+       printf("\tevp_cipher == %p ==? &d_cbc_ede_cipher3\n", c);
+       printf("\tevp_cipher: nid, blksz= %d, %d, keylen=%d, ivlen=%d\n",
+                c->nid,c->block_size,c->key_len,c->iv_len);
        printf("\tkey_block: len= %d, data= ", s->s3->tmp.key_block_length);
        {
-        int ki;
-        for (ki=0; ki<s->s3->tmp.key_block_length; ki++)
-               printf("%02x", key_block[ki]);  printf("\n");
+        int i;
+        for (i=0; i<s->s3->tmp.key_block_length; i++)
+               printf("%02x", key_block[i]);  printf("\n");
         }
 #endif /* KSSL_DEBUG */
 
@@ -485,13 +487,11 @@ printf("which = %04X\nmac key=",which);
        s->session->key_arg_length=0;
 #ifdef KSSL_DEBUG
        {
-        int ki;
+        int i;
        printf("EVP_CipherInit_ex(dd,c,key=,iv=,which)\n");
-       printf("\tkey= ");
-       for (ki=0; ki<c->key_len; ki++) printf("%02x", key[ki]);
+       printf("\tkey= "); for (i=0; i<c->key_len; i++) printf("%02x", key[i]);
        printf("\n");
-       printf("\t iv= ");
-       for (ki=0; ki<c->iv_len; ki++) printf("%02x", iv[ki]);
+       printf("\t iv= "); for (i=0; i<c->iv_len; i++) printf("%02x", iv[i]);
        printf("\n");
        }
 #endif /* KSSL_DEBUG */
@@ -608,7 +608,10 @@ int tls1_enc(SSL *s, int send)
        if (send)
                {
                if (EVP_MD_CTX_md(s->write_hash))
+                       {
                        n=EVP_MD_CTX_size(s->write_hash);
+                       OPENSSL_assert(n >= 0);
+                       }
                ds=s->enc_write_ctx;
                rec= &(s->s3->wrec);
                if (s->enc_write_ctx == NULL)
@@ -619,7 +622,10 @@ int tls1_enc(SSL *s, int send)
        else
                {
                if (EVP_MD_CTX_md(s->read_hash))
+                       {
                        n=EVP_MD_CTX_size(s->read_hash);
+                       OPENSSL_assert(n >= 0);
+                       }
                ds=s->enc_read_ctx;
                rec= &(s->s3->rrec);
                if (s->enc_read_ctx == NULL)
@@ -666,13 +672,11 @@ int tls1_enc(SSL *s, int send)
                {
                 unsigned long ui;
                printf("EVP_Cipher(ds=%p,rec->data=%p,rec->input=%p,l=%ld) ==>\n",
-                        (void *)ds,rec->data,rec->input,l);
-               printf("\tEVP_CIPHER_CTX: %ld buf_len, %ld key_len [%ld %ld], %ld iv_len\n",
-                        (unsigned long)ds->buf_len,
-                       (unsigned long)ds->cipher->key_len,
-                        (unsigned long)DES_KEY_SZ,
-                       (unsigned long)DES_SCHEDULE_SZ,
-                        (unsigned long)ds->cipher->iv_len);
+                        ds,rec->data,rec->input,l);
+               printf("\tEVP_CIPHER_CTX: %d buf_len, %d key_len [%d %d], %d iv_len\n",
+                        ds->buf_len, ds->cipher->key_len,
+                        DES_KEY_SZ, DES_SCHEDULE_SZ,
+                        ds->cipher->iv_len);
                printf("\t\tIV: ");
                for (i=0; i<ds->cipher->iv_len; i++) printf("%02X", ds->iv[i]);
                printf("\n");
@@ -696,10 +700,10 @@ int tls1_enc(SSL *s, int send)
 
 #ifdef KSSL_DEBUG
                {
-                unsigned long ki;
+                unsigned long i;
                 printf("\trec->data=");
-               for (ki=0; ki<l; i++)
-                        printf(" %02x", rec->data[ki]);  printf("\n");
+               for (i=0; i<l; i++)
+                        printf(" %02x", rec->data[i]);  printf("\n");
                 }
 #endif /* KSSL_DEBUG */
 
@@ -754,7 +758,9 @@ int tls1_cert_verify_mac(SSL *s, int md_nid, unsigned char *out)
        int i;
 
        if (s->s3->handshake_buffer) 
-               ssl3_digest_cached_records(s);
+               if (!ssl3_digest_cached_records(s))
+                       return 0;
+
        for (i=0;i<SSL_MAX_DIGEST;i++) 
                {
                  if (s->s3->handshake_dgst[i]&&EVP_MD_CTX_type(s->s3->handshake_dgst[i])==md_nid) 
@@ -789,17 +795,18 @@ int tls1_final_finish_mac(SSL *s,
 
        q=buf;
 
-       EVP_MD_CTX_init(&ctx);
-
        if (s->s3->handshake_buffer) 
-               ssl3_digest_cached_records(s);
+               if (!ssl3_digest_cached_records(s))
+                       return 0;
+
+       EVP_MD_CTX_init(&ctx);
 
        for (idx=0;ssl_get_handshake_digest(idx,&mask,&md);idx++)
                {
                if (mask & s->s3->tmp.new_cipher->algorithm2)
                        {
-                       unsigned int hashsize = EVP_MD_size(md);
-                       if (hashsize > (sizeof buf - (size_t)(q-buf)))
+                       int hashsize = EVP_MD_size(md);
+                       if (hashsize < 0 || hashsize > (int)(sizeof buf - (size_t)(q-buf)))
                                {
                                /* internal error: 'buf' is too small for this cipersuite! */
                                err = 1;
@@ -808,7 +815,7 @@ int tls1_final_finish_mac(SSL *s,
                                {
                                EVP_MD_CTX_copy_ex(&ctx,s->s3->handshake_dgst[idx]);
                                EVP_DigestFinal_ex(&ctx,q,&i);
-                               if (i != hashsize) /* can't really happen */
+                               if (i != (unsigned int)hashsize) /* can't really happen */
                                        err = 1;
                                q+=i;
                                }
@@ -837,6 +844,7 @@ int tls1_mac(SSL *ssl, unsigned char *md, int send)
        EVP_MD_CTX hmac, *mac_ctx;
        unsigned char buf[5]; 
        int stream_mac = (send?(ssl->mac_flags & SSL_MAC_FLAG_WRITE_MAC_STREAM):(ssl->mac_flags&SSL_MAC_FLAG_READ_MAC_STREAM));
+       int t;
 
        if (send)
                {
@@ -853,7 +861,9 @@ int tls1_mac(SSL *ssl, unsigned char *md, int send)
                hash=ssl->read_hash;
                }
 
-       md_size=EVP_MD_CTX_size(hash);
+       t=EVP_MD_CTX_size(hash);
+       OPENSSL_assert(t >= 0);
+       md_size=t;
 
        buf[0]=rec->type;
        buf[1]=(unsigned char)(ssl->version>>8);
@@ -872,7 +882,7 @@ int tls1_mac(SSL *ssl, unsigned char *md, int send)
                        mac_ctx = &hmac;
                }
 
-       if (ssl->version == DTLS1_VERSION)
+       if (ssl->version == DTLS1_VERSION || ssl->version == DTLS1_BAD_VER)
                {
                unsigned char dtlsseq[8],*p=dtlsseq;
 
@@ -886,7 +896,9 @@ int tls1_mac(SSL *ssl, unsigned char *md, int send)
 
        EVP_DigestSignUpdate(mac_ctx,buf,5);
        EVP_DigestSignUpdate(mac_ctx,rec->input,rec->length);
-       EVP_DigestSignFinal(mac_ctx,md,&md_size);
+       t=EVP_DigestSignFinal(mac_ctx,md,&md_size);
+       OPENSSL_assert(t > 0);
+               
        if (!stream_mac) EVP_MD_CTX_cleanup(&hmac);
 #ifdef TLS_DEBUG
 printf("sec=");
@@ -899,7 +911,7 @@ printf("rec=");
 {unsigned int z; for (z=0; z<rec->length; z++) printf("%02X ",buf[z]); printf("\n"); }
 #endif
 
-       if (ssl->version != DTLS1_VERSION)
+       if (ssl->version != DTLS1_VERSION && ssl->version != DTLS1_BAD_VER)
                {
                for (i=7; i>=0; i--)
                        {
@@ -922,7 +934,7 @@ int tls1_generate_master_secret(SSL *s, unsigned char *out, unsigned char *p,
        int col = 0, sol = 0;
 
 #ifdef KSSL_DEBUG
-       printf ("tls1_generate_master_secret(%p,%p, %p, %d)\n", (void *)s,out, p,len);
+       printf ("tls1_generate_master_secret(%p,%p, %p, %d)\n", s,out, p,len);
 #endif /* KSSL_DEBUG */
 
 #ifdef TLSEXT_TYPE_opaque_prf_input