Use C syntax, not FORTRAN or whatever that was :-)
[openssl.git] / ssl / s3_enc.c
index f1cd25e3d35bab4498c2d989736fd66e2227a2b2..012a4b8740772e56241f03013cb97cc0e5a18503 100644 (file)
@@ -1,5 +1,5 @@
 /* ssl/s3_enc.c */
-/* Copyright (C) 1995-1997 Eric Young (eay@cryptsoft.com)
+/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
  * This package is an SSL implementation written
@@ -57,7 +57,9 @@
  */
 
 #include <stdio.h>
-#include "evp.h"
+#include <openssl/md5.h>
+#include <openssl/sha.h>
+#include <openssl/evp.h>
 #include "ssl_locl.h"
 
 static unsigned char ssl3_pad_1[48]={
@@ -76,36 +78,102 @@ static unsigned char ssl3_pad_2[48]={
        0x5c,0x5c,0x5c,0x5c,0x5c,0x5c,0x5c,0x5c,
        0x5c,0x5c,0x5c,0x5c,0x5c,0x5c,0x5c,0x5c };
 
-int ssl3_change_cipher_state(s,which)
-SSL *s;
-int which;
+static int ssl3_handshake_mac(SSL *s, EVP_MD_CTX *in_ctx,
+       const char *sender, int len, unsigned char *p);
+
+static void ssl3_generate_key_block(SSL *s, unsigned char *km, int num)
+       {
+       MD5_CTX m5;
+       SHA_CTX s1;
+       unsigned char buf[8],smd[SHA_DIGEST_LENGTH];
+       unsigned char c='A';
+       int i,j,k;
+
+#ifdef CHARSET_EBCDIC
+       c = os_toascii[c]; /*'A' in ASCII */
+#endif
+       k=0;
+       for (i=0; i<num; i+=MD5_DIGEST_LENGTH)
+               {
+               k++;
+               for (j=0; j<k; j++)
+                       buf[j]=c;
+               c++;
+               SHA1_Init(  &s1);
+               SHA1_Update(&s1,buf,k);
+               SHA1_Update(&s1,s->session->master_key,
+                       s->session->master_key_length);
+               SHA1_Update(&s1,s->s3->server_random,SSL3_RANDOM_SIZE);
+               SHA1_Update(&s1,s->s3->client_random,SSL3_RANDOM_SIZE);
+               SHA1_Final( smd,&s1);
+
+               MD5_Init(  &m5);
+               MD5_Update(&m5,s->session->master_key,
+                       s->session->master_key_length);
+               MD5_Update(&m5,smd,SHA_DIGEST_LENGTH);
+               if ((i+MD5_DIGEST_LENGTH) > num)
+                       {
+                       MD5_Final(smd,&m5);
+                       memcpy(km,smd,(num-i));
+                       }
+               else
+                       MD5_Final(km,&m5);
+
+               km+=MD5_DIGEST_LENGTH;
+               }
+       memset(smd,0,SHA_DIGEST_LENGTH);
+       }
+
+int ssl3_change_cipher_state(SSL *s, int which)
        {
        unsigned char *p,*key_block,*mac_secret;
        unsigned char exp_key[EVP_MAX_KEY_LENGTH];
        unsigned char exp_iv[EVP_MAX_KEY_LENGTH];
        unsigned char *ms,*key,*iv,*er1,*er2;
        EVP_CIPHER_CTX *dd;
-       EVP_CIPHER *c;
-       SSL_COMPRESSION *comp;
-       EVP_MD *m;
+       const EVP_CIPHER *c;
+       COMP_METHOD *comp;
+       const EVP_MD *m;
        MD5_CTX md;
-       int exp,n,i,j,k;
+       int exp,n,i,j,k,cl;
 
-       exp=(s->s3->tmp.new_cipher->algorithms & SSL_EXPORT)?1:0;
+       exp=SSL_C_IS_EXPORT(s->s3->tmp.new_cipher);
        c=s->s3->tmp.new_sym_enc;
        m=s->s3->tmp.new_hash;
-       comp=s->s3->tmp.new_compression;
+       if (s->s3->tmp.new_compression == NULL)
+               comp=NULL;
+       else
+               comp=s->s3->tmp.new_compression->method;
        key_block=s->s3->tmp.key_block;
 
        if (which & SSL3_CC_READ)
                {
                if ((s->enc_read_ctx == NULL) &&
                        ((s->enc_read_ctx=(EVP_CIPHER_CTX *)
-                       Malloc(sizeof(EVP_CIPHER_CTX))) == NULL))
+                       OPENSSL_malloc(sizeof(EVP_CIPHER_CTX))) == NULL))
                        goto err;
                dd= s->enc_read_ctx;
                s->read_hash=m;
-               s->read_compression=comp;
+               /* COMPRESS */
+               if (s->expand != NULL)
+                       {
+                       COMP_CTX_free(s->expand);
+                       s->expand=NULL;
+                       }
+               if (comp != NULL)
+                       {
+                       s->expand=COMP_CTX_new(comp);
+                       if (s->expand == NULL)
+                               {
+                               SSLerr(SSL_F_SSL3_CHANGE_CIPHER_STATE,SSL_R_COMPRESSION_LIBRARY_ERROR);
+                               goto err2;
+                               }
+                       if (s->s3->rrec.comp == NULL)
+                               s->s3->rrec.comp=(unsigned char *)
+                                       OPENSSL_malloc(SSL3_RT_MAX_PLAIN_LENGTH);
+                       if (s->s3->rrec.comp == NULL)
+                               goto err;
+                       }
                memset(&(s->s3->read_sequence[0]),0,8);
                mac_secret= &(s->s3->read_mac_secret[0]);
                }
@@ -113,18 +181,37 @@ int which;
                {
                if ((s->enc_write_ctx == NULL) &&
                        ((s->enc_write_ctx=(EVP_CIPHER_CTX *)
-                       Malloc(sizeof(EVP_CIPHER_CTX))) == NULL))
+                       OPENSSL_malloc(sizeof(EVP_CIPHER_CTX))) == NULL))
                        goto err;
                dd= s->enc_write_ctx;
                s->write_hash=m;
-               s->write_compression=comp;
+               /* COMPRESS */
+               if (s->compress != NULL)
+                       {
+                       COMP_CTX_free(s->compress);
+                       s->compress=NULL;
+                       }
+               if (comp != NULL)
+                       {
+                       s->compress=COMP_CTX_new(comp);
+                       if (s->compress == NULL)
+                               {
+                               SSLerr(SSL_F_SSL3_CHANGE_CIPHER_STATE,SSL_R_COMPRESSION_LIBRARY_ERROR);
+                               goto err2;
+                               }
+                       }
                memset(&(s->s3->write_sequence[0]),0,8);
                mac_secret= &(s->s3->write_mac_secret[0]);
                }
 
+       EVP_CIPHER_CTX_init(dd);
+
        p=s->s3->tmp.key_block;
        i=EVP_MD_size(m);
-       j=(exp)?5:EVP_CIPHER_key_length(c);
+       cl=EVP_CIPHER_key_length(c);
+       j=exp ? (cl < SSL_C_EXPORT_KEYLENGTH(s->s3->tmp.new_cipher) ?
+                cl : SSL_C_EXPORT_KEYLENGTH(s->s3->tmp.new_cipher)) : cl;
+       /* Was j=(exp)?5:EVP_CIPHER_key_length(c); */
        k=EVP_CIPHER_iv_length(c);
        if (    (which == SSL3_CHANGE_CIPHER_CLIENT_WRITE) ||
                (which == SSL3_CHANGE_CIPHER_SERVER_READ))
@@ -164,18 +251,20 @@ int which;
                MD5_Final(&(exp_key[0]),&md);
                key= &(exp_key[0]);
 
-               MD5_Init(&md);
-               MD5_Update(&md,er1,SSL3_RANDOM_SIZE);
-               MD5_Update(&md,er2,SSL3_RANDOM_SIZE);
-               MD5_Final(&(exp_iv[0]),&md);
-               iv= &(exp_iv[0]);
+               if (k > 0)
+                       {
+                       MD5_Init(&md);
+                       MD5_Update(&md,er1,SSL3_RANDOM_SIZE);
+                       MD5_Update(&md,er2,SSL3_RANDOM_SIZE);
+                       MD5_Final(&(exp_iv[0]),&md);
+                       iv= &(exp_iv[0]);
+                       }
                }
 
-       s->session->key_arg_length=k;
-       if (k > 0)
-               memcpy(&(s->session->key_arg[0]),iv,k);
+       s->session->key_arg_length=0;
 
        EVP_CipherInit(dd,c,key,iv,(which & SSL3_CC_WRITE));
+
        memset(&(exp_key[0]),0,sizeof(exp_key));
        memset(&(exp_iv[0]),0,sizeof(exp_iv));
        return(1);
@@ -185,18 +274,18 @@ err2:
        return(0);
        }
 
-int ssl3_setup_key_block(s)
-SSL *s;
+int ssl3_setup_key_block(SSL *s)
        {
        unsigned char *p;
-       EVP_CIPHER *c;
-       EVP_MD *hash;
-       int num,exp;
+       const EVP_CIPHER *c;
+       const EVP_MD *hash;
+       int num;
+       SSL_COMP *comp;
 
        if (s->s3->tmp.key_block_length != 0)
                return(1);
 
-       if (!ssl_cipher_get_evp(s->session->cipher,&c,&hash))
+       if (!ssl_cipher_get_evp(s->session,&c,&hash,&comp))
                {
                SSLerr(SSL_F_SSL3_SETUP_KEY_BLOCK,SSL_R_CIPHER_OR_HASH_UNAVAILABLE);
                return(0);
@@ -204,15 +293,14 @@ SSL *s;
 
        s->s3->tmp.new_sym_enc=c;
        s->s3->tmp.new_hash=hash;
-
-       exp=(s->session->cipher->algorithms & SSL_EXPORT)?1:0;
+       s->s3->tmp.new_compression=comp;
 
        num=EVP_CIPHER_key_length(c)+EVP_MD_size(hash)+EVP_CIPHER_iv_length(c);
        num*=2;
 
        ssl3_cleanup_key_block(s);
 
-       if ((p=(unsigned char *)Malloc(num)) == NULL)
+       if ((p=OPENSSL_malloc(num)) == NULL)
                goto err;
 
        s->s3->tmp.key_block_length=num;
@@ -226,57 +314,47 @@ err:
        return(0);
        }
 
-void ssl3_cleanup_key_block(s)
-SSL *s;
+void ssl3_cleanup_key_block(SSL *s)
        {
        if (s->s3->tmp.key_block != NULL)
                {
                memset(s->s3->tmp.key_block,0,
                        s->s3->tmp.key_block_length);
-               Free(s->s3->tmp.key_block);
+               OPENSSL_free(s->s3->tmp.key_block);
                s->s3->tmp.key_block=NULL;
                }
        s->s3->tmp.key_block_length=0;
        }
 
-int ssl3_enc(s,send)
-SSL *s;
-int send;
+int ssl3_enc(SSL *s, int send)
        {
        SSL3_RECORD *rec;
        EVP_CIPHER_CTX *ds;
        unsigned long l;
        int bs,i;
-       EVP_CIPHER *enc;
-       SSL_COMPRESSION *comp;
+       const EVP_CIPHER *enc;
 
        if (send)
                {
                ds=s->enc_write_ctx;
                rec= &(s->s3->wrec);
                if (s->enc_write_ctx == NULL)
-                       { enc=NULL; comp=NULL; }
+                       enc=NULL;
                else
-                       {
                        enc=EVP_CIPHER_CTX_cipher(s->enc_write_ctx);
-                       comp=s->write_compression;
-                       }
                }
        else
                {
                ds=s->enc_read_ctx;
                rec= &(s->s3->rrec);
                if (s->enc_read_ctx == NULL)
-                       { enc=NULL; comp=NULL; }
+                       enc=NULL;
                else
-                       {
                        enc=EVP_CIPHER_CTX_cipher(s->enc_read_ctx);
-                       comp=s->read_compression;
-                       }
                }
 
        if ((s->session == NULL) || (ds == NULL) ||
-               ((enc == NULL) && (comp == NULL)))
+               (enc == NULL))
                {
                memcpy(rec->data,rec->input,rec->length);
                rec->input=rec->data;
@@ -286,6 +364,8 @@ int send;
                l=rec->length;
                bs=EVP_CIPHER_block_size(ds->cipher);
 
+               /* COMPRESS */
+
                /* This should be using (bs-1) and bs instead of 7 and 8 */
                if ((bs != 1) && send)
                        {
@@ -305,7 +385,7 @@ int send;
                        if (i > bs)
                                {
                                SSLerr(SSL_F_SSL3_ENC,SSL_R_BLOCK_CIPHER_PAD_IS_WRONG);
-                               ssl3_send_alert(s,SSL3_AL_FATAL,SSL3_AD_BAD_RECORD_MAC);
+                               ssl3_send_alert(s,SSL3_AL_FATAL,SSL_AD_DECRYPT_ERROR);
                                return(0);
                                }
                        rec->length-=i;
@@ -314,27 +394,36 @@ int send;
        return(1);
        }
 
-void ssl3_init_finished_mac(s)
-SSL *s;
+void ssl3_init_finished_mac(SSL *s)
        {
-       EVP_DigestInit(&(s->s3->finish_dgst1),EVP_md5());
-       EVP_DigestInit(&(s->s3->finish_dgst2),EVP_sha1());
+       EVP_DigestInit(&(s->s3->finish_dgst1),s->ctx->md5);
+       EVP_DigestInit(&(s->s3->finish_dgst2),s->ctx->sha1);
        }
 
-void ssl3_finish_mac(s,buf,len)
-SSL *s;
-unsigned char *buf;
-int len;
+void ssl3_finish_mac(SSL *s, const unsigned char *buf, int len)
        {
        EVP_DigestUpdate(&(s->s3->finish_dgst1),buf,len);
        EVP_DigestUpdate(&(s->s3->finish_dgst2),buf,len);
        }
 
-int ssl3_final_finish_mac(s,in_ctx,sender,p)
-SSL *s;
-EVP_MD_CTX *in_ctx;
-unsigned char *sender;
-unsigned char *p;
+int ssl3_cert_verify_mac(SSL *s, EVP_MD_CTX *ctx, unsigned char *p)
+       {
+       return(ssl3_handshake_mac(s,ctx,NULL,0,p));
+       }
+
+int ssl3_final_finish_mac(SSL *s, EVP_MD_CTX *ctx1, EVP_MD_CTX *ctx2,
+            const char *sender, int len, unsigned char *p)
+       {
+       int ret;
+
+       ret=ssl3_handshake_mac(s,ctx1,sender,len,p);
+       p+=ret;
+       ret+=ssl3_handshake_mac(s,ctx2,sender,len,p);
+       return(ret);
+       }
+
+static int ssl3_handshake_mac(SSL *s, EVP_MD_CTX *in_ctx,
+            const char *sender, int len, unsigned char *p)
        {
        unsigned int ret;
        int npad,n;
@@ -342,19 +431,19 @@ unsigned char *p;
        unsigned char md_buf[EVP_MAX_MD_SIZE];
        EVP_MD_CTX ctx;
 
-       memcpy(&ctx,in_ctx,sizeof(EVP_MD_CTX));
+       EVP_MD_CTX_copy(&ctx,in_ctx);
 
        n=EVP_MD_CTX_size(&ctx);
        npad=(48/n)*n;
 
        if (sender != NULL)
-               EVP_DigestUpdate(&ctx,sender,4);
+               EVP_DigestUpdate(&ctx,sender,len);
        EVP_DigestUpdate(&ctx,s->session->master_key,
                s->session->master_key_length);
        EVP_DigestUpdate(&ctx,ssl3_pad_1,npad);
        EVP_DigestFinal(&ctx,md_buf,&i);
 
-       EVP_DigestInit(&ctx,EVP_MD_CTX_type(&ctx));
+       EVP_DigestInit(&ctx,EVP_MD_CTX_md(&ctx));
        EVP_DigestUpdate(&ctx,s->session->master_key,
                s->session->master_key_length);
        EVP_DigestUpdate(&ctx,ssl3_pad_2,npad);
@@ -366,15 +455,12 @@ unsigned char *p;
        return((int)ret);
        }
 
-int ssl3_mac(ssl,md,send)
-SSL *ssl;
-unsigned char *md;
-int send;
+int ssl3_mac(SSL *ssl, unsigned char *md, int send)
        {
        SSL3_RECORD *rec;
        unsigned char *mac_sec,*seq;
        EVP_MD_CTX md_ctx;
-       EVP_MD *hash;
+       const EVP_MD *hash;
        unsigned char *p,rec_char;
        unsigned int md_size;
        int npad,i;
@@ -397,19 +483,6 @@ int send;
        md_size=EVP_MD_size(hash);
        npad=(48/md_size)*md_size;
 
-#ifdef MAC_DEBUG
-printf("npad=%d md_size=%d",npad,md_size);
-printf("\nmsec=");
-for (i=0; i<md_size; i++) printf("%02X ",mac_sec[i]);
-printf("\npad1=");
-for (i=0; i<npad; i++) printf("%02X ",ssl3_pad_1[i]);
-printf("\nseq =");
-for (i=0; i<8; i++) printf("%02X ",seq[i]);
-printf("\nreqt=%02X len=%04X\n",rec->type,rec->length);
-for (i=0; i<rec->length; i++) printf("%02X",rec->input[i]);
-printf("\n");
-#endif
-
        /* Chop the digest off the end :-) */
 
        EVP_DigestInit(  &md_ctx,hash);
@@ -433,25 +506,22 @@ printf("\n");
        for (i=7; i>=0; i--)
                if (++seq[i]) break; 
 
-#ifdef MAC_DEBUG
-printf("md=");
-for (i=0; i<md_size; i++) printf("%02X ",md[i]);
-printf("\n");
-#endif
-
        return(md_size);
        }
 
-int ssl3_generate_master_secret(s,out,p,len)
-SSL *s;
-unsigned char *out;
-unsigned char *p;
-int len;
+int ssl3_generate_master_secret(SSL *s, unsigned char *out, unsigned char *p,
+            int len)
        {
-       static unsigned char *salt[3]={
-               (unsigned char *)"A",
-               (unsigned char *)"BB",
-               (unsigned char *)"CCC",
+       static const unsigned char *salt[3]={
+#ifndef CHARSET_EBCDIC
+               (const unsigned char *)"A",
+               (const unsigned char *)"BB",
+               (const unsigned char *)"CCC",
+#else
+               (const unsigned char *)"\x41",
+               (const unsigned char *)"\x42\x42",
+               (const unsigned char *)"\x43\x43\x43",
+#endif
                };
        unsigned char buf[EVP_MAX_MD_SIZE];
        EVP_MD_CTX ctx;
@@ -460,8 +530,8 @@ int len;
 
        for (i=0; i<3; i++)
                {
-               EVP_DigestInit(&ctx,EVP_sha1());
-               EVP_DigestUpdate(&ctx,salt[i],strlen((char *)salt[i]));
+               EVP_DigestInit(&ctx,s->ctx->sha1);
+               EVP_DigestUpdate(&ctx,salt[i],strlen((const char *)salt[i]));
                EVP_DigestUpdate(&ctx,p,len);
                EVP_DigestUpdate(&ctx,&(s->s3->client_random[0]),
                        SSL3_RANDOM_SIZE);
@@ -469,7 +539,7 @@ int len;
                        SSL3_RANDOM_SIZE);
                EVP_DigestFinal(&ctx,buf,&n);
 
-               EVP_DigestInit(&ctx,EVP_md5());
+               EVP_DigestInit(&ctx,s->ctx->md5);
                EVP_DigestUpdate(&ctx,p,len);
                EVP_DigestUpdate(&ctx,buf,n);
                EVP_DigestFinal(&ctx,out,&n);
@@ -479,3 +549,35 @@ int len;
        return(ret);
        }
 
+int ssl3_alert_code(int code)
+       {
+       switch (code)
+               {
+       case SSL_AD_CLOSE_NOTIFY:       return(SSL3_AD_CLOSE_NOTIFY);
+       case SSL_AD_UNEXPECTED_MESSAGE: return(SSL3_AD_UNEXPECTED_MESSAGE);
+       case SSL_AD_BAD_RECORD_MAC:     return(SSL3_AD_BAD_RECORD_MAC);
+       case SSL_AD_DECRYPTION_FAILED:  return(SSL3_AD_BAD_RECORD_MAC);
+       case SSL_AD_RECORD_OVERFLOW:    return(SSL3_AD_BAD_RECORD_MAC);
+       case SSL_AD_DECOMPRESSION_FAILURE:return(SSL3_AD_DECOMPRESSION_FAILURE);
+       case SSL_AD_HANDSHAKE_FAILURE:  return(SSL3_AD_HANDSHAKE_FAILURE);
+       case SSL_AD_NO_CERTIFICATE:     return(SSL3_AD_NO_CERTIFICATE);
+       case SSL_AD_BAD_CERTIFICATE:    return(SSL3_AD_BAD_CERTIFICATE);
+       case SSL_AD_UNSUPPORTED_CERTIFICATE:return(SSL3_AD_UNSUPPORTED_CERTIFICATE);
+       case SSL_AD_CERTIFICATE_REVOKED:return(SSL3_AD_CERTIFICATE_REVOKED);
+       case SSL_AD_CERTIFICATE_EXPIRED:return(SSL3_AD_CERTIFICATE_EXPIRED);
+       case SSL_AD_CERTIFICATE_UNKNOWN:return(SSL3_AD_CERTIFICATE_UNKNOWN);
+       case SSL_AD_ILLEGAL_PARAMETER:  return(SSL3_AD_ILLEGAL_PARAMETER);
+       case SSL_AD_UNKNOWN_CA:         return(SSL3_AD_BAD_CERTIFICATE);
+       case SSL_AD_ACCESS_DENIED:      return(SSL3_AD_HANDSHAKE_FAILURE);
+       case SSL_AD_DECODE_ERROR:       return(SSL3_AD_HANDSHAKE_FAILURE);
+       case SSL_AD_DECRYPT_ERROR:      return(SSL3_AD_HANDSHAKE_FAILURE);
+       case SSL_AD_EXPORT_RESTRICTION: return(SSL3_AD_HANDSHAKE_FAILURE);
+       case SSL_AD_PROTOCOL_VERSION:   return(SSL3_AD_HANDSHAKE_FAILURE);
+       case SSL_AD_INSUFFICIENT_SECURITY:return(SSL3_AD_HANDSHAKE_FAILURE);
+       case SSL_AD_INTERNAL_ERROR:     return(SSL3_AD_HANDSHAKE_FAILURE);
+       case SSL_AD_USER_CANCELLED:     return(SSL3_AD_HANDSHAKE_FAILURE);
+       case SSL_AD_NO_RENEGOTIATION:   return(-1); /* Don't send it :-) */
+       default:                        return(-1);
+               }
+       }
+