Allow code which calls RSA temp key callback to cope
[openssl.git] / ssl / s3_enc.c
index 116b09615524fc3b8db40347a0d788a0cc4a7015..df4acab3d02817bdf1d862628f37ab88050b596c 100644 (file)
@@ -57,7 +57,9 @@
  */
 
 #include <stdio.h>
-#include "evp.h"
+#include <openssl/md5.h>
+#include <openssl/sha.h>
+#include <openssl/evp.h>
 #include "ssl_locl.h"
 
 static unsigned char ssl3_pad_1[48]={
@@ -76,17 +78,10 @@ static unsigned char ssl3_pad_2[48]={
        0x5c,0x5c,0x5c,0x5c,0x5c,0x5c,0x5c,0x5c,
        0x5c,0x5c,0x5c,0x5c,0x5c,0x5c,0x5c,0x5c };
 
-#ifndef NO_PROTO
 static int ssl3_handshake_mac(SSL *s, EVP_MD_CTX *in_ctx,
-       unsigned char *sender, int len, unsigned char *p);
-#else
-static int ssl3_handshake_mac();
-#endif
+       const char *sender, int len, unsigned char *p);
 
-static void ssl3_generate_key_block(s,km,num)
-SSL *s;
-unsigned char *km;
-int num;
+static void ssl3_generate_key_block(SSL *s, unsigned char *km, int num)
        {
        MD5_CTX m5;
        SHA_CTX s1;
@@ -94,6 +89,9 @@ int num;
        unsigned char c='A';
        int i,j,k;
 
+#ifdef CHARSET_EBCDIC
+       c = os_toascii[c]; /*'A' in ASCII */
+#endif
        k=0;
        for (i=0; i<num; i+=MD5_DIGEST_LENGTH)
                {
@@ -126,25 +124,26 @@ int num;
        memset(smd,0,SHA_DIGEST_LENGTH);
        }
 
-int ssl3_change_cipher_state(s,which)
-SSL *s;
-int which;
+int ssl3_change_cipher_state(SSL *s, int which)
        {
        unsigned char *p,*key_block,*mac_secret;
        unsigned char exp_key[EVP_MAX_KEY_LENGTH];
        unsigned char exp_iv[EVP_MAX_KEY_LENGTH];
        unsigned char *ms,*key,*iv,*er1,*er2;
        EVP_CIPHER_CTX *dd;
-       EVP_CIPHER *c;
+       const EVP_CIPHER *c;
        COMP_METHOD *comp;
-       EVP_MD *m;
+       const EVP_MD *m;
        MD5_CTX md;
-       int exp,n,i,j,k;
+       int exp,n,i,j,k,cl;
 
-       exp=(s->s3->tmp.new_cipher->algorithms & SSL_EXPORT)?1:0;
+       exp=SSL_C_IS_EXPORT(s->s3->tmp.new_cipher);
        c=s->s3->tmp.new_sym_enc;
        m=s->s3->tmp.new_hash;
-       comp=s->s3->tmp.new_compression;
+       if (s->s3->tmp.new_compression == NULL)
+               comp=NULL;
+       else
+               comp=s->s3->tmp.new_compression->method;
        key_block=s->s3->tmp.key_block;
 
        if (which & SSL3_CC_READ)
@@ -169,8 +168,9 @@ int which;
                                SSLerr(SSL_F_SSL3_CHANGE_CIPHER_STATE,SSL_R_COMPRESSION_LIBRARY_ERROR);
                                goto err2;
                                }
-                       s->s3->rrec.comp=(unsigned char *)
-                               Malloc(SSL3_RT_MAX_PLAIN_LENGTH);
+                       if (s->s3->rrec.comp == NULL)
+                               s->s3->rrec.comp=(unsigned char *)
+                                       Malloc(SSL3_RT_MAX_PLAIN_LENGTH);
                        if (s->s3->rrec.comp == NULL)
                                goto err;
                        }
@@ -208,7 +208,10 @@ int which;
 
        p=s->s3->tmp.key_block;
        i=EVP_MD_size(m);
-       j=(exp)?5:EVP_CIPHER_key_length(c);
+       cl=EVP_CIPHER_key_length(c);
+       j=exp ? (cl < SSL_C_EXPORT_KEYLENGTH(s->s3->tmp.new_cipher) ?
+                cl : SSL_C_EXPORT_KEYLENGTH(s->s3->tmp.new_cipher)) : cl;
+       /* Was j=(exp)?5:EVP_CIPHER_key_length(c); */
        k=EVP_CIPHER_iv_length(c);
        if (    (which == SSL3_CHANGE_CIPHER_CLIENT_WRITE) ||
                (which == SSL3_CHANGE_CIPHER_SERVER_READ))
@@ -271,18 +274,18 @@ err2:
        return(0);
        }
 
-int ssl3_setup_key_block(s)
-SSL *s;
+int ssl3_setup_key_block(SSL *s)
        {
        unsigned char *p;
-       EVP_CIPHER *c;
-       EVP_MD *hash;
-       int num,exp;
+       const EVP_CIPHER *c;
+       const EVP_MD *hash;
+       int num;
+       SSL_COMP *comp;
 
        if (s->s3->tmp.key_block_length != 0)
                return(1);
 
-       if (!ssl_cipher_get_evp(s->session->cipher,&c,&hash))
+       if (!ssl_cipher_get_evp(s->session,&c,&hash,&comp))
                {
                SSLerr(SSL_F_SSL3_SETUP_KEY_BLOCK,SSL_R_CIPHER_OR_HASH_UNAVAILABLE);
                return(0);
@@ -290,20 +293,14 @@ SSL *s;
 
        s->s3->tmp.new_sym_enc=c;
        s->s3->tmp.new_hash=hash;
-#ifdef ZLIB
-       s->s3->tmp.new_compression=COMP_zlib();
-#endif
-/*     s->s3->tmp.new_compression=COMP_rle(); */
-/*     s->session->compress_meth= xxxxx */
-
-       exp=(s->session->cipher->algorithms & SSL_EXPORT)?1:0;
+       s->s3->tmp.new_compression=comp;
 
        num=EVP_CIPHER_key_length(c)+EVP_MD_size(hash)+EVP_CIPHER_iv_length(c);
        num*=2;
 
        ssl3_cleanup_key_block(s);
 
-       if ((p=(unsigned char *)Malloc(num)) == NULL)
+       if ((p=Malloc(num)) == NULL)
                goto err;
 
        s->s3->tmp.key_block_length=num;
@@ -317,8 +314,7 @@ err:
        return(0);
        }
 
-void ssl3_cleanup_key_block(s)
-SSL *s;
+void ssl3_cleanup_key_block(SSL *s)
        {
        if (s->s3->tmp.key_block != NULL)
                {
@@ -330,15 +326,13 @@ SSL *s;
        s->s3->tmp.key_block_length=0;
        }
 
-int ssl3_enc(s,send)
-SSL *s;
-int send;
+int ssl3_enc(SSL *s, int send)
        {
        SSL3_RECORD *rec;
        EVP_CIPHER_CTX *ds;
        unsigned long l;
        int bs,i;
-       EVP_CIPHER *enc;
+       const EVP_CIPHER *enc;
 
        if (send)
                {
@@ -400,36 +394,25 @@ int send;
        return(1);
        }
 
-void ssl3_init_finished_mac(s)
-SSL *s;
+void ssl3_init_finished_mac(SSL *s)
        {
        EVP_DigestInit(&(s->s3->finish_dgst1),s->ctx->md5);
        EVP_DigestInit(&(s->s3->finish_dgst2),s->ctx->sha1);
        }
 
-void ssl3_finish_mac(s,buf,len)
-SSL *s;
-unsigned char *buf;
-int len;
+void ssl3_finish_mac(SSL *s, const unsigned char *buf, int len)
        {
        EVP_DigestUpdate(&(s->s3->finish_dgst1),buf,len);
        EVP_DigestUpdate(&(s->s3->finish_dgst2),buf,len);
        }
 
-int ssl3_cert_verify_mac(s,ctx,p)
-SSL *s;
-EVP_MD_CTX *ctx;
-unsigned char *p;
+int ssl3_cert_verify_mac(SSL *s, EVP_MD_CTX *ctx, unsigned char *p)
        {
        return(ssl3_handshake_mac(s,ctx,NULL,0,p));
        }
 
-int ssl3_final_finish_mac(s,ctx1,ctx2,sender,len,p)
-SSL *s;
-EVP_MD_CTX *ctx1,*ctx2;
-unsigned char *sender;
-int len;
-unsigned char *p;
+int ssl3_final_finish_mac(SSL *s, EVP_MD_CTX *ctx1, EVP_MD_CTX *ctx2,
+            const char *sender, int len, unsigned char *p)
        {
        int ret;
 
@@ -439,12 +422,8 @@ unsigned char *p;
        return(ret);
        }
 
-static int ssl3_handshake_mac(s,in_ctx,sender,len,p)
-SSL *s;
-EVP_MD_CTX *in_ctx;
-unsigned char *sender;
-int len;
-unsigned char *p;
+static int ssl3_handshake_mac(SSL *s, EVP_MD_CTX *in_ctx,
+            const char *sender, int len, unsigned char *p)
        {
        unsigned int ret;
        int npad,n;
@@ -452,7 +431,7 @@ unsigned char *p;
        unsigned char md_buf[EVP_MAX_MD_SIZE];
        EVP_MD_CTX ctx;
 
-       memcpy(&ctx,in_ctx,sizeof(EVP_MD_CTX));
+       EVP_MD_CTX_copy(&ctx,in_ctx);
 
        n=EVP_MD_CTX_size(&ctx);
        npad=(48/n)*n;
@@ -464,7 +443,7 @@ unsigned char *p;
        EVP_DigestUpdate(&ctx,ssl3_pad_1,npad);
        EVP_DigestFinal(&ctx,md_buf,&i);
 
-       EVP_DigestInit(&ctx,EVP_MD_CTX_type(&ctx));
+       EVP_DigestInit(&ctx,EVP_MD_CTX_md(&ctx));
        EVP_DigestUpdate(&ctx,s->session->master_key,
                s->session->master_key_length);
        EVP_DigestUpdate(&ctx,ssl3_pad_2,npad);
@@ -476,15 +455,12 @@ unsigned char *p;
        return((int)ret);
        }
 
-int ssl3_mac(ssl,md,send)
-SSL *ssl;
-unsigned char *md;
-int send;
+int ssl3_mac(SSL *ssl, unsigned char *md, int send)
        {
        SSL3_RECORD *rec;
        unsigned char *mac_sec,*seq;
        EVP_MD_CTX md_ctx;
-       EVP_MD *hash;
+       const EVP_MD *hash;
        unsigned char *p,rec_char;
        unsigned int md_size;
        int npad,i;
@@ -533,16 +509,19 @@ int send;
        return(md_size);
        }
 
-int ssl3_generate_master_secret(s,out,p,len)
-SSL *s;
-unsigned char *out;
-unsigned char *p;
-int len;
+int ssl3_generate_master_secret(SSL *s, unsigned char *out, unsigned char *p,
+            int len)
        {
-       static unsigned char *salt[3]={
-               (unsigned char *)"A",
-               (unsigned char *)"BB",
-               (unsigned char *)"CCC",
+       static const unsigned char *salt[3]={
+#ifndef CHARSET_EBCDIC
+               (const unsigned char *)"A",
+               (const unsigned char *)"BB",
+               (const unsigned char *)"CCC",
+#else
+               (const unsigned char *)"\x41",
+               (const unsigned char *)"\x42\x42",
+               (const unsigned char *)"\x43\x43\x43",
+#endif
                };
        unsigned char buf[EVP_MAX_MD_SIZE];
        EVP_MD_CTX ctx;
@@ -552,7 +531,7 @@ int len;
        for (i=0; i<3; i++)
                {
                EVP_DigestInit(&ctx,s->ctx->sha1);
-               EVP_DigestUpdate(&ctx,salt[i],strlen((char *)salt[i]));
+               EVP_DigestUpdate(&ctx,salt[i],strlen((const char *)salt[i]));
                EVP_DigestUpdate(&ctx,p,len);
                EVP_DigestUpdate(&ctx,&(s->s3->client_random[0]),
                        SSL3_RANDOM_SIZE);
@@ -570,8 +549,7 @@ int len;
        return(ret);
        }
 
-int ssl3_alert_code(code)
-int code;
+int ssl3_alert_code(int code)
        {
        switch (code)
                {
@@ -593,11 +571,11 @@ int code;
        case SSL_AD_ACCESS_DENIED:      return(SSL3_AD_HANDSHAKE_FAILURE);
        case SSL_AD_DECODE_ERROR:       return(SSL3_AD_HANDSHAKE_FAILURE);
        case SSL_AD_DECRYPT_ERROR:      return(SSL3_AD_HANDSHAKE_FAILURE);
-       case SSL_AD_EXPORT_RESTRICION:  return(SSL3_AD_HANDSHAKE_FAILURE);
+       case SSL_AD_EXPORT_RESTRICTION: return(SSL3_AD_HANDSHAKE_FAILURE);
        case SSL_AD_PROTOCOL_VERSION:   return(SSL3_AD_HANDSHAKE_FAILURE);
        case SSL_AD_INSUFFICIENT_SECURITY:return(SSL3_AD_HANDSHAKE_FAILURE);
        case SSL_AD_INTERNAL_ERROR:     return(SSL3_AD_HANDSHAKE_FAILURE);
-       case SSL_AD_USER_CANCLED:       return(SSL3_AD_HANDSHAKE_FAILURE);
+       case SSL_AD_USER_CANCELLED:     return(SSL3_AD_HANDSHAKE_FAILURE);
        case SSL_AD_NO_RENEGOTIATION:   return(-1); /* Don't send it :-) */
        default:                        return(-1);
                }