Fix error codes for memory-saving patch.
[openssl.git] / ssl / s3_both.c
index 8e840ec7c97c0c37202ee2b257b0f776661ba9e3..06e761b4b276294a5b8a5b885b08081f99097854 100644 (file)
@@ -56,7 +56,7 @@
  * [including the GNU Public Licence.]
  */
 /* ====================================================================
- * Copyright (c) 1998-2000 The OpenSSL Project.  All rights reserved.
+ * Copyright (c) 1998-2002 The OpenSSL Project.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * Hudson (tjh@cryptsoft.com).
  *
  */
+/* ====================================================================
+ * Copyright 2002 Sun Microsystems, Inc. ALL RIGHTS RESERVED.
+ * ECC cipher suite support in OpenSSL originally developed by 
+ * SUN MICROSYSTEMS, INC., and contributed to the OpenSSL project.
+ */
 
 #include <limits.h>
 #include <string.h>
 #include <stdio.h>
+#include "ssl_locl.h"
 #include <openssl/buffer.h>
 #include <openssl/rand.h>
 #include <openssl/objects.h>
 #include <openssl/evp.h>
 #include <openssl/x509.h>
-#include "ssl_locl.h"
 
-/* send s->init_buf in records of type 'type' */
+/* send s->init_buf in records of type 'type' (SSL3_RT_HANDSHAKE or SSL3_RT_CHANGE_CIPHER_SPEC) */
 int ssl3_do_write(SSL *s, int type)
        {
        int ret;
@@ -133,7 +138,11 @@ int ssl3_do_write(SSL *s, int type)
                ssl3_finish_mac(s,(unsigned char *)&s->init_buf->data[s->init_off],ret);
        
        if (ret == s->init_num)
+               {
+               if (s->msg_callback)
+                       s->msg_callback(1, s->version, type, s->init_buf->data, (size_t)(s->init_off + s->init_num), s, s->msg_callback_arg);
                return(1);
+               }
        s->init_off+=ret;
        s->init_num-=ret;
        return(0);
@@ -151,8 +160,6 @@ int ssl3_send_finished(SSL *s, int a, int b, const char *sender, int slen)
                p= &(d[4]);
 
                i=s->method->ssl3_enc->final_finish_mac(s,
-                       &(s->s3->finish_dgst1),
-                       &(s->s3->finish_dgst2),
                        sender,slen,s->s3->tmp.finish_md);
                s->s3->tmp.finish_md_len = i;
                memcpy(p, s->s3->tmp.finish_md, i);
@@ -188,7 +195,7 @@ int ssl3_get_finished(SSL *s, int a, int b)
         * change cipher spec message and is in s->s3->tmp.peer_finish_md
         */ 
 
-       n=ssl3_get_message(s,
+       n=s->method->ssl_get_message(s,
                a,
                b,
                SSL3_MT_FINISHED,
@@ -264,16 +271,23 @@ unsigned long ssl3_output_cert_chain(SSL *s, X509 *x)
        X509_STORE_CTX xs_ctx;
        X509_OBJECT obj;
 
+       int no_chain;
+
+       if ((s->mode & SSL_MODE_NO_AUTO_CHAIN) || s->ctx->extra_certs)
+               no_chain = 1;
+       else
+               no_chain = 0;
+
        /* TLSv1 sends a chain with nothing in it, instead of an alert */
        buf=s->init_buf;
-       if (!BUF_MEM_grow(buf,(int)(10)))
+       if (!BUF_MEM_grow_clean(buf,10))
                {
                SSLerr(SSL_F_SSL3_OUTPUT_CERT_CHAIN,ERR_R_BUF_LIB);
                return(0);
                }
        if (x != NULL)
                {
-               if(!X509_STORE_CTX_init(&xs_ctx,s->ctx->cert_store,NULL,NULL))
+               if(!no_chain && !X509_STORE_CTX_init(&xs_ctx,s->ctx->cert_store,NULL,NULL))
                        {
                        SSLerr(SSL_F_SSL3_OUTPUT_CERT_CHAIN,ERR_R_X509_LIB);
                        return(0);
@@ -282,7 +296,7 @@ unsigned long ssl3_output_cert_chain(SSL *s, X509 *x)
                for (;;)
                        {
                        n=i2d_X509(x,NULL);
-                       if (!BUF_MEM_grow(buf,(int)(n+l+3)))
+                       if (!BUF_MEM_grow_clean(buf,(int)(n+l+3)))
                                {
                                SSLerr(SSL_F_SSL3_OUTPUT_CERT_CHAIN,ERR_R_BUF_LIB);
                                return(0);
@@ -291,6 +305,10 @@ unsigned long ssl3_output_cert_chain(SSL *s, X509 *x)
                        l2n3(n,p);
                        i2d_X509(x,&p);
                        l+=n+3;
+
+                       if (no_chain)
+                               break;
+
                        if (X509_NAME_cmp(X509_get_subject_name(x),
                                X509_get_issuer_name(x)) == 0) break;
 
@@ -302,8 +320,8 @@ unsigned long ssl3_output_cert_chain(SSL *s, X509 *x)
                         * ref count */
                        X509_free(x);
                        }
-
-               X509_STORE_CTX_cleanup(&xs_ctx);
+               if (!no_chain)
+                       X509_STORE_CTX_cleanup(&xs_ctx);
                }
 
        /* Thawte special :-) */
@@ -312,7 +330,7 @@ unsigned long ssl3_output_cert_chain(SSL *s, X509 *x)
                {
                x=sk_X509_value(s->ctx->extra_certs,i);
                n=i2d_X509(x,NULL);
-               if (!BUF_MEM_grow(buf,(int)(n+l+3)))
+               if (!BUF_MEM_grow_clean(buf,(int)(n+l+3)))
                        {
                        SSLerr(SSL_F_SSL3_OUTPUT_CERT_CHAIN,ERR_R_BUF_LIB);
                        return(0);
@@ -371,8 +389,8 @@ long ssl3_get_message(SSL *s, int st1, int stn, int mt, long max, int *ok)
                        {
                        while (s->init_num < 4)
                                {
-                               i=ssl3_read_bytes(s,SSL3_RT_HANDSHAKE,&p[s->init_num],
-                                       4 - s->init_num, 0);
+                               i=s->method->ssl_read_bytes(s,SSL3_RT_HANDSHAKE,
+                                       &p[s->init_num],4 - s->init_num, 0);
                                if (i <= 0)
                                        {
                                        s->rwstate=SSL_READING;
@@ -393,8 +411,10 @@ long ssl3_get_message(SSL *s, int st1, int stn, int mt, long max, int *ok)
                                                {
                                                s->init_num = 0;
                                                skip_message = 1;
+
+                                               if (s->msg_callback)
+                                                       s->msg_callback(0, s->version, SSL3_RT_HANDSHAKE, p, 4, s, s->msg_callback_arg);
                                                }
-                       
                        }
                while (skip_message);
 
@@ -433,7 +453,7 @@ long ssl3_get_message(SSL *s, int st1, int stn, int mt, long max, int *ok)
                        SSLerr(SSL_F_SSL3_GET_MESSAGE,SSL_R_EXCESSIVE_MESSAGE_SIZE);
                        goto f_err;
                        }
-               if (l && !BUF_MEM_grow(s->init_buf,(int)l+4))
+               if (l && !BUF_MEM_grow_clean(s->init_buf,(int)l+4))
                        {
                        SSLerr(SSL_F_SSL3_GET_MESSAGE,ERR_R_BUF_LIB);
                        goto err;
@@ -450,7 +470,7 @@ long ssl3_get_message(SSL *s, int st1, int stn, int mt, long max, int *ok)
        n = s->s3->tmp.message_size - s->init_num;
        while (n > 0)
                {
-               i=ssl3_read_bytes(s,SSL3_RT_HANDSHAKE,&p[s->init_num],n,0);
+               i=s->method->ssl_read_bytes(s,SSL3_RT_HANDSHAKE,&p[s->init_num],n,0);
                if (i <= 0)
                        {
                        s->rwstate=SSL_READING;
@@ -461,6 +481,8 @@ long ssl3_get_message(SSL *s, int st1, int stn, int mt, long max, int *ok)
                n -= i;
                }
        ssl3_finish_mac(s, (unsigned char *)s->init_buf->data, s->init_num + 4);
+       if (s->msg_callback)
+               s->msg_callback(0, s->version, SSL3_RT_HANDSHAKE, s->init_buf->data, (size_t)s->init_num + 4, s, s->msg_callback_arg);
        *ok=1;
        return s->init_num;
 f_err:
@@ -473,7 +495,7 @@ err:
 int ssl_cert_type(X509 *x, EVP_PKEY *pkey)
        {
        EVP_PKEY *pk;
-       int ret= -1,i,j;
+       int ret= -1,i;
 
        if (pkey == NULL)
                pk=X509_get_pubkey(x);
@@ -485,36 +507,25 @@ int ssl_cert_type(X509 *x, EVP_PKEY *pkey)
        if (i == EVP_PKEY_RSA)
                {
                ret=SSL_PKEY_RSA_ENC;
-               if (x != NULL)
-                       {
-                       j=X509_get_ext_count(x);
-                       /* check to see if this is a signing only certificate */
-                       /* EAY EAY EAY EAY */
-                       }
                }
        else if (i == EVP_PKEY_DSA)
                {
                ret=SSL_PKEY_DSA_SIGN;
                }
-       else if (i == EVP_PKEY_DH)
+#ifndef OPENSSL_NO_EC
+       else if (i == EVP_PKEY_EC)
                {
-               /* if we just have a key, we needs to be guess */
-
-               if (x == NULL)
-                       ret=SSL_PKEY_DH_DSA;
-               else
-                       {
-                       j=X509_get_signature_type(x);
-                       if (j == EVP_PKEY_RSA)
-                               ret=SSL_PKEY_DH_RSA;
-                       else if (j== EVP_PKEY_DSA)
-                               ret=SSL_PKEY_DH_DSA;
-                       else ret= -1;
-                       }
+               ret = SSL_PKEY_ECC;
+               }       
+#endif
+       else if (i == NID_id_GostR3410_94 || i == NID_id_GostR3410_94_cc) 
+               {
+               ret = SSL_PKEY_GOST94;
+               }
+       else if (i == NID_id_GostR3410_2001 || i == NID_id_GostR3410_2001_cc) 
+               {
+               ret = SSL_PKEY_GOST01;
                }
-       else
-               ret= -1;
-
 err:
        if(!pkey) EVP_PKEY_free(pk);
        return(ret);
@@ -540,6 +551,8 @@ int ssl_verify_alarm_type(long type)
        case X509_V_ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD:
        case X509_V_ERR_CERT_NOT_YET_VALID:
        case X509_V_ERR_CRL_NOT_YET_VALID:
+       case X509_V_ERR_CERT_UNTRUSTED:
+       case X509_V_ERR_CERT_REJECTED:
                al=SSL_AD_BAD_CERTIFICATE;
                break;
        case X509_V_ERR_CERT_SIGNATURE_FAILURE:
@@ -561,11 +574,16 @@ int ssl_verify_alarm_type(long type)
        case X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY:
        case X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE:
        case X509_V_ERR_CERT_CHAIN_TOO_LONG:
+       case X509_V_ERR_PATH_LENGTH_EXCEEDED:
+       case X509_V_ERR_INVALID_CA:
                al=SSL_AD_UNKNOWN_CA;
                break;
        case X509_V_ERR_APPLICATION_VERIFICATION:
                al=SSL_AD_HANDSHAKE_FAILURE;
                break;
+       case X509_V_ERR_INVALID_PURPOSE:
+               al=SSL_AD_UNSUPPORTED_CERTIFICATE;
+               break;
        default:
                al=SSL_AD_CERTIFICATE_UNKNOWN;
                break;
@@ -573,33 +591,179 @@ int ssl_verify_alarm_type(long type)
        return(al);
        }
 
-int ssl3_setup_buffers(SSL *s)
+#ifndef OPENSSL_NO_BUF_FREELISTS
+/* On some platforms, malloc() performance is bad enough that you can't just
+ * free() and malloc() buffers all the time, so we need to use freelists from
+ * unused buffers.  Currently, each freelist holds memory chunks of only a
+ * given size (list->chunklen); other sized chunks are freed and malloced.
+ * This doesn't help much if you're using many different SSL option settings
+ * with a given context.  (The options affecting buffer size are
+ * max_send_fragment, read buffer vs write buffer,
+ * SSL_OP_MICROSOFT_BIG_WRITE_BUFFER, SSL_OP_NO_COMPRESSION, and
+ * SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS.)  Using a separate freelist for every
+ * possible size is not an option, since max_send_fragment can take on many
+ * different values.
+ *
+ * If you are on a platform with a slow malloc(), and you're using SSL
+ * connections with many different settings for these options, and you need to
+ * use the SSL_MOD_RELEASE_BUFFERS feature, you have a few options:
+ *    - Link against a faster malloc implementation.
+ *    - Use a separate SSL_CTX for each option set.
+ *    - Improve this code.
+ */
+static void *
+freelist_extract(SSL_CTX *ctx, int for_read, int sz)
+       {
+       SSL3_BUF_FREELIST *list;
+       SSL3_BUF_FREELIST_ENTRY *ent = NULL;
+       void *result = NULL;
+
+       CRYPTO_w_lock(CRYPTO_LOCK_SSL_CTX);
+       list = for_read ? ctx->rbuf_freelist : ctx->wbuf_freelist;
+       if (list != NULL && sz == list->chunklen)
+               ent = list->head;
+       if (ent != NULL)
+               {
+               list->head = ent->next;
+               result = ent;
+               if (--list->len == 0)
+                       list->chunklen = 0;
+               }
+       CRYPTO_w_unlock(CRYPTO_LOCK_SSL_CTX);
+       if (!result)
+               result = OPENSSL_malloc(sz);
+       return result;
+}
+
+static void
+freelist_insert(SSL_CTX *ctx, int for_read, size_t sz, void *mem)
+       {
+       SSL3_BUF_FREELIST *list;
+       SSL3_BUF_FREELIST_ENTRY *ent;
+
+       CRYPTO_w_lock(CRYPTO_LOCK_SSL_CTX);
+       list = for_read ? ctx->rbuf_freelist : ctx->wbuf_freelist;
+       if (list != NULL &&
+           (sz == list->chunklen || list->chunklen == 0) &&
+           list->len < ctx->freelist_max_len &&
+           sz >= sizeof(*ent))
+               {
+               list->chunklen = sz;
+               ent = mem;
+               ent->next = list->head;
+               list->head = ent;
+               ++list->len;
+               mem = NULL;
+               }
+
+       CRYPTO_w_unlock(CRYPTO_LOCK_SSL_CTX);
+       if (mem)
+               OPENSSL_free(mem);
+       }
+#else
+#define freelist_extract(c,fr,sz) OPENSSL_malloc(sz)
+#define freelist_insert(c,fr,sz,m) OPENSSL_free(m)
+#endif
+
+int ssl3_setup_read_buffer(SSL *s)
        {
        unsigned char *p;
-       unsigned int extra;
+       size_t len,align=0;
+
+#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD!=0
+       align = (-SSL3_RT_HEADER_LENGTH)&(SSL3_ALIGN_PAYLOAD-1);
+#endif
 
        if (s->s3->rbuf.buf == NULL)
                {
+               len = SSL3_RT_MAX_PLAIN_LENGTH
+                       + SSL3_RT_MAX_ENCRYPTED_OVERHEAD
+                       + SSL3_RT_HEADER_LENGTH + align;
                if (s->options & SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER)
-                       extra=SSL3_RT_MAX_EXTRA;
-               else
-                       extra=0;
-               if ((p=OPENSSL_malloc(SSL3_RT_MAX_PACKET_SIZE+extra))
-                       == NULL)
+                       {
+                       s->s3->init_extra = 1;
+                       len += SSL3_RT_MAX_EXTRA;
+                       }
+#ifndef OPENSSL_NO_COMP
+               if (!(s->options & SSL_OP_NO_COMPRESSION))
+                       len += SSL3_RT_MAX_COMPRESSED_OVERHEAD;
+#endif
+               if ((p=freelist_extract(s->ctx, 1, len)) == NULL)
                        goto err;
-               s->s3->rbuf.buf=p;
+               s->s3->rbuf.buf = p;
+               s->s3->rbuf.len = len;
                }
 
+       s->packet= &(s->s3->rbuf.buf[0]);
+       return 1;
+
+err:
+       SSLerr(SSL_F_SSL3_SETUP_READ_BUFFER,ERR_R_MALLOC_FAILURE);
+       return 0;
+       }
+
+int ssl3_setup_write_buffer(SSL *s)
+       {
+       unsigned char *p;
+       size_t len,align=0;
+
+#if defined(SSL3_ALIGN_PAYLOAD) && SSL3_ALIGN_PAYLOAD!=0
+       align = (-SSL3_RT_HEADER_LENGTH)&(SSL3_ALIGN_PAYLOAD-1);
+#endif
+
        if (s->s3->wbuf.buf == NULL)
                {
-               if ((p=OPENSSL_malloc(SSL3_RT_MAX_PACKET_SIZE))
-                       == NULL)
+               len = s->max_send_fragment
+                       + SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD
+                       + SSL3_RT_HEADER_LENGTH + align;
+#ifndef OPENSSL_NO_COMP
+               if (!(s->options & SSL_OP_NO_COMPRESSION))
+                       len += SSL3_RT_MAX_COMPRESSED_OVERHEAD;
+#endif
+               if (!(s->options & SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS))
+                       len += SSL3_RT_HEADER_LENGTH + align
+                               + SSL3_RT_SEND_MAX_ENCRYPTED_OVERHEAD;
+
+               if ((p=freelist_extract(s->ctx, 0, len)) == NULL)
                        goto err;
-               s->s3->wbuf.buf=p;
+               s->s3->wbuf.buf = p;
+               s->s3->wbuf.len = len;
                }
-       s->packet= &(s->s3->rbuf.buf[0]);
-       return(1);
+
+       return 1;
+
 err:
-       SSLerr(SSL_F_SSL3_SETUP_BUFFERS,ERR_R_MALLOC_FAILURE);
-       return(0);
+       SSLerr(SSL_F_SSL3_SETUP_WRITE_BUFFER,ERR_R_MALLOC_FAILURE);
+       return 0;
+       }
+
+
+int ssl3_setup_buffers(SSL *s)
+       {
+       if (!ssl3_setup_read_buffer(s))
+               return 0;
+       if (!ssl3_setup_write_buffer(s))
+               return 0;
+       return 1;
        }
+
+int ssl3_release_write_buffer(SSL *s)
+       {
+       if (s->s3->wbuf.buf != NULL)
+               {
+               freelist_insert(s->ctx, 0, s->s3->wbuf.len, s->s3->wbuf.buf);
+               s->s3->wbuf.buf = NULL;
+               }
+       return 1;
+       }
+
+int ssl3_release_read_buffer(SSL *s)
+       {
+       if (s->s3->rbuf.buf != NULL)
+               {
+               freelist_insert(s->ctx, 1, s->s3->rbuf.len, s->s3->rbuf.buf);
+               s->s3->rbuf.buf = NULL;
+               }
+       return 1;
+       }
+