Fix DTLS handshake message size checks.

[openssl.git] / ssl / d1_both.c

diff --git a/ssl/d1_both.c b/ssl/d1_both.c
index 7de9ae4b5fd86c9533d54183d6c536f5f1d2bf61..b9e15dfb3866024759b18a96fcde49bcab9ac4e4 100644 (file)
--- a/ssl/d1_both.c
+++ b/ssl/d1_both.c
@@ -587,6 +587,16 @@ dtls1_retrieve_buffered_fragment(SSL *s, long max, int *ok)
                return 0;
        }
 
+/* dtls1_max_handshake_message_len returns the maximum number of bytes
+ * permitted in a DTLS handshake message for |s|. The minimum is 16KB, but may
+ * be greater if the maximum certificate list size requires it. */
+static unsigned long dtls1_max_handshake_message_len(const SSL *s)
+       {
+       unsigned long max_len = DTLS1_HM_HEADER_LENGTH + SSL3_RT_MAX_ENCRYPTED_LENGTH;
+       if (max_len < (unsigned long)s->max_cert_list)
+               return s->max_cert_list;
+       return max_len;
+       }
 
 static int
 dtls1_reassemble_fragment(SSL *s, struct hm_header_st* msg_hdr, int *ok)
@@ -595,20 +605,10 @@ dtls1_reassemble_fragment(SSL *s, struct hm_header_st* msg_hdr, int *ok)
        pitem *item = NULL;
        int i = -1, is_complete;
        unsigned char seq64be[8];
-       unsigned long frag_len = msg_hdr->frag_len, max_len;
-
-       if ((msg_hdr->frag_off+frag_len) > msg_hdr->msg_len)
-               goto err;
-
-       /* Determine maximum allowed message size. Depends on (user set)
-        * maximum certificate length, but 16k is minimum.
-        */
-       if (DTLS1_HM_HEADER_LENGTH + SSL3_RT_MAX_ENCRYPTED_LENGTH < s->max_cert_list)
-               max_len = s->max_cert_list;
-       else
-               max_len = DTLS1_HM_HEADER_LENGTH + SSL3_RT_MAX_ENCRYPTED_LENGTH;
+       unsigned long frag_len = msg_hdr->frag_len;
 
-       if ((msg_hdr->frag_off+frag_len) > max_len)
+       if ((msg_hdr->frag_off+frag_len) > msg_hdr->msg_len ||
+           msg_hdr->msg_len > dtls1_max_handshake_message_len(s))
                goto err;
 
        /* Try to find item in queue */
@@ -639,7 +639,8 @@ dtls1_reassemble_fragment(SSL *s, struct hm_header_st* msg_hdr, int *ok)
 
 
        /* If message is already reassembled, this must be a
-        * retransmit and can be dropped.
+        * retransmit and can be dropped. In this case item != NULL and so frag
+        * does not need to be freed.
         */
        if (frag->reassembly == NULL)
                {
@@ -693,8 +694,7 @@ dtls1_reassemble_fragment(SSL *s, struct hm_header_st* msg_hdr, int *ok)
        return DTLS1_HM_FRAGMENT_RETRY;
 
 err:
-       if (frag != NULL) dtls1_hm_fragment_free(frag);
-       if (item != NULL) OPENSSL_free(item);
+       if (frag != NULL && item == NULL) dtls1_hm_fragment_free(frag);
        *ok = 0;
        return i;
        }
@@ -749,6 +749,9 @@ dtls1_process_out_of_seq_message(SSL *s, struct hm_header_st* msg_hdr, int *ok)
                if (frag_len && frag_len < msg_hdr->msg_len)
                        return dtls1_reassemble_fragment(s, msg_hdr, ok);
 
+               if (frag_len > dtls1_max_handshake_message_len(s))
+                       goto err;
+
                frag = dtls1_hm_fragment_new(frag_len, 0);
                if ( frag == NULL)
                        goto err;
@@ -778,8 +781,7 @@ dtls1_process_out_of_seq_message(SSL *s, struct hm_header_st* msg_hdr, int *ok)
        return DTLS1_HM_FRAGMENT_RETRY;
 
 err:
-       if ( frag != NULL) dtls1_hm_fragment_free(frag);
-       if ( item != NULL) OPENSSL_free(item);
+       if (frag != NULL && item == NULL) dtls1_hm_fragment_free(frag);
        *ok = 0;
        return i;
        }
@@ -793,6 +795,7 @@ dtls1_get_message_fragment(SSL *s, int st1, int stn, long max, int *ok)
        int i,al;
        struct hm_header_st msg_hdr;
 
+       redo:
        /* see if we have the required fragment already */
        if ((frag_len = dtls1_retrieve_buffered_fragment(s,max,ok)) || *ok)
                {
@@ -851,8 +854,7 @@ dtls1_get_message_fragment(SSL *s, int st1, int stn, long max, int *ok)
                                        s->msg_callback_arg);
                        
                        s->init_num = 0;
-                       return dtls1_get_message_fragment(s, st1, stn,
-                               max, ok);
+                       goto redo;
                        }
                else /* Incorrectly formated Hello request */
                        {
@@ -1180,6 +1182,8 @@ dtls1_buffer_message(SSL *s, int is_ccs)
        OPENSSL_assert(s->init_off == 0);
 
        frag = dtls1_hm_fragment_new(s->init_num, 0);
+       if (!frag)
+               return 0;
 
        memcpy(frag->fragment, s->init_buf->data, s->init_num);