Move ossl_assert
[openssl.git] / ssl / statem / statem_dtls.c
index a651e65fab8b8d28578c818a21655cbee3a84643..7fb6cdb8e2ab761d256dc609d796e5a559537c85 100644 (file)
@@ -12,6 +12,7 @@
 #include <stdio.h>
 #include "../ssl_locl.h"
 #include "statem_locl.h"
+#include "internal/cryptlib.h"
 #include <openssl/buffer.h>
 #include <openssl/objects.h>
 #include <openssl/evp.h>
@@ -32,7 +33,6 @@
 
 #define RSMBLY_BITMASK_IS_COMPLETE(bitmask, msg_len, is_complete) { \
                         long ii; \
-                        OPENSSL_assert((msg_len) > 0); \
                         is_complete = 1; \
                         if (bitmask[(((msg_len) - 1) >> 3)] != bitmask_end_values[((msg_len) & 7)]) is_complete = 0; \
                         if (is_complete) for (ii = (((msg_len) - 1) >> 3) - 1; ii >= 0 ; ii--) \
@@ -122,9 +122,11 @@ int dtls1_do_write(SSL *s, int type)
         /* should have something reasonable now */
         return -1;
 
-    if (s->init_off == 0 && type == SSL3_RT_HANDSHAKE)
-        OPENSSL_assert(s->init_num ==
-                       s->d1->w_msg_hdr.msg_len + DTLS1_HM_HEADER_LENGTH);
+    if (s->init_off == 0 && type == SSL3_RT_HANDSHAKE) {
+        if (!ossl_assert(s->init_num ==
+                         s->d1->w_msg_hdr.msg_len + DTLS1_HM_HEADER_LENGTH))
+            return -1;
+    }
 
     if (s->write_hash) {
         if (s->enc_write_ctx
@@ -214,10 +216,8 @@ int dtls1_do_write(SSL *s, int type)
         else
             len = s->init_num;
 
-        /* Shouldn't ever happen */
-        /* TODO(size_t): can this go now? */
-        if (len > INT_MAX)
-            len = INT_MAX;
+        if (len > s->max_send_fragment)
+            len = s->max_send_fragment;
 
         /*
          * XDTLS: this function is too long.  split out the CCS part
@@ -256,7 +256,7 @@ int dtls1_do_write(SSL *s, int type)
                 } else
                     return -1;
             } else {
-                return (-1);
+                return -1;
             }
         } else {
 
@@ -264,7 +264,8 @@ int dtls1_do_write(SSL *s, int type)
              * bad if this assert fails, only part of the handshake message
              * got sent.  but why would this happen?
              */
-            OPENSSL_assert(len == written);
+            if (!ossl_assert(len == written))
+                return -1;
 
             if (type == SSL3_RT_HANDSHAKE && !s->d1->retransmitting) {
                 /*
@@ -395,11 +396,10 @@ int dtls_get_message(SSL *s, int *mt, size_t *len)
  * permitted in a DTLS handshake message for |s|. The minimum is 16KB, but
  * may be greater if the maximum certificate list size requires it.
  */
-static unsigned long dtls1_max_handshake_message_len(const SSL *s)
+static size_t dtls1_max_handshake_message_len(const SSL *s)
 {
-    unsigned long max_len =
-        DTLS1_HM_HEADER_LENGTH + SSL3_RT_MAX_ENCRYPTED_LENGTH;
-    if (max_len < (unsigned long)s->max_cert_list)
+    size_t max_len = DTLS1_HM_HEADER_LENGTH + SSL3_RT_MAX_ENCRYPTED_LENGTH;
+    if (max_len < s->max_cert_list)
         return s->max_cert_list;
     return max_len;
 }
@@ -516,7 +516,7 @@ dtls1_reassemble_fragment(SSL *s, const struct hm_header_st *msg_hdr)
     int i = -1, is_complete;
     unsigned char seq64be[8];
     size_t frag_len = msg_hdr->frag_len;
-    size_t read;
+    size_t readbytes;
 
     if ((msg_hdr->frag_off + frag_len) > msg_hdr->msg_len ||
         msg_hdr->msg_len > dtls1_max_handshake_message_len(s))
@@ -561,10 +561,10 @@ dtls1_reassemble_fragment(SSL *s, const struct hm_header_st *msg_hdr)
                                           devnull,
                                           frag_len >
                                           sizeof(devnull) ? sizeof(devnull) :
-                                          frag_len, 0, &read);
+                                          frag_len, 0, &readbytes);
             if (i <= 0)
                 goto err;
-            frag_len -= read;
+            frag_len -= readbytes;
         }
         return DTLS1_HM_FRAGMENT_RETRY;
     }
@@ -572,8 +572,8 @@ dtls1_reassemble_fragment(SSL *s, const struct hm_header_st *msg_hdr)
     /* read the body of the fragment (header has already been read */
     i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, NULL,
                                   frag->fragment + msg_hdr->frag_off,
-                                  frag_len, 0, &read);
-    if (i <= 0 || read != frag_len)
+                                  frag_len, 0, &readbytes);
+    if (i <= 0 || readbytes != frag_len)
         i = -1;
     if (i <= 0)
         goto err;
@@ -581,6 +581,8 @@ dtls1_reassemble_fragment(SSL *s, const struct hm_header_st *msg_hdr)
     RSMBLY_BITMASK_MARK(frag->reassembly, (long)msg_hdr->frag_off,
                         (long)(msg_hdr->frag_off + frag_len));
 
+    if (!ossl_assert(msg_hdr->msg_len > 0))
+        goto err;
     RSMBLY_BITMASK_IS_COMPLETE(frag->reassembly, (long)msg_hdr->msg_len,
                                is_complete);
 
@@ -603,7 +605,8 @@ dtls1_reassemble_fragment(SSL *s, const struct hm_header_st *msg_hdr)
          * would have returned it and control would never have reached this
          * branch.
          */
-        OPENSSL_assert(item != NULL);
+        if (!ossl_assert(item != NULL))
+            goto err;
     }
 
     return DTLS1_HM_FRAGMENT_RETRY;
@@ -622,7 +625,7 @@ dtls1_process_out_of_seq_message(SSL *s, const struct hm_header_st *msg_hdr)
     pitem *item = NULL;
     unsigned char seq64be[8];
     size_t frag_len = msg_hdr->frag_len;
-    size_t read;
+    size_t readbytes;
 
     if ((msg_hdr->frag_off + frag_len) > msg_hdr->msg_len)
         goto err;
@@ -655,14 +658,14 @@ dtls1_process_out_of_seq_message(SSL *s, const struct hm_header_st *msg_hdr)
                                           devnull,
                                           frag_len >
                                           sizeof(devnull) ? sizeof(devnull) :
-                                          frag_len, 0, &read);
+                                          frag_len, 0, &readbytes);
             if (i <= 0)
                 goto err;
-            frag_len -= read;
+            frag_len -= readbytes;
         }
     } else {
         if (frag_len != msg_hdr->msg_len) {
-            return dtls1_reassemble_fragment(s, msg_hdr);;
+            return dtls1_reassemble_fragment(s, msg_hdr);
         }
 
         if (frag_len > dtls1_max_handshake_message_len(s))
@@ -679,8 +682,9 @@ dtls1_process_out_of_seq_message(SSL *s, const struct hm_header_st *msg_hdr)
              * read the body of the fragment (header has already been read
              */
             i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, NULL,
-                                          frag->fragment, frag_len, 0, &read);
-            if (i<=0 || read != frag_len)
+                                          frag->fragment, frag_len, 0,
+                                          &readbytes);
+            if (i<=0 || readbytes != frag_len)
                 i = -1;
             if (i <= 0)
                 goto err;
@@ -699,7 +703,8 @@ dtls1_process_out_of_seq_message(SSL *s, const struct hm_header_st *msg_hdr)
          * have been processed with |dtls1_reassemble_fragment|, above, or
          * the record will have been discarded.
          */
-        OPENSSL_assert(item != NULL);
+        if (!ossl_assert(item != NULL))
+            goto err;
     }
 
     return DTLS1_HM_FRAGMENT_RETRY;
@@ -716,7 +721,7 @@ static int dtls_get_reassembled_message(SSL *s, int *errtype, size_t *len)
     size_t mlen, frag_off, frag_len;
     int i, al, recvd_type;
     struct hm_header_st msg_hdr;
-    size_t read;
+    size_t readbytes;
 
     *errtype = 0;
 
@@ -730,7 +735,7 @@ static int dtls_get_reassembled_message(SSL *s, int *errtype, size_t *len)
 
     /* read handshake message header */
     i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, &recvd_type, wire,
-                                  DTLS1_HM_HEADER_LENGTH, 0, &read);
+                                  DTLS1_HM_HEADER_LENGTH, 0, &readbytes);
     if (i <= 0) {               /* nbio, or an error */
         s->rwstate = SSL_READING;
         *len = 0;
@@ -744,17 +749,17 @@ static int dtls_get_reassembled_message(SSL *s, int *errtype, size_t *len)
             goto f_err;
         }
 
-        memcpy(s->init_buf->data, wire, read);
-        s->init_num = read - 1;
+        memcpy(s->init_buf->data, wire, readbytes);
+        s->init_num = readbytes - 1;
         s->init_msg = s->init_buf->data + 1;
         s->s3->tmp.message_type = SSL3_MT_CHANGE_CIPHER_SPEC;
-        s->s3->tmp.message_size = read - 1;
-        *len = read - 1;
+        s->s3->tmp.message_size = readbytes - 1;
+        *len = readbytes - 1;
         return 1;
     }
 
     /* Handshake fails if message header is incomplete */
-    if (read != DTLS1_HM_HEADER_LENGTH) {
+    if (readbytes != DTLS1_HM_HEADER_LENGTH) {
         al = SSL_AD_UNEXPECTED_MESSAGE;
         SSLerr(SSL_F_DTLS_GET_REASSEMBLED_MESSAGE, SSL_R_UNEXPECTED_MESSAGE);
         goto f_err;
@@ -772,7 +777,7 @@ static int dtls_get_reassembled_message(SSL *s, int *errtype, size_t *len)
      * Fragments must not span records.
      */
     if (frag_len > RECORD_LAYER_get_rrec_length(&s->rlayer)) {
-        al = SSL3_AD_ILLEGAL_PARAMETER;
+        al = SSL_AD_ILLEGAL_PARAMETER;
         SSLerr(SSL_F_DTLS_GET_REASSEMBLED_MESSAGE, SSL_R_BAD_LENGTH);
         goto f_err;
     }
@@ -793,8 +798,10 @@ static int dtls_get_reassembled_message(SSL *s, int *errtype, size_t *len)
         return 0;
     }
 
-    if (!s->server && s->d1->r_msg_hdr.frag_off == 0 &&
-        wire[0] == SSL3_MT_HELLO_REQUEST) {
+    if (!s->server
+            && s->d1->r_msg_hdr.frag_off == 0
+            && s->statem.hand_state != TLS_ST_OK
+            && wire[0] == SSL3_MT_HELLO_REQUEST) {
         /*
          * The server may always send 'Hello Request' messages -- we are
          * doing a handshake anyway now, so ignore them if their format is
@@ -825,7 +832,7 @@ static int dtls_get_reassembled_message(SSL *s, int *errtype, size_t *len)
             (unsigned char *)s->init_buf->data + DTLS1_HM_HEADER_LENGTH;
 
         i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, NULL,
-                                      &p[frag_off], frag_len, 0, &read);
+                                      &p[frag_off], frag_len, 0, &readbytes);
 
         /*
          * This shouldn't ever fail due to NBIO because we already checked
@@ -837,16 +844,16 @@ static int dtls_get_reassembled_message(SSL *s, int *errtype, size_t *len)
             return 0;
         }
     } else {
-        read = 0;
+        readbytes = 0;
     }
 
     /*
      * XDTLS: an incorrectly formatted fragment should cause the handshake
      * to fail
      */
-    if (read != frag_len) {
-        al = SSL3_AD_ILLEGAL_PARAMETER;
-        SSLerr(SSL_F_DTLS_GET_REASSEMBLED_MESSAGE, SSL3_AD_ILLEGAL_PARAMETER);
+    if (readbytes != frag_len) {
+        al = SSL_AD_ILLEGAL_PARAMETER;
+        SSLerr(SSL_F_DTLS_GET_REASSEMBLED_MESSAGE, SSL_R_BAD_LENGTH);
         goto f_err;
     }
 
@@ -924,13 +931,8 @@ int dtls1_read_failed(SSL *s, int code)
          */
         return code;
     }
-#ifndef OPENSSL_NO_HEARTBEATS
-    /* done, no need to send a retransmit */
-    if (!SSL_in_init(s) && !s->tlsext_hb_pending)
-#else
     /* done, no need to send a retransmit */
     if (!SSL_in_init(s))
-#endif
     {
         BIO_set_flags(SSL_get_rbio(s), BIO_FLAGS_READ);
         return code;
@@ -986,7 +988,8 @@ int dtls1_buffer_message(SSL *s, int is_ccs)
      * this function is called immediately after a message has been
      * serialized
      */
-    OPENSSL_assert(s->init_off == 0);
+    if (!ossl_assert(s->init_off == 0))
+        return 0;
 
     frag = dtls1_hm_fragment_new(s->init_num, 0);
     if (frag == NULL)
@@ -996,13 +999,15 @@ int dtls1_buffer_message(SSL *s, int is_ccs)
 
     if (is_ccs) {
         /* For DTLS1_BAD_VER the header length is non-standard */
-        OPENSSL_assert(s->d1->w_msg_hdr.msg_len +
-                       ((s->version ==
-                         DTLS1_BAD_VER) ? 3 : DTLS1_CCS_HEADER_LENGTH)
-                       == (unsigned int)s->init_num);
+        if (!ossl_assert(s->d1->w_msg_hdr.msg_len +
+                         ((s->version ==
+                           DTLS1_BAD_VER) ? 3 : DTLS1_CCS_HEADER_LENGTH)
+                         == (unsigned int)s->init_num))
+            return 0;
     } else {
-        OPENSSL_assert(s->d1->w_msg_hdr.msg_len +
-                       DTLS1_HM_HEADER_LENGTH == (unsigned int)s->init_num);
+        if (!ossl_assert(s->d1->w_msg_hdr.msg_len +
+                         DTLS1_HM_HEADER_LENGTH == (unsigned int)s->init_num))
+            return 0;
     }
 
     frag->msg_header.msg_len = s->d1->w_msg_hdr.msg_len;
@@ -1050,11 +1055,6 @@ int dtls1_retransmit_message(SSL *s, unsigned short seq, int *found)
     unsigned char seq64be[8];
     struct dtls1_retransmit_state saved_state;
 
-    /*-
-      OPENSSL_assert(s->init_num == 0);
-      OPENSSL_assert(s->init_off == 0);
-     */
-
     /* XDTLS:  the requested message ought to be found, otherwise error */
     memset(seq64be, 0, sizeof(seq64be));
     seq64be[6] = (unsigned char)(seq >> 8);