Always use uint8_t for TLS record type
authorTomas Mraz <tomas@openssl.org>
Thu, 24 Aug 2023 08:34:53 +0000 (10:34 +0200)
committerHugo Landau <hlandau@openssl.org>
Fri, 25 Aug 2023 11:04:00 +0000 (12:04 +0100)
Reviewed-by: Hugo Landau <hlandau@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/21823)

17 files changed:
doc/designs/quic-design/record-layer.md
include/internal/recordmethod.h
ssl/d1_msg.c
ssl/quic/quic_tls.c
ssl/record/methods/dtls_meth.c
ssl/record/methods/ktls_meth.c
ssl/record/methods/recmethod_local.h
ssl/record/methods/tls13_meth.c
ssl/record/methods/tls_common.c
ssl/record/methods/tls_multib.c
ssl/record/rec_layer_d1.c
ssl/record/rec_layer_s3.c
ssl/record/record.h
ssl/ssl_local.h
ssl/statem/statem_dtls.c
ssl/statem/statem_lib.c
test/recordlentest.c

index 8db4b319c54297c9ca93335284e6ea5e2ef69e50..172098b54756fe426193df43e08a74918e6c5b09 100644 (file)
@@ -455,7 +455,7 @@ struct ossl_record_method_st {
      * exit the record layer may update this to an alternative fragment size to
      * be used. This must always be less than or equal to |maxfrag|.
      */
-    size_t (*get_max_records)(OSSL_RECORD_LAYER *rl, int type, size_t len,
+    size_t (*get_max_records)(OSSL_RECORD_LAYER *rl, uint8_t type, size_t len,
                               size_t maxfrag, size_t *preffrag);
 
     /*
index c49b5da4ee5b452fb745eb878164d5c2a20a2153..23eee7956648e409e1f62f6afc92a0f8c3f484d7 100644 (file)
@@ -179,7 +179,7 @@ struct ossl_record_method_st {
      * exit the record layer may update this to an alternative fragment size to
      * be used. This must always be less than or equal to |maxfrag|.
      */
-    size_t (*get_max_records)(OSSL_RECORD_LAYER *rl, int type, size_t len,
+    size_t (*get_max_records)(OSSL_RECORD_LAYER *rl, uint8_t type, size_t len,
                               size_t maxfrag, size_t *preffrag);
 
     /*
index eb84ed64701d55936659696a0cf8486c140f5f07..23cfa150fc99030e41c8a5f97f0f316e61e079bc 100644 (file)
@@ -9,8 +9,8 @@
 
 #include "ssl_local.h"
 
-int dtls1_write_app_data_bytes(SSL *s, int type, const void *buf_, size_t len,
-                               size_t *written)
+int dtls1_write_app_data_bytes(SSL *s, uint8_t type, const void *buf_,
+                               size_t len, size_t *written)
 {
     int i;
     SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s);
index 586042bcd71c86da1bc5adbec3f684883b1d472d..8b9f812253d139cdf88b6265085345c2418799ec 100644 (file)
@@ -230,7 +230,8 @@ static int quic_processed_read_pending(OSSL_RECORD_LAYER *rl)
     return 0;
 }
 
-static size_t quic_get_max_records(OSSL_RECORD_LAYER *rl, int type, size_t len,
+static size_t quic_get_max_records(OSSL_RECORD_LAYER *rl, uint8_t type,
+                                   size_t len,
                                    size_t maxfrag, size_t *preffrag)
 {
     return 1;
index 7b16f42c47a80f933304958fcf725d9e1be200df..c5477b9b75e36ff9487b4531c0f094ebc9e02e95 100644 (file)
@@ -693,7 +693,7 @@ dtls_new_record_layer(OSSL_LIB_CTX *libctx, const char *propq, int vers,
 int dtls_prepare_record_header(OSSL_RECORD_LAYER *rl,
                                WPACKET *thispkt,
                                OSSL_RECORD_TEMPLATE *templ,
-                               unsigned int rectype,
+                               uint8_t rectype,
                                unsigned char **recdata)
 {
     size_t maxcomplen;
index 5cd833d070dbbb346a55910d15a9b14f96d0778e..2d3c33bf7a8ccd496b8cdd11d48c04dbf189bae4 100644 (file)
@@ -493,7 +493,7 @@ static int ktls_initialise_write_packets(OSSL_RECORD_LAYER *rl,
 static int ktls_prepare_record_header(OSSL_RECORD_LAYER *rl,
                                       WPACKET *thispkt,
                                       OSSL_RECORD_TEMPLATE *templ,
-                                      unsigned int rectype,
+                                      uint8_t rectype,
                                       unsigned char **recdata)
 {
     /* The kernel writes the record header, so nothing to do */
index cf0af62f3a19c61d1e5daf019dcb568d46308ea4..86517f00626ab3f661ab1346f5acc980b325e62b 100644 (file)
@@ -138,7 +138,7 @@ struct record_functions_st
 
     /* Write related functions */
 
-    size_t (*get_max_records)(OSSL_RECORD_LAYER *rl, int type, size_t len,
+    size_t (*get_max_records)(OSSL_RECORD_LAYER *rl, uint8_t type, size_t len,
                               size_t maxfrag, size_t *preffrag);
 
     /* Return 1 for success or 0 for error */
@@ -167,13 +167,13 @@ struct record_functions_st
                                     size_t *wpinited);
 
     /* Get the actual record type to be used for a given template */
-    unsigned int (*get_record_type)(OSSL_RECORD_LAYER *rl,
-                                    OSSL_RECORD_TEMPLATE *template);
+    uint8_t (*get_record_type)(OSSL_RECORD_LAYER *rl,
+                               OSSL_RECORD_TEMPLATE *template);
 
     /* Write the record header data to the WPACKET */
     int (*prepare_record_header)(OSSL_RECORD_LAYER *rl, WPACKET *thispkt,
                                  OSSL_RECORD_TEMPLATE *templ,
-                                 unsigned int rectype,
+                                 uint8_t rectype,
                                  unsigned char **recdata);
 
     int (*add_record_padding)(OSSL_RECORD_LAYER *rl,
@@ -419,7 +419,7 @@ int dtls_get_more_records(OSSL_RECORD_LAYER *rl);
 int dtls_prepare_record_header(OSSL_RECORD_LAYER *rl,
                                WPACKET *thispkt,
                                OSSL_RECORD_TEMPLATE *templ,
-                               unsigned int rectype,
+                               uint8_t rectype,
                                unsigned char **recdata);
 int dtls_post_encryption_processing(OSSL_RECORD_LAYER *rl,
                                     size_t mac_size,
@@ -451,7 +451,7 @@ int tls_free(OSSL_RECORD_LAYER *rl);
 int tls_unprocessed_read_pending(OSSL_RECORD_LAYER *rl);
 int tls_processed_read_pending(OSSL_RECORD_LAYER *rl);
 size_t tls_app_data_pending(OSSL_RECORD_LAYER *rl);
-size_t tls_get_max_records(OSSL_RECORD_LAYER *rl, int type, size_t len,
+size_t tls_get_max_records(OSSL_RECORD_LAYER *rl, uint8_t type, size_t len,
                            size_t maxfrag, size_t *preffrag);
 int tls_write_records(OSSL_RECORD_LAYER *rl, OSSL_RECORD_TEMPLATE *templates,
                       size_t numtempl);
@@ -480,9 +480,10 @@ int tls_write_records_multiblock(OSSL_RECORD_LAYER *rl,
                                  OSSL_RECORD_TEMPLATE *templates,
                                  size_t numtempl);
 
-size_t tls_get_max_records_default(OSSL_RECORD_LAYER *rl, int type, size_t len,
+size_t tls_get_max_records_default(OSSL_RECORD_LAYER *rl, uint8_t type,
+                                   size_t len,
                                    size_t maxfrag, size_t *preffrag);
-size_t tls_get_max_records_multiblock(OSSL_RECORD_LAYER *rl, int type,
+size_t tls_get_max_records_multiblock(OSSL_RECORD_LAYER *rl, uint8_t type,
                                       size_t len, size_t maxfrag,
                                       size_t *preffrag);
 int tls_allocate_write_buffers_default(OSSL_RECORD_LAYER *rl,
@@ -508,7 +509,7 @@ int tls1_initialise_write_packets(OSSL_RECORD_LAYER *rl,
 int tls_prepare_record_header_default(OSSL_RECORD_LAYER *rl,
                                       WPACKET *thispkt,
                                       OSSL_RECORD_TEMPLATE *templ,
-                                      unsigned int rectype,
+                                      uint8_t rectype,
                                       unsigned char **recdata);
 int tls_prepare_for_encryption_default(OSSL_RECORD_LAYER *rl,
                                        size_t mac_size,
index 84db78a8f67b1244b68efecf4c0d1f739fa0b0f6..8704c7260354f8feab09eb804dc5bd416c4c7970 100644 (file)
@@ -233,11 +233,11 @@ static int tls13_post_process_record(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec)
     return 1;
 }
 
-static unsigned int tls13_get_record_type(OSSL_RECORD_LAYER *rl,
-                                          OSSL_RECORD_TEMPLATE *template)
+static uint8_t tls13_get_record_type(OSSL_RECORD_LAYER *rl,
+                                     OSSL_RECORD_TEMPLATE *template)
 {
     if (rl->allow_plain_alerts && template->type == SSL3_RT_ALERT)
-        return  SSL3_RT_ALERT;
+        return SSL3_RT_ALERT;
 
     /*
      * Aside from the above case we always use the application data record type
index 576b651aa7c138b4c53d57e1ce7bbf41c896c5df..6ea090df244ba2f94bb40f5ef063e4c65a663069 100644 (file)
@@ -1487,7 +1487,8 @@ size_t tls_app_data_pending(OSSL_RECORD_LAYER *rl)
     return num;
 }
 
-size_t tls_get_max_records_default(OSSL_RECORD_LAYER *rl, int type, size_t len,
+size_t tls_get_max_records_default(OSSL_RECORD_LAYER *rl, uint8_t type,
+                                   size_t len,
                                    size_t maxfrag, size_t *preffrag)
 {
     /*
@@ -1511,7 +1512,7 @@ size_t tls_get_max_records_default(OSSL_RECORD_LAYER *rl, int type, size_t len,
     return 1;
 }
 
-size_t tls_get_max_records(OSSL_RECORD_LAYER *rl, int type, size_t len,
+size_t tls_get_max_records(OSSL_RECORD_LAYER *rl, uint8_t type, size_t len,
                            size_t maxfrag, size_t *preffrag)
 {
     return rl->funcs->get_max_records(rl, type, len, maxfrag, preffrag);
@@ -1574,7 +1575,7 @@ int tls_initialise_write_packets_default(OSSL_RECORD_LAYER *rl,
 int tls_prepare_record_header_default(OSSL_RECORD_LAYER *rl,
                                       WPACKET *thispkt,
                                       OSSL_RECORD_TEMPLATE *templ,
-                                      unsigned int rectype,
+                                      uint8_t rectype,
                                       unsigned char **recdata)
 {
     size_t maxcomplen;
@@ -1755,7 +1756,7 @@ int tls_write_records_default(OSSL_RECORD_LAYER *rl,
     memset(wr, 0, sizeof(wr));
     for (j = 0; j < numtempl + prefix; j++) {
         unsigned char *compressdata = NULL;
-        unsigned int rectype;
+        uint8_t rectype;
 
         thispkt = &pkt[j];
         thiswr = &wr[j];
index fb5a2b3e0ef11614c4a0d7ac5687aac0d13f6b9c..8aa89f86b28437bd9cb72266f145a72a53270df0 100644 (file)
@@ -20,7 +20,7 @@
 # define EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK 0
 #endif
 
-static int tls_is_multiblock_capable(OSSL_RECORD_LAYER *rl, int type,
+static int tls_is_multiblock_capable(OSSL_RECORD_LAYER *rl, uint8_t type,
                                      size_t len, size_t fraglen)
 {
 #if !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK
@@ -38,7 +38,7 @@ static int tls_is_multiblock_capable(OSSL_RECORD_LAYER *rl, int type,
     return 0;
 }
 
-size_t tls_get_max_records_multiblock(OSSL_RECORD_LAYER *rl, int type,
+size_t tls_get_max_records_multiblock(OSSL_RECORD_LAYER *rl, uint8_t type,
                                       size_t len, size_t maxfrag,
                                       size_t *preffrag)
 {
index fed57b65cd51401408f2c664dc85cd86499c783a..75ed6775cbce40485c86c4def464d10e6ddb679a 100644 (file)
@@ -195,8 +195,9 @@ static void dtls_unbuffer_record(SSL_CONNECTION *s)
  *     Application data protocol
  *             none of our business
  */
-int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
-                     size_t len, int peek, size_t *readbytes)
+int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type,
+                     unsigned char *buf, size_t len,
+                     int peek, size_t *readbytes)
 {
     int i, j, ret;
     size_t n;
@@ -603,7 +604,7 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
  * Call this to write data in records of type 'type' It will return <= 0 if
  * not all data has been sent or non-blocking IO.
  */
-int dtls1_write_bytes(SSL_CONNECTION *s, int type, const void *buf,
+int dtls1_write_bytes(SSL_CONNECTION *s, uint8_t type, const void *buf,
                       size_t len, size_t *written)
 {
     int i;
@@ -617,7 +618,7 @@ int dtls1_write_bytes(SSL_CONNECTION *s, int type, const void *buf,
     return i;
 }
 
-int do_dtls1_write(SSL_CONNECTION *sc, int type, const unsigned char *buf,
+int do_dtls1_write(SSL_CONNECTION *sc, uint8_t type, const unsigned char *buf,
                    size_t len, size_t *written)
 {
     int i;
index 16726019f9d6c988bc64632f405ba8c3b1868ed7..bf92a8e6ab6d17ba56400833892fd7753919a407 100644 (file)
@@ -200,7 +200,7 @@ const char *SSL_rstate_string(const SSL *s)
     return shrt;
 }
 
-static int tls_write_check_pending(SSL_CONNECTION *s, int type,
+static int tls_write_check_pending(SSL_CONNECTION *s, uint8_t type,
                                    const unsigned char *buf, size_t len)
 {
     if (s->rlayer.wpend_tot == 0)
@@ -221,7 +221,7 @@ static int tls_write_check_pending(SSL_CONNECTION *s, int type,
  * Call this to write data in records of type 'type' It will return <= 0 if
  * not all data has been sent or non-blocking IO.
  */
-int ssl3_write_bytes(SSL *ssl, int type, const void *buf_, size_t len,
+int ssl3_write_bytes(SSL *ssl, uint8_t type, const void *buf_, size_t len,
                      size_t *written)
 {
     const unsigned char *buf = buf_;
@@ -558,8 +558,9 @@ int ssl_release_record(SSL_CONNECTION *s, TLS_RECORD *rr, size_t length)
  *     Application data protocol
  *             none of our business
  */
-int ssl3_read_bytes(SSL *ssl, int type, int *recvd_type, unsigned char *buf,
-                    size_t len, int peek, size_t *readbytes)
+int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type,
+                    unsigned char *buf, size_t len,
+                    int peek, size_t *readbytes)
 {
     int i, j, ret;
     size_t n, curr_rec, totalbytes;
index a8fa6833e90f82b6d172ee99ddde0acf9f56aec9..e4d1e34036299ca2de5d7efdce7aa6ddb8cfa0ad 100644 (file)
@@ -107,7 +107,7 @@ typedef struct record_layer_st {
     /* partial write - check the numbers match */
     /* number bytes written */
     size_t wpend_tot;
-    int wpend_type;
+    uint8_t wpend_type;
     /* number of bytes submitted */
     size_t wpend_ret;
     const unsigned char *wpend_buf;
@@ -148,21 +148,21 @@ int RECORD_LAYER_processed_read_pending(const RECORD_LAYER *rl);
 int RECORD_LAYER_write_pending(const RECORD_LAYER *rl);
 int RECORD_LAYER_is_sslv2_record(RECORD_LAYER *rl);
 __owur size_t ssl3_pending(const SSL *s);
-__owur int ssl3_write_bytes(SSL *s, int type, const void *buf, size_t len,
+__owur int ssl3_write_bytes(SSL *s, uint8_t type, const void *buf, size_t len,
                             size_t *written);
-__owur int ssl3_read_bytes(SSL *s, int type, int *recvd_type,
+__owur int ssl3_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type,
                            unsigned char *buf, size_t len, int peek,
                            size_t *readbytes);
 
 int DTLS_RECORD_LAYER_new(RECORD_LAYER *rl);
 void DTLS_RECORD_LAYER_free(RECORD_LAYER *rl);
 void DTLS_RECORD_LAYER_clear(RECORD_LAYER *rl);
-__owur int dtls1_read_bytes(SSL *s, int type, int *recvd_type,
+__owur int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type,
                             unsigned char *buf, size_t len, int peek,
                             size_t *readbytes);
-__owur int dtls1_write_bytes(SSL_CONNECTION *s, int type, const void *buf,
+__owur int dtls1_write_bytes(SSL_CONNECTION *s, uint8_t type, const void *buf,
                              size_t len, size_t *written);
-int do_dtls1_write(SSL_CONNECTION *s, int type, const unsigned char *buf,
+int do_dtls1_write(SSL_CONNECTION *s, uint8_t type, const unsigned char *buf,
                    size_t len, size_t *written);
 void dtls1_increment_epoch(SSL_CONNECTION *s, int rw);
 int ssl_release_record(SSL_CONNECTION *s, TLS_RECORD *rr, size_t length);
index d628f4026263cc78f120e77fb7490128c39f46a2..4da83ab6928ea6e0f4e2a6e4068f5fa1ec75dee7 100644 (file)
@@ -433,10 +433,10 @@ struct ssl_method_st {
     int (*ssl_shutdown) (SSL *s);
     int (*ssl_renegotiate) (SSL *s);
     int (*ssl_renegotiate_check) (SSL *s, int);
-    int (*ssl_read_bytes) (SSL *s, int type, int *recvd_type,
+    int (*ssl_read_bytes) (SSL *s, uint8_t type, uint8_t *recvd_type,
                            unsigned char *buf, size_t len, int peek,
                            size_t *readbytes);
-    int (*ssl_write_bytes) (SSL *s, int type, const void *buf_, size_t len,
+    int (*ssl_write_bytes) (SSL *s, uint8_t type, const void *buf_, size_t len,
                             size_t *written);
     int (*ssl_dispatch_alert) (SSL *s);
     long (*ssl_ctrl) (SSL *s, int cmd, long larg, void *parg);
@@ -2587,7 +2587,7 @@ int ssl3_init_finished_mac(SSL_CONNECTION *s);
 __owur int ssl3_setup_key_block(SSL_CONNECTION *s);
 __owur int ssl3_change_cipher_state(SSL_CONNECTION *s, int which);
 void ssl3_cleanup_key_block(SSL_CONNECTION *s);
-__owur int ssl3_do_write(SSL_CONNECTION *s, int type);
+__owur int ssl3_do_write(SSL_CONNECTION *s, uint8_t type);
 int ssl3_send_alert(SSL_CONNECTION *s, int level, int desc);
 __owur int ssl3_generate_master_secret(SSL_CONNECTION *s, unsigned char *out,
                                        unsigned char *p, size_t len,
@@ -2650,14 +2650,14 @@ __owur int ssl_get_min_max_version(const SSL_CONNECTION *s, int *min_version,
                                    int *max_version, int *real_max);
 
 __owur OSSL_TIME tls1_default_timeout(void);
-__owur int dtls1_do_write(SSL_CONNECTION *s, int type);
+__owur int dtls1_do_write(SSL_CONNECTION *s, uint8_t type);
 void dtls1_set_message_header(SSL_CONNECTION *s,
                               unsigned char mt,
                               size_t len,
                               size_t frag_off, size_t frag_len);
 
-int dtls1_write_app_data_bytes(SSL *s, int type, const void *buf_, size_t len,
-                               size_t *written);
+int dtls1_write_app_data_bytes(SSL *s, uint8_t type, const void *buf_,
+                               size_t len, size_t *written);
 
 __owur int dtls1_read_failed(SSL_CONNECTION *s, int code);
 __owur int dtls1_buffer_message(SSL_CONNECTION *s, int ccs);
index 2e26a3f3df3d4cb01f452ea6271fff0ebb205a75..31dc9f3511ea2985607220057414454a55cf689f 100644 (file)
@@ -111,7 +111,7 @@ void dtls1_hm_fragment_free(hm_fragment *frag)
  * send s->init_buf in records of type 'type' (SSL3_RT_HANDSHAKE or
  * SSL3_RT_CHANGE_CIPHER_SPEC)
  */
-int dtls1_do_write(SSL_CONNECTION *s, int type)
+int dtls1_do_write(SSL_CONNECTION *s, uint8_t type)
 {
     int ret;
     size_t written;
@@ -809,7 +809,8 @@ static int dtls_get_reassembled_message(SSL_CONNECTION *s, int *errtype,
 {
     unsigned char wire[DTLS1_HM_HEADER_LENGTH];
     size_t mlen, frag_off, frag_len;
-    int i, ret, recvd_type;
+    int i, ret;
+    uint8_t recvd_type;
     struct hm_header_st msg_hdr;
     size_t readbytes;
     SSL *ssl = SSL_CONNECTION_GET_SSL(s);
index f339b2694c673d681e9aecc7461a29786bd5d0eb..d979b6d5fc437714b93abc1e75f79f4cfdcc1742 100644 (file)
@@ -58,7 +58,7 @@ int ossl_statem_set_mutator(SSL *s,
  * send s->init_buf in records of type 'type' (SSL3_RT_HANDSHAKE or
  * SSL3_RT_CHANGE_CIPHER_SPEC)
  */
-int ssl3_do_write(SSL_CONNECTION *s, int type)
+int ssl3_do_write(SSL_CONNECTION *s, uint8_t type)
 {
     int ret;
     size_t written = 0;
@@ -1530,7 +1530,8 @@ WORK_STATE tls_finish_handshake(SSL_CONNECTION *s, ossl_unused WORK_STATE wst,
 int tls_get_message_header(SSL_CONNECTION *s, int *mt)
 {
     /* s->init_num < SSL3_HM_HEADER_LENGTH */
-    int skip_message, i, recvd_type;
+    int skip_message, i;
+    uint8_t recvd_type;
     unsigned char *p;
     size_t l, readbytes;
     SSL *ssl = SSL_CONNECTION_GET_SSL(s);
index c693b089139f481ad523f44013346cb8b4d201e9..d46b75f686d25fe1d795e79b7cf8434d73dc91f4 100644 (file)
@@ -24,7 +24,7 @@ static char *privkey = NULL;
 
 #define TOTAL_RECORD_OVERFLOW_TESTS 6
 
-static int write_record(BIO *b, size_t len, int rectype, int recversion)
+static int write_record(BIO *b, size_t len, uint8_t rectype, int recversion)
 {
     unsigned char header[SSL3_RT_HEADER_LENGTH];
     size_t written;