From c35e921ffa58a84be7f68a37e5799ebefecf7326 Mon Sep 17 00:00:00 2001 From: Boris Pismenny Date: Thu, 21 Feb 2019 16:39:36 +0200 Subject: [PATCH] ssl: Linux TLS Rx Offload This patch adds support for the Linux TLS Rx socket option. It completes the previous patch for TLS Tx offload. If the socket option is successful, then the receive data-path of the TCP socket is implemented by the kernel. We choose to set this option at the earliest - just after CCS is complete. Change-Id: I59741e04d89dddca7fb138e88fffcc1259b30132 Signed-off-by: Boris Pismenny Reviewed-by: Bernd Edlinger Reviewed-by: Matt Caswell (Merged from https://github.com/openssl/openssl/pull/7848) --- include/openssl/ssl.h | 6 ++- ssl/record/rec_layer_s3.c | 10 +++-- ssl/record/ssl3_record.c | 47 ++++++++++++++++++--- ssl/t1_enc.c | 87 +++++++++++++++++++++++++++++++++------ 4 files changed, 129 insertions(+), 21 deletions(-) diff --git a/include/openssl/ssl.h b/include/openssl/ssl.h index 72c9d060f3..f4b17f1beb 100644 --- a/include/openssl/ssl.h +++ b/include/openssl/ssl.h @@ -500,7 +500,7 @@ typedef int (*SSL_async_callback_fn)(SSL *s, void *arg); */ # define SSL_MODE_ASYNC 0x00000100U /* - * Use the kernel TLS transmission data-path. + * Don't use the kernel TLS data-path for sending. */ # define SSL_MODE_NO_KTLS_TX 0x00000200U /* @@ -515,6 +515,10 @@ typedef int (*SSL_async_callback_fn)(SSL *s, void *arg); * - OpenSSL 1.1.1 and 1.1.1a */ # define SSL_MODE_DTLS_SCTP_LABEL_LENGTH_BUG 0x00000400U +/* + * Don't use the kernel TLS data-path for receiving. + */ +# define SSL_MODE_NO_KTLS_RX 0x00000800U /* Cert related flags */ /* diff --git a/ssl/record/rec_layer_s3.c b/ssl/record/rec_layer_s3.c index b21227765a..8b2320d2dc 100644 --- a/ssl/record/rec_layer_s3.c +++ b/ssl/record/rec_layer_s3.c @@ -268,11 +268,15 @@ int ssl3_read_n(SSL *s, size_t n, size_t max, int extend, int clearold, return -1; } - /* We always act like read_ahead is set for DTLS */ - if (!s->rlayer.read_ahead && !SSL_IS_DTLS(s)) + /* + * Ktls always reads full records. + * Also, we always act like read_ahead is set for DTLS. + */ + if (!BIO_get_ktls_recv(s->rbio) && !s->rlayer.read_ahead + && !SSL_IS_DTLS(s)) { /* ignore max parameter */ max = n; - else { + } else { if (max < n) max = n; if (max > rb->len - rb->offset) diff --git a/ssl/record/ssl3_record.c b/ssl/record/ssl3_record.c index e1231d2453..24694b3ffa 100644 --- a/ssl/record/ssl3_record.c +++ b/ssl/record/ssl3_record.c @@ -187,9 +187,11 @@ int ssl3_get_record(SSL *s) size_t num_recs = 0, max_recs, j; PACKET pkt, sslv2pkt; size_t first_rec_len; + int is_ktls_left; rr = RECORD_LAYER_get_rrec(&s->rlayer); rbuf = RECORD_LAYER_get_rbuf(&s->rlayer); + is_ktls_left = (rbuf->left > 0); max_recs = s->max_pipelines; if (max_recs == 0) max_recs = 1; @@ -208,8 +210,32 @@ int ssl3_get_record(SSL *s) rret = ssl3_read_n(s, SSL3_RT_HEADER_LENGTH, SSL3_BUFFER_get_len(rbuf), 0, num_recs == 0 ? 1 : 0, &n); - if (rret <= 0) - return rret; /* error or non-blocking */ + if (rret <= 0) { + if (!BIO_get_ktls_recv(s->rbio)) + return rret; /* error or non-blocking */ +#ifndef OPENSSL_NO_KTLS + switch (errno) { + case EBADMSG: + SSLfatal(s, SSL_AD_BAD_RECORD_MAC, + SSL_F_SSL3_GET_RECORD, + SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC); + break; + case EMSGSIZE: + SSLfatal(s, SSL_AD_RECORD_OVERFLOW, + SSL_F_SSL3_GET_RECORD, + SSL_R_PACKET_LENGTH_TOO_LONG); + break; + case EINVAL: + SSLfatal(s, SSL_AD_PROTOCOL_VERSION, + SSL_F_SSL3_GET_RECORD, + SSL_R_WRONG_VERSION_NUMBER); + break; + default: + break; + } + return rret; +#endif + } RECORD_LAYER_set_rstate(&s->rlayer, SSL_ST_READ_BODY); p = RECORD_LAYER_get_packet(&s->rlayer); @@ -387,7 +413,7 @@ int ssl3_get_record(SSL *s) len -= SSL3_RT_MAX_COMPRESSED_OVERHEAD; #endif - if (thisrr->length > len) { + if (thisrr->length > len && !BIO_get_ktls_recv(s->rbio)) { SSLfatal(s, SSL_AD_RECORD_OVERFLOW, SSL_F_SSL3_GET_RECORD, SSL_R_ENCRYPTED_LENGTH_TOO_LONG); return -1; @@ -405,6 +431,7 @@ int ssl3_get_record(SSL *s) } else { more = thisrr->length; } + if (more > 0) { /* now s->packet_length == SSL3_RT_HEADER_LENGTH */ @@ -492,6 +519,13 @@ int ssl3_get_record(SSL *s) return 1; } + /* + * KTLS reads full records. If there is any data left, + * then it is from before enabling ktls + */ + if (BIO_get_ktls_recv(s->rbio) && !is_ktls_left) + goto skip_decryption; + /* * If in encrypt-then-mac mode calculate mac from encrypted record. All * the details below are public so no timing details can leak. @@ -674,6 +708,8 @@ int ssl3_get_record(SSL *s) return -1; } + skip_decryption: + for (j = 0; j < num_recs; j++) { thisrr = &rr[j]; @@ -735,7 +771,7 @@ int ssl3_get_record(SSL *s) return -1; } - if (thisrr->length > SSL3_RT_MAX_PLAIN_LENGTH) { + if (thisrr->length > SSL3_RT_MAX_PLAIN_LENGTH && !BIO_get_ktls_recv(s->rbio)) { SSLfatal(s, SSL_AD_RECORD_OVERFLOW, SSL_F_SSL3_GET_RECORD, SSL_R_DATA_LENGTH_TOO_LONG); return -1; @@ -743,7 +779,8 @@ int ssl3_get_record(SSL *s) /* If received packet overflows current Max Fragment Length setting */ if (s->session != NULL && USE_MAX_FRAGMENT_LENGTH_EXT(s->session) - && thisrr->length > GET_MAX_FRAGMENT_LENGTH(s->session)) { + && thisrr->length > GET_MAX_FRAGMENT_LENGTH(s->session) + && !BIO_get_ktls_recv(s->rbio)) { SSLfatal(s, SSL_AD_RECORD_OVERFLOW, SSL_F_SSL3_GET_RECORD, SSL_R_DATA_LENGTH_TOO_LONG); return -1; diff --git a/ssl/t1_enc.c b/ssl/t1_enc.c index fe4ba9386d..5925e6a1b8 100644 --- a/ssl/t1_enc.c +++ b/ssl/t1_enc.c @@ -83,6 +83,39 @@ static int tls1_generate_key_block(SSL *s, unsigned char *km, size_t num) return ret; } +#ifndef OPENSSL_NO_KTLS + /* + * Count the number of records that were not processed yet from record boundary. + * + * This function assumes that there are only fully formed records read in the + * record layer. If read_ahead is enabled, then this might be false and this + * function will fail. + */ +static int count_unprocessed_records(SSL *s) +{ + SSL3_BUFFER *rbuf = RECORD_LAYER_get_rbuf(&s->rlayer); + PACKET pkt, subpkt; + int count = 0; + + if (!PACKET_buf_init(&pkt, rbuf->buf + rbuf->offset, rbuf->left)) + return -1; + + while (PACKET_remaining(&pkt) > 0) { + /* Skip record type and version */ + if (!PACKET_forward(&pkt, 3)) + return -1; + + /* Read until next record */ + if (PACKET_get_length_prefixed_2(&pkt, &subpkt)) + return -1; + + count += 1; + } + + return count; +} +#endif + int tls1_change_cipher_state(SSL *s, int which) { unsigned char *p, *mac_secret; @@ -101,8 +134,10 @@ int tls1_change_cipher_state(SSL *s, int which) int reuse_dd = 0; #ifndef OPENSSL_NO_KTLS struct tls12_crypto_info_aes_gcm_128 crypto_info; - BIO *wbio; + BIO *bio; unsigned char geniv[12]; + int count_unprocessed; + int bit; #endif c = s->s3->tmp.new_sym_enc; @@ -326,8 +361,8 @@ int tls1_change_cipher_state(SSL *s, int which) if (s->compress) goto skip_ktls; - if ((which & SSL3_CC_READ) || - ((which & SSL3_CC_WRITE) && (s->mode & SSL_MODE_NO_KTLS_TX))) + if (((which & SSL3_CC_READ) && (s->mode & SSL_MODE_NO_KTLS_RX)) + || ((which & SSL3_CC_WRITE) && (s->mode & SSL_MODE_NO_KTLS_TX))) goto skip_ktls; /* ktls supports only the maximum fragment size */ @@ -344,19 +379,26 @@ int tls1_change_cipher_state(SSL *s, int which) if (s->version != TLS1_2_VERSION) goto skip_ktls; - wbio = s->wbio; - if (!ossl_assert(wbio != NULL)) { + if (which & SSL3_CC_WRITE) + bio = s->wbio; + else + bio = s->rbio; + + if (!ossl_assert(bio != NULL)) { SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_TLS1_CHANGE_CIPHER_STATE, ERR_R_INTERNAL_ERROR); goto err; } /* All future data will get encrypted by ktls. Flush the BIO or skip ktls */ - if (BIO_flush(wbio) <= 0) - goto skip_ktls; + if (which & SSL3_CC_WRITE) { + if (BIO_flush(bio) <= 0) + goto skip_ktls; + } /* ktls doesn't support renegotiation */ - if (BIO_get_ktls_send(s->wbio)) { + if ((BIO_get_ktls_send(s->wbio) && (which & SSL3_CC_WRITE)) || + (BIO_get_ktls_recv(s->rbio) && (which & SSL3_CC_READ))) { SSLfatal(s, SSL_AD_NO_RENEGOTIATION, SSL_F_TLS1_CHANGE_CIPHER_STATE, ERR_R_INTERNAL_ERROR); goto err; @@ -373,12 +415,33 @@ int tls1_change_cipher_state(SSL *s, int which) TLS_CIPHER_AES_GCM_128_IV_SIZE); memcpy(crypto_info.salt, geniv, TLS_CIPHER_AES_GCM_128_SALT_SIZE); memcpy(crypto_info.key, key, EVP_CIPHER_key_length(c)); - memcpy(crypto_info.rec_seq, &s->rlayer.write_sequence, - TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE); + if (which & SSL3_CC_WRITE) + memcpy(crypto_info.rec_seq, &s->rlayer.write_sequence, + TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE); + else + memcpy(crypto_info.rec_seq, &s->rlayer.read_sequence, + TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE); + + if (which & SSL3_CC_READ) { + count_unprocessed = count_unprocessed_records(s); + if (count_unprocessed < 0) + goto skip_ktls; + + /* increment the crypto_info record sequence */ + while (count_unprocessed) { + for (bit = 7; bit >= 0; bit--) { /* increment */ + ++crypto_info.rec_seq[bit]; + if (crypto_info.rec_seq[bit] != 0) + break; + } + count_unprocessed--; + } + } /* ktls works with user provided buffers directly */ - if (BIO_set_ktls(wbio, &crypto_info, which & SSL3_CC_WRITE)) { - ssl3_release_write_buffer(s); + if (BIO_set_ktls(bio, &crypto_info, which & SSL3_CC_WRITE)) { + if (which & SSL3_CC_WRITE) + ssl3_release_write_buffer(s); SSL_set_options(s, SSL_OP_NO_RENEGOTIATION); } -- 2.34.1