Add support for KTLS zerocopy sendfile on Linux

TLS device offload allows to perform zerocopy sendfile transmissions.
FreeBSD provides this feature by default, and Linux 5.19 introduced it
as an opt-in. Zerocopy improves the TX rate significantly, but has a
side effect: if the underlying file is changed while being transmitted,
and a TCP retransmission happens, the receiver may get a TLS record
containing both new and old data, which leads to an authentication
failure and termination of connection. This effect is the reason Linux
makes a copy on sendfile by default.

This commit adds support for TLS zerocopy sendfile on Linux disabled by
default to avoid any unlikely backward compatibility issues on Linux,
although sacrificing consistency in OpenSSL's behavior on Linux and
FreeBSD. A new option called KTLSTxZerocopySendfile is added to enable
the new zerocopy behavior on Linux. This option should be used when the
the application guarantees that the file is not modified during
transmission, or it doesn't care about breaking the connection.

The related documentation is also added in this commit. The unit test
added doesn't test the actual functionality (it would require specific
hardware and a non-local peer), but solely checks that it's possible to
set the new option flag.

Signed-off-by: Maxim Mikityanskiy <maximmi@nvidia.com>
Reviewed-by: Tariq Toukan <tariqt@nvidia.com>
Reviewed-by: Boris Pismenny <borisp@nvidia.com>
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Todd Short <todd.short@me.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/18650)

diff --git a/CHANGES.md b/CHANGES.md
index 2c12daf151f6713eb357dd468f7acbc9a83872d8..ede13f7d79e0ff4fe3bde9a0aeea7767b270ca80 100644 (file)
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -226,6 +226,10 @@ OpenSSL 3.2
 
    *Tianjia Zhang*
 
+ * Zerocopy KTLS sendfile() support on Linux.
+
+   *Maxim Mikityanskiy*
+
 OpenSSL 3.0
 -----------
 

diff --git a/apps/s_server.c b/apps/s_server.c
index f519505ade0566a21faaeb9a7a2baf3263ee1b30..46af6b87dad7f2cb97a274225f5426ed0bf959f3 100644 (file)
--- a/apps/s_server.c
+++ b/apps/s_server.c
@@ -96,6 +96,7 @@ static int keymatexportlen = 20;
 static int async = 0;
 
 static int use_sendfile = 0;
+static int use_zc_sendfile = 0;
 
 static const char *session_id_prefix = NULL;
 
@@ -716,6 +717,7 @@ typedef enum OPTION_choice {
     OPT_KEYLOG_FILE, OPT_MAX_EARLY, OPT_RECV_MAX_EARLY, OPT_EARLY_DATA,
     OPT_S_NUM_TICKETS, OPT_ANTI_REPLAY, OPT_NO_ANTI_REPLAY, OPT_SCTP_LABEL_BUG,
     OPT_HTTP_SERVER_BINMODE, OPT_NOCANAMES, OPT_IGNORE_UNEXPECTED_EOF, OPT_KTLS,
+    OPT_USE_ZC_SENDFILE,
     OPT_TFO, OPT_CERT_COMP,
     OPT_R_ENUM,
     OPT_S_ENUM,
@@ -966,6 +968,7 @@ const OPTIONS s_server_options[] = {
 #ifndef OPENSSL_NO_KTLS
     {"ktls", OPT_KTLS, '-', "Enable Kernel TLS for sending and receiving"},
     {"sendfile", OPT_SENDFILE, '-', "Use sendfile to response file with -WWW"},
+    {"zerocopy_sendfile", OPT_USE_ZC_SENDFILE, '-', "Use zerocopy mode of KTLS sendfile"},
 #endif
 
     OPT_R_OPTIONS,
@@ -1080,6 +1083,7 @@ int s_server_main(int argc, char *argv[])
     s_brief = 0;
     async = 0;
     use_sendfile = 0;
+    use_zc_sendfile = 0;
 
     port = OPENSSL_strdup(PORT);
     cctx = SSL_CONF_CTX_new();
@@ -1654,6 +1658,11 @@ int s_server_main(int argc, char *argv[])
         case OPT_SENDFILE:
 #ifndef OPENSSL_NO_KTLS
             use_sendfile = 1;
+#endif
+            break;
+        case OPT_USE_ZC_SENDFILE:
+#ifndef OPENSSL_NO_KTLS
+            use_zc_sendfile = 1;
 #endif
             break;
         case OPT_IGNORE_UNEXPECTED_EOF:
@@ -1728,6 +1737,11 @@ int s_server_main(int argc, char *argv[])
 #endif
 
 #ifndef OPENSSL_NO_KTLS
+    if (use_zc_sendfile && !use_sendfile) {
+        BIO_printf(bio_out, "Warning: -zerocopy_sendfile depends on -sendfile, enabling -sendfile now.\n");
+        use_sendfile = 1;
+    }
+
     if (use_sendfile && enable_ktls == 0) {
         BIO_printf(bio_out, "Warning: -sendfile depends on -ktls, enabling -ktls now.\n");
         enable_ktls = 1;
@@ -1933,6 +1947,8 @@ int s_server_main(int argc, char *argv[])
 #ifndef OPENSSL_NO_KTLS
     if (enable_ktls)
         SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS);
+    if (use_zc_sendfile)
+        SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE);
 #endif
 
     if (max_send_fragment > 0

diff --git a/crypto/bio/bss_conn.c b/crypto/bio/bss_conn.c
index 75bfe645662c9f7fdaa3a80449a038196ee4b894..f494b14000fa168ac299f6a096cd1a86e5438f2c 100644 (file)
--- a/crypto/bio/bss_conn.c
+++ b/crypto/bio/bss_conn.c
@@ -598,6 +598,11 @@ static long conn_ctrl(BIO *b, int cmd, long num, void *ptr)
         BIO_clear_ktls_ctrl_msg_flag(b);
         ret = 0;
         break;
+    case BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE:
+        ret = ktls_enable_tx_zerocopy_sendfile(b->num);
+        if (ret)
+            BIO_set_ktls_zerocopy_sendfile_flag(b);
+        break;
 # endif
     default:
         ret = 0;

diff --git a/crypto/bio/bss_sock.c b/crypto/bio/bss_sock.c
index 69dfd37bfe11b142de94589e8f61f5446aff8670..f64eb8c843a4ba110ebf658446b6f754f4bcdf2c 100644 (file)
--- a/crypto/bio/bss_sock.c
+++ b/crypto/bio/bss_sock.c
@@ -235,6 +235,11 @@ static long sock_ctrl(BIO *b, int cmd, long num, void *ptr)
         BIO_clear_ktls_ctrl_msg_flag(b);
         ret = 0;
         break;
+    case BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE:
+        ret = ktls_enable_tx_zerocopy_sendfile(b->num);
+        if (ret)
+            BIO_set_ktls_zerocopy_sendfile_flag(b);
+        break;
 # endif
     case BIO_CTRL_EOF:
         ret = (b->flags & BIO_FLAGS_IN_EOF) != 0;

diff --git a/doc/man1/openssl-s_server.pod.in b/doc/man1/openssl-s_server.pod.in
index 94f3b4b46ce6da9d98a0b539b76e68fb7462742c..a1e354908c912951d0474f6ab8d2fc8c715910bc 100644 (file)
--- a/doc/man1/openssl-s_server.pod.in
+++ b/doc/man1/openssl-s_server.pod.in
@@ -132,6 +132,7 @@ B<openssl> B<s_server>
 [B<-alpn> I<val>]
 [B<-ktls>]
 [B<-sendfile>]
+[B<-zerocopy_sendfile>]
 [B<-keylogfile> I<outfile>]
 [B<-recv_max_early_data> I<int>]
 [B<-max_early_data> I<int>]
@@ -792,6 +793,15 @@ instead of BIO_write() to send the HTTP response requested by a client.
 This option is only valid when B<-ktls> along with B<-WWW> or B<-HTTP>
 are specified.
 
+=item B<-zerocopy_sendfile>
+
+If this option is set, SSL_sendfile() will use the zerocopy TX mode, which gives
+a performance boost when used with KTLS hardware offload. Note that invalid
+TLS records might be transmitted if the file is changed while being sent.
+This option depends on B<-sendfile>; when used alone, B<-sendfile> is implied,
+and a warning is shown. Note that KTLS sendfile on FreeBSD always runs in the
+zerocopy mode.
+
 =item B<-keylogfile> I<outfile>
 
 Appends TLS secrets to the specified keylog file such that external programs

diff --git a/doc/man3/SSL_CONF_cmd.pod b/doc/man3/SSL_CONF_cmd.pod
index c20df37e3b203597bdba49b9de091a0ffb1e6e59..3717c202bd6017898587317c230cef518f53d18f 100644 (file)
--- a/doc/man3/SSL_CONF_cmd.pod
+++ b/doc/man3/SSL_CONF_cmd.pod
@@ -561,6 +561,14 @@ B<RxCertificateCompression>: support receiving compressed certificates, enabled
 default. Inverse of B<SSL_OP_NO_RX_CERTIFICATE_COMPRESSION>: that is,
 B<-RxCertificateCompression> is the same as setting B<SSL_OP_NO_RX_CERTIFICATE_COMPRESSION>.
 
+B<KTLSTxZerocopySendfile>: use the zerocopy TX mode of sendfile(), which gives
+a performance boost when used with KTLS hardware offload. Note that invalid TLS
+records might be transmitted if the file is changed while being sent. This
+option has no effect if B<KTLS> is not enabled. Equivalent to
+B<SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE>. This option only applies to Linux.
+KTLS sendfile on FreeBSD doesn't offer an option to disable zerocopy and
+always runs in this mode.
+
 =item B<VerifyMode>
 
 The B<value> argument is a comma separated list of flags to set.

diff --git a/doc/man3/SSL_CTX_set_options.pod b/doc/man3/SSL_CTX_set_options.pod
index 1da057adb83721025d43662fec8f7a67b27d98fc..b72973f8d06616dde20f78255127ace68d6206f8 100644 (file)
--- a/doc/man3/SSL_CTX_set_options.pod
+++ b/doc/man3/SSL_CTX_set_options.pod
@@ -175,6 +175,16 @@ by the kernel directly and not via any available OpenSSL Providers. This might
 be undesirable if, for example, the application requires all cryptographic
 operations to be performed by the FIPS provider.
 
+=item SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE
+
+With this option, sendfile() will use the zerocopy mode, which gives a
+performance boost when used with KTLS hardware offload. Note that invalid TLS
+records might be transmitted if the file is changed while being sent. This
+option has no effect if B<SSL_OP_ENABLE_KTLS> is not enabled.
+
+This option only applies to Linux. KTLS sendfile on FreeBSD doesn't offer an
+option to disable zerocopy and always runs in this mode.
+
 =item SSL_OP_ENABLE_MIDDLEBOX_COMPAT
 
 If set then dummy Change Cipher Spec (CCS) messages are sent in TLSv1.3. This

diff --git a/include/internal/bio.h b/include/internal/bio.h
index 40218e1fb05c0ab6d7968bf4b770fe6e05919629..9481f4c985e00ae69173507f17ad46b10c349af4 100644 (file)
--- a/include/internal/bio.h
+++ b/include/internal/bio.h
@@ -43,16 +43,20 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read);
 # define BIO_CTRL_SET_KTLS                      72
 # define BIO_CTRL_SET_KTLS_TX_SEND_CTRL_MSG     74
 # define BIO_CTRL_CLEAR_KTLS_TX_CTRL_MSG        75
+# define BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE 90
 
 /*
  * This is used with socket BIOs:
  * BIO_FLAGS_KTLS_TX means we are using ktls with this BIO for sending.
  * BIO_FLAGS_KTLS_TX_CTRL_MSG means we are about to send a ctrl message next.
  * BIO_FLAGS_KTLS_RX means we are using ktls with this BIO for receiving.
+ * BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE means we are using the zerocopy mode with
+ * this BIO for sending using sendfile.
  */
 # define BIO_FLAGS_KTLS_TX_CTRL_MSG 0x1000
 # define BIO_FLAGS_KTLS_RX          0x2000
 # define BIO_FLAGS_KTLS_TX          0x4000
+# define BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE 0x8000
 
 /* KTLS related controls and flags */
 # define BIO_set_ktls_flag(b, is_tx) \
@@ -65,6 +69,8 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read);
     BIO_test_flags(b, BIO_FLAGS_KTLS_TX_CTRL_MSG)
 # define BIO_clear_ktls_ctrl_msg_flag(b) \
     BIO_clear_flags(b, BIO_FLAGS_KTLS_TX_CTRL_MSG)
+# define BIO_set_ktls_zerocopy_sendfile_flag(b) \
+    BIO_set_flags(b, BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE)
 
 # define BIO_set_ktls(b, keyblob, is_tx)   \
      BIO_ctrl(b, BIO_CTRL_SET_KTLS, is_tx, keyblob)
@@ -72,6 +78,8 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read);
      BIO_ctrl(b, BIO_CTRL_SET_KTLS_TX_SEND_CTRL_MSG, record_type, NULL)
 # define BIO_clear_ktls_ctrl_msg(b) \
      BIO_ctrl(b, BIO_CTRL_CLEAR_KTLS_TX_CTRL_MSG, 0, NULL)
+# define BIO_set_ktls_tx_zerocopy_sendfile(b) \
+     BIO_ctrl(b, BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE, 0, NULL)
 
 /* Functions to allow the core to offer the CORE_BIO type to providers */
 OSSL_CORE_BIO *ossl_core_bio_new_from_bio(BIO *bio);

diff --git a/include/internal/ktls.h b/include/internal/ktls.h
index efa4451d7c80f3e4dd3a59df092e9c10c4e1f62d..af27a32569713efc29ec22862797a24d5226e50e 100644 (file)
--- a/include/internal/ktls.h
+++ b/include/internal/ktls.h
@@ -214,6 +214,13 @@ static ossl_inline ossl_ssize_t ktls_sendfile(int s, int fd, off_t off,
 #     warning "Skipping Compilation of KTLS receive data path"
 #    endif
 #   endif
+#   if LINUX_VERSION_CODE < KERNEL_VERSION(5, 19, 0)
+#    define OPENSSL_NO_KTLS_ZC_TX
+#    ifndef PEDANTIC
+#     warning "KTLS requires Kernel Headers >= 5.19.0 for zerocopy sendfile"
+#     warning "Skipping Compilation of KTLS zerocopy sendfile"
+#    endif
+#   endif
 #   define OPENSSL_KTLS_AES_GCM_128
 #   if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0)
 #    define OPENSSL_KTLS_AES_GCM_256
@@ -293,6 +300,18 @@ static ossl_inline int ktls_start(int fd, ktls_crypto_info_t *crypto_info,
                       crypto_info, crypto_info->tls_crypto_info_len) ? 0 : 1;
 }
 
+static ossl_inline int ktls_enable_tx_zerocopy_sendfile(int fd)
+{
+#ifndef OPENSSL_NO_KTLS_ZC_TX
+    int enable = 1;
+
+    return setsockopt(fd, SOL_TLS, TLS_TX_ZEROCOPY_RO,
+                      &enable, sizeof(enable)) ? 0 : 1;
+#else
+    return 0;
+#endif
+}
+
 /*
  * Send a TLS record using the crypto_info provided in ktls_start and use
  * record_type instead of the default SSL3_RT_APPLICATION_DATA.

diff --git a/include/openssl/bio.h.in b/include/openssl/bio.h.in
index 6105c602fd8ceeeb91582d0787483e9acd835cd7..89c88c67e7947e3ea7e8d7e50c6b7a0fe0ece378 100644 (file)
--- a/include/openssl/bio.h.in
+++ b/include/openssl/bio.h.in
@@ -182,6 +182,11 @@ extern "C" {
 # define BIO_CTRL_DGRAM_GET_NO_TRUNC            88
 # define BIO_CTRL_DGRAM_SET_NO_TRUNC            89
 
+/*
+ * internal BIO:
+ * # define BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE 90
+ */
+
 # define BIO_DGRAM_CAP_NONE                 0U
 # define BIO_DGRAM_CAP_HANDLES_SRC_ADDR     (1U << 0)
 # define BIO_DGRAM_CAP_HANDLES_DST_ADDR     (1U << 1)
@@ -225,7 +230,7 @@ extern "C" {
 # define BIO_FLAGS_NONCLEAR_RST  0x400
 # define BIO_FLAGS_IN_EOF        0x800
 
-/* the BIO FLAGS values 0x1000 to 0x4000 are reserved for internal KTLS flags */
+/* the BIO FLAGS values 0x1000 to 0x8000 are reserved for internal KTLS flags */
 
 typedef union bio_addr_st BIO_ADDR;
 typedef struct bio_addrinfo_st BIO_ADDRINFO;

diff --git a/include/openssl/ssl.h.in b/include/openssl/ssl.h.in
index 2224b3269b60c39953e56dfc17956fdb6fcf0201..871ad265c54611318fb903ca6c1f6b739da8fc08 100644 (file)
--- a/include/openssl/ssl.h.in
+++ b/include/openssl/ssl.h.in
@@ -420,6 +420,8 @@ typedef int (*SSL_async_callback_fn)(SSL *s, void *arg);
  */
 # define SSL_OP_NO_TX_CERTIFICATE_COMPRESSION            SSL_OP_BIT(32)
 # define SSL_OP_NO_RX_CERTIFICATE_COMPRESSION            SSL_OP_BIT(33)
+    /* Enable KTLS TX zerocopy on Linux */
+# define SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE         SSL_OP_BIT(34)
 
 /*
  * Option "collections."

diff --git a/ssl/record/methods/ktls_meth.c b/ssl/record/methods/ktls_meth.c
index 5c94837dc0b98787beba1b9cc184ed43d9d17e7c..2f9b11a512ac5d55c26a430a6a291c57bb6c625a 100644 (file)
--- a/ssl/record/methods/ktls_meth.c
+++ b/ssl/record/methods/ktls_meth.c
@@ -334,6 +334,14 @@ static int ktls_set_crypto_state(OSSL_RECORD_LAYER *rl, int level,
     if (!BIO_set_ktls(rl->bio, &crypto_info, rl->direction))
         return OSSL_RECORD_RETURN_NON_FATAL_ERR;
 
+    if (rl->direction == OSSL_RECORD_DIRECTION_WRITE &&
+        (rl->options & SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE) != 0)
+        /* Ignore errors. The application opts in to using the zerocopy
+         * optimization. If the running kernel doesn't support it, just
+         * continue without the optimization.
+         */
+        BIO_set_ktls_tx_zerocopy_sendfile(rl->bio);
+
     return OSSL_RECORD_RETURN_SUCCESS;
 }
 

diff --git a/ssl/ssl_conf.c b/ssl/ssl_conf.c
index bebfc501a91412a52a94e94ac7a4ff301efc9776..0bea29fb66cf31884c9183989950ddcbe38552f9 100644 (file)
--- a/ssl/ssl_conf.c
+++ b/ssl/ssl_conf.c
@@ -400,6 +400,7 @@ static int cmd_Options(SSL_CONF_CTX *cctx, const char *value)
         SSL_FLAG_TBL_CERT("StrictCertCheck", SSL_CERT_FLAG_TLS_STRICT),
         SSL_FLAG_TBL_INV("TxCertificateCompression", SSL_OP_NO_TX_CERTIFICATE_COMPRESSION),
         SSL_FLAG_TBL_INV("RxCertificateCompression", SSL_OP_NO_RX_CERTIFICATE_COMPRESSION),
+        SSL_FLAG_TBL("KTLSTxZerocopySendfile", SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE),
     };
     if (value == NULL)
         return -3;

diff --git a/test/sslapitest.c b/test/sslapitest.c
index 8f14381b56f0d844a30e5454fedaaa6933941a3e..a26f6286f301fed7efedeb92293bafb9be96109b 100644 (file)
--- a/test/sslapitest.c
+++ b/test/sslapitest.c
@@ -1293,7 +1293,8 @@ end:
 #define SENDFILE_CHUNK                  (4 * 4096)
 #define min(a,b)                        ((a) > (b) ? (b) : (a))
 
-static int execute_test_ktls_sendfile(int tls_version, const char *cipher)
+static int execute_test_ktls_sendfile(int tls_version, const char *cipher,
+                                      int zerocopy)
 {
     SSL_CTX *cctx = NULL, *sctx = NULL;
     SSL *clientssl = NULL, *serverssl = NULL;
@@ -1350,6 +1351,12 @@ static int execute_test_ktls_sendfile(int tls_version, const char *cipher)
     if (!TEST_true(SSL_set_options(serverssl, SSL_OP_ENABLE_KTLS)))
         goto end;
 
+    if (zerocopy) {
+        if (!TEST_true(SSL_set_options(serverssl,
+                                       SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE)))
+            goto end;
+    }
+
     if (!TEST_true(create_ssl_connection(serverssl, clientssl,
                                          SSL_ERROR_NONE)))
         goto end;
@@ -1480,14 +1487,16 @@ static int test_ktls(int test)
                              cipher->cipher);
 }
 
-static int test_ktls_sendfile(int tst)
+static int test_ktls_sendfile(int test)
 {
     struct ktls_test_cipher *cipher;
+    int tst = test >> 1;
 
     OPENSSL_assert(tst < (int)NUM_KTLS_TEST_CIPHERS);
     cipher = &ktls_test_ciphers[tst];
 
-    return execute_test_ktls_sendfile(cipher->tls_version, cipher->cipher);
+    return execute_test_ktls_sendfile(cipher->tls_version, cipher->cipher,
+                                      test & 1);
 }
 #endif
 
@@ -10544,7 +10553,7 @@ int setup_tests(void)
 #if !defined(OPENSSL_NO_KTLS) && !defined(OPENSSL_NO_SOCK)
 # if !defined(OPENSSL_NO_TLS1_2) || !defined(OSSL_NO_USABLE_TLS1_3)
     ADD_ALL_TESTS(test_ktls, NUM_KTLS_TEST_CIPHERS * 4);
-    ADD_ALL_TESTS(test_ktls_sendfile, NUM_KTLS_TEST_CIPHERS);
+    ADD_ALL_TESTS(test_ktls_sendfile, NUM_KTLS_TEST_CIPHERS * 2);
 # endif
 #endif
     ADD_TEST(test_large_message_tls);