S390x: Support ME and CRT offloading

S390x has to ability to offload modular exponentiation and CRT operations to
Crypto Express Adapters.  This possible performance optimization was not yet
used by OpenSSL.  Add support for offloading and implement an optimized
version of RSA and DH with it.

The environment variable OPENSSL_s390xcap now recognizes the token "nocex" to
prevent offloading.

Signed-off-by: Juergen Christ <jchrist@linux.ibm.com>
Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/20113)

diff --git a/crypto/bn/bn_s390x.c b/crypto/bn/bn_s390x.c
new file mode 100644 (file)
index 0000000..da69b02
--- /dev/null
+++ b/crypto/bn/bn_s390x.c
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2023-2023 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "crypto/bn.h"
+#include "crypto/s390x_arch.h"
+
+#ifdef S390X_MOD_EXP
+
+# include <sys/types.h>
+# include <sys/stat.h>
+# include <fcntl.h>
+# include <asm/zcrypt.h>
+# include <sys/ioctl.h>
+# include <unistd.h>
+# include <errno.h>
+
+static int s390x_mod_exp_hw(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
+                            const BIGNUM *m)
+{
+    struct ica_rsa_modexpo me;
+    unsigned char *buffer;
+    size_t size;
+    int res = 0;
+
+    if (OPENSSL_s390xcex == -1)
+        return 0;
+    size = BN_num_bytes(m);
+    buffer = OPENSSL_zalloc(4 * size);
+    if (buffer == NULL)
+        return 0;
+    me.inputdata = buffer;
+    me.inputdatalength = size;
+    me.outputdata = buffer + size;
+    me.outputdatalength = size;
+    me.b_key = buffer + 2 * size;
+    me.n_modulus = buffer + 3 * size;
+    if (BN_bn2binpad(a, me.inputdata, size) == -1
+        || BN_bn2binpad(p, me.b_key, size) == -1
+        || BN_bn2binpad(m, me.n_modulus, size) == -1)
+        goto dealloc;
+    if (ioctl(OPENSSL_s390xcex, ICARSAMODEXPO, &me) != -1) {
+        if (BN_bin2bn(me.outputdata, size, r) != NULL)
+            res = 1;
+    } else if (errno == EBADF) {
+        /*-
+         * In this cases, someone (e.g. a sandbox) closed the fd.
+         * Make sure to not further use this hardware acceleration.
+         */
+        OPENSSL_s390xcex = -1;
+    }
+ dealloc:
+    OPENSSL_clear_free(buffer, 4 * size);
+    return res;
+}
+
+int s390x_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
+                  const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx)
+{
+    if (s390x_mod_exp_hw(r, a, p, m) == 1)
+        return 1;
+    return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx);
+}
+
+int s390x_crt(BIGNUM *r, const BIGNUM *i, const BIGNUM *p, const BIGNUM *q,
+              const BIGNUM *dmp, const BIGNUM *dmq, const BIGNUM *iqmp)
+{
+    struct ica_rsa_modexpo_crt crt;
+    unsigned char *buffer, *part;
+    size_t size, plen, qlen;
+    int res = 0;
+
+    if (OPENSSL_s390xcex == -1)
+        return 0;
+    /*-
+     * Hardware-accelerated CRT can only deal with p>q.  Fall back to
+     * software in the (hopefully rare) other cases.
+     */
+    if (BN_ucmp(p, q) != 1)
+        return 0;
+    plen = BN_num_bytes(p);
+    qlen = BN_num_bytes(q);
+    size = (plen > qlen ? plen : qlen);
+    buffer = OPENSSL_zalloc(9 * size + 24);
+    if (buffer == NULL)
+        return 0;
+    part = buffer;
+    crt.inputdata = part;
+    crt.inputdatalength = 2 * size;
+    part += 2 * size;
+    crt.outputdata = part;
+    crt.outputdatalength = 2 * size;
+    part += 2 * size;
+    crt.bp_key = part;
+    part += size + 8;
+    crt.bq_key = part;
+    part += size;
+    crt.np_prime = part;
+    part += size + 8;
+    crt.nq_prime = part;
+    part += size;
+    crt.u_mult_inv = part;
+    if (BN_bn2binpad(i, crt.inputdata, crt.inputdatalength) == -1
+        || BN_bn2binpad(p, crt.np_prime, size + 8) == -1
+        || BN_bn2binpad(q, crt.nq_prime, size) == -1
+        || BN_bn2binpad(dmp, crt.bp_key, size + 8) == -1
+        || BN_bn2binpad(dmq, crt.bq_key, size) == -1
+        || BN_bn2binpad(iqmp, crt.u_mult_inv, size + 8) == -1)
+        goto dealloc;
+    if (ioctl(OPENSSL_s390xcex, ICARSACRT, &crt) != -1) {
+        if (BN_bin2bn(crt.outputdata, crt.outputdatalength, r) != NULL)
+            res = 1;
+    } else if (errno == EBADF) {
+        /*-
+         * In this cases, someone (e.g. a sandbox) closed the fd.
+         * Make sure to not further use this hardware acceleration.
+         */
+        OPENSSL_s390xcex = -1;
+    }
+ dealloc:
+    OPENSSL_clear_free(buffer, 9 * size + 24);
+    return res;
+}
+
+#else
+int s390x_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
+                  const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx)
+{
+    return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx);
+}
+
+int s390x_crt(BIGNUM *r, const BIGNUM *i, const BIGNUM *p, const BIGNUM *q,
+              const BIGNUM *dmp, const BIGNUM *dmq, const BIGNUM *iqmp)
+{
+    return 0;
+}
+
+#endif

diff --git a/crypto/bn/build.info b/crypto/bn/build.info
index 0732519a11a6011c3f62f3dd1cf696e4959d4fca..77ec6d4721e8bca88f85b5e2042888a466254410 100644 (file)
--- a/crypto/bn/build.info
+++ b/crypto/bn/build.info
@@ -58,7 +58,7 @@ IF[{- !$disabled{asm} -}]
   IF[{- ($target{perlasm_scheme} // '') eq '31' -}]
     $BNASM_s390x=bn_asm.c s390x-mont.S
   ELSE
-    $BNASM_s390x=asm/s390x.S s390x-mont.S
+    $BNASM_s390x=asm/s390x.S s390x-mont.S bn_s390x.c
   ENDIF
   $BNDEF_s390x=OPENSSL_BN_ASM_MONT
   $BNASM_s390x_ec2m=s390x-gf2m.s

diff --git a/crypto/dh/dh_key.c b/crypto/dh/dh_key.c
index 0854395fc5cc5bececeee8cc69bb78cdc47380a6..328780b9a9f1f9d6d93f91e354370fb0a16d95f0 100644 (file)
--- a/crypto/dh/dh_key.c
+++ b/crypto/dh/dh_key.c
@@ -184,7 +184,11 @@ static int dh_bn_mod_exp(const DH *dh, BIGNUM *r,
                          const BIGNUM *a, const BIGNUM *p,
                          const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx)
 {
+#ifdef S390X_MOD_EXP
+    return s390x_mod_exp(r, a, p, m, ctx, m_ctx);
+#else
     return BN_mod_exp_mont(r, a, p, m, ctx, m_ctx);
+#endif
 }
 
 static int dh_init(DH *dh)

diff --git a/crypto/rsa/rsa_ossl.c b/crypto/rsa/rsa_ossl.c
index 10d08ebc352a2ff179d7ff0f6b0ac2ad58e72a8d..ec69076e6349e5280afcb084970614723c8e7c2d 100644 (file)
--- a/crypto/rsa/rsa_ossl.c
+++ b/crypto/rsa/rsa_ossl.c
@@ -33,6 +33,27 @@ static int rsa_ossl_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa,
                            BN_CTX *ctx);
 static int rsa_ossl_init(RSA *rsa);
 static int rsa_ossl_finish(RSA *rsa);
+#ifdef S390X_MOD_EXP
+static int rsa_ossl_s390x_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa,
+                                BN_CTX *ctx);
+static RSA_METHOD rsa_pkcs1_ossl_meth = {
+    "OpenSSL PKCS#1 RSA",
+    rsa_ossl_public_encrypt,
+    rsa_ossl_public_decrypt,     /* signature verification */
+    rsa_ossl_private_encrypt,    /* signing */
+    rsa_ossl_private_decrypt,
+    rsa_ossl_s390x_mod_exp,
+    s390x_mod_exp,
+    rsa_ossl_init,
+    rsa_ossl_finish,
+    RSA_FLAG_FIPS_METHOD,       /* flags */
+    NULL,
+    0,                          /* rsa_sign */
+    0,                          /* rsa_verify */
+    NULL,                       /* rsa_keygen */
+    NULL                        /* rsa_multi_prime_keygen */
+};
+#else
 static RSA_METHOD rsa_pkcs1_ossl_meth = {
     "OpenSSL PKCS#1 RSA",
     rsa_ossl_public_encrypt,
@@ -51,6 +72,7 @@ static RSA_METHOD rsa_pkcs1_ossl_meth = {
     NULL,                       /* rsa_keygen */
     NULL                        /* rsa_multi_prime_keygen */
 };
+#endif
 
 static const RSA_METHOD *default_RSA_meth = &rsa_pkcs1_ossl_meth;
 
@@ -1118,3 +1140,16 @@ static int rsa_ossl_finish(RSA *rsa)
     BN_MONT_CTX_free(rsa->_method_mod_q);
     return 1;
 }
+
+#ifdef S390X_MOD_EXP
+static int rsa_ossl_s390x_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa,
+                                BN_CTX *ctx)
+{
+    if (rsa->version != RSA_ASN1_VERSION_MULTI) {
+        if (s390x_crt(r0, i, rsa->p, rsa->q, rsa->dmp1, rsa->dmq1, rsa->iqmp) == 1)
+            return 1;
+    }
+    return rsa_ossl_mod_exp(r0, i, rsa, ctx);
+}
+
+#endif

diff --git a/crypto/s390x_arch.h b/crypto/s390x_arch.h
index c48cde8d9d3f2981d31f4f877c3d405b0f709bde..cfb3460d5bafdfbfe221a8fcca48afb557aeba3d 100644 (file)
--- a/crypto/s390x_arch.h
+++ b/crypto/s390x_arch.h
@@ -12,6 +12,8 @@
 
 # ifndef __ASSEMBLER__
 
+#include "crypto/bn.h"
+
 void s390x_kimd(const unsigned char *in, size_t len, unsigned int fc,
                 void *param);
 void s390x_klmd(const unsigned char *in, size_t inlen, unsigned char *out,
@@ -77,6 +79,13 @@ __attribute__ ((visibility("hidden")))
 #endif
 extern struct OPENSSL_s390xcap_st OPENSSL_s390xcap_P;
 
+#ifdef S390X_MOD_EXP
+# if defined(__GNUC__) && defined(__linux)
+__attribute__ ((visibility("hidden")))
+# endif
+extern int OPENSSL_s390xcex;
+#endif
+
 /* Max number of 64-bit words currently returned by STFLE */
 #  define S390X_STFLE_MAX       3
 

diff --git a/crypto/s390xcap.c b/crypto/s390xcap.c
index 59f0f3ef452a3d9d9848194550988c86928fde1a..5d2e96805c6be520e51793804ddf6c7465f2097a 100644 (file)
--- a/crypto/s390xcap.c
+++ b/crypto/s390xcap.c
@@ -16,6 +16,15 @@
 #include "crypto/ctype.h"
 #include "s390x_arch.h"
 
+#if defined(OPENSSL_SYS_LINUX) && !defined(FIPS_MODULE)
+# include <sys/types.h>
+# include <sys/stat.h>
+# include <fcntl.h>
+# include <asm/zcrypt.h>
+# include <sys/ioctl.h>
+# include <unistd.h>
+#endif
+
 #if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
 # if __GLIBC_PREREQ(2, 16)
 #  include <sys/auxv.h>
@@ -67,19 +76,41 @@ void OPENSSL_vx_probe(void);
 #endif
 
 static const char *env;
-static int parse_env(struct OPENSSL_s390xcap_st *cap);
+static int parse_env(struct OPENSSL_s390xcap_st *cap, int *cex);
 
 void OPENSSL_s390x_facilities(void);
 void OPENSSL_s390x_functions(void);
 
 struct OPENSSL_s390xcap_st OPENSSL_s390xcap_P;
 
+#ifdef S390X_MOD_EXP
+static int probe_cex(void);
+int OPENSSL_s390xcex;
+
+#if defined(__GNUC__)
+__attribute__ ((visibility("hidden")))
+#endif
+void OPENSSL_s390x_cleanup(void);
+
+#if defined(__GNUC__)
+__attribute__ ((visibility("hidden")))
+#endif
+void OPENSSL_s390x_cleanup(void)
+{
+    if (OPENSSL_s390xcex != -1) {
+        (void)close(OPENSSL_s390xcex);
+        OPENSSL_s390xcex = -1;
+    }
+}
+#endif
+
 #if defined(__GNUC__) && defined(__linux)
 __attribute__ ((visibility("hidden")))
 #endif
 void OPENSSL_cpuid_setup(void)
 {
     struct OPENSSL_s390xcap_st cap;
+    int cex = 1;
 
     if (OPENSSL_s390xcap_P.stfle[0])
         return;
@@ -140,7 +171,7 @@ void OPENSSL_cpuid_setup(void)
 
     env = getenv("OPENSSL_s390xcap");
     if (env != NULL) {
-        if (!parse_env(&cap))
+        if (!parse_env(&cap, &cex))
             env = NULL;
     }
 
@@ -178,9 +209,52 @@ void OPENSSL_cpuid_setup(void)
         OPENSSL_s390xcap_P.kdsa[0] &= cap.kdsa[0];
         OPENSSL_s390xcap_P.kdsa[1] &= cap.kdsa[1];
     }
+
+#ifdef S390X_MOD_EXP
+    if (cex == 0) {
+        OPENSSL_s390xcex = -1;
+    } else {
+        OPENSSL_s390xcex = open("/dev/z90crypt", O_RDWR | O_CLOEXEC);
+        if (probe_cex() == 1)
+            OPENSSL_atexit(OPENSSL_s390x_cleanup);
+    }
+#endif
 }
 
-static int parse_env(struct OPENSSL_s390xcap_st *cap)
+#ifdef S390X_MOD_EXP
+static int probe_cex(void)
+{
+    struct ica_rsa_modexpo me;
+    const unsigned char inval[16] = {
+        0,0,0,0,0,0,0,0,
+        0,0,0,0,0,0,0,2
+    };
+    const unsigned char modulus[16] = {
+        0,0,0,0,0,0,0,0,
+        0,0,0,0,0,0,0,3
+    };
+    unsigned char res[16];
+    int olderrno;
+    int rc = 1;
+
+    me.inputdata = (unsigned char *)inval;
+    me.inputdatalength = sizeof(inval);
+    me.outputdata = (unsigned char *)res;
+    me.outputdatalength = sizeof(res);
+    me.b_key = (unsigned char *)inval;
+    me.n_modulus = (unsigned char *)modulus;
+    olderrno = errno;
+    if (ioctl(OPENSSL_s390xcex, ICARSAMODEXPO, &me) == -1) {
+        (void)close(OPENSSL_s390xcex);
+        OPENSSL_s390xcex = -1;
+        rc = 0;
+    }
+    errno = olderrno;
+    return rc;
+}
+#endif
+
+static int parse_env(struct OPENSSL_s390xcap_st *cap, int *cex)
 {
     /*-
      * CPU model data
@@ -732,6 +806,13 @@ static int parse_env(struct OPENSSL_s390xcap_st *cap)
         else if TOK_CPU(z15)
         else if TOK_CPU(z16)
 
+        /* nocex to deactivate cex support */
+        else if (sscanf(tok_begin, " %" STR(LEN) "s %" STR(LEN) "s ",
+                        tok[0], tok[1]) == 1
+                && !strcmp(tok[0], "nocex")) {
+            *cex = 0;
+        }
+
         /* whitespace(ignored) or invalid tokens */
         else {
             while (*tok_begin != '\0') {

diff --git a/doc/man3/OPENSSL_s390xcap.pod b/doc/man3/OPENSSL_s390xcap.pod
index 2cddec5eafbdc81357bc47f1b0156154c247db5a..a8ade1b612e09fe9bd17a95a448b63b4a0345e0c 100644 (file)
--- a/doc/man3/OPENSSL_s390xcap.pod
+++ b/doc/man3/OPENSSL_s390xcap.pod
@@ -25,7 +25,7 @@ processed from left to right (whitespace is ignored):
 
  OPENSSL_s390xcap="<tok1>;<tok2>;..."
 
-There are three types of tokens:
+There are four types of tokens:
 
 =over 4
 
@@ -51,6 +51,11 @@ Store-facility-list-extended (stfle) followed by three 64-bit masks. The
 part of the environment variable's mask corresponding to the stfle
 instruction is set to the specified 192-bit mask.
 
+=item nocex
+
+Deactivate modular exponentiation and CRT operation offloading to
+Crypto Express Adapters.
+
 =back
 
 The 64-bit masks are specified in hexadecimal notation. The 0x prefix is

diff --git a/include/crypto/bn.h b/include/crypto/bn.h
index 2453cb814e944f280219441b3bd815242a806106..b354578ee097407506d59271cbfe7787600ff843 100644 (file)
--- a/include/crypto/bn.h
+++ b/include/crypto/bn.h
@@ -122,4 +122,13 @@ int ossl_bn_rsa_do_unblind(const BIGNUM *intermediate,
                            const BIGNUM *to_mod, BN_CTX *ctx,
                            unsigned char *buf, int num);
 
+#if defined(OPENSSL_SYS_LINUX) && !defined(FIPS_MODULE) && defined (__s390x__)
+# define S390X_MOD_EXP
+#endif
+
+int s390x_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
+                const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx);
+int s390x_crt(BIGNUM *r, const BIGNUM *i, const BIGNUM *p, const BIGNUM *q,
+            const BIGNUM *dmp, const BIGNUM *dmq, const BIGNUM *iqmp);
+
 #endif