signature: Clamp PSS salt len to MD len

FIPS 186-4 section 5 "The RSA Digital Signature Algorithm", subsection 5.5 "PKCS #1" says: "For RSASSA-PSS […] the length (in bytes) of the salt (sLen) shall satisfy 0 <= sLen <= hLen, where hLen is the length of the hash function output block (in bytes)." Introduce a new option RSA_PSS_SALTLEN_AUTO_DIGEST_MAX and make it the default. The new value will behave like RSA_PSS_SALTLEN_AUTO, but will not use more than the digest length when signing, so that FIPS 186-4 is not violated. This value has two advantages when compared with RSA_PSS_SALTLEN_DIGEST: (1) It will continue to do auto-detection when verifying signatures for maximum compatibility, where RSA_PSS_SALTLEN_DIGEST would fail for other digest sizes. (2) It will work for combinations where the maximum salt length is smaller than the digest size, which typically happens with large digest sizes (e.g., SHA-512) and small RSA keys. J.-S. Coron shows in "Optimal Security Proofs for PSS and Other Signature Schemes. Advances in Cryptology – Eurocrypt 2002, volume 2332 of Lecture Notes in Computer Science, pp. 272 – 287. Springer Verlag, 2002." that longer salts than the output size of modern hash functions do not increase security: "For example,for an application in which at most one billion signatures will be generated, k0 = 30 bits of random salt are actually sufficient to guarantee the same level of security as RSA, and taking a larger salt does not increase the security level." Signed-off-by: Clemens Lang <cllang@redhat.com> Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com> Reviewed-by: Tomas Mraz <tomas@openssl.org> (cherry picked from commit 6c73ca4) (Merged from #19862)
openssl · Dec 9, 2022 · db2fc00 · db2fc00
1 parent fea151a
commit db2fc00
Show file tree

Hide file tree

Showing 13 changed files with 223 additions and 21 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -85,6 +85,15 @@ OpenSSL 3.1
 
    *Tomáš Mráz*
 
+ * Change the default salt length for PKCS#1 RSASSA-PSS signatures to the
+   maximum size that is smaller or equal to the digest length to comply with
+   FIPS 186-4 section 5. This is implemented by a new option
+   `OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO_DIGEST_MAX` ("auto-digestmax") for the
+   `rsa_pss_saltlen` parameter, which is now the default. Signature
+   verification is not affected by this change and continues to work as before.
+
+   *Clemens Lang*
+
 OpenSSL 3.0
 -----------
 

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
@@ -450,21 +450,35 @@ static RSA_PSS_PARAMS *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
     const EVP_MD *sigmd, *mgf1md;
     EVP_PKEY *pk = EVP_PKEY_CTX_get0_pkey(pkctx);
     int saltlen;
+    int saltlenMax = -1;
 
     if (EVP_PKEY_CTX_get_signature_md(pkctx, &sigmd) <= 0)
         return NULL;
     if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
         return NULL;
     if (EVP_PKEY_CTX_get_rsa_pss_saltlen(pkctx, &saltlen) <= 0)
         return NULL;
-    if (saltlen == -1) {
+    if (saltlen == RSA_PSS_SALTLEN_DIGEST) {
         saltlen = EVP_MD_get_size(sigmd);
-    } else if (saltlen == -2 || saltlen == -3) {
+    } else if (saltlen == RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) {
+        /* FIPS 186-4 section 5 "The RSA Digital Signature Algorithm",
+         * subsection 5.5 "PKCS #1" says: "For RSASSA-PSS […] the length (in
+         * bytes) of the salt (sLen) shall satisfy 0 <= sLen <= hLen, where
+         * hLen is the length of the hash function output block (in bytes)."
+         *
+         * Provide a way to use at most the digest length, so that the default
+         * does not violate FIPS 186-4. */
+        saltlen = RSA_PSS_SALTLEN_MAX;
+        saltlenMax = EVP_MD_get_size(sigmd);
+    }
+    if (saltlen == RSA_PSS_SALTLEN_MAX || saltlen == RSA_PSS_SALTLEN_AUTO) {
         saltlen = EVP_PKEY_get_size(pk) - EVP_MD_get_size(sigmd) - 2;
         if ((EVP_PKEY_get_bits(pk) & 0x7) == 1)
             saltlen--;
         if (saltlen < 0)
             return NULL;
+        if (saltlenMax >= 0 && saltlen > saltlenMax)
+            saltlen = saltlenMax;
     }
 
     return ossl_rsa_pss_params_create(sigmd, mgf1md, saltlen);

diff --git a/crypto/rsa/rsa_pss.c b/crypto/rsa/rsa_pss.c
@@ -61,11 +61,12 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
      *      -1      sLen == hLen
      *      -2      salt length is autorecovered from signature
      *      -3      salt length is maximized
+     *      -4      salt length is autorecovered from signature
      *      -N      reserved
      */
     if (sLen == RSA_PSS_SALTLEN_DIGEST) {
         sLen = hLen;
-    } else if (sLen < RSA_PSS_SALTLEN_MAX) {
+    } else if (sLen < RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) {
         ERR_raise(ERR_LIB_RSA, RSA_R_SLEN_CHECK_FAILED);
         goto err;
     }
@@ -112,7 +113,9 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
         ERR_raise(ERR_LIB_RSA, RSA_R_SLEN_RECOVERY_FAILED);
         goto err;
     }
-    if (sLen != RSA_PSS_SALTLEN_AUTO && (maskedDBLen - i) != sLen) {
+    if (sLen != RSA_PSS_SALTLEN_AUTO
+            && sLen != RSA_PSS_SALTLEN_AUTO_DIGEST_MAX
+            && (maskedDBLen - i) != sLen) {
         ERR_raise_data(ERR_LIB_RSA, RSA_R_SLEN_CHECK_FAILED,
                        "expected: %d retrieved: %d", sLen,
                        maskedDBLen - i);
@@ -160,6 +163,7 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
     int hLen, maskedDBLen, MSBits, emLen;
     unsigned char *H, *salt = NULL, *p;
     EVP_MD_CTX *ctx = NULL;
+    int sLenMax = -1;
 
     if (mgf1Hash == NULL)
         mgf1Hash = Hash;
@@ -172,13 +176,25 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
      *      -1      sLen == hLen
      *      -2      salt length is maximized
      *      -3      same as above (on signing)
+     *      -4      salt length is min(hLen, maximum salt length)
      *      -N      reserved
      */
+    /* FIPS 186-4 section 5 "The RSA Digital Signature Algorithm", subsection
+     * 5.5 "PKCS #1" says: "For RSASSA-PSS […] the length (in bytes) of the
+     * salt (sLen) shall satisfy 0 <= sLen <= hLen, where hLen is the length of
+     * the hash function output block (in bytes)."
+     *
+     * Provide a way to use at most the digest length, so that the default does
+     * not violate FIPS 186-4. */
     if (sLen == RSA_PSS_SALTLEN_DIGEST) {
         sLen = hLen;
-    } else if (sLen == RSA_PSS_SALTLEN_MAX_SIGN) {
+    } else if (sLen == RSA_PSS_SALTLEN_MAX_SIGN
+            || sLen == RSA_PSS_SALTLEN_AUTO) {
         sLen = RSA_PSS_SALTLEN_MAX;
-    } else if (sLen < RSA_PSS_SALTLEN_MAX) {
+    } else if (sLen == RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) {
+        sLen = RSA_PSS_SALTLEN_MAX;
+        sLenMax = hLen;
+    } else if (sLen < RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) {
         ERR_raise(ERR_LIB_RSA, RSA_R_SLEN_CHECK_FAILED);
         goto err;
     }
@@ -195,6 +211,8 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
     }
     if (sLen == RSA_PSS_SALTLEN_MAX) {
         sLen = emLen - hLen - 2;
+        if (sLenMax >= 0 && sLen > sLenMax)
+            sLen = sLenMax;
     } else if (sLen > emLen - hLen - 2) {
         ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
         goto err;

diff --git a/doc/man3/EVP_PKEY_CTX_ctrl.pod b/doc/man3/EVP_PKEY_CTX_ctrl.pod
@@ -270,8 +270,8 @@ EVP_PKEY_CTX_get_rsa_padding() gets the RSA padding mode for I<ctx>.
 
 EVP_PKEY_CTX_set_rsa_pss_saltlen() sets the RSA PSS salt length to I<saltlen>.
 As its name implies it is only supported for PSS padding. If this function is
-not called then the maximum salt length is used when signing and auto detection
-when verifying. Three special values are supported:
+not called then the salt length is maximized up to the digest length when
+signing and auto detection when verifying. Four special values are supported:
 
 =over 4
 
@@ -289,6 +289,13 @@ causes the salt length to be automatically determined based on the
 B<PSS> block structure when verifying.  When signing, it has the same
 meaning as B<RSA_PSS_SALTLEN_MAX>.
 
+=item B<RSA_PSS_SALTLEN_AUTO_DIGEST_MAX>
+
+causes the salt length to be automatically determined based on the B<PSS> block
+structure when verifying, like B<RSA_PSS_SALTLEN_AUTO>.  When signing, the salt
+length is maximized up to a maximum of the digest length to comply with FIPS
+186-4 section 5.5.
+
 =back
 
 EVP_PKEY_CTX_get_rsa_pss_saltlen() gets the RSA PSS salt length for I<ctx>.

diff --git a/doc/man7/EVP_SIGNATURE-RSA.pod b/doc/man7/EVP_SIGNATURE-RSA.pod
@@ -68,6 +68,11 @@ Use the maximum salt length.
 
 Auto detect the salt length.
 
+=item "auto-digestmax" (B<OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO_DIGEST_MAX>)
+
+Auto detect the salt length when verifying.  Maximize the salt length up to the
+digest size when signing to comply with FIPS 186-4 section 5.5.
+
 =back
 
 =back

diff --git a/include/openssl/core_names.h b/include/openssl/core_names.h
@@ -398,6 +398,7 @@ extern "C" {
 #define OSSL_PKEY_RSA_PSS_SALT_LEN_DIGEST "digest"
 #define OSSL_PKEY_RSA_PSS_SALT_LEN_MAX    "max"
 #define OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO   "auto"
+#define OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO_DIGEST_MAX "auto-digestmax"
 
 /* Key generation parameters */
 #define OSSL_PKEY_PARAM_RSA_BITS             OSSL_PKEY_PARAM_BITS

diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
@@ -140,6 +140,9 @@ int EVP_PKEY_CTX_set_rsa_keygen_pubexp(EVP_PKEY_CTX *ctx, BIGNUM *pubexp);
 # define RSA_PSS_SALTLEN_AUTO   -2
 /* Set salt length to maximum possible */
 # define RSA_PSS_SALTLEN_MAX    -3
+/* Auto-detect on verify, set salt length to min(maximum possible, digest
+ * length) on sign */
+# define RSA_PSS_SALTLEN_AUTO_DIGEST_MAX  -4
 /* Old compatible max salt length for sign only */
 # define RSA_PSS_SALTLEN_MAX_SIGN    -2
 

diff --git a/providers/implementations/signature/rsa_sig.c b/providers/implementations/signature/rsa_sig.c
@@ -189,22 +189,36 @@ static void *rsa_newctx(void *provctx, const char *propq)
     prsactx->libctx = PROV_LIBCTX_OF(provctx);
     prsactx->flag_allow_md = 1;
     prsactx->propq = propq_copy;
-    /* Maximum for sign, auto for verify */
-    prsactx->saltlen = RSA_PSS_SALTLEN_AUTO;
+    /* Maximum up to digest length for sign, auto for verify */
+    prsactx->saltlen = RSA_PSS_SALTLEN_AUTO_DIGEST_MAX;
     prsactx->min_saltlen = -1;
     return prsactx;
 }
 
 static int rsa_pss_compute_saltlen(PROV_RSA_CTX *ctx)
 {
     int saltlen = ctx->saltlen;
-
+    int saltlenMax = -1;
+
+    /* FIPS 186-4 section 5 "The RSA Digital Signature Algorithm", subsection
+     * 5.5 "PKCS #1" says: "For RSASSA-PSS […] the length (in bytes) of the
+     * salt (sLen) shall satisfy 0 <= sLen <= hLen, where hLen is the length of
+     * the hash function output block (in bytes)."
+     *
+     * Provide a way to use at most the digest length, so that the default does
+     * not violate FIPS 186-4. */
     if (saltlen == RSA_PSS_SALTLEN_DIGEST) {
         saltlen = EVP_MD_get_size(ctx->md);
-    } else if (saltlen == RSA_PSS_SALTLEN_AUTO || saltlen == RSA_PSS_SALTLEN_MAX) {
+    } else if (saltlen == RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) {
+        saltlen = RSA_PSS_SALTLEN_MAX;
+        saltlenMax = EVP_MD_get_size(ctx->md);
+    }
+    if (saltlen == RSA_PSS_SALTLEN_MAX || saltlen == RSA_PSS_SALTLEN_AUTO) {
         saltlen = RSA_size(ctx->rsa) - EVP_MD_get_size(ctx->md) - 2;
         if ((RSA_bits(ctx->rsa) & 0x7) == 1)
             saltlen--;
+        if (saltlenMax >= 0 && saltlen > saltlenMax)
+            saltlen = saltlenMax;
     }
     if (saltlen < 0) {
         ERR_raise(ERR_LIB_PROV, ERR_R_INTERNAL_ERROR);
@@ -408,8 +422,8 @@ static int rsa_signverify_init(void *vprsactx, void *vrsa,
 
     prsactx->operation = operation;
 
-    /* Maximum for sign, auto for verify */
-    prsactx->saltlen = RSA_PSS_SALTLEN_AUTO;
+    /* Maximize up to digest length for sign, auto for verify */
+    prsactx->saltlen = RSA_PSS_SALTLEN_AUTO_DIGEST_MAX;
     prsactx->min_saltlen = -1;
 
     switch (RSA_test_flags(prsactx->rsa, RSA_FLAG_TYPE_MASK)) {
@@ -1107,6 +1121,9 @@ static int rsa_get_ctx_params(void *vprsactx, OSSL_PARAM *params)
             case RSA_PSS_SALTLEN_AUTO:
                 value = OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO;
                 break;
+            case RSA_PSS_SALTLEN_AUTO_DIGEST_MAX:
+                value = OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO_DIGEST_MAX;
+                break;
             default:
                 {
                     int len = BIO_snprintf(p->data, p->data_size, "%d",
@@ -1270,6 +1287,8 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
                 saltlen = RSA_PSS_SALTLEN_MAX;
             else if (strcmp(p->data, OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO) == 0)
                 saltlen = RSA_PSS_SALTLEN_AUTO;
+            else if (strcmp(p->data, OSSL_PKEY_RSA_PSS_SALT_LEN_AUTO_DIGEST_MAX) == 0)
+                saltlen = RSA_PSS_SALTLEN_AUTO_DIGEST_MAX;
             else
                 saltlen = atoi(p->data);
             break;
@@ -1278,18 +1297,19 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
         }
 
         /*
-         * RSA_PSS_SALTLEN_MAX seems curiously named in this check.
-         * Contrary to what it's name suggests, it's the currently
-         * lowest saltlen number possible.
+         * RSA_PSS_SALTLEN_AUTO_DIGEST_MAX seems curiously named in this check.
+         * Contrary to what it's name suggests, it's the currently lowest
+         * saltlen number possible.
          */
-        if (saltlen < RSA_PSS_SALTLEN_MAX) {
+        if (saltlen < RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) {
             ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_SALT_LENGTH);
             return 0;
         }
 
         if (rsa_pss_restricted(prsactx)) {
             switch (saltlen) {
             case RSA_PSS_SALTLEN_AUTO:
+            case RSA_PSS_SALTLEN_AUTO_DIGEST_MAX:
                 if (prsactx->operation == EVP_PKEY_OP_VERIFY) {
                     ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_SALT_LENGTH,
                                    "Cannot use autodetected salt length");

diff --git a/test/recipes/15-test_rsapss.t b/test/recipes/15-test_rsapss.t
@@ -16,7 +16,7 @@ use OpenSSL::Test::Utils;
 
 setup("test_rsapss");
 
-plan tests => 11;
+plan tests => 16;
 
 #using test/testrsa.pem which happens to be a 512 bit RSA
 ok(run(app(['openssl', 'dgst', '-sign', srctop_file('test', 'testrsa.pem'), '-sha1',
@@ -67,13 +67,48 @@ ok(run(app(['openssl', 'dgst', '-prverify', srctop_file('test', 'testrsa.pem'),
             srctop_file('test', 'testrsa.pem')])),
    "openssl dgst -sign rsa512bit.pem -sha1 -sigopt rsa_pss_saltlen:max produces 42 bits of PSS salt");
 
+ok(run(app(['openssl', 'dgst', '-prverify', srctop_file('test', 'testrsa.pem'),
+            '-sha1',
+            '-sigopt', 'rsa_padding_mode:pss',
+            '-sigopt', 'rsa_pss_saltlen:auto-digestmax',
+            '-sigopt', 'rsa_mgf1_md:sha512',
+            '-signature', 'testrsapss-restricted.sig',
+            srctop_file('test', 'testrsa.pem')])),
+   "openssl dgst -prverify rsa512bit.pem -sha1 -sigopt rsa_pss_saltlen:auto-digestmax verifies signatures with saltlen > digestlen");
+
 ok(run(app(['openssl', 'dgst', '-prverify', srctop_file('test', 'testrsa.pem'),
             '-sha1',
             '-sigopt', 'rsa_padding_mode:pss',
             '-signature', 'testrsapss-unrestricted.sig',
             srctop_file('test', 'testrsa.pem')])),
    "openssl dgst -prverify [plain RSA key, PSS padding mode, no PSS restrictions]");
 
+ok(run(app(['openssl', 'dgst', '-sign', srctop_file('test', 'testrsa.pem'), '-sha1',
+            '-sigopt', 'rsa_padding_mode:pss',
+            '-sigopt', 'rsa_pss_saltlen:auto-digestmax',
+            '-out', 'testrsapss-sha1-autodigestmax.sig',
+            srctop_file('test', 'testrsa.pem')])),
+   "openssl dgst -sign -sha1 -rsa_pss_saltlen:auto-digestmax");
+ok(run(app(['openssl', 'dgst', '-prverify', srctop_file('test', 'testrsa.pem'), '-sha1',
+            '-sigopt', 'rsa_padding_mode:pss',
+            '-sigopt', 'rsa_pss_saltlen:20',
+            '-signature', 'testrsapss-sha1-autodigestmax.sig',
+            srctop_file('test', 'testrsa.pem')])),
+   "openssl dgst -sign -sha1 -rsa_padding_mode:auto-digestmax produces 20 (i.e., digestlen) bits of PSS salt");
+
+ok(run(app(['openssl', 'dgst', '-sign', srctop_file('test', 'testrsa.pem'), '-sha256',
+            '-sigopt', 'rsa_padding_mode:pss',
+            '-sigopt', 'rsa_pss_saltlen:auto-digestmax',
+            '-out', 'testrsapss-sha256-autodigestmax.sig',
+            srctop_file('test', 'testrsa.pem')])),
+   "openssl dgst -sign -sha256 -rsa_pss_saltlen:auto-digestmax");
+ok(run(app(['openssl', 'dgst', '-prverify', srctop_file('test', 'testrsa.pem'), '-sha256',
+            '-sigopt', 'rsa_padding_mode:pss',
+            '-sigopt', 'rsa_pss_saltlen:30',
+            '-signature', 'testrsapss-sha256-autodigestmax.sig',
+            srctop_file('test', 'testrsa.pem')])),
+   "openssl dgst -sign rsa512bit.pem -sha256 -rsa_padding_mode:auto-digestmax produces 30 bits of PSS salt (due to 512bit key)");
+
 # Test that RSA-PSS keys are supported by genpkey and rsa commands.
 {
    my $rsapss = "rsapss.key";

diff --git a/test/recipes/25-test_req.t b/test/recipes/25-test_req.t
@@ -199,7 +199,7 @@ subtest "generating certificate requests with RSA-PSS" => sub {
         ok(!run(app(["openssl", "req",
                      "-config", srctop_file("test", "test.cnf"),
                      "-new", "-out", "testreq-rsapss3.pem", "-utf8",
-                     "-sigopt", "rsa_pss_saltlen:-4",
+                     "-sigopt", "rsa_pss_saltlen:-5",
                      "-key", srctop_file("test", "testrsapss.pem")])),
            "Generating request with expected failure");