Defines and strings for special salt length values, add tests

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2236)

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index ae844eaf1f3238995afbda6de9a8aceaa109e03e..20a27be7e21281f792a05afb4c21369388d39fdb 100644 (file)
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -540,7 +540,7 @@ static RSA_PSS_PARAMS *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
         saltlen = EVP_MD_size(sigmd);
     else if (saltlen == -2) {
         saltlen = EVP_PKEY_size(pk) - EVP_MD_size(sigmd) - 2;
-        if (((EVP_PKEY_bits(pk) - 1) & 0x7) == 0)
+        if ((EVP_PKEY_bits(pk) & 0x7) == 1)
             saltlen--;
     }
 

diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index c31b9a3cb8a92e78640845d00d3a3e8d1fa9ee9d..d4b278ba5064c531f519c86cba791cf60826b05b 100644 (file)
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -58,7 +58,8 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
         rctx->pad_mode = RSA_PKCS1_PSS_PADDING;
     else
         rctx->pad_mode = RSA_PKCS1_PADDING;
-    rctx->saltlen = -2;
+    /* Maximum for sign, auto for verify */
+    rctx->saltlen = RSA_PSS_SALTLEN_AUTO;
     rctx->min_saltlen = -1;
     ctx->data = rctx;
     ctx->keygen_info = rctx->gentmp;
@@ -430,14 +431,16 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
         if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN) {
             *(int *)p2 = rctx->saltlen;
         } else {
-            if (p1 < -2)
+            if (p1 < RSA_PSS_SALTLEN_MAX)
                 return -2;
             if (rsa_pss_restricted(rctx)) {
-                if (p1 == -2 && ctx->operation == EVP_PKEY_OP_VERIFY) {
+                if (p1 == RSA_PSS_SALTLEN_AUTO
+                    && ctx->operation == EVP_PKEY_OP_VERIFY) {
                     RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
                     return -2;
                 }
-                if ((p1 == -1 && rctx->min_saltlen > EVP_MD_size(rctx->md))
+                if ((p1 == RSA_PSS_SALTLEN_DIGEST
+                     && rctx->min_saltlen > EVP_MD_size(rctx->md))
                     || (p1 >= 0 && p1 < rctx->min_saltlen)) {
                     RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_PSS_SALTLEN_TOO_SMALL);
                     return 0;
@@ -596,7 +599,14 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
 
     if (strcmp(type, "rsa_pss_saltlen") == 0) {
         int saltlen;
-        saltlen = atoi(value);
+        if (!strcmp(value, "digest"))
+            saltlen = RSA_PSS_SALTLEN_DIGEST;
+        else if (!strcmp(value, "max"))
+            saltlen = RSA_PSS_SALTLEN_MAX;
+        else if (!strcmp(value, "auto"))
+            saltlen = RSA_PSS_SALTLEN_AUTO;
+        else
+            saltlen = atoi(value);
         return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
     }
 

diff --git a/crypto/rsa/rsa_pss.c b/crypto/rsa/rsa_pss.c
index 0ec63b2ec7e7ad2c6740e1e244415f3d9da632b2..0a6178b0c40bbf745174f4d106e87cd81b91ef92 100644 (file)
--- a/crypto/rsa/rsa_pss.c
+++ b/crypto/rsa/rsa_pss.c
@@ -41,7 +41,6 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
     EVP_MD_CTX *ctx = EVP_MD_CTX_new();
     unsigned char H_[EVP_MAX_MD_SIZE];
 
-
     if (ctx == NULL)
         goto err;
 
@@ -57,11 +56,9 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
      *      -2      salt length is autorecovered from signature
      *      -N      reserved
      */
-    if (sLen == -1)
+    if (sLen == RSA_PSS_SALTLEN_DIGEST)
         sLen = hLen;
-    else if (sLen == -2)
-        sLen = -2;
-    else if (sLen < -2) {
+    else if (sLen < RSA_PSS_SALTLEN_MAX) {
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);
         goto err;
     }
@@ -76,7 +73,9 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
         EM++;
         emLen--;
     }
-    if (emLen < (hLen + sLen + 2)) { /* sLen can be small negative */
+    if (sLen == RSA_PSS_SALTLEN_MAX) {
+        sLen = emLen - hLen - 2;
+    } else if (emLen < (hLen + sLen + 2)) { /* sLen can be small negative */
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_DATA_TOO_LARGE);
         goto err;
     }
@@ -102,7 +101,7 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_RECOVERY_FAILED);
         goto err;
     }
-    if (sLen >= 0 && (maskedDBLen - i) != sLen) {
+    if (sLen != RSA_PSS_SALTLEN_AUTO && (maskedDBLen - i) != sLen) {
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);
         goto err;
     }
@@ -160,11 +159,11 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
      *      -2      salt length is maximized
      *      -N      reserved
      */
-    if (sLen == -1)
+    if (sLen == RSA_PSS_SALTLEN_DIGEST)
         sLen = hLen;
-    else if (sLen == -2)
-        sLen = -2;
-    else if (sLen < -2) {
+    else if (sLen == RSA_PSS_SALTLEN_MAX_SIGN)
+        sLen = RSA_PSS_SALTLEN_MAX;
+    else if (sLen < RSA_PSS_SALTLEN_MAX) {
         RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);
         goto err;
     }
@@ -175,7 +174,7 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
         *EM++ = 0;
         emLen--;
     }
-    if (sLen == -2) {
+    if (sLen == RSA_PSS_SALTLEN_MAX) {
         sLen = emLen - hLen - 2;
     } else if (emLen < (hLen + sLen + 2)) {
         RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS_MGF1,

diff --git a/doc/man1/pkeyutl.pod b/doc/man1/pkeyutl.pod
index 6fc03254642a7e72abacab5d99cb88ffedececef..310c5ccdcb27b874290580eaf8e5884c69f7e8ec 100644 (file)
--- a/doc/man1/pkeyutl.pod
+++ b/doc/man1/pkeyutl.pod
@@ -215,11 +215,11 @@ specified.
 
 =item B<rsa_pss_saltlen:len>
 
-For B<pss> mode only this option specifies the salt length. Two special values
-are supported: -1 sets the salt length to the digest length. When signing -2
-sets the salt length to the maximum permissible value. When verifying -2 causes
-the salt length to be automatically determined based on the B<PSS> block
-structure.
+For B<pss> mode only this option specifies the salt length. Three special
+values are supported: "digest" sets the salt length to the digest length,
+"max" sets the salt length to the maximum permissible value. When verifying
+"auto" causes the salt length to be automatically determined based on the
+B<PSS> block structure.
 
 =item B<rsa_mgf1_md:digest>
 

diff --git a/doc/man3/EVP_PKEY_CTX_ctrl.pod b/doc/man3/EVP_PKEY_CTX_ctrl.pod
index a30450bb4651e9627a92e0dfc09388413827b3f1..0732a05779693f999544e72f1124ae799e3f09dc 100644 (file)
--- a/doc/man3/EVP_PKEY_CTX_ctrl.pod
+++ b/doc/man3/EVP_PKEY_CTX_ctrl.pod
@@ -82,12 +82,13 @@ if this control is called. If it is not called then the first byte of the plaint
 buffer is expected to be the algorithm identifier byte.
 
 The EVP_PKEY_CTX_set_rsa_pss_saltlen() macro sets the RSA PSS salt length to
-B<len> as its name implies it is only supported for PSS padding.  Two special
-values are supported: -1 sets the salt length to the digest length. When
-signing -2 sets the salt length to the maximum permissible value. When
-verifying -2 causes the salt length to be automatically determined based on the
-B<PSS> block structure. If this macro is not called a salt length value of -2
-is used by default.
+B<len> as its name implies it is only supported for PSS padding.  Three special
+values are supported: RSA_PSS_SALTLEN_DIGEST sets the salt length to the
+digest length, RSA_PSS_SALTLEN_MAX sets the salt length to the maximum
+permissible value. When verifying RSA_PSS_SALTLEN_AUTO causes the salt length
+to be automatically determined based on the B<PSS> block structure. If this
+macro is not called maximum salt length is used when signing and auto detection
+when verifying is used by default.
 
 The EVP_PKEY_CTX_set_rsa_rsa_keygen_bits() macro sets the RSA key length for
 RSA key generation to B<bits>. If not specified 1024 bits is used.

diff --git a/doc/man3/EVP_PKEY_CTX_set_rsa_pss_keygen_md.pod b/doc/man3/EVP_PKEY_CTX_set_rsa_pss_keygen_md.pod
index 853d4b8d365153284aecd0589105b5bfd130e96c..eb9641433eb25bda9353392f00d9768ef19fbbc5 100644 (file)
--- a/doc/man3/EVP_PKEY_CTX_set_rsa_pss_keygen_md.pod
+++ b/doc/man3/EVP_PKEY_CTX_set_rsa_pss_keygen_md.pod
@@ -44,8 +44,9 @@ than B<PSS>. It is otherwise similar to the B<RSA> version.
 The EVP_PKEY_CTX_set_rsa_pss_saltlen() macro is used to set the salt length.
 If the key has usage restrictions then an error is returned if an attempt is
 made to set the salt length below the minimum value. It is otherwise similar
-to the B<RSA> operation except detection of the salt length (using -2) is
-not supported for verification if the key has usage restrictions.
+to the B<RSA> operation except detection of the salt length (using
+RSA_PSS_SALTLEN_AUTO is not supported for verification if the key has
+usage restrictions.
 
 The EVP_PKEY_CTX_set_signature_md() and EVP_PKEY_CTX_set_rsa_mgf1_md() macros
 are used to set the digest and MGF1 algorithms respectively. If the key has

diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
index b9179b36ffc0f99ab1deba000c7a6348837436b1..8ad4cdaf1ea157f1935ee24c904e0b3c303456d7 100644 (file)
--- a/include/openssl/rsa.h
+++ b/include/openssl/rsa.h
@@ -94,6 +94,14 @@ extern "C" {
 # define EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, len) \
         RSA_pkey_ctx_ctrl(ctx, (EVP_PKEY_OP_SIGN|EVP_PKEY_OP_VERIFY), \
                           EVP_PKEY_CTRL_RSA_PSS_SALTLEN, len, NULL)
+/* Salt length matches digest */
+# define RSA_PSS_SALTLEN_DIGEST -1
+/* Verify only: auto detect salt length */
+# define RSA_PSS_SALTLEN_AUTO   -2
+/* Set salt length to maximum possible */
+# define RSA_PSS_SALTLEN_MAX    -3
+/* Old compatible max salt length for sign only */
+# define RSA_PSS_SALTLEN_MAX_SIGN    -2
 
 # define EVP_PKEY_CTX_set_rsa_pss_keygen_saltlen(ctx, len) \
         EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA_PSS, EVP_PKEY_OP_KEYGEN, \

diff --git a/test/evptests.txt b/test/evptests.txt
index 566b2f0e8df2418bd0da35a292c3b3e97ea51410..91830bcd7b51b154cd5d60c1fc0866b4dbf55d35 100644 (file)
--- a/test/evptests.txt
+++ b/test/evptests.txt
@@ -2931,6 +2931,13 @@ Ctrl = digest:sha256
 Input="0123456789ABCDEF0123456789ABCDEF"
 Output=4DE433D5844043EF08D354DA03CB29068780D52706D7D1E4D50EFB7D58C9D547D83A747DDD0635A96B28F854E50145518482CB49E963054621B53C60C498D07C16E9C2789C893CF38D4D86900DE71BDE463BD2761D1271E358C7480A1AC0BAB930DDF39602AD1BC165B5D7436B516B7A7858E8EB7AB1C420EEB482F4D207F0E462B1724959320A084E13848D11D10FB593E66BF680BF6D3F345FC3E9C3DE60ABBAC37E1C6EC80A268C8D9FC49626C679097AA690BC1AA662B95EB8DB70390861AA0898229F9349B4B5FDD030D4928C47084708A933144BE23BD3C6E661B85B2C0EF9ED36D498D5B7320E8194D363D4AD478C059BAE804181965E0B81B663158A
 
+# Verify using salt length auto detect
+Verify = RSA-2048-PUBLIC
+Ctrl = rsa_padding_mode:pss
+Ctrl = rsa_pss_saltlen:auto
+Input="0123456789ABCDEF0123"
+Output = 6BF7EDC63A0BA184EEEC7F3020FEC8F5EBF38C2B76481881F48BCCE5796E7AB294548BA9AE810457C7723CABD1BDE94CF59CF7C0FC7461B22760C8ED703DD98E97BFDD61FA8D1181C411F6DEE5FF159F4850746D78EDEE385A363DC28E2CB373D5CAD7953F3BD5E639BE345732C03A1BDEA268814DA036EB1891C82D4012F3B903D86636055F87B96FC98806AD1B217685A4D754046A5DE0B0D7870664BE07902153EC85BA457BE7D7F89D7FE0F626D02A9CBBB2BB479DDA1A5CAE75247FB7BF6BFB15C1D3FD9E6B1573CCDBC72011C3B97716058BB11C7EA2E4E56ADAFE1F5DE6A7FD405AC5890100F9C3408EFFB5C73BF73F48177FF743B4B819D0699D507B
+
 # Digest too short
 Verify = RSA-2048-PUBLIC
 Ctrl = rsa_padding_mode:pss
@@ -3049,10 +3056,10 @@ Ctrl = digest:sha1
 Input="0123456789ABCDEF0123"
 Output = 3EFE09D88509027D837BFA5F8471CF7B69E6DF395DD999BB9CA42021F15722D9AC76670507C6BCFB73F64FB2211B611B8F140E76EBDB064BD762FDBA89D019E304A0D6B274E1C2FE1DF50005598A0306AF805416094E2A5BA60BC72BDE38CE061E853ED40F14967A8B9CA4DC739B462F89558F12FDF2D8D19FBEF16AD66FE2DDDA8BEE983ECBD873064244849D8D94B5B33F45E076871A47ED653E73257A2BE2DB3C0878094B0D2B6B682C8007DFD989425FB39A1FEEC9EED5876414601A49176EC344F5E3EDEE81CA2DDD29B7364F4638112CB3A547E2BC170E28CB66BDABE863754BE8AD5BA230567B575266F4B6B4CF81F28310ABF05351CC9E2DB85D00BF
 
-# Verify using default parameters, explicitly setting parameters -1 salt length
+# Verify explicitly setting parameters "digest" salt length
 Verify = RSA-PSS-DEFAULT
 Ctrl = rsa_padding_mode:pss
-Ctrl = rsa_pss_saltlen:-1
+Ctrl = rsa_pss_saltlen:digest
 Ctrl = digest:sha1
 Input="0123456789ABCDEF0123"
 Output = 3EFE09D88509027D837BFA5F8471CF7B69E6DF395DD999BB9CA42021F15722D9AC76670507C6BCFB73F64FB2211B611B8F140E76EBDB064BD762FDBA89D019E304A0D6B274E1C2FE1DF50005598A0306AF805416094E2A5BA60BC72BDE38CE061E853ED40F14967A8B9CA4DC739B462F89558F12FDF2D8D19FBEF16AD66FE2DDDA8BEE983ECBD873064244849D8D94B5B33F45E076871A47ED653E73257A2BE2DB3C0878094B0D2B6B682C8007DFD989425FB39A1FEEC9EED5876414601A49176EC344F5E3EDEE81CA2DDD29B7364F4638112CB3A547E2BC170E28CB66BDABE863754BE8AD5BA230567B575266F4B6B4CF81F28310ABF05351CC9E2DB85D00BF
@@ -3063,6 +3070,12 @@ Ctrl = rsa_pss_saltlen:30
 Input="0123456789ABCDEF0123"
 Output = 6BF7EDC63A0BA184EEEC7F3020FEC8F5EBF38C2B76481881F48BCCE5796E7AB294548BA9AE810457C7723CABD1BDE94CF59CF7C0FC7461B22760C8ED703DD98E97BFDD61FA8D1181C411F6DEE5FF159F4850746D78EDEE385A363DC28E2CB373D5CAD7953F3BD5E639BE345732C03A1BDEA268814DA036EB1891C82D4012F3B903D86636055F87B96FC98806AD1B217685A4D754046A5DE0B0D7870664BE07902153EC85BA457BE7D7F89D7FE0F626D02A9CBBB2BB479DDA1A5CAE75247FB7BF6BFB15C1D3FD9E6B1573CCDBC72011C3B97716058BB11C7EA2E4E56ADAFE1F5DE6A7FD405AC5890100F9C3408EFFB5C73BF73F48177FF743B4B819D0699D507B
 
+# Verify using maximum salt length
+Verify = RSA-PSS-DEFAULT
+Ctrl = rsa_pss_saltlen:max
+Input="0123456789ABCDEF0123"
+Output = 4470DCFE812DEE2E58E4301D4ED274AB348FE040B724B2CD1D8CD0914BFF375F0B86FCB32BFA8AEA9BD22BD7C4F1ADD4F3D215A5CFCC99055BAFECFC23800E9BECE19A08C66BEBC5802122D13A732E5958FC228DCC0B49B5B4B1154F032D8FA2F3564AA949C1310CC9266B0C47F86D449AC9D2E7678347E7266E2D7C888CCE1ADF44A109A293F8516AE2BD94CE220F26E137DB8E7A66BB9FCE052CDC1D0BE24D8CEBB20D10125F26B069F117044B9E1D16FDDAABCA5340AE1702F37D0E1C08A2E93801C0A41035C6C73DA02A0E32227EAFB0B85E79107B59650D0EE7DC32A6772CCCE90F06369B2880FE87ED76997BA61F5EA818091EE88F8B0D6F24D02A3FC6
+
 # Attempt to change salt length below minimum
 Verify = RSA-PSS-DEFAULT
 Ctrl = rsa_pss_saltlen:0