From: Dr. Stephen Henson <steve@openssl.org>
Date: Tue, 17 Jan 2017 17:51:24 +0000 (+0000)
Subject: Defines and strings for special salt length values, add tests
X-Git-Tag: OpenSSL_1_1_1-pre1~2653
X-Git-Url: https://git.openssl.org/gitweb/?p=openssl.git;a=commitdiff_plain;h=137096a7ead3738a0035b9e760b7c3f74b7555a3

Defines and strings for special salt length values, add tests

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2236)
---

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index ae844eaf1f..20a27be7e2 100644
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -540,7 +540,7 @@ static RSA_PSS_PARAMS *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
         saltlen = EVP_MD_size(sigmd);
     else if (saltlen == -2) {
         saltlen = EVP_PKEY_size(pk) - EVP_MD_size(sigmd) - 2;
-        if (((EVP_PKEY_bits(pk) - 1) & 0x7) == 0)
+        if ((EVP_PKEY_bits(pk) & 0x7) == 1)
             saltlen--;
     }
 
diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index c31b9a3cb8..d4b278ba50 100644
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -58,7 +58,8 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
         rctx->pad_mode = RSA_PKCS1_PSS_PADDING;
     else
         rctx->pad_mode = RSA_PKCS1_PADDING;
-    rctx->saltlen = -2;
+    /* Maximum for sign, auto for verify */
+    rctx->saltlen = RSA_PSS_SALTLEN_AUTO;
     rctx->min_saltlen = -1;
     ctx->data = rctx;
     ctx->keygen_info = rctx->gentmp;
@@ -430,14 +431,16 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
         if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN) {
             *(int *)p2 = rctx->saltlen;
         } else {
-            if (p1 < -2)
+            if (p1 < RSA_PSS_SALTLEN_MAX)
                 return -2;
             if (rsa_pss_restricted(rctx)) {
-                if (p1 == -2 && ctx->operation == EVP_PKEY_OP_VERIFY) {
+                if (p1 == RSA_PSS_SALTLEN_AUTO
+                    && ctx->operation == EVP_PKEY_OP_VERIFY) {
                     RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
                     return -2;
                 }
-                if ((p1 == -1 && rctx->min_saltlen > EVP_MD_size(rctx->md))
+                if ((p1 == RSA_PSS_SALTLEN_DIGEST
+                     && rctx->min_saltlen > EVP_MD_size(rctx->md))
                     || (p1 >= 0 && p1 < rctx->min_saltlen)) {
                     RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_PSS_SALTLEN_TOO_SMALL);
                     return 0;
@@ -596,7 +599,14 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
 
     if (strcmp(type, "rsa_pss_saltlen") == 0) {
         int saltlen;
-        saltlen = atoi(value);
+        if (!strcmp(value, "digest"))
+            saltlen = RSA_PSS_SALTLEN_DIGEST;
+        else if (!strcmp(value, "max"))
+            saltlen = RSA_PSS_SALTLEN_MAX;
+        else if (!strcmp(value, "auto"))
+            saltlen = RSA_PSS_SALTLEN_AUTO;
+        else
+            saltlen = atoi(value);
         return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
     }
 
diff --git a/crypto/rsa/rsa_pss.c b/crypto/rsa/rsa_pss.c
index 0ec63b2ec7..0a6178b0c4 100644
--- a/crypto/rsa/rsa_pss.c
+++ b/crypto/rsa/rsa_pss.c
@@ -41,7 +41,6 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
     EVP_MD_CTX *ctx = EVP_MD_CTX_new();
     unsigned char H_[EVP_MAX_MD_SIZE];
 
-
     if (ctx == NULL)
         goto err;
 
@@ -57,11 +56,9 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
      *      -2      salt length is autorecovered from signature
      *      -N      reserved
      */
-    if (sLen == -1)
+    if (sLen == RSA_PSS_SALTLEN_DIGEST)
         sLen = hLen;
-    else if (sLen == -2)
-        sLen = -2;
-    else if (sLen < -2) {
+    else if (sLen < RSA_PSS_SALTLEN_MAX) {
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);
         goto err;
     }
@@ -76,7 +73,9 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
         EM++;
         emLen--;
     }
-    if (emLen < (hLen + sLen + 2)) { /* sLen can be small negative */
+    if (sLen == RSA_PSS_SALTLEN_MAX) {
+        sLen = emLen - hLen - 2;
+    } else if (emLen < (hLen + sLen + 2)) { /* sLen can be small negative */
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_DATA_TOO_LARGE);
         goto err;
     }
@@ -102,7 +101,7 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_RECOVERY_FAILED);
         goto err;
     }
-    if (sLen >= 0 && (maskedDBLen - i) != sLen) {
+    if (sLen != RSA_PSS_SALTLEN_AUTO && (maskedDBLen - i) != sLen) {
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);
         goto err;
     }
@@ -160,11 +159,11 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
      *      -2      salt length is maximized
      *      -N      reserved
      */
-    if (sLen == -1)
+    if (sLen == RSA_PSS_SALTLEN_DIGEST)
         sLen = hLen;
-    else if (sLen == -2)
-        sLen = -2;
-    else if (sLen < -2) {
+    else if (sLen == RSA_PSS_SALTLEN_MAX_SIGN)
+        sLen = RSA_PSS_SALTLEN_MAX;
+    else if (sLen < RSA_PSS_SALTLEN_MAX) {
         RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);
         goto err;
     }
@@ -175,7 +174,7 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
         *EM++ = 0;
         emLen--;
     }
-    if (sLen == -2) {
+    if (sLen == RSA_PSS_SALTLEN_MAX) {
         sLen = emLen - hLen - 2;
     } else if (emLen < (hLen + sLen + 2)) {
         RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS_MGF1,
diff --git a/doc/man1/pkeyutl.pod b/doc/man1/pkeyutl.pod
index 6fc0325464..310c5ccdcb 100644
--- a/doc/man1/pkeyutl.pod
+++ b/doc/man1/pkeyutl.pod
@@ -215,11 +215,11 @@ specified.
 
 =item B<rsa_pss_saltlen:len>
 
-For B<pss> mode only this option specifies the salt length. Two special values
-are supported: -1 sets the salt length to the digest length. When signing -2
-sets the salt length to the maximum permissible value. When verifying -2 causes
-the salt length to be automatically determined based on the B<PSS> block
-structure.
+For B<pss> mode only this option specifies the salt length. Three special
+values are supported: "digest" sets the salt length to the digest length,
+"max" sets the salt length to the maximum permissible value. When verifying
+"auto" causes the salt length to be automatically determined based on the
+B<PSS> block structure.
 
 =item B<rsa_mgf1_md:digest>
 
diff --git a/doc/man3/EVP_PKEY_CTX_ctrl.pod b/doc/man3/EVP_PKEY_CTX_ctrl.pod
index a30450bb46..0732a05779 100644
--- a/doc/man3/EVP_PKEY_CTX_ctrl.pod
+++ b/doc/man3/EVP_PKEY_CTX_ctrl.pod
@@ -82,12 +82,13 @@ if this control is called. If it is not called then the first byte of the plaint
 buffer is expected to be the algorithm identifier byte.
 
 The EVP_PKEY_CTX_set_rsa_pss_saltlen() macro sets the RSA PSS salt length to
-B<len> as its name implies it is only supported for PSS padding.  Two special
-values are supported: -1 sets the salt length to the digest length. When
-signing -2 sets the salt length to the maximum permissible value. When
-verifying -2 causes the salt length to be automatically determined based on the
-B<PSS> block structure. If this macro is not called a salt length value of -2
-is used by default.
+B<len> as its name implies it is only supported for PSS padding.  Three special
+values are supported: RSA_PSS_SALTLEN_DIGEST sets the salt length to the
+digest length, RSA_PSS_SALTLEN_MAX sets the salt length to the maximum
+permissible value. When verifying RSA_PSS_SALTLEN_AUTO causes the salt length
+to be automatically determined based on the B<PSS> block structure. If this
+macro is not called maximum salt length is used when signing and auto detection
+when verifying is used by default.
 
 The EVP_PKEY_CTX_set_rsa_rsa_keygen_bits() macro sets the RSA key length for
 RSA key generation to B<bits>. If not specified 1024 bits is used.
diff --git a/doc/man3/EVP_PKEY_CTX_set_rsa_pss_keygen_md.pod b/doc/man3/EVP_PKEY_CTX_set_rsa_pss_keygen_md.pod
index 853d4b8d36..eb9641433e 100644
--- a/doc/man3/EVP_PKEY_CTX_set_rsa_pss_keygen_md.pod
+++ b/doc/man3/EVP_PKEY_CTX_set_rsa_pss_keygen_md.pod
@@ -44,8 +44,9 @@ than B<PSS>. It is otherwise similar to the B<RSA> version.
 The EVP_PKEY_CTX_set_rsa_pss_saltlen() macro is used to set the salt length.
 If the key has usage restrictions then an error is returned if an attempt is
 made to set the salt length below the minimum value. It is otherwise similar
-to the B<RSA> operation except detection of the salt length (using -2) is
-not supported for verification if the key has usage restrictions.
+to the B<RSA> operation except detection of the salt length (using
+RSA_PSS_SALTLEN_AUTO is not supported for verification if the key has
+usage restrictions.
 
 The EVP_PKEY_CTX_set_signature_md() and EVP_PKEY_CTX_set_rsa_mgf1_md() macros
 are used to set the digest and MGF1 algorithms respectively. If the key has
diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
index b9179b36ff..8ad4cdaf1e 100644
--- a/include/openssl/rsa.h
+++ b/include/openssl/rsa.h
@@ -94,6 +94,14 @@ extern "C" {
 # define EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, len) \
         RSA_pkey_ctx_ctrl(ctx, (EVP_PKEY_OP_SIGN|EVP_PKEY_OP_VERIFY), \
                           EVP_PKEY_CTRL_RSA_PSS_SALTLEN, len, NULL)
+/* Salt length matches digest */
+# define RSA_PSS_SALTLEN_DIGEST -1
+/* Verify only: auto detect salt length */
+# define RSA_PSS_SALTLEN_AUTO   -2
+/* Set salt length to maximum possible */
+# define RSA_PSS_SALTLEN_MAX    -3
+/* Old compatible max salt length for sign only */
+# define RSA_PSS_SALTLEN_MAX_SIGN    -2
 
 # define EVP_PKEY_CTX_set_rsa_pss_keygen_saltlen(ctx, len) \
         EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA_PSS, EVP_PKEY_OP_KEYGEN, \
diff --git a/test/evptests.txt b/test/evptests.txt
index 566b2f0e8d..91830bcd7b 100644
--- a/test/evptests.txt
+++ b/test/evptests.txt
@@ -2931,6 +2931,13 @@ Ctrl = digest:sha256
 Input="0123456789ABCDEF0123456789ABCDEF"
 Output=4DE433D5844043EF08D354DA03CB29068780D52706D7D1E4D50EFB7D58C9D547D83A747DDD0635A96B28F854E50145518482CB49E963054621B53C60C498D07C16E9C2789C893CF38D4D86900DE71BDE463BD2761D1271E358C7480A1AC0BAB930DDF39602AD1BC165B5D7436B516B7A7858E8EB7AB1C420EEB482F4D207F0E462B1724959320A084E13848D11D10FB593E66BF680BF6D3F345FC3E9C3DE60ABBAC37E1C6EC80A268C8D9FC49626C679097AA690BC1AA662B95EB8DB70390861AA0898229F9349B4B5FDD030D4928C47084708A933144BE23BD3C6E661B85B2C0EF9ED36D498D5B7320E8194D363D4AD478C059BAE804181965E0B81B663158A
 
+# Verify using salt length auto detect
+Verify = RSA-2048-PUBLIC
+Ctrl = rsa_padding_mode:pss
+Ctrl = rsa_pss_saltlen:auto
+Input="0123456789ABCDEF0123"
+Output = 6BF7EDC63A0BA184EEEC7F3020FEC8F5EBF38C2B76481881F48BCCE5796E7AB294548BA9AE810457C7723CABD1BDE94CF59CF7C0FC7461B22760C8ED703DD98E97BFDD61FA8D1181C411F6DEE5FF159F4850746D78EDEE385A363DC28E2CB373D5CAD7953F3BD5E639BE345732C03A1BDEA268814DA036EB1891C82D4012F3B903D86636055F87B96FC98806AD1B217685A4D754046A5DE0B0D7870664BE07902153EC85BA457BE7D7F89D7FE0F626D02A9CBBB2BB479DDA1A5CAE75247FB7BF6BFB15C1D3FD9E6B1573CCDBC72011C3B97716058BB11C7EA2E4E56ADAFE1F5DE6A7FD405AC5890100F9C3408EFFB5C73BF73F48177FF743B4B819D0699D507B
+
 # Digest too short
 Verify = RSA-2048-PUBLIC
 Ctrl = rsa_padding_mode:pss
@@ -3049,10 +3056,10 @@ Ctrl = digest:sha1
 Input="0123456789ABCDEF0123"
 Output = 3EFE09D88509027D837BFA5F8471CF7B69E6DF395DD999BB9CA42021F15722D9AC76670507C6BCFB73F64FB2211B611B8F140E76EBDB064BD762FDBA89D019E304A0D6B274E1C2FE1DF50005598A0306AF805416094E2A5BA60BC72BDE38CE061E853ED40F14967A8B9CA4DC739B462F89558F12FDF2D8D19FBEF16AD66FE2DDDA8BEE983ECBD873064244849D8D94B5B33F45E076871A47ED653E73257A2BE2DB3C0878094B0D2B6B682C8007DFD989425FB39A1FEEC9EED5876414601A49176EC344F5E3EDEE81CA2DDD29B7364F4638112CB3A547E2BC170E28CB66BDABE863754BE8AD5BA230567B575266F4B6B4CF81F28310ABF05351CC9E2DB85D00BF
 
-# Verify using default parameters, explicitly setting parameters -1 salt length
+# Verify explicitly setting parameters "digest" salt length
 Verify = RSA-PSS-DEFAULT
 Ctrl = rsa_padding_mode:pss
-Ctrl = rsa_pss_saltlen:-1
+Ctrl = rsa_pss_saltlen:digest
 Ctrl = digest:sha1
 Input="0123456789ABCDEF0123"
 Output = 3EFE09D88509027D837BFA5F8471CF7B69E6DF395DD999BB9CA42021F15722D9AC76670507C6BCFB73F64FB2211B611B8F140E76EBDB064BD762FDBA89D019E304A0D6B274E1C2FE1DF50005598A0306AF805416094E2A5BA60BC72BDE38CE061E853ED40F14967A8B9CA4DC739B462F89558F12FDF2D8D19FBEF16AD66FE2DDDA8BEE983ECBD873064244849D8D94B5B33F45E076871A47ED653E73257A2BE2DB3C0878094B0D2B6B682C8007DFD989425FB39A1FEEC9EED5876414601A49176EC344F5E3EDEE81CA2DDD29B7364F4638112CB3A547E2BC170E28CB66BDABE863754BE8AD5BA230567B575266F4B6B4CF81F28310ABF05351CC9E2DB85D00BF
@@ -3063,6 +3070,12 @@ Ctrl = rsa_pss_saltlen:30
 Input="0123456789ABCDEF0123"
 Output = 6BF7EDC63A0BA184EEEC7F3020FEC8F5EBF38C2B76481881F48BCCE5796E7AB294548BA9AE810457C7723CABD1BDE94CF59CF7C0FC7461B22760C8ED703DD98E97BFDD61FA8D1181C411F6DEE5FF159F4850746D78EDEE385A363DC28E2CB373D5CAD7953F3BD5E639BE345732C03A1BDEA268814DA036EB1891C82D4012F3B903D86636055F87B96FC98806AD1B217685A4D754046A5DE0B0D7870664BE07902153EC85BA457BE7D7F89D7FE0F626D02A9CBBB2BB479DDA1A5CAE75247FB7BF6BFB15C1D3FD9E6B1573CCDBC72011C3B97716058BB11C7EA2E4E56ADAFE1F5DE6A7FD405AC5890100F9C3408EFFB5C73BF73F48177FF743B4B819D0699D507B
 
+# Verify using maximum salt length
+Verify = RSA-PSS-DEFAULT
+Ctrl = rsa_pss_saltlen:max
+Input="0123456789ABCDEF0123"
+Output = 4470DCFE812DEE2E58E4301D4ED274AB348FE040B724B2CD1D8CD0914BFF375F0B86FCB32BFA8AEA9BD22BD7C4F1ADD4F3D215A5CFCC99055BAFECFC23800E9BECE19A08C66BEBC5802122D13A732E5958FC228DCC0B49B5B4B1154F032D8FA2F3564AA949C1310CC9266B0C47F86D449AC9D2E7678347E7266E2D7C888CCE1ADF44A109A293F8516AE2BD94CE220F26E137DB8E7A66BB9FCE052CDC1D0BE24D8CEBB20D10125F26B069F117044B9E1D16FDDAABCA5340AE1702F37D0E1C08A2E93801C0A41035C6C73DA02A0E32227EAFB0B85E79107B59650D0EE7DC32A6772CCCE90F06369B2880FE87ED76997BA61F5EA818091EE88F8B0D6F24D02A3FC6
+
 # Attempt to change salt length below minimum
 Verify = RSA-PSS-DEFAULT
 Ctrl = rsa_pss_saltlen:0