Replace RSA specific PKCS7_RECIP_INFO set up with an public key algorithm

[openssl.git] / crypto / rsa / rsa_oaep.c

diff --git a/crypto/rsa/rsa_oaep.c b/crypto/rsa/rsa_oaep.c
index 62d2ed7e9fdbdfb9afe812c37bd15cadcff588e4..45d6f6ef8a5eaa5435cc3e313cc7415ab1b405ff 100644 (file)
--- a/crypto/rsa/rsa_oaep.c
+++ b/crypto/rsa/rsa_oaep.c
@@ -43,33 +43,33 @@ int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
                {
                RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP,
                   RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
-               return (0);
+               return 0;
                }
 
        if (emlen < 2 * SHA_DIGEST_LENGTH + 1)
                {
                RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, RSA_R_KEY_SIZE_TOO_SMALL);
-               return (0);
+               return 0;
                }
-       
+
        dbmask = OPENSSL_malloc(emlen - SHA_DIGEST_LENGTH);
        if (dbmask == NULL)
                {
                RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
-               return (0);
+               return 0;
                }
 
        to[0] = 0;
        seed = to + 1;
        db = to + SHA_DIGEST_LENGTH + 1;
 
-       EVP_Digest((void *)param, plen, db, NULL, EVP_sha1());
+       EVP_Digest((void *)param, plen, db, NULL, EVP_sha1(), NULL);
        memset(db + SHA_DIGEST_LENGTH, 0,
                emlen - flen - 2 * SHA_DIGEST_LENGTH - 1);
        db[emlen - flen - SHA_DIGEST_LENGTH - 1] = 0x01;
        memcpy(db + emlen - flen - SHA_DIGEST_LENGTH, from, (unsigned int) flen);
        if (RAND_bytes(seed, SHA_DIGEST_LENGTH) <= 0)
-               return (0);
+               return 0;
 #ifdef PKCS_TESTVECT
        memcpy(seed,
           "\xaa\xfd\x12\xf6\x59\xca\xe6\x34\x89\xb4\x79\xe5\x07\x6d\xde\xc2\xf0\x6c\xb5\x8f",
@@ -79,7 +79,7 @@ int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
        MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed, SHA_DIGEST_LENGTH);
        for (i = 0; i < emlen - SHA_DIGEST_LENGTH; i++)
                db[i] ^= dbmask[i];
-       
+
        MGF1(seedmask, SHA_DIGEST_LENGTH, db, emlen - SHA_DIGEST_LENGTH);
        for (i = 0; i < SHA_DIGEST_LENGTH; i++)
                seed[i] ^= seedmask[i];
@@ -96,21 +96,34 @@ int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
        const unsigned char *maskeddb;
        int lzero;
        unsigned char *db = NULL, seed[SHA_DIGEST_LENGTH], phash[SHA_DIGEST_LENGTH];
+       int bad = 0;
 
        if (--num < 2 * SHA_DIGEST_LENGTH + 1)
+               /* 'num' is the length of the modulus, i.e. does not depend on the
+                * particular ciphertext. */
                goto decoding_err;
 
        lzero = num - flen;
        if (lzero < 0)
-               goto decoding_err;
+               {
+               /* lzero == -1 */
+
+               /* signalling this error immediately after detection might allow
+                * for side-channel attacks (e.g. timing if 'plen' is huge
+                * -- cf. James H. Manger, "A Chosen Ciphertext Attack on RSA Optimal
+                * Asymmetric Encryption Padding (OAEP) [...]", CRYPTO 2001),
+                * so we use a 'bad' flag */
+               bad = 1;
+               lzero = 0;
+               }
        maskeddb = from - lzero + SHA_DIGEST_LENGTH;
-       
+
        dblen = num - SHA_DIGEST_LENGTH;
        db = OPENSSL_malloc(dblen);
        if (db == NULL)
                {
-               RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
-               return (-1);
+               RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
+               return -1;
                }
 
        MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen);
@@ -121,9 +134,9 @@ int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
        for (i = 0; i < dblen; i++)
                db[i] ^= maskeddb[i];
 
-       EVP_Digest((void *)param, plen, phash, NULL, EVP_sha1());
-       
-       if (memcmp(db, phash, SHA_DIGEST_LENGTH) != 0)
+       EVP_Digest((void *)param, plen, phash, NULL, EVP_sha1(), NULL);
+
+       if (memcmp(db, phash, SHA_DIGEST_LENGTH) != 0 || bad)
                goto decoding_err;
        else
                {
@@ -134,6 +147,8 @@ int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
                        goto decoding_err;
                else
                        {
+                       /* everything looks OK */
+
                        mlen = dblen - i;
                        if (tlen < mlen)
                                {
@@ -146,7 +161,7 @@ int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
                }
        OPENSSL_free(db);
        return mlen;
-       
+
 decoding_err:
        /* to avoid chosen ciphertext attacks, the error message should not reveal
         * which kind of decoding error happened */
@@ -155,32 +170,34 @@ decoding_err:
        return -1;
        }
 
-int MGF1(unsigned char *mask, long len,
-       const unsigned char *seed, long seedlen)
+int PKCS1_MGF1(unsigned char *mask, long len,
+       const unsigned char *seed, long seedlen, const EVP_MD *dgst)
        {
        long i, outlen = 0;
        unsigned char cnt[4];
        EVP_MD_CTX c;
-       unsigned char md[SHA_DIGEST_LENGTH];
+       unsigned char md[EVP_MAX_MD_SIZE];
+       int mdlen;
 
        EVP_MD_CTX_init(&c);
+       mdlen = EVP_MD_size(dgst);
        for (i = 0; outlen < len; i++)
                {
                cnt[0] = (unsigned char)((i >> 24) & 255);
                cnt[1] = (unsigned char)((i >> 16) & 255);
                cnt[2] = (unsigned char)((i >> 8)) & 255;
                cnt[3] = (unsigned char)(i & 255);
-               EVP_DigestInit(&c,EVP_sha1());
+               EVP_DigestInit_ex(&c,dgst, NULL);
                EVP_DigestUpdate(&c, seed, seedlen);
                EVP_DigestUpdate(&c, cnt, 4);
-               if (outlen + SHA_DIGEST_LENGTH <= len)
+               if (outlen + mdlen <= len)
                        {
-                       EVP_DigestFinal(&c, mask + outlen, NULL);
-                       outlen += SHA_DIGEST_LENGTH;
+                       EVP_DigestFinal_ex(&c, mask + outlen, NULL);
+                       outlen += mdlen;
                        }
                else
                        {
-                       EVP_DigestFinal(&c, md, NULL);
+                       EVP_DigestFinal_ex(&c, md, NULL);
                        memcpy(mask + outlen, md, len - outlen);
                        outlen = len;
                        }
@@ -188,4 +205,9 @@ int MGF1(unsigned char *mask, long len,
        EVP_MD_CTX_cleanup(&c);
        return 0;
        }
+
+int MGF1(unsigned char *mask, long len, const unsigned char *seed, long seedlen)
+       {
+       return PKCS1_MGF1(mask, len, seed, seedlen, EVP_sha1());
+       }
 #endif