Update to OAEP support.
[openssl.git] / crypto / rsa / rsa_pmeth.c
index 023917ea25ff37de4ea2555bdf00600c1d8b43bf..ed1eb4922cfee095089affdc04acfeecfed5f114 100644 (file)
 #include <openssl/rsa.h>
 #include <openssl/bn.h>
 #include <openssl/evp.h>
+#include <openssl/x509v3.h>
+#ifndef OPENSSL_NO_CMS
+#include <openssl/cms.h>
+#endif
 #include "evp_locl.h"
 #include "rsa_locl.h"
 
@@ -81,10 +85,13 @@ typedef struct
        const EVP_MD *md;
        /* message digest for MGF1 */
        const EVP_MD *mgf1md;
-       /* PSS/OAEP salt length */
+       /* PSS salt length */
        int saltlen;
        /* Temp buffer */
        unsigned char *tbuf;
+       /* OAEP label */
+       unsigned char *oaep_label;
+       size_t oaep_labellen;
        } RSA_PKEY_CTX;
 
 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
@@ -102,6 +109,9 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
 
        rctx->saltlen = -2;
 
+       rctx->oaep_label = NULL;
+       rctx->oaep_labellen = 0;
+
        ctx->data = rctx;
        ctx->keygen_info = rctx->gentmp;
        ctx->keygen_info_count = 2;
@@ -125,6 +135,17 @@ static int pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
                }
        dctx->pad_mode = sctx->pad_mode;
        dctx->md = sctx->md;
+       dctx->mgf1md = sctx->mgf1md;
+       if (sctx->oaep_label)
+               {
+               if (dctx->oaep_label)
+                       OPENSSL_free(dctx->oaep_label);
+               dctx->oaep_label = BUF_memdup(sctx->oaep_label,
+                                               sctx->oaep_labellen);
+               if (!dctx->oaep_label)
+                       return 0;
+               dctx->oaep_labellen = sctx->oaep_labellen;
+               }
        return 1;
        }
 
@@ -147,6 +168,8 @@ static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
                        BN_free(rctx->pub_exp);
                if (rctx->tbuf)
                        OPENSSL_free(rctx->tbuf);
+               if (rctx->oaep_label)
+                       OPENSSL_free(rctx->oaep_label);
                OPENSSL_free(rctx);
                }
        }
@@ -166,7 +189,20 @@ static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
                                        RSA_R_INVALID_DIGEST_LENGTH);
                        return -1;
                        }
-               if (rctx->pad_mode == RSA_X931_PADDING)
+
+               if (EVP_MD_type(rctx->md) == NID_mdc2)
+                       {
+                       unsigned int sltmp;
+                       if (rctx->pad_mode != RSA_PKCS1_PADDING)
+                               return -1;
+                       ret = RSA_sign_ASN1_OCTET_STRING(NID_mdc2,
+                                               tbs, tbslen, sig, &sltmp, rsa);
+
+                       if (ret <= 0)
+                               return ret;
+                       ret = sltmp;
+                       }
+               else if (rctx->pad_mode == RSA_X931_PADDING)
                        {
                        if (!setup_tbuf(rctx, ctx))
                                return -1;
@@ -251,6 +287,8 @@ static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
                        ret = int_rsa_verify(EVP_MD_type(rctx->md),
                                                NULL, 0, rout, &sltmp,
                                        sig, siglen, ctx->pkey->pkey.rsa);
+                       if (ret <= 0)
+                               return 0;
                        ret = sltmp;
                        }
                else
@@ -318,7 +356,6 @@ static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
        return 1;
                        
        }
-       
 
 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
                                        unsigned char *out, size_t *outlen,
@@ -326,7 +363,23 @@ static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
        {
        int ret;
        RSA_PKEY_CTX *rctx = ctx->data;
-       ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
+       if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING)
+               {
+               int klen = RSA_size(ctx->pkey->pkey.rsa);
+               if (!setup_tbuf(rctx, ctx))
+                       return -1;
+               if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen,
+                                                       in, inlen,
+                                                       rctx->oaep_label,
+                                                       rctx->oaep_labellen,
+                                                       rctx->md, rctx->mgf1md))
+                       return -1;
+               ret = RSA_public_encrypt(klen, rctx->tbuf, out,
+                                               ctx->pkey->pkey.rsa,
+                                               RSA_NO_PADDING);
+               }
+       else
+               ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
                                                        rctx->pad_mode);
        if (ret < 0)
                return ret;
@@ -340,7 +393,29 @@ static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
        {
        int ret;
        RSA_PKEY_CTX *rctx = ctx->data;
-       ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
+       if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING)
+               {
+               int i;
+               if (!setup_tbuf(rctx, ctx))
+                       return -1;
+               ret = RSA_private_decrypt(inlen, in, rctx->tbuf,
+                                                       ctx->pkey->pkey.rsa,
+                                                       RSA_NO_PADDING);
+               if (ret <= 0)
+                       return ret;
+               for (i = 0; i < ret; i++)
+                       {
+                       if (rctx->tbuf[i])
+                               break;
+                       }
+               ret = RSA_padding_check_PKCS1_OAEP_mgf1(out,ret,rctx->tbuf + i,
+                                                       ret - i, ret,
+                                                       rctx->oaep_label,
+                                                       rctx->oaep_labellen,
+                                                       rctx->md, rctx->mgf1md);
+               }
+       else 
+               ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
                                                        rctx->pad_mode);
        if (ret < 0)
                return ret;
@@ -407,15 +482,25 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
                                RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
                return -2;
 
+               case EVP_PKEY_CTRL_GET_RSA_PADDING:
+               *(int *)p2 = rctx->pad_mode;
+               return 1;
+
                case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
-               if (p1 < -2)
-                       return -2;
+               case EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN:
                if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
                        {
                        RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
                        return -2;
                        }
-               rctx->saltlen = p1;
+               if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN)
+                       *(int *)p2 = rctx->saltlen;
+               else
+                       {
+                       if (p1 < -2)
+                               return -2;
+                       rctx->saltlen = p1;
+                       }
                return 1;
 
                case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
@@ -430,29 +515,102 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
                case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
                if (!p2)
                        return -2;
+               BN_free(rctx->pub_exp);
                rctx->pub_exp = p2;
                return 1;
 
+               case EVP_PKEY_CTRL_RSA_OAEP_MD:
+               case EVP_PKEY_CTRL_GET_RSA_OAEP_MD:
+               if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING)
+                       {
+                       RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
+                       return -2;
+                       }
+               if (type == EVP_PKEY_CTRL_GET_RSA_OAEP_MD)
+                       *(const EVP_MD **)p2 = rctx->md;
+               else
+                       rctx->md = p2;
+               return 1;
+
                case EVP_PKEY_CTRL_MD:
                if (!check_padding_md(p2, rctx->pad_mode))
                        return 0;
                rctx->md = p2;
                return 1;
 
+               case EVP_PKEY_CTRL_GET_MD:
+               *(const EVP_MD **)p2 = rctx->md;
+               return 1;
+
                case EVP_PKEY_CTRL_RSA_MGF1_MD:
-               rctx->mgf1md = p2;
+               case EVP_PKEY_CTRL_GET_RSA_MGF1_MD:
+               if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING
+                       && rctx->pad_mode != RSA_PKCS1_OAEP_PADDING)
+                       {
+                       RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_MGF1_MD);
+                       return -2;
+                       }
+               if (type == EVP_PKEY_CTRL_GET_RSA_MGF1_MD)
+                       {
+                       if (rctx->mgf1md)
+                               *(const EVP_MD **)p2 = rctx->mgf1md;
+                       else
+                               *(const EVP_MD **)p2 = rctx->md;
+                       }
+               else
+                       rctx->mgf1md = p2;
+               return 1;
+
+               case EVP_PKEY_CTRL_RSA_OAEP_LABEL:
+               if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING)
+                       {
+                       RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
+                       return -2;
+                       }
+               if (rctx->oaep_label)
+                       OPENSSL_free(rctx->oaep_label);
+               if (p2 && p1 > 0)
+                       {
+                       rctx->oaep_label = p2;
+                       rctx->oaep_labellen = p1;
+                       }
+               else
+                       {
+                       rctx->oaep_label = NULL;
+                       rctx->oaep_labellen = 0;
+                       }
                return 1;
 
+               case EVP_PKEY_CTRL_GET_RSA_OAEP_LABEL:
+               if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING)
+                       {
+                       RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
+                       return -2;
+                       }
+               *(unsigned char **)p2 = rctx->oaep_label;
+               return rctx->oaep_labellen;
+
                case EVP_PKEY_CTRL_DIGESTINIT:
                case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
                case EVP_PKEY_CTRL_PKCS7_DECRYPT:
                case EVP_PKEY_CTRL_PKCS7_SIGN:
+               return 1;
 #ifndef OPENSSL_NO_CMS
-               case EVP_PKEY_CTRL_CMS_ENCRYPT:
                case EVP_PKEY_CTRL_CMS_DECRYPT:
+               {
+               X509_ALGOR *alg = NULL;
+               ASN1_OBJECT *encalg = NULL;
+               if (p2)
+                       CMS_RecipientInfo_ktri_get0_algs(p2, NULL, NULL, &alg);
+               if (alg)
+                       X509_ALGOR_get0(&encalg, NULL, NULL, alg);
+               if (encalg && OBJ_obj2nid(encalg) == NID_rsaesOaep)
+                       rctx->pad_mode = RSA_PKCS1_OAEP_PADDING;
+               }
+               case EVP_PKEY_CTRL_CMS_ENCRYPT:
                case EVP_PKEY_CTRL_CMS_SIGN:
-#endif
                return 1;
+#endif
                case EVP_PKEY_CTRL_PEER_KEY:
                        RSAerr(RSA_F_PKEY_RSA_CTRL,
                        RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
@@ -483,6 +641,8 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
                        pm = RSA_NO_PADDING;
                else if (!strcmp(value, "oeap"))
                        pm = RSA_PKCS1_OAEP_PADDING;
+               else if (!strcmp(value, "oaep"))
+                       pm = RSA_PKCS1_OAEP_PADDING;
                else if (!strcmp(value, "x931"))
                        pm = RSA_X931_PADDING;
                else if (!strcmp(value, "pss"))
@@ -522,6 +682,43 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
                return ret;
                }
 
+       if (!strcmp(type, "rsa_mgf1_md"))
+               {
+               const EVP_MD *md;
+               if (!(md = EVP_get_digestbyname(value)))
+                       {
+                       RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
+                                               RSA_R_INVALID_DIGEST);
+                       return 0;
+                       }
+               return EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md);
+               }
+
+       if (!strcmp(type, "rsa_oaep_md"))
+               {
+               const EVP_MD *md;
+               if (!(md = EVP_get_digestbyname(value)))
+                       {
+                       RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
+                                               RSA_R_INVALID_DIGEST);
+                       return 0;
+                       }
+               return EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md);
+               }
+       if (!strcmp(type, "rsa_oaep_label"))
+               {
+               unsigned char *lab;
+               long lablen;
+               int ret;
+               lab = string_to_hex(value, &lablen);
+               if (!lab)
+                       return 0;
+               ret = EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, lab, lablen);
+               if (ret <= 0)
+                       OPENSSL_free(lab);
+               return ret;
+               }
+
        return -2;
        }