crypto/rsa/rsa_pmeth.c

   1 /* crypto/rsa/rsa_pmeth.c */
   2 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
   3  * project 2006.
   4  */
   5 /* ====================================================================
   6  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  *
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in
  17  *    the documentation and/or other materials provided with the
  18  *    distribution.
  19  *
  20  * 3. All advertising materials mentioning features or use of this
  21  *    software must display the following acknowledgment:
  22  *    "This product includes software developed by the OpenSSL Project
  23  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
  24  *
  25  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
  26  *    endorse or promote products derived from this software without
  27  *    prior written permission. For written permission, please contact
  28  *    licensing@OpenSSL.org.
  29  *
  30  * 5. Products derived from this software may not be called "OpenSSL"
  31  *    nor may "OpenSSL" appear in their names without prior written
  32  *    permission of the OpenSSL Project.
  33  *
  34  * 6. Redistributions of any form whatsoever must retain the following
  35  *    acknowledgment:
  36  *    "This product includes software developed by the OpenSSL Project
  37  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
  38  *
  39  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
  40  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  41  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  42  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
  43  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  44  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  45  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  46  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  48  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  49  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
  50  * OF THE POSSIBILITY OF SUCH DAMAGE.
  51  * ====================================================================
  52  *
  53  * This product includes cryptographic software written by Eric Young
  54  * (eay@cryptsoft.com).  This product includes software written by Tim
  55  * Hudson (tjh@cryptsoft.com).
  56  *
  57  */
  58
  59 #include <stdio.h>
  60 #include "cryptlib.h"
  61 #include <openssl/asn1t.h>
  62 #include <openssl/x509.h>
  63 #include <openssl/rsa.h>
  64 #include <openssl/bn.h>
  65 #include <openssl/evp.h>
  66 #include <openssl/x509v3.h>
  67 #ifndef OPENSSL_NO_CMS
  68 #include <openssl/cms.h>
  69 #endif
  70 #ifdef OPENSSL_FIPS
  71 #include <openssl/fips.h>
  72 #endif
  73 #include "evp_locl.h"
  74 #include "rsa_locl.h"
  75
  76 /* RSA pkey context structure */
  77
  78 typedef struct
  79         {
  80         /* Key gen parameters */
  81         int nbits;
  82         BIGNUM *pub_exp;
  83         /* Keygen callback info */
  84         int gentmp[2];
  85         /* RSA padding mode */
  86         int pad_mode;
  87         /* message digest */
  88         const EVP_MD *md;
  89         /* message digest for MGF1 */
  90         const EVP_MD *mgf1md;
  91         /* PSS salt length */
  92         int saltlen;
  93         /* Temp buffer */
  94         unsigned char *tbuf;
  95         /* OAEP label */
  96         unsigned char *oaep_label;
  97         size_t oaep_labellen;
  98         } RSA_PKEY_CTX;
  99
 100 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
 101         {
 102         RSA_PKEY_CTX *rctx;
 103         rctx = OPENSSL_malloc(sizeof(RSA_PKEY_CTX));
 104         if (!rctx)
 105                 return 0;
 106         rctx->nbits = 1024;
 107         rctx->pub_exp = NULL;
 108         rctx->pad_mode = RSA_PKCS1_PADDING;
 109         rctx->md = NULL;
 110         rctx->mgf1md = NULL;
 111         rctx->tbuf = NULL;
 112
 113         rctx->saltlen = -2;
 114
 115         rctx->oaep_label = NULL;
 116         rctx->oaep_labellen = 0;
 117
 118         ctx->data = rctx;
 119         ctx->keygen_info = rctx->gentmp;
 120         ctx->keygen_info_count = 2;
 121
 122         return 1;
 123         }
 124
 125 static int pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
 126         {
 127         RSA_PKEY_CTX *dctx, *sctx;
 128         if (!pkey_rsa_init(dst))
 129                 return 0;
 130         sctx = src->data;
 131         dctx = dst->data;
 132         dctx->nbits = sctx->nbits;
 133         if (sctx->pub_exp)
 134                 {
 135                 dctx->pub_exp = BN_dup(sctx->pub_exp);
 136                 if (!dctx->pub_exp)
 137                         return 0;
 138                 }
 139         dctx->pad_mode = sctx->pad_mode;
 140         dctx->md = sctx->md;
 141         dctx->mgf1md = sctx->mgf1md;
 142         if (sctx->oaep_label)
 143                 {
 144                 if (dctx->oaep_label)
 145                         OPENSSL_free(dctx->oaep_label);
 146                 dctx->oaep_label = BUF_memdup(sctx->oaep_label,
 147                                                 sctx->oaep_labellen);
 148                 if (!dctx->oaep_label)
 149                         return 0;
 150                 dctx->oaep_labellen = sctx->oaep_labellen;
 151                 }
 152         return 1;
 153         }
 154
 155 static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
 156         {
 157         if (ctx->tbuf)
 158                 return 1;
 159         ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));
 160         if (!ctx->tbuf)
 161                 return 0;
 162         return 1;
 163         }
 164
 165 static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
 166         {
 167         RSA_PKEY_CTX *rctx = ctx->data;
 168         if (rctx)
 169                 {
 170                 if (rctx->pub_exp)
 171                         BN_free(rctx->pub_exp);
 172                 if (rctx->tbuf)
 173                         OPENSSL_free(rctx->tbuf);
 174                 if (rctx->oaep_label)
 175                         OPENSSL_free(rctx->oaep_label);
 176                 OPENSSL_free(rctx);
 177                 }
 178         }
 179 #ifdef OPENSSL_FIPS
 180 /* FIP checker. Return value indicates status of context parameters:
 181  * 1  : redirect to FIPS.
 182  * 0  : don't redirect to FIPS.
 183  * -1 : illegal operation in FIPS mode.
 184  */
 185
 186 static int pkey_fips_check_ctx(EVP_PKEY_CTX *ctx)
 187         {
 188         RSA_PKEY_CTX *rctx = ctx->data;
 189         RSA *rsa = ctx->pkey->pkey.rsa;
 190         int rv = -1;
 191         if (!FIPS_mode())
 192                 return 0;
 193         if (rsa->flags & RSA_FLAG_NON_FIPS_ALLOW)
 194                 rv = 0;
 195         if (!(rsa->meth->flags & RSA_FLAG_FIPS_METHOD) && rv)
 196                 return -1;
 197         if (rctx->md)
 198                 {
 199                 const EVP_MD *fmd;
 200                 fmd = FIPS_get_digestbynid(EVP_MD_type(rctx->md));
 201                 if (!fmd || !(fmd->flags & EVP_MD_FLAG_FIPS))
 202                         return rv;
 203                 }
 204         if (rctx->mgf1md && !(rctx->mgf1md->flags & EVP_MD_FLAG_FIPS))
 205                 {
 206                 const EVP_MD *fmd;
 207                 fmd = FIPS_get_digestbynid(EVP_MD_type(rctx->mgf1md));
 208                 if (!fmd || !(fmd->flags & EVP_MD_FLAG_FIPS))
 209                         return rv;
 210                 }
 211         return 1;
 212         }
 213 #endif
 214
 215 static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
 216                                         const unsigned char *tbs, size_t tbslen)
 217         {
 218         int ret;
 219         RSA_PKEY_CTX *rctx = ctx->data;
 220         RSA *rsa = ctx->pkey->pkey.rsa;
 221
 222 #ifdef OPENSSL_FIPS
 223         ret = pkey_fips_check_ctx(ctx);
 224         if (ret < 0)
 225                 {
 226                 RSAerr(RSA_F_PKEY_RSA_SIGN, RSA_R_OPERATION_NOT_ALLOWED_IN_FIPS_MODE);
 227                 return -1;
 228                 }
 229 #endif
 230
 231         if (rctx->md)
 232                 {
 233                 if (tbslen != (size_t)EVP_MD_size(rctx->md))
 234                         {
 235                         RSAerr(RSA_F_PKEY_RSA_SIGN,
 236                                         RSA_R_INVALID_DIGEST_LENGTH);
 237                         return -1;
 238                         }
 239 #ifdef OPENSSL_FIPS
 240                 if (ret > 0)
 241                         {
 242                         unsigned int slen;
 243                         ret = FIPS_rsa_sign_digest(rsa, tbs, tbslen, rctx->md,
 244                                                         rctx->pad_mode,
 245                                                         rctx->saltlen,
 246                                                         rctx->mgf1md,
 247                                                         sig, &slen);
 248                         if (ret > 0)
 249                                 *siglen = slen;
 250                         else
 251                                 *siglen = 0;
 252                         return ret;
 253                         }
 254 #endif
 255
 256                 if (EVP_MD_type(rctx->md) == NID_mdc2)
 257                         {
 258                         unsigned int sltmp;
 259                         if (rctx->pad_mode != RSA_PKCS1_PADDING)
 260                                 return -1;
 261                         ret = RSA_sign_ASN1_OCTET_STRING(NID_mdc2,
 262                                                 tbs, tbslen, sig, &sltmp, rsa);
 263
 264                         if (ret <= 0)
 265                                 return ret;
 266                         ret = sltmp;
 267                         }
 268                 else if (rctx->pad_mode == RSA_X931_PADDING)
 269                         {
 270                         if (!setup_tbuf(rctx, ctx))
 271                                 return -1;
 272                         memcpy(rctx->tbuf, tbs, tbslen);
 273                         rctx->tbuf[tbslen] =
 274                                 RSA_X931_hash_id(EVP_MD_type(rctx->md));
 275                         ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
 276                                                 sig, rsa, RSA_X931_PADDING);
 277                         }
 278                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
 279                         {
 280                         unsigned int sltmp;
 281                         ret = RSA_sign(EVP_MD_type(rctx->md),
 282                                                 tbs, tbslen, sig, &sltmp, rsa);
 283                         if (ret <= 0)
 284                                 return ret;
 285                         ret = sltmp;
 286                         }
 287                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
 288                         {
 289                         if (!setup_tbuf(rctx, ctx))
 290                                 return -1;
 291                         if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa,
 292                                                 rctx->tbuf, tbs,
 293                                                 rctx->md, rctx->mgf1md,
 294                                                 rctx->saltlen))
 295                                 return -1;
 296                         ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
 297                                                 sig, rsa, RSA_NO_PADDING);
 298                         }
 299                 else
 300                         return -1;
 301                 }
 302         else
 303                 ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
 304                                                         rctx->pad_mode);
 305         if (ret < 0)
 306                 return ret;
 307         *siglen = ret;
 308         return 1;
 309         }
 310
 311
 312 static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
 313                                         unsigned char *rout, size_t *routlen,
 314                                         const unsigned char *sig, size_t siglen)
 315         {
 316         int ret;
 317         RSA_PKEY_CTX *rctx = ctx->data;
 318
 319         if (rctx->md)
 320                 {
 321                 if (rctx->pad_mode == RSA_X931_PADDING)
 322                         {
 323                         if (!setup_tbuf(rctx, ctx))
 324                                 return -1;
 325                         ret = RSA_public_decrypt(siglen, sig,
 326                                                 rctx->tbuf, ctx->pkey->pkey.rsa,
 327                                                 RSA_X931_PADDING);
 328                         if (ret < 1)
 329                                 return 0;
 330                         ret--;
 331                         if (rctx->tbuf[ret] !=
 332                                 RSA_X931_hash_id(EVP_MD_type(rctx->md)))
 333                                 {
 334                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
 335                                                 RSA_R_ALGORITHM_MISMATCH);
 336                                 return 0;
 337                                 }
 338                         if (ret != EVP_MD_size(rctx->md))
 339                                 {
 340                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
 341                                         RSA_R_INVALID_DIGEST_LENGTH);
 342                                 return 0;
 343                                 }
 344                         if (rout)
 345                                 memcpy(rout, rctx->tbuf, ret);
 346                         }
 347                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
 348                         {
 349                         size_t sltmp;
 350                         ret = int_rsa_verify(EVP_MD_type(rctx->md),
 351                                                 NULL, 0, rout, &sltmp,
 352                                         sig, siglen, ctx->pkey->pkey.rsa);
 353                         if (ret <= 0)
 354                                 return 0;
 355                         ret = sltmp;
 356                         }
 357                 else
 358                         return -1;
 359                 }
 360         else
 361                 ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
 362                                                         rctx->pad_mode);
 363         if (ret < 0)
 364                 return ret;
 365         *routlen = ret;
 366         return 1;
 367         }
 368
 369 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
 370                                         const unsigned char *sig, size_t siglen,
 371                                         const unsigned char *tbs, size_t tbslen)
 372         {
 373         RSA_PKEY_CTX *rctx = ctx->data;
 374         RSA *rsa = ctx->pkey->pkey.rsa;
 375         size_t rslen;
 376 #ifdef OPENSSL_FIPS
 377         int rv;
 378         rv = pkey_fips_check_ctx(ctx);
 379         if (rv < 0)
 380                 {
 381                 RSAerr(RSA_F_PKEY_RSA_VERIFY, RSA_R_OPERATION_NOT_ALLOWED_IN_FIPS_MODE);
 382                 return -1;
 383                 }
 384 #endif
 385         if (rctx->md)
 386                 {
 387 #ifdef OPENSSL_FIPS
 388                 if (rv > 0)
 389                         {
 390                         return FIPS_rsa_verify_digest(rsa,
 391                                                         tbs, tbslen,
 392                                                         rctx->md,
 393                                                         rctx->pad_mode,
 394                                                         rctx->saltlen,
 395                                                         rctx->mgf1md,
 396                                                         sig, siglen);
 397
 398                         }
 399 #endif
 400                 if (rctx->pad_mode == RSA_PKCS1_PADDING)
 401                         return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
 402                                         sig, siglen, rsa);
 403                 if (rctx->pad_mode == RSA_X931_PADDING)
 404                         {
 405                         if (pkey_rsa_verifyrecover(ctx, NULL, &rslen,
 406                                         sig, siglen) <= 0)
 407                                 return 0;
 408                         }
 409                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
 410                         {
 411                         int ret;
 412                         if (!setup_tbuf(rctx, ctx))
 413                                 return -1;
 414                         ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
 415                                                         rsa, RSA_NO_PADDING);
 416                         if (ret <= 0)
 417                                 return 0;
 418                         ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs,
 419                                                 rctx->md, rctx->mgf1md,
 420                                                 rctx->tbuf, rctx->saltlen);
 421                         if (ret <= 0)
 422                                 return 0;
 423                         return 1;
 424                         }
 425                 else
 426                         return -1;
 427                 }
 428         else
 429                 {
 430                 if (!setup_tbuf(rctx, ctx))
 431                         return -1;
 432                 rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
 433                                                 rsa, rctx->pad_mode);
 434                 if (rslen == 0)
 435                         return 0;
 436                 }
 437
 438         if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))
 439                 return 0;
 440
 441         return 1;
 442
 443         }
 444
 445 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
 446                                         unsigned char *out, size_t *outlen,
 447                                         const unsigned char *in, size_t inlen)
 448         {
 449         int ret;
 450         RSA_PKEY_CTX *rctx = ctx->data;
 451         if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING)
 452                 {
 453                 int klen = RSA_size(ctx->pkey->pkey.rsa);
 454                 if (!setup_tbuf(rctx, ctx))
 455                         return -1;
 456                 if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen,
 457                                                         in, inlen,
 458                                                         rctx->oaep_label,
 459                                                         rctx->oaep_labellen,
 460                                                         rctx->md, rctx->mgf1md))
 461                         return -1;
 462                 ret = RSA_public_encrypt(klen, rctx->tbuf, out,
 463                                                 ctx->pkey->pkey.rsa,
 464                                                 RSA_NO_PADDING);
 465                 }
 466         else
 467                 ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 468                                                         rctx->pad_mode);
 469         if (ret < 0)
 470                 return ret;
 471         *outlen = ret;
 472         return 1;
 473         }
 474
 475 static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
 476                                         unsigned char *out, size_t *outlen,
 477                                         const unsigned char *in, size_t inlen)
 478         {
 479         int ret;
 480         RSA_PKEY_CTX *rctx = ctx->data;
 481         if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING)
 482                 {
 483                 int i;
 484                 if (!setup_tbuf(rctx, ctx))
 485                         return -1;
 486                 ret = RSA_private_decrypt(inlen, in, rctx->tbuf,
 487                                                         ctx->pkey->pkey.rsa,
 488                                                         RSA_NO_PADDING);
 489                 if (ret <= 0)
 490                         return ret;
 491                 for (i = 0; i < ret; i++)
 492                         {
 493                         if (rctx->tbuf[i])
 494                                 break;
 495                         }
 496                 ret = RSA_padding_check_PKCS1_OAEP_mgf1(out,ret,rctx->tbuf + i,
 497                                                         ret - i, ret,
 498                                                         rctx->oaep_label,
 499                                                         rctx->oaep_labellen,
 500                                                         rctx->md, rctx->mgf1md);
 501                 }
 502         else
 503                 ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 504                                                         rctx->pad_mode);
 505         if (ret < 0)
 506                 return ret;
 507         *outlen = ret;
 508         return 1;
 509         }
 510
 511 static int check_padding_md(const EVP_MD *md, int padding)
 512         {
 513         if (!md)
 514                 return 1;
 515
 516         if (padding == RSA_NO_PADDING)
 517                 {
 518                 RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
 519                 return 0;
 520                 }
 521
 522         if (padding == RSA_X931_PADDING)
 523                 {
 524                 if (RSA_X931_hash_id(EVP_MD_type(md)) == -1)
 525                         {
 526                         RSAerr(RSA_F_CHECK_PADDING_MD,
 527                                                 RSA_R_INVALID_X931_DIGEST);
 528                         return 0;
 529                         }
 530                 return 1;
 531                 }
 532
 533         return 1;
 534         }
 535
 536
 537 static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 538         {
 539         RSA_PKEY_CTX *rctx = ctx->data;
 540         switch (type)
 541                 {
 542                 case EVP_PKEY_CTRL_RSA_PADDING:
 543                 if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING))
 544                         {
 545                         if (!check_padding_md(rctx->md, p1))
 546                                 return 0;
 547                         if (p1 == RSA_PKCS1_PSS_PADDING)
 548                                 {
 549                                 if (!(ctx->operation &
 550                                      (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
 551                                         goto bad_pad;
 552                                 if (!rctx->md)
 553                                         rctx->md = EVP_sha1();
 554                                 }
 555                         if (p1 == RSA_PKCS1_OAEP_PADDING)
 556                                 {
 557                                 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
 558                                         goto bad_pad;
 559                                 if (!rctx->md)
 560                                         rctx->md = EVP_sha1();
 561                                 }
 562                         rctx->pad_mode = p1;
 563                         return 1;
 564                         }
 565                 bad_pad:
 566                 RSAerr(RSA_F_PKEY_RSA_CTRL,
 567                                 RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
 568                 return -2;
 569
 570                 case EVP_PKEY_CTRL_GET_RSA_PADDING:
 571                 *(int *)p2 = rctx->pad_mode;
 572                 return 1;
 573
 574                 case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
 575                 case EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN:
 576                 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
 577                         {
 578                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
 579                         return -2;
 580                         }
 581                 if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN)
 582                         *(int *)p2 = rctx->saltlen;
 583                 else
 584                         {
 585                         if (p1 < -2)
 586                                 return -2;
 587                         rctx->saltlen = p1;
 588                         }
 589                 return 1;
 590
 591                 case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
 592                 if (p1 < 256)
 593                         {
 594                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_KEYBITS);
 595                         return -2;
 596                         }
 597                 rctx->nbits = p1;
 598                 return 1;
 599
 600                 case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
 601                 if (!p2)
 602                         return -2;
 603                 BN_free(rctx->pub_exp);
 604                 rctx->pub_exp = p2;
 605                 return 1;
 606
 607                 case EVP_PKEY_CTRL_RSA_OAEP_MD:
 608                 case EVP_PKEY_CTRL_GET_RSA_OAEP_MD:
 609                 if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING)
 610                         {
 611                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
 612                         return -2;
 613                         }
 614                 if (type == EVP_PKEY_CTRL_GET_RSA_OAEP_MD)
 615                         *(const EVP_MD **)p2 = rctx->md;
 616                 else
 617                         rctx->md = p2;
 618                 return 1;
 619
 620                 case EVP_PKEY_CTRL_MD:
 621                 if (!check_padding_md(p2, rctx->pad_mode))
 622                         return 0;
 623                 rctx->md = p2;
 624                 return 1;
 625
 626                 case EVP_PKEY_CTRL_GET_MD:
 627                 *(const EVP_MD **)p2 = rctx->md;
 628                 return 1;
 629
 630                 case EVP_PKEY_CTRL_RSA_MGF1_MD:
 631                 case EVP_PKEY_CTRL_GET_RSA_MGF1_MD:
 632                 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING
 633                         && rctx->pad_mode != RSA_PKCS1_OAEP_PADDING)
 634                         {
 635                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_MGF1_MD);
 636                         return -2;
 637                         }
 638                 if (type == EVP_PKEY_CTRL_GET_RSA_MGF1_MD)
 639                         {
 640                         if (rctx->mgf1md)
 641                                 *(const EVP_MD **)p2 = rctx->mgf1md;
 642                         else
 643                                 *(const EVP_MD **)p2 = rctx->md;
 644                         }
 645                 else
 646                         rctx->mgf1md = p2;
 647                 return 1;
 648
 649                 case EVP_PKEY_CTRL_RSA_OAEP_LABEL:
 650                 if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING)
 651                         {
 652                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
 653                         return -2;
 654                         }
 655                 if (rctx->oaep_label)
 656                         OPENSSL_free(rctx->oaep_label);
 657                 if (p2 && p1 > 0)
 658                         {
 659                         rctx->oaep_label = p2;
 660                         rctx->oaep_labellen = p1;
 661                         }
 662                 else
 663                         {
 664                         rctx->oaep_label = NULL;
 665                         rctx->oaep_labellen = 0;
 666                         }
 667                 return 1;
 668
 669                 case EVP_PKEY_CTRL_GET_RSA_OAEP_LABEL:
 670                 if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING)
 671                         {
 672                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PADDING_MODE);
 673                         return -2;
 674                         }
 675                 *(unsigned char **)p2 = rctx->oaep_label;
 676                 return rctx->oaep_labellen;
 677
 678                 case EVP_PKEY_CTRL_DIGESTINIT:
 679                 case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
 680                 case EVP_PKEY_CTRL_PKCS7_DECRYPT:
 681                 case EVP_PKEY_CTRL_PKCS7_SIGN:
 682                 return 1;
 683 #ifndef OPENSSL_NO_CMS
 684                 case EVP_PKEY_CTRL_CMS_DECRYPT:
 685                 {
 686                 X509_ALGOR *alg = NULL;
 687                 ASN1_OBJECT *encalg = NULL;
 688                 if (p2)
 689                         CMS_RecipientInfo_ktri_get0_algs(p2, NULL, NULL, &alg);
 690                 if (alg)
 691                         X509_ALGOR_get0(&encalg, NULL, NULL, alg);
 692                 if (encalg && OBJ_obj2nid(encalg) == NID_rsaesOaep)
 693                         rctx->pad_mode = RSA_PKCS1_OAEP_PADDING;
 694                 }
 695                 case EVP_PKEY_CTRL_CMS_ENCRYPT:
 696                 case EVP_PKEY_CTRL_CMS_SIGN:
 697                 return 1;
 698 #endif
 699                 case EVP_PKEY_CTRL_PEER_KEY:
 700                         RSAerr(RSA_F_PKEY_RSA_CTRL,
 701                         RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
 702                         return -2;
 703
 704                 default:
 705                 return -2;
 706
 707                 }
 708         }
 709
 710 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
 711                         const char *type, const char *value)
 712         {
 713         if (!value)
 714                 {
 715                 RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
 716                 return 0;
 717                 }
 718         if (!strcmp(type, "rsa_padding_mode"))
 719                 {
 720                 int pm;
 721                 if (!strcmp(value, "pkcs1"))
 722                         pm = RSA_PKCS1_PADDING;
 723                 else if (!strcmp(value, "sslv23"))
 724                         pm = RSA_SSLV23_PADDING;
 725                 else if (!strcmp(value, "none"))
 726                         pm = RSA_NO_PADDING;
 727                 else if (!strcmp(value, "oeap"))
 728                         pm = RSA_PKCS1_OAEP_PADDING;
 729                 else if (!strcmp(value, "oaep"))
 730                         pm = RSA_PKCS1_OAEP_PADDING;
 731                 else if (!strcmp(value, "x931"))
 732                         pm = RSA_X931_PADDING;
 733                 else if (!strcmp(value, "pss"))
 734                         pm = RSA_PKCS1_PSS_PADDING;
 735                 else
 736                         {
 737                         RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
 738                                                 RSA_R_UNKNOWN_PADDING_TYPE);
 739                         return -2;
 740                         }
 741                 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
 742                 }
 743
 744         if (!strcmp(type, "rsa_pss_saltlen"))
 745                 {
 746                 int saltlen;
 747                 saltlen = atoi(value);
 748                 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
 749                 }
 750
 751         if (!strcmp(type, "rsa_keygen_bits"))
 752                 {
 753                 int nbits;
 754                 nbits = atoi(value);
 755                 return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
 756                 }
 757
 758         if (!strcmp(type, "rsa_keygen_pubexp"))
 759                 {
 760                 int ret;
 761                 BIGNUM *pubexp = NULL;
 762                 if (!BN_asc2bn(&pubexp, value))
 763                         return 0;
 764                 ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
 765                 if (ret <= 0)
 766                         BN_free(pubexp);
 767                 return ret;
 768                 }
 769
 770         if (!strcmp(type, "rsa_mgf1_md"))
 771                 {
 772                 const EVP_MD *md;
 773                 if (!(md = EVP_get_digestbyname(value)))
 774                         {
 775                         RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
 776                                                 RSA_R_INVALID_DIGEST);
 777                         return 0;
 778                         }
 779                 return EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md);
 780                 }
 781
 782         if (!strcmp(type, "rsa_oaep_md"))
 783                 {
 784                 const EVP_MD *md;
 785                 if (!(md = EVP_get_digestbyname(value)))
 786                         {
 787                         RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
 788                                                 RSA_R_INVALID_DIGEST);
 789                         return 0;
 790                         }
 791                 return EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md);
 792                 }
 793         if (!strcmp(type, "rsa_oaep_label"))
 794                 {
 795                 unsigned char *lab;
 796                 long lablen;
 797                 int ret;
 798                 lab = string_to_hex(value, &lablen);
 799                 if (!lab)
 800                         return 0;
 801                 ret = EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, lab, lablen);
 802                 if (ret <= 0)
 803                         OPENSSL_free(lab);
 804                 return ret;
 805                 }
 806
 807         return -2;
 808         }
 809
 810 static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
 811         {
 812         RSA *rsa = NULL;
 813         RSA_PKEY_CTX *rctx = ctx->data;
 814         BN_GENCB *pcb, cb;
 815         int ret;
 816         if (!rctx->pub_exp)
 817                 {
 818                 rctx->pub_exp = BN_new();
 819                 if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
 820                         return 0;
 821                 }
 822         rsa = RSA_new();
 823         if (!rsa)
 824                 return 0;
 825         if (ctx->pkey_gencb)
 826                 {
 827                 pcb = &cb;
 828                 evp_pkey_set_cb_translate(pcb, ctx);
 829                 }
 830         else
 831                 pcb = NULL;
 832         ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
 833         if (ret > 0)
 834                 EVP_PKEY_assign_RSA(pkey, rsa);
 835         else
 836                 RSA_free(rsa);
 837         return ret;
 838         }
 839
 840 const EVP_PKEY_METHOD rsa_pkey_meth =
 841         {
 842         EVP_PKEY_RSA,
 843         EVP_PKEY_FLAG_AUTOARGLEN,
 844         pkey_rsa_init,
 845         pkey_rsa_copy,
 846         pkey_rsa_cleanup,
 847
 848         0,0,
 849
 850         0,
 851         pkey_rsa_keygen,
 852
 853         0,
 854         pkey_rsa_sign,
 855
 856         0,
 857         pkey_rsa_verify,
 858
 859         0,
 860         pkey_rsa_verifyrecover,
 861
 862
 863         0,0,0,0,
 864
 865         0,
 866         pkey_rsa_encrypt,
 867
 868         0,
 869         pkey_rsa_decrypt,
 870
 871         0,0,
 872
 873         pkey_rsa_ctrl,
 874         pkey_rsa_ctrl_str
 875
 876
 877         };