crypto/rsa/rsa_pmeth.c

   1 /* crypto/rsa/rsa_pmeth.c */
   2 /* Written by Dr Stephen N Henson (shenson@bigfoot.com) for the OpenSSL
   3  * project 2006.
   4  */
   5 /* ====================================================================
   6  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  *
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in
  17  *    the documentation and/or other materials provided with the
  18  *    distribution.
  19  *
  20  * 3. All advertising materials mentioning features or use of this
  21  *    software must display the following acknowledgment:
  22  *    "This product includes software developed by the OpenSSL Project
  23  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
  24  *
  25  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
  26  *    endorse or promote products derived from this software without
  27  *    prior written permission. For written permission, please contact
  28  *    licensing@OpenSSL.org.
  29  *
  30  * 5. Products derived from this software may not be called "OpenSSL"
  31  *    nor may "OpenSSL" appear in their names without prior written
  32  *    permission of the OpenSSL Project.
  33  *
  34  * 6. Redistributions of any form whatsoever must retain the following
  35  *    acknowledgment:
  36  *    "This product includes software developed by the OpenSSL Project
  37  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
  38  *
  39  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
  40  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  41  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  42  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
  43  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  44  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  45  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  46  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  48  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  49  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
  50  * OF THE POSSIBILITY OF SUCH DAMAGE.
  51  * ====================================================================
  52  *
  53  * This product includes cryptographic software written by Eric Young
  54  * (eay@cryptsoft.com).  This product includes software written by Tim
  55  * Hudson (tjh@cryptsoft.com).
  56  *
  57  */
  58
  59 #include <stdio.h>
  60 #include "cryptlib.h"
  61 #include <openssl/asn1t.h>
  62 #include <openssl/x509.h>
  63 #include <openssl/rsa.h>
  64 #include <openssl/evp.h>
  65 #include "evp_locl.h"
  66
  67 extern int int_rsa_verify(int dtype, const unsigned char *m, unsigned int m_len,
  68                 unsigned char *rm, unsigned int *prm_len,
  69                 const unsigned char *sigbuf, unsigned int siglen,
  70                 RSA *rsa);
  71
  72 /* RSA pkey context structure */
  73
  74 typedef struct
  75         {
  76         /* Key gen parameters */
  77         int nbits;
  78         BIGNUM *pub_exp;
  79         /* Keygen callback info */
  80         int gentmp[2];
  81         /* RSA padding mode */
  82         int pad_mode;
  83         /* message digest */
  84         const EVP_MD *md;
  85         /* PSS/OAEP salt length */
  86         int saltlen;
  87         /* Temp buffer */
  88         unsigned char *tbuf;
  89         } RSA_PKEY_CTX;
  90
  91 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
  92         {
  93         RSA_PKEY_CTX *rctx;
  94         rctx = OPENSSL_malloc(sizeof(RSA_PKEY_CTX));
  95         if (!rctx)
  96                 return 0;
  97         rctx->nbits = 1024;
  98         rctx->pub_exp = NULL;
  99         rctx->pad_mode = RSA_PKCS1_PADDING;
 100         rctx->md = NULL;
 101         rctx->tbuf = NULL;
 102
 103         rctx->saltlen = -2;
 104
 105         ctx->data = rctx;
 106         ctx->keygen_info = rctx->gentmp;
 107         ctx->keygen_info_count = 2;
 108
 109         return 1;
 110         }
 111
 112 static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
 113         {
 114         if (ctx->tbuf)
 115                 return 1;
 116         ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));
 117         if (!ctx->tbuf)
 118                 return 0;
 119         return 1;
 120         }
 121
 122 static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
 123         {
 124         RSA_PKEY_CTX *rctx = ctx->data;
 125         if (rctx)
 126                 {
 127                 if (rctx->pub_exp)
 128                         BN_free(rctx->pub_exp);
 129                 if (rctx->tbuf)
 130                         OPENSSL_free(rctx->tbuf);
 131                 OPENSSL_free(rctx);
 132                 }
 133         }
 134
 135 static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, int *siglen,
 136                                         const unsigned char *tbs, int tbslen)
 137         {
 138         int ret;
 139         RSA_PKEY_CTX *rctx = ctx->data;
 140         RSA *rsa = ctx->pkey->pkey.rsa;
 141
 142         if (rctx->md)
 143                 {
 144                 if (tbslen != EVP_MD_size(rctx->md))
 145                         {
 146                         RSAerr(RSA_F_PKEY_RSA_SIGN,
 147                                         RSA_R_INVALID_DIGEST_LENGTH);
 148                         return -1;
 149                         }
 150                 if (rctx->pad_mode == RSA_X931_PADDING)
 151                         {
 152                         if (!setup_tbuf(rctx, ctx))
 153                                 return -1;
 154                         memcpy(rctx->tbuf, tbs, tbslen);
 155                         rctx->tbuf[tbslen] =
 156                                 RSA_X931_hash_id(EVP_MD_type(rctx->md));
 157                         ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
 158                                                 sig, rsa, RSA_X931_PADDING);
 159                         }
 160                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
 161                         {
 162                         unsigned int sltmp;
 163                         ret = RSA_sign(EVP_MD_type(rctx->md),
 164                                                 tbs, tbslen, sig, &sltmp, rsa);
 165                         if (ret <= 0)
 166                                 return ret;
 167                         ret = sltmp;
 168                         }
 169                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
 170                         {
 171                         if (!setup_tbuf(rctx, ctx))
 172                                 return -1;
 173                         if (!RSA_padding_add_PKCS1_PSS(rsa, rctx->tbuf, tbs,
 174                                                 rctx->md, rctx->saltlen))
 175                                 return -1;
 176                         ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
 177                                                 sig, rsa, RSA_NO_PADDING);
 178                         }
 179                 else
 180                         return -1;
 181                 }
 182         else
 183                 ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
 184                                                         rctx->pad_mode);
 185         if (ret < 0)
 186                 return ret;
 187         *siglen = ret;
 188         return 1;
 189         }
 190
 191
 192 static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
 193                                         unsigned char *rout, int *routlen,
 194                                         const unsigned char *sig, int siglen)
 195         {
 196         int ret;
 197         RSA_PKEY_CTX *rctx = ctx->data;
 198
 199         if (rctx->md)
 200                 {
 201                 if (rctx->pad_mode == RSA_X931_PADDING)
 202                         {
 203                         if (!setup_tbuf(rctx, ctx))
 204                                 return -1;
 205                         ret = RSA_public_decrypt(siglen, sig,
 206                                                 rctx->tbuf, ctx->pkey->pkey.rsa,
 207                                                 RSA_X931_PADDING);
 208                         if (ret < 1)
 209                                 return 0;
 210                         ret--;
 211                         if (rctx->tbuf[ret] !=
 212                                 RSA_X931_hash_id(EVP_MD_type(rctx->md)))
 213                                 {
 214                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
 215                                                 RSA_R_ALGORITHM_MISMATCH);
 216                                 return 0;
 217                                 }
 218                         if (ret != EVP_MD_size(rctx->md))
 219                                 {
 220                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
 221                                         RSA_R_INVALID_DIGEST_LENGTH);
 222                                 return 0;
 223                                 }
 224                         if (rout)
 225                                 memcpy(rout, rctx->tbuf, ret);
 226                         }
 227                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
 228                         {
 229                         unsigned int sltmp;
 230                         ret = int_rsa_verify(EVP_MD_type(rctx->md),
 231                                                 NULL, 0, rout, &sltmp,
 232                                         sig, siglen, ctx->pkey->pkey.rsa);
 233                         ret = sltmp;
 234                         }
 235                 else
 236                         return -1;
 237                 }
 238         else
 239                 ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
 240                                                         rctx->pad_mode);
 241         if (ret < 0)
 242                 return ret;
 243         *routlen = ret;
 244         return 1;
 245         }
 246
 247 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
 248                                         const unsigned char *sig, int siglen,
 249                                         const unsigned char *tbs, int tbslen)
 250         {
 251         RSA_PKEY_CTX *rctx = ctx->data;
 252         RSA *rsa = ctx->pkey->pkey.rsa;
 253         int rslen;
 254         if (rctx->md)
 255                 {
 256                 if (rctx->pad_mode == RSA_PKCS1_PADDING)
 257                         return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
 258                                         sig, siglen, rsa);
 259                 if (rctx->pad_mode == RSA_X931_PADDING)
 260                         {
 261                         if (pkey_rsa_verifyrecover(ctx, NULL, &rslen,
 262                                         sig, siglen) <= 0)
 263                                 return 0;
 264                         }
 265                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
 266                         {
 267                         int ret;
 268                         if (!setup_tbuf(rctx, ctx))
 269                                 return -1;
 270                         ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
 271                                                         rsa, RSA_NO_PADDING);
 272                         if (ret <= 0)
 273                                 return 0;
 274                         ret = RSA_verify_PKCS1_PSS(rsa, tbs, rctx->md,
 275                                                 rctx->tbuf, rctx->saltlen);
 276                         if (ret <= 0)
 277                                 return 0;
 278                         return 1;
 279                         }
 280                 else
 281                         return -1;
 282                 }
 283         else
 284                 {
 285                 if (!setup_tbuf(rctx, ctx))
 286                         return -1;
 287                 rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
 288                                                 rsa, rctx->pad_mode);
 289                 if (rslen <= 0)
 290                         return 0;
 291                 }
 292
 293         if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))
 294                 return 0;
 295
 296         return 1;
 297
 298         }
 299
 300
 301 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
 302                                         const unsigned char *in, int inlen)
 303         {
 304         int ret;
 305         RSA_PKEY_CTX *rctx = ctx->data;
 306         ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 307                                                         rctx->pad_mode);
 308         if (ret < 0)
 309                 return ret;
 310         *outlen = ret;
 311         return 1;
 312         }
 313
 314 static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
 315                                         const unsigned char *in, int inlen)
 316         {
 317         int ret;
 318         RSA_PKEY_CTX *rctx = ctx->data;
 319         ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 320                                                         rctx->pad_mode);
 321         if (ret < 0)
 322                 return ret;
 323         *outlen = ret;
 324         return 1;
 325         }
 326
 327 static int check_padding_md(const EVP_MD *md, int padding)
 328         {
 329         if (!md)
 330                 return 1;
 331
 332         if (padding == RSA_NO_PADDING)
 333                 {
 334                 RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
 335                 return 0;
 336                 }
 337
 338         if (padding == RSA_X931_PADDING)
 339                 {
 340                 if (RSA_X931_hash_id(EVP_MD_type(md)) == -1)
 341                         {
 342                         RSAerr(RSA_F_CHECK_PADDING_MD,
 343                                                 RSA_R_INVALID_X931_DIGEST);
 344                         return 0;
 345                         }
 346                 return 1;
 347                 }
 348
 349         return 1;
 350         }
 351
 352
 353 static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 354         {
 355         RSA_PKEY_CTX *rctx = ctx->data;
 356         switch (type)
 357                 {
 358                 case EVP_PKEY_CTRL_RSA_PADDING:
 359                 if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING))
 360                         {
 361                         if (!check_padding_md(rctx->md, p1))
 362                                 return 0;
 363                         if (p1 == RSA_PKCS1_PSS_PADDING)
 364                                 {
 365                                 if (!(ctx->operation &
 366                                      (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
 367                                         goto bad_pad;
 368                                 if (!rctx->md)
 369                                         rctx->md = EVP_sha1();
 370                                 }
 371                         if (p1 == RSA_PKCS1_OAEP_PADDING)
 372                                 {
 373                                 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
 374                                         goto bad_pad;
 375                                 if (!rctx->md)
 376                                         rctx->md = EVP_sha1();
 377                                 }
 378                         rctx->pad_mode = p1;
 379                         return 1;
 380                         }
 381                 bad_pad:
 382                 RSAerr(RSA_F_PKEY_RSA_CTRL,
 383                                 RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
 384                 return -2;
 385
 386                 case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
 387                 if (p1 < -2)
 388                         return -2;
 389                 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
 390                         {
 391                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
 392                         return -2;
 393                         }
 394                 rctx->saltlen = p1;
 395                 return 1;
 396
 397                 case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
 398                 if (p1 < 256)
 399                         {
 400                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_KEYBITS);
 401                         return -2;
 402                         }
 403                 rctx->nbits = p1;
 404                 return 1;
 405
 406                 case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
 407                 if (!p2)
 408                         return -2;
 409                 rctx->pub_exp = p2;
 410                 return 1;
 411
 412                 case EVP_PKEY_CTRL_MD:
 413                 if (!check_padding_md(p2, rctx->pad_mode))
 414                         return 0;
 415                 rctx->md = p2;
 416                 return 1;
 417
 418                 case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
 419                 case EVP_PKEY_CTRL_PKCS7_DECRYPT:
 420                 return 1;
 421
 422                 default:
 423                 return -2;
 424
 425                 }
 426         }
 427
 428 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
 429                         const char *type, const char *value)
 430         {
 431         if (!value)
 432                 {
 433                 RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
 434                 return 0;
 435                 }
 436         if (!strcmp(type, "rsa_padding_mode"))
 437                 {
 438                 int pm;
 439                 if (!strcmp(value, "pkcs1"))
 440                         pm = RSA_PKCS1_PADDING;
 441                 else if (!strcmp(value, "sslv23"))
 442                         pm = RSA_SSLV23_PADDING;
 443                 else if (!strcmp(value, "none"))
 444                         pm = RSA_NO_PADDING;
 445                 else if (!strcmp(value, "oeap"))
 446                         pm = RSA_PKCS1_OAEP_PADDING;
 447                 else if (!strcmp(value, "x931"))
 448                         pm = RSA_X931_PADDING;
 449                 else if (!strcmp(value, "pss"))
 450                         pm = RSA_PKCS1_PSS_PADDING;
 451                 else
 452                         {
 453                         RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
 454                                                 RSA_R_UNKNOWN_PADDING_TYPE);
 455                         return -2;
 456                         }
 457                 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
 458                 }
 459
 460         if (!strcmp(type, "rsa_pss_saltlen"))
 461                 {
 462                 int saltlen;
 463                 saltlen = atoi(value);
 464                 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
 465                 }
 466
 467         if (!strcmp(type, "rsa_keygen_bits"))
 468                 {
 469                 int nbits;
 470                 nbits = atoi(value);
 471                 return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
 472                 }
 473
 474         if (!strcmp(type, "rsa_keygen_pubexp"))
 475                 {
 476                 int ret;
 477                 BIGNUM *pubexp = NULL;
 478                 if (!BN_asc2bn(&pubexp, value))
 479                         return 0;
 480                 ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
 481                 if (ret <= 0)
 482                         BN_free(pubexp);
 483                 return ret;
 484                 }
 485
 486         return -2;
 487         }
 488
 489 static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
 490         {
 491         RSA *rsa = NULL;
 492         RSA_PKEY_CTX *rctx = ctx->data;
 493         BN_GENCB *pcb, cb;
 494         int ret;
 495         if (!rctx->pub_exp)
 496                 {
 497                 rctx->pub_exp = BN_new();
 498                 if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
 499                         return 0;
 500                 }
 501         rsa = RSA_new();
 502         if (!rsa)
 503                 return 0;
 504         if (ctx->pkey_gencb)
 505                 {
 506                 pcb = &cb;
 507                 evp_pkey_set_cb_translate(pcb, ctx);
 508                 }
 509         else
 510                 pcb = NULL;
 511         ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
 512         if (ret > 0)
 513                 EVP_PKEY_assign_RSA(pkey, rsa);
 514         else
 515                 RSA_free(rsa);
 516         return ret;
 517         }
 518
 519 const EVP_PKEY_METHOD rsa_pkey_meth =
 520         {
 521         EVP_PKEY_RSA,
 522         EVP_PKEY_FLAG_AUTOARGLEN,
 523         pkey_rsa_init,
 524         pkey_rsa_cleanup,
 525
 526         0,0,
 527
 528         0,
 529         pkey_rsa_keygen,
 530
 531         0,
 532         pkey_rsa_sign,
 533
 534         0,
 535         pkey_rsa_verify,
 536
 537         0,
 538         pkey_rsa_verifyrecover,
 539
 540
 541         0,0,0,0,
 542
 543         0,
 544         pkey_rsa_encrypt,
 545
 546         0,
 547         pkey_rsa_decrypt,
 548
 549         0,0,
 550
 551         pkey_rsa_ctrl,
 552         pkey_rsa_ctrl_str
 553
 554
 555         };