crypto/rsa/rsa_pmeth.c

   1 /* Written by Dr Stephen N Henson (shenson@bigfoot.com) for the OpenSSL
   2  * project 2006.
   3  */
   4 /* ====================================================================
   5  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  *
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in
  16  *    the documentation and/or other materials provided with the
  17  *    distribution.
  18  *
  19  * 3. All advertising materials mentioning features or use of this
  20  *    software must display the following acknowledgment:
  21  *    "This product includes software developed by the OpenSSL Project
  22  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
  23  *
  24  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
  25  *    endorse or promote products derived from this software without
  26  *    prior written permission. For written permission, please contact
  27  *    licensing@OpenSSL.org.
  28  *
  29  * 5. Products derived from this software may not be called "OpenSSL"
  30  *    nor may "OpenSSL" appear in their names without prior written
  31  *    permission of the OpenSSL Project.
  32  *
  33  * 6. Redistributions of any form whatsoever must retain the following
  34  *    acknowledgment:
  35  *    "This product includes software developed by the OpenSSL Project
  36  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
  37  *
  38  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
  39  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  40  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  41  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
  42  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  43  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  44  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  45  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  46  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  47  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  48  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
  49  * OF THE POSSIBILITY OF SUCH DAMAGE.
  50  * ====================================================================
  51  *
  52  * This product includes cryptographic software written by Eric Young
  53  * (eay@cryptsoft.com).  This product includes software written by Tim
  54  * Hudson (tjh@cryptsoft.com).
  55  *
  56  */
  57
  58 #include <stdio.h>
  59 #include "cryptlib.h"
  60 #include <openssl/asn1t.h>
  61 #include <openssl/x509.h>
  62 #include <openssl/rsa.h>
  63 #include <openssl/evp.h>
  64 #include "evp_locl.h"
  65
  66 extern int int_rsa_verify(int dtype, const unsigned char *m, unsigned int m_len,
  67                 unsigned char *rm, unsigned int *prm_len,
  68                 const unsigned char *sigbuf, unsigned int siglen,
  69                 RSA *rsa);
  70
  71 /* RSA pkey context structure */
  72
  73 typedef struct
  74         {
  75         /* Key gen parameters */
  76         int nbits;
  77         BIGNUM *pub_exp;
  78         /* RSA padding mode */
  79         int pad_mode;
  80         /* message digest */
  81         const EVP_MD *md;
  82         /* PSS/OAEP salt length */
  83         int saltlen;
  84         /* Temp buffer */
  85         unsigned char *tbuf;
  86         } RSA_PKEY_CTX;
  87
  88 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
  89         {
  90         RSA_PKEY_CTX *rctx;
  91         rctx = OPENSSL_malloc(sizeof(RSA_PKEY_CTX));
  92         if (!rctx)
  93                 return 0;
  94         rctx->nbits = 1024;
  95         rctx->pub_exp = NULL;
  96         rctx->pad_mode = RSA_PKCS1_PADDING;
  97         rctx->md = NULL;
  98         rctx->tbuf = NULL;
  99
 100         rctx->saltlen = -2;
 101
 102         ctx->data = rctx;
 103
 104         return 1;
 105         }
 106
 107 static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
 108         {
 109         if (ctx->tbuf)
 110                 return 1;
 111         ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));
 112         if (!ctx->tbuf)
 113                 return 0;
 114         return 1;
 115         }
 116
 117 static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
 118         {
 119         RSA_PKEY_CTX *rctx = ctx->data;
 120         if (rctx)
 121                 {
 122                 if (rctx->pub_exp)
 123                         BN_free(rctx->pub_exp);
 124                 if (rctx->tbuf)
 125                         OPENSSL_free(rctx->tbuf);
 126                 }
 127         OPENSSL_free(rctx);
 128         }
 129
 130 static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, int *siglen,
 131                                         const unsigned char *tbs, int tbslen)
 132         {
 133         int ret;
 134         RSA_PKEY_CTX *rctx = ctx->data;
 135         RSA *rsa = ctx->pkey->pkey.rsa;
 136
 137         if (rctx->md)
 138                 {
 139                 if (tbslen != EVP_MD_size(rctx->md))
 140                         {
 141                         RSAerr(RSA_F_PKEY_RSA_SIGN,
 142                                         RSA_R_INVALID_DIGEST_LENGTH);
 143                         return -1;
 144                         }
 145                 if (rctx->pad_mode == RSA_X931_PADDING)
 146                         {
 147                         if (!setup_tbuf(rctx, ctx))
 148                                 return -1;
 149                         memcpy(rctx->tbuf, tbs, tbslen);
 150                         rctx->tbuf[tbslen] =
 151                                 RSA_X931_hash_id(EVP_MD_type(rctx->md));
 152                         ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
 153                                                 sig, rsa, RSA_X931_PADDING);
 154                         }
 155                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
 156                         {
 157                         unsigned int sltmp;
 158                         ret = RSA_sign(EVP_MD_type(rctx->md),
 159                                                 tbs, tbslen, sig, &sltmp, rsa);
 160                         if (ret <= 0)
 161                                 return ret;
 162                         ret = sltmp;
 163                         }
 164                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
 165                         {
 166                         if (!setup_tbuf(rctx, ctx))
 167                                 return -1;
 168                         if (!RSA_padding_add_PKCS1_PSS(rsa, rctx->tbuf, tbs,
 169                                                 rctx->md, rctx->saltlen))
 170                                 return -1;
 171                         ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
 172                                                 sig, rsa, RSA_NO_PADDING);
 173                         }
 174                 else
 175                         return -1;
 176                 }
 177         else
 178                 ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
 179                                                         rctx->pad_mode);
 180         if (ret < 0)
 181                 return ret;
 182         *siglen = ret;
 183         return 1;
 184         }
 185
 186
 187 static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
 188                                         unsigned char *rout, int *routlen,
 189                                         const unsigned char *sig, int siglen)
 190         {
 191         int ret;
 192         RSA_PKEY_CTX *rctx = ctx->data;
 193
 194         if (rctx->md)
 195                 {
 196                 if (rctx->pad_mode == RSA_X931_PADDING)
 197                         {
 198                         if (!setup_tbuf(rctx, ctx))
 199                                 return -1;
 200                         ret = RSA_public_decrypt(siglen, sig,
 201                                                 rctx->tbuf, ctx->pkey->pkey.rsa,
 202                                                 RSA_X931_PADDING);
 203                         if (ret < 1)
 204                                 return 0;
 205                         ret--;
 206                         if (rctx->tbuf[ret] !=
 207                                 RSA_X931_hash_id(EVP_MD_type(rctx->md)))
 208                                 {
 209                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
 210                                                 RSA_R_ALGORITHM_MISMATCH);
 211                                 return 0;
 212                                 }
 213                         if (ret != EVP_MD_size(rctx->md))
 214                                 {
 215                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
 216                                         RSA_R_INVALID_DIGEST_LENGTH);
 217                                 return 0;
 218                                 }
 219                         if (rout)
 220                                 memcpy(rout, rctx->tbuf, ret);
 221                         }
 222                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
 223                         {
 224                         unsigned int sltmp;
 225                         ret = int_rsa_verify(EVP_MD_type(rctx->md),
 226                                                 NULL, 0, rout, &sltmp,
 227                                         sig, siglen, ctx->pkey->pkey.rsa);
 228                         ret = sltmp;
 229                         }
 230                 else
 231                         return -1;
 232                 }
 233         else
 234                 ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
 235                                                         rctx->pad_mode);
 236         if (ret < 0)
 237                 return ret;
 238         *routlen = ret;
 239         return 1;
 240         }
 241
 242 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
 243                                         const unsigned char *sig, int siglen,
 244                                         const unsigned char *tbs, int tbslen)
 245         {
 246         RSA_PKEY_CTX *rctx = ctx->data;
 247         RSA *rsa = ctx->pkey->pkey.rsa;
 248         int rslen;
 249         if (rctx->md)
 250                 {
 251                 if (rctx->pad_mode == RSA_PKCS1_PADDING)
 252                         return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
 253                                         sig, siglen, rsa);
 254                 if (rctx->pad_mode == RSA_X931_PADDING)
 255                         {
 256                         if (pkey_rsa_verifyrecover(ctx, NULL, &rslen,
 257                                         sig, siglen) <= 0)
 258                                 return 0;
 259                         }
 260                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
 261                         {
 262                         int ret;
 263                         if (!setup_tbuf(rctx, ctx))
 264                                 return -1;
 265                         ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
 266                                                         rsa, RSA_NO_PADDING);
 267                         if (ret <= 0)
 268                                 return 0;
 269                         ret = RSA_verify_PKCS1_PSS(rsa, tbs, rctx->md,
 270                                                 rctx->tbuf, rctx->saltlen);
 271                         if (ret <= 0)
 272                                 return 0;
 273                         return 1;
 274                         }
 275                 else
 276                         return -1;
 277                 }
 278         else
 279                 {
 280                 if (!setup_tbuf(rctx, ctx))
 281                         return -1;
 282                 rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
 283                                                 rsa, rctx->pad_mode);
 284                 if (rslen <= 0)
 285                         return 0;
 286                 }
 287
 288         if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))
 289                 return 0;
 290
 291         return 1;
 292
 293         }
 294
 295
 296 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
 297                                         const unsigned char *in, int inlen)
 298         {
 299         int ret;
 300         RSA_PKEY_CTX *rctx = ctx->data;
 301         ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 302                                                         rctx->pad_mode);
 303         if (ret < 0)
 304                 return ret;
 305         *outlen = ret;
 306         return 1;
 307         }
 308
 309 static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
 310                                         const unsigned char *in, int inlen)
 311         {
 312         int ret;
 313         RSA_PKEY_CTX *rctx = ctx->data;
 314         ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
 315                                                         rctx->pad_mode);
 316         if (ret < 0)
 317                 return ret;
 318         *outlen = ret;
 319         return 1;
 320         }
 321
 322 static int check_padding_md(const EVP_MD *md, int padding)
 323         {
 324         if (!md)
 325                 return 1;
 326
 327         if (padding == RSA_NO_PADDING)
 328                 {
 329                 RSAerr(RSA_F_CHECK_PADDING_NID, RSA_R_INVALID_PADDING_MODE);
 330                 return 0;
 331                 }
 332
 333         if (padding == RSA_X931_PADDING)
 334                 {
 335                 if (RSA_X931_hash_id(EVP_MD_type(md)) == -1)
 336                         {
 337                         RSAerr(RSA_F_CHECK_PADDING_NID,
 338                                                 RSA_R_INVALID_X931_DIGEST);
 339                         return 0;
 340                         }
 341                 return 1;
 342                 }
 343
 344         return 1;
 345         }
 346
 347
 348 static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 349         {
 350         RSA_PKEY_CTX *rctx = ctx->data;
 351         switch (type)
 352                 {
 353                 case EVP_PKEY_CTRL_RSA_PADDING:
 354                 if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING))
 355                         {
 356                         if (!check_padding_md(rctx->md, p1))
 357                                 return 0;
 358                         if (p1 == RSA_PKCS1_PSS_PADDING)
 359                                 {
 360                                 if (ctx->operation == EVP_PKEY_OP_VERIFYRECOVER)
 361                                         return -2;
 362                                 if (!rctx->md)
 363                                         rctx->md = EVP_sha1();
 364                                 }
 365                         if (p1 == RSA_PKCS1_OAEP_PADDING)
 366                                 {
 367                                 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
 368                                         return -2;
 369                                 if (!rctx->md)
 370                                         rctx->md = EVP_sha1();
 371                                 }
 372                         rctx->pad_mode = p1;
 373                         return 1;
 374                         }
 375                 return -2;
 376
 377                 case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
 378                 if (p1 < -2)
 379                         return -2;
 380                 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
 381                         return -2;
 382                 rctx->saltlen = p1;
 383                 return 1;
 384
 385                 case EVP_PKEY_CTRL_MD:
 386                 if (!check_padding_md(p2, rctx->pad_mode))
 387                         return 0;
 388                 rctx->md = p2;
 389                 return 1;
 390
 391                 default:
 392                 return -2;
 393
 394                 }
 395         }
 396
 397 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
 398                         const char *type, const char *value)
 399         {
 400         if (!strcmp(type, "rsa_padding_mode"))
 401                 {
 402                 int pm;
 403                 if (!value)
 404                         return 0;
 405                 if (!strcmp(value, "pkcs1"))
 406                         pm = RSA_PKCS1_PADDING;
 407                 else if (!strcmp(value, "sslv23"))
 408                         pm = RSA_SSLV23_PADDING;
 409                 else if (!strcmp(value, "none"))
 410                         pm = RSA_NO_PADDING;
 411                 else if (!strcmp(value, "oeap"))
 412                         pm = RSA_PKCS1_OAEP_PADDING;
 413                 else if (!strcmp(value, "x931"))
 414                         pm = RSA_X931_PADDING;
 415                 else if (!strcmp(value, "pss"))
 416                         pm = RSA_PKCS1_PSS_PADDING;
 417                 else
 418                         return -2;
 419                 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
 420                 }
 421         if (!strcmp(type, "rsa_pss_saltlen"))
 422                 {
 423                 int saltlen;
 424                 saltlen = atoi(value);
 425                 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
 426                 }
 427         return -2;
 428         }
 429
 430 const EVP_PKEY_METHOD rsa_pkey_meth =
 431         {
 432         EVP_PKEY_RSA,
 433         0,
 434         pkey_rsa_init,
 435         pkey_rsa_cleanup,
 436
 437         0,0,
 438
 439         0,0,
 440
 441         0,
 442         pkey_rsa_sign,
 443
 444         0,
 445         pkey_rsa_verify,
 446
 447         0,
 448         pkey_rsa_verifyrecover,
 449
 450
 451         0,0,0,0,
 452
 453         0,
 454         pkey_rsa_encrypt,
 455
 456         0,
 457         pkey_rsa_decrypt,
 458
 459         pkey_rsa_ctrl,
 460         pkey_rsa_ctrl_str
 461
 462
 463         };