Allow digests to supply S/MIME micalg values from a ctrl.
[openssl.git] / crypto / rsa / rsa_pmeth.c
1 /* crypto/rsa/rsa_pmeth.c */
2 /* Written by Dr Stephen N Henson (shenson@bigfoot.com) for the OpenSSL
3  * project 2006.
4  */
5 /* ====================================================================
6  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer. 
14  *
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in
17  *    the documentation and/or other materials provided with the
18  *    distribution.
19  *
20  * 3. All advertising materials mentioning features or use of this
21  *    software must display the following acknowledgment:
22  *    "This product includes software developed by the OpenSSL Project
23  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
24  *
25  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
26  *    endorse or promote products derived from this software without
27  *    prior written permission. For written permission, please contact
28  *    licensing@OpenSSL.org.
29  *
30  * 5. Products derived from this software may not be called "OpenSSL"
31  *    nor may "OpenSSL" appear in their names without prior written
32  *    permission of the OpenSSL Project.
33  *
34  * 6. Redistributions of any form whatsoever must retain the following
35  *    acknowledgment:
36  *    "This product includes software developed by the OpenSSL Project
37  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
38  *
39  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
40  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
41  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
42  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
43  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
44  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
45  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
46  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
48  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
49  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
50  * OF THE POSSIBILITY OF SUCH DAMAGE.
51  * ====================================================================
52  *
53  * This product includes cryptographic software written by Eric Young
54  * (eay@cryptsoft.com).  This product includes software written by Tim
55  * Hudson (tjh@cryptsoft.com).
56  *
57  */
58
59 #include <stdio.h>
60 #include "cryptlib.h"
61 #include <openssl/asn1t.h>
62 #include <openssl/x509.h>
63 #include <openssl/rsa.h>
64 #include <openssl/evp.h>
65 #include "evp_locl.h"
66
67 extern int int_rsa_verify(int dtype, const unsigned char *m, size_t m_len,
68                 unsigned char *rm, size_t *prm_len,
69                 const unsigned char *sigbuf, size_t siglen,
70                 RSA *rsa);
71
72 /* RSA pkey context structure */
73
74 typedef struct
75         {
76         /* Key gen parameters */
77         int nbits;
78         BIGNUM *pub_exp;
79         /* Keygen callback info */
80         int gentmp[2];
81         /* RSA padding mode */
82         int pad_mode;
83         /* message digest */
84         const EVP_MD *md;
85         /* PSS/OAEP salt length */
86         int saltlen;
87         /* Temp buffer */
88         unsigned char *tbuf;
89         } RSA_PKEY_CTX;
90
91 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
92         {
93         RSA_PKEY_CTX *rctx;
94         rctx = OPENSSL_malloc(sizeof(RSA_PKEY_CTX));
95         if (!rctx)
96                 return 0;
97         rctx->nbits = 1024;
98         rctx->pub_exp = NULL;
99         rctx->pad_mode = RSA_PKCS1_PADDING;
100         rctx->md = NULL;
101         rctx->tbuf = NULL;
102
103         rctx->saltlen = -2;
104
105         ctx->data = rctx;
106         ctx->keygen_info = rctx->gentmp;
107         ctx->keygen_info_count = 2;
108         
109         return 1;
110         }
111
112 static int pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
113         {
114         RSA_PKEY_CTX *dctx, *sctx;
115         if (!pkey_rsa_init(dst))
116                 return 0;
117         sctx = src->data;
118         dctx = dst->data;
119         dctx->nbits = sctx->nbits;
120         if (sctx->pub_exp)
121                 {
122                 dctx->pub_exp = BN_dup(sctx->pub_exp);
123                 if (!dctx->pub_exp)
124                         return 0;
125                 }
126         dctx->pad_mode = sctx->pad_mode;
127         dctx->md = sctx->md;
128         return 1;
129         }
130
131 static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
132         {
133         if (ctx->tbuf)
134                 return 1;
135         ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));
136         if (!ctx->tbuf)
137                 return 0;
138         return 1;
139         }
140
141 static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
142         {
143         RSA_PKEY_CTX *rctx = ctx->data;
144         if (rctx)
145                 {
146                 if (rctx->pub_exp)
147                         BN_free(rctx->pub_exp);
148                 if (rctx->tbuf)
149                         OPENSSL_free(rctx->tbuf);
150                 OPENSSL_free(rctx);
151                 }
152         }
153
154 static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
155                                         const unsigned char *tbs, size_t tbslen)
156         {
157         int ret;
158         RSA_PKEY_CTX *rctx = ctx->data;
159         RSA *rsa = ctx->pkey->pkey.rsa;
160
161         if (rctx->md)
162                 {
163                 if (tbslen != (size_t)EVP_MD_size(rctx->md))
164                         {
165                         RSAerr(RSA_F_PKEY_RSA_SIGN,
166                                         RSA_R_INVALID_DIGEST_LENGTH);
167                         return -1;
168                         }
169                 if (rctx->pad_mode == RSA_X931_PADDING)
170                         {
171                         if (!setup_tbuf(rctx, ctx))
172                                 return -1;
173                         memcpy(rctx->tbuf, tbs, tbslen);
174                         rctx->tbuf[tbslen] =
175                                 RSA_X931_hash_id(EVP_MD_type(rctx->md));
176                         ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
177                                                 sig, rsa, RSA_X931_PADDING);
178                         }
179                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
180                         {
181                         unsigned int sltmp;
182                         ret = RSA_sign(EVP_MD_type(rctx->md),
183                                                 tbs, tbslen, sig, &sltmp, rsa);
184                         if (ret <= 0)
185                                 return ret;
186                         ret = sltmp;
187                         }
188                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
189                         {
190                         if (!setup_tbuf(rctx, ctx))
191                                 return -1;
192                         if (!RSA_padding_add_PKCS1_PSS(rsa, rctx->tbuf, tbs,
193                                                 rctx->md, rctx->saltlen))
194                                 return -1;
195                         ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
196                                                 sig, rsa, RSA_NO_PADDING);
197                         }
198                 else
199                         return -1;
200                 }
201         else
202                 ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
203                                                         rctx->pad_mode);
204         if (ret < 0)
205                 return ret;
206         *siglen = ret;
207         return 1;
208         }
209
210
211 static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
212                                         unsigned char *rout, size_t *routlen,
213                                         const unsigned char *sig, size_t siglen)
214         {
215         int ret;
216         RSA_PKEY_CTX *rctx = ctx->data;
217
218         if (rctx->md)
219                 {
220                 if (rctx->pad_mode == RSA_X931_PADDING)
221                         {
222                         if (!setup_tbuf(rctx, ctx))
223                                 return -1;
224                         ret = RSA_public_decrypt(siglen, sig,
225                                                 rctx->tbuf, ctx->pkey->pkey.rsa,
226                                                 RSA_X931_PADDING);
227                         if (ret < 1)
228                                 return 0;
229                         ret--;
230                         if (rctx->tbuf[ret] !=
231                                 RSA_X931_hash_id(EVP_MD_type(rctx->md)))
232                                 {
233                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
234                                                 RSA_R_ALGORITHM_MISMATCH);
235                                 return 0;
236                                 }
237                         if (ret != EVP_MD_size(rctx->md))
238                                 {
239                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
240                                         RSA_R_INVALID_DIGEST_LENGTH);
241                                 return 0;
242                                 }
243                         if (rout)
244                                 memcpy(rout, rctx->tbuf, ret);
245                         }
246                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
247                         {
248                         unsigned int sltmp;
249                         ret = int_rsa_verify(EVP_MD_type(rctx->md),
250                                                 NULL, 0, rout, &sltmp,
251                                         sig, siglen, ctx->pkey->pkey.rsa);
252                         ret = sltmp;
253                         }
254                 else
255                         return -1;
256                 }
257         else
258                 ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
259                                                         rctx->pad_mode);
260         if (ret < 0)
261                 return ret;
262         *routlen = ret;
263         return 1;
264         }
265
266 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
267                                         const unsigned char *sig, size_t siglen,
268                                         const unsigned char *tbs, size_t tbslen)
269         {
270         RSA_PKEY_CTX *rctx = ctx->data;
271         RSA *rsa = ctx->pkey->pkey.rsa;
272         size_t rslen;
273         if (rctx->md)
274                 {
275                 if (rctx->pad_mode == RSA_PKCS1_PADDING)
276                         return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
277                                         sig, siglen, rsa);
278                 if (rctx->pad_mode == RSA_X931_PADDING)
279                         {
280                         if (pkey_rsa_verifyrecover(ctx, NULL, &rslen,
281                                         sig, siglen) <= 0)
282                                 return 0;
283                         }
284                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
285                         {
286                         int ret;
287                         if (!setup_tbuf(rctx, ctx))
288                                 return -1;
289                         ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
290                                                         rsa, RSA_NO_PADDING);
291                         if (ret <= 0)
292                                 return 0;
293                         ret = RSA_verify_PKCS1_PSS(rsa, tbs, rctx->md,
294                                                 rctx->tbuf, rctx->saltlen);
295                         if (ret <= 0)
296                                 return 0;
297                         return 1;
298                         }
299                 else
300                         return -1;
301                 }
302         else
303                 {
304                 if (!setup_tbuf(rctx, ctx))
305                         return -1;
306                 rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
307                                                 rsa, rctx->pad_mode);
308                 if (rslen == 0)
309                         return 0;
310                 }
311
312         if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))
313                 return 0;
314
315         return 1;
316                         
317         }
318         
319
320 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
321                                         unsigned char *out, size_t *outlen,
322                                         const unsigned char *in, size_t inlen)
323         {
324         int ret;
325         RSA_PKEY_CTX *rctx = ctx->data;
326         ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
327                                                         rctx->pad_mode);
328         if (ret < 0)
329                 return ret;
330         *outlen = ret;
331         return 1;
332         }
333
334 static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
335                                         unsigned char *out, size_t *outlen,
336                                         const unsigned char *in, size_t inlen)
337         {
338         int ret;
339         RSA_PKEY_CTX *rctx = ctx->data;
340         ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
341                                                         rctx->pad_mode);
342         if (ret < 0)
343                 return ret;
344         *outlen = ret;
345         return 1;
346         }
347
348 static int check_padding_md(const EVP_MD *md, int padding)
349         {
350         if (!md)
351                 return 1;
352
353         if (padding == RSA_NO_PADDING)
354                 {
355                 RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
356                 return 0;
357                 }
358
359         if (padding == RSA_X931_PADDING)
360                 {
361                 if (RSA_X931_hash_id(EVP_MD_type(md)) == -1)
362                         {
363                         RSAerr(RSA_F_CHECK_PADDING_MD,
364                                                 RSA_R_INVALID_X931_DIGEST);
365                         return 0;
366                         }
367                 return 1;
368                 }
369
370         return 1;
371         }
372                         
373
374 static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
375         {
376         RSA_PKEY_CTX *rctx = ctx->data;
377         switch (type)
378                 {
379                 case EVP_PKEY_CTRL_RSA_PADDING:
380                 if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING))
381                         {
382                         if (!check_padding_md(rctx->md, p1))
383                                 return 0;
384                         if (p1 == RSA_PKCS1_PSS_PADDING) 
385                                 {
386                                 if (!(ctx->operation &
387                                      (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
388                                         goto bad_pad;
389                                 if (!rctx->md)
390                                         rctx->md = EVP_sha1();
391                                 }
392                         if (p1 == RSA_PKCS1_OAEP_PADDING) 
393                                 {
394                                 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
395                                         goto bad_pad;
396                                 if (!rctx->md)
397                                         rctx->md = EVP_sha1();
398                                 }
399                         rctx->pad_mode = p1;
400                         return 1;
401                         }
402                 bad_pad:
403                 RSAerr(RSA_F_PKEY_RSA_CTRL,
404                                 RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
405                 return -2;
406
407                 case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
408                 if (p1 < -2)
409                         return -2;
410                 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
411                         {
412                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
413                         return -2;
414                         }
415                 rctx->saltlen = p1;
416                 return 1;
417
418                 case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
419                 if (p1 < 256)
420                         {
421                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_KEYBITS);
422                         return -2;
423                         }
424                 rctx->nbits = p1;
425                 return 1;
426
427                 case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
428                 if (!p2)
429                         return -2;
430                 rctx->pub_exp = p2;
431                 return 1;
432
433                 case EVP_PKEY_CTRL_MD:
434                 if (!check_padding_md(p2, rctx->pad_mode))
435                         return 0;
436                 rctx->md = p2;
437                 return 1;
438
439                 case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
440                 case EVP_PKEY_CTRL_PKCS7_DECRYPT:
441                 case EVP_PKEY_CTRL_PKCS7_SIGN:
442                 return 1;
443
444                 default:
445                 return -2;
446
447                 }
448         }
449                         
450 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
451                         const char *type, const char *value)
452         {
453         if (!value)
454                 {
455                 RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
456                 return 0;
457                 }
458         if (!strcmp(type, "rsa_padding_mode"))
459                 {
460                 int pm;
461                 if (!strcmp(value, "pkcs1"))
462                         pm = RSA_PKCS1_PADDING;
463                 else if (!strcmp(value, "sslv23"))
464                         pm = RSA_SSLV23_PADDING;
465                 else if (!strcmp(value, "none"))
466                         pm = RSA_NO_PADDING;
467                 else if (!strcmp(value, "oeap"))
468                         pm = RSA_PKCS1_OAEP_PADDING;
469                 else if (!strcmp(value, "x931"))
470                         pm = RSA_X931_PADDING;
471                 else if (!strcmp(value, "pss"))
472                         pm = RSA_PKCS1_PSS_PADDING;
473                 else
474                         {
475                         RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
476                                                 RSA_R_UNKNOWN_PADDING_TYPE);
477                         return -2;
478                         }
479                 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
480                 }
481
482         if (!strcmp(type, "rsa_pss_saltlen"))
483                 {
484                 int saltlen;
485                 saltlen = atoi(value);
486                 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
487                 }
488
489         if (!strcmp(type, "rsa_keygen_bits"))
490                 {
491                 int nbits;
492                 nbits = atoi(value);
493                 return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
494                 }
495
496         if (!strcmp(type, "rsa_keygen_pubexp"))
497                 {
498                 int ret;
499                 BIGNUM *pubexp = NULL;
500                 if (!BN_asc2bn(&pubexp, value))
501                         return 0;
502                 ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
503                 if (ret <= 0)
504                         BN_free(pubexp);
505                 return ret;
506                 }
507
508         return -2;
509         }
510
511 static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
512         {
513         RSA *rsa = NULL;
514         RSA_PKEY_CTX *rctx = ctx->data;
515         BN_GENCB *pcb, cb;
516         int ret;
517         if (!rctx->pub_exp)
518                 {
519                 rctx->pub_exp = BN_new();
520                 if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
521                         return 0;
522                 }
523         rsa = RSA_new();
524         if (!rsa)
525                 return 0;
526         if (ctx->pkey_gencb)
527                 {
528                 pcb = &cb;
529                 evp_pkey_set_cb_translate(pcb, ctx);
530                 }
531         else
532                 pcb = NULL;
533         ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
534         if (ret > 0)
535                 EVP_PKEY_assign_RSA(pkey, rsa);
536         else
537                 RSA_free(rsa);
538         return ret;
539         }
540
541 const EVP_PKEY_METHOD rsa_pkey_meth = 
542         {
543         EVP_PKEY_RSA,
544         EVP_PKEY_FLAG_AUTOARGLEN,
545         pkey_rsa_init,
546         pkey_rsa_copy,
547         pkey_rsa_cleanup,
548
549         0,0,
550
551         0,
552         pkey_rsa_keygen,
553
554         0,
555         pkey_rsa_sign,
556
557         0,
558         pkey_rsa_verify,
559
560         0,
561         pkey_rsa_verifyrecover,
562
563
564         0,0,0,0,
565
566         0,
567         pkey_rsa_encrypt,
568
569         0,
570         pkey_rsa_decrypt,
571
572         0,0,
573
574         pkey_rsa_ctrl,
575         pkey_rsa_ctrl_str
576
577
578         };