Fix error codes.
[openssl.git] / crypto / rsa / rsa_pmeth.c
1 /* crypto/rsa/rsa_pmeth.c */
2 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
3  * project 2006.
4  */
5 /* ====================================================================
6  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer. 
14  *
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in
17  *    the documentation and/or other materials provided with the
18  *    distribution.
19  *
20  * 3. All advertising materials mentioning features or use of this
21  *    software must display the following acknowledgment:
22  *    "This product includes software developed by the OpenSSL Project
23  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
24  *
25  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
26  *    endorse or promote products derived from this software without
27  *    prior written permission. For written permission, please contact
28  *    licensing@OpenSSL.org.
29  *
30  * 5. Products derived from this software may not be called "OpenSSL"
31  *    nor may "OpenSSL" appear in their names without prior written
32  *    permission of the OpenSSL Project.
33  *
34  * 6. Redistributions of any form whatsoever must retain the following
35  *    acknowledgment:
36  *    "This product includes software developed by the OpenSSL Project
37  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
38  *
39  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
40  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
41  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
42  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
43  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
44  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
45  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
46  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
48  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
49  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
50  * OF THE POSSIBILITY OF SUCH DAMAGE.
51  * ====================================================================
52  *
53  * This product includes cryptographic software written by Eric Young
54  * (eay@cryptsoft.com).  This product includes software written by Tim
55  * Hudson (tjh@cryptsoft.com).
56  *
57  */
58
59 #include <stdio.h>
60 #include "cryptlib.h"
61 #include <openssl/asn1t.h>
62 #include <openssl/x509.h>
63 #include <openssl/rsa.h>
64 #include <openssl/bn.h>
65 #include <openssl/evp.h>
66 #include "evp_locl.h"
67 #include "rsa_locl.h"
68
69 /* RSA pkey context structure */
70
71 typedef struct
72         {
73         /* Key gen parameters */
74         int nbits;
75         BIGNUM *pub_exp;
76         /* Keygen callback info */
77         int gentmp[2];
78         /* RSA padding mode */
79         int pad_mode;
80         /* message digest */
81         const EVP_MD *md;
82         /* message digest for MGF1 */
83         const EVP_MD *mgf1md;
84         /* PSS/OAEP salt length */
85         int saltlen;
86         /* Temp buffer */
87         unsigned char *tbuf;
88         } RSA_PKEY_CTX;
89
90 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
91         {
92         RSA_PKEY_CTX *rctx;
93         rctx = OPENSSL_malloc(sizeof(RSA_PKEY_CTX));
94         if (!rctx)
95                 return 0;
96         rctx->nbits = 1024;
97         rctx->pub_exp = NULL;
98         rctx->pad_mode = RSA_PKCS1_PADDING;
99         rctx->md = NULL;
100         rctx->mgf1md = NULL;
101         rctx->tbuf = NULL;
102
103         rctx->saltlen = -2;
104
105         ctx->data = rctx;
106         ctx->keygen_info = rctx->gentmp;
107         ctx->keygen_info_count = 2;
108         
109         return 1;
110         }
111
112 static int pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
113         {
114         RSA_PKEY_CTX *dctx, *sctx;
115         if (!pkey_rsa_init(dst))
116                 return 0;
117         sctx = src->data;
118         dctx = dst->data;
119         dctx->nbits = sctx->nbits;
120         if (sctx->pub_exp)
121                 {
122                 dctx->pub_exp = BN_dup(sctx->pub_exp);
123                 if (!dctx->pub_exp)
124                         return 0;
125                 }
126         dctx->pad_mode = sctx->pad_mode;
127         dctx->md = sctx->md;
128         return 1;
129         }
130
131 static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
132         {
133         if (ctx->tbuf)
134                 return 1;
135         ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));
136         if (!ctx->tbuf)
137                 return 0;
138         return 1;
139         }
140
141 static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
142         {
143         RSA_PKEY_CTX *rctx = ctx->data;
144         if (rctx)
145                 {
146                 if (rctx->pub_exp)
147                         BN_free(rctx->pub_exp);
148                 if (rctx->tbuf)
149                         OPENSSL_free(rctx->tbuf);
150                 OPENSSL_free(rctx);
151                 }
152         }
153
154 static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
155                                         const unsigned char *tbs, size_t tbslen)
156         {
157         int ret;
158         RSA_PKEY_CTX *rctx = ctx->data;
159         RSA *rsa = ctx->pkey->pkey.rsa;
160
161         if (rctx->md)
162                 {
163                 if (tbslen != (size_t)EVP_MD_size(rctx->md))
164                         {
165                         RSAerr(RSA_F_PKEY_RSA_SIGN,
166                                         RSA_R_INVALID_DIGEST_LENGTH);
167                         return -1;
168                         }
169                 if (rctx->pad_mode == RSA_X931_PADDING)
170                         {
171                         if (!setup_tbuf(rctx, ctx))
172                                 return -1;
173                         memcpy(rctx->tbuf, tbs, tbslen);
174                         rctx->tbuf[tbslen] =
175                                 RSA_X931_hash_id(EVP_MD_type(rctx->md));
176                         ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
177                                                 sig, rsa, RSA_X931_PADDING);
178                         }
179                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
180                         {
181                         unsigned int sltmp;
182                         ret = RSA_sign(EVP_MD_type(rctx->md),
183                                                 tbs, tbslen, sig, &sltmp, rsa);
184                         if (ret <= 0)
185                                 return ret;
186                         ret = sltmp;
187                         }
188                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
189                         {
190                         if (!setup_tbuf(rctx, ctx))
191                                 return -1;
192                         if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa,
193                                                 rctx->tbuf, tbs,
194                                                 rctx->md, rctx->mgf1md,
195                                                 rctx->saltlen))
196                                 return -1;
197                         ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
198                                                 sig, rsa, RSA_NO_PADDING);
199                         }
200                 else
201                         return -1;
202                 }
203         else
204                 ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
205                                                         rctx->pad_mode);
206         if (ret < 0)
207                 return ret;
208         *siglen = ret;
209         return 1;
210         }
211
212
213 static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
214                                         unsigned char *rout, size_t *routlen,
215                                         const unsigned char *sig, size_t siglen)
216         {
217         int ret;
218         RSA_PKEY_CTX *rctx = ctx->data;
219
220         if (rctx->md)
221                 {
222                 if (rctx->pad_mode == RSA_X931_PADDING)
223                         {
224                         if (!setup_tbuf(rctx, ctx))
225                                 return -1;
226                         ret = RSA_public_decrypt(siglen, sig,
227                                                 rctx->tbuf, ctx->pkey->pkey.rsa,
228                                                 RSA_X931_PADDING);
229                         if (ret < 1)
230                                 return 0;
231                         ret--;
232                         if (rctx->tbuf[ret] !=
233                                 RSA_X931_hash_id(EVP_MD_type(rctx->md)))
234                                 {
235                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
236                                                 RSA_R_ALGORITHM_MISMATCH);
237                                 return 0;
238                                 }
239                         if (ret != EVP_MD_size(rctx->md))
240                                 {
241                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
242                                         RSA_R_INVALID_DIGEST_LENGTH);
243                                 return 0;
244                                 }
245                         if (rout)
246                                 memcpy(rout, rctx->tbuf, ret);
247                         }
248                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
249                         {
250                         size_t sltmp;
251                         ret = int_rsa_verify(EVP_MD_type(rctx->md),
252                                                 NULL, 0, rout, &sltmp,
253                                         sig, siglen, ctx->pkey->pkey.rsa);
254                         if (ret <= 0)
255                                 return 0;
256                         ret = sltmp;
257                         }
258                 else
259                         return -1;
260                 }
261         else
262                 ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
263                                                         rctx->pad_mode);
264         if (ret < 0)
265                 return ret;
266         *routlen = ret;
267         return 1;
268         }
269
270 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
271                                         const unsigned char *sig, size_t siglen,
272                                         const unsigned char *tbs, size_t tbslen)
273         {
274         RSA_PKEY_CTX *rctx = ctx->data;
275         RSA *rsa = ctx->pkey->pkey.rsa;
276         size_t rslen;
277         if (rctx->md)
278                 {
279                 if (rctx->pad_mode == RSA_PKCS1_PADDING)
280                         return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
281                                         sig, siglen, rsa);
282                 if (rctx->pad_mode == RSA_X931_PADDING)
283                         {
284                         if (pkey_rsa_verifyrecover(ctx, NULL, &rslen,
285                                         sig, siglen) <= 0)
286                                 return 0;
287                         }
288                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
289                         {
290                         int ret;
291                         if (!setup_tbuf(rctx, ctx))
292                                 return -1;
293                         ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
294                                                         rsa, RSA_NO_PADDING);
295                         if (ret <= 0)
296                                 return 0;
297                         ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs,
298                                                 rctx->md, rctx->mgf1md,
299                                                 rctx->tbuf, rctx->saltlen);
300                         if (ret <= 0)
301                                 return 0;
302                         return 1;
303                         }
304                 else
305                         return -1;
306                 }
307         else
308                 {
309                 if (!setup_tbuf(rctx, ctx))
310                         return -1;
311                 rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
312                                                 rsa, rctx->pad_mode);
313                 if (rslen == 0)
314                         return 0;
315                 }
316
317         if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))
318                 return 0;
319
320         return 1;
321                         
322         }
323         
324
325 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
326                                         unsigned char *out, size_t *outlen,
327                                         const unsigned char *in, size_t inlen)
328         {
329         int ret;
330         RSA_PKEY_CTX *rctx = ctx->data;
331         ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
332                                                         rctx->pad_mode);
333         if (ret < 0)
334                 return ret;
335         *outlen = ret;
336         return 1;
337         }
338
339 static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
340                                         unsigned char *out, size_t *outlen,
341                                         const unsigned char *in, size_t inlen)
342         {
343         int ret;
344         RSA_PKEY_CTX *rctx = ctx->data;
345         ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
346                                                         rctx->pad_mode);
347         if (ret < 0)
348                 return ret;
349         *outlen = ret;
350         return 1;
351         }
352
353 static int check_padding_md(const EVP_MD *md, int padding)
354         {
355         if (!md)
356                 return 1;
357
358         if (padding == RSA_NO_PADDING)
359                 {
360                 RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
361                 return 0;
362                 }
363
364         if (padding == RSA_X931_PADDING)
365                 {
366                 if (RSA_X931_hash_id(EVP_MD_type(md)) == -1)
367                         {
368                         RSAerr(RSA_F_CHECK_PADDING_MD,
369                                                 RSA_R_INVALID_X931_DIGEST);
370                         return 0;
371                         }
372                 return 1;
373                 }
374
375         return 1;
376         }
377                         
378
379 static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
380         {
381         RSA_PKEY_CTX *rctx = ctx->data;
382         switch (type)
383                 {
384                 case EVP_PKEY_CTRL_RSA_PADDING:
385                 if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING))
386                         {
387                         if (!check_padding_md(rctx->md, p1))
388                                 return 0;
389                         if (p1 == RSA_PKCS1_PSS_PADDING) 
390                                 {
391                                 if (!(ctx->operation &
392                                      (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
393                                         goto bad_pad;
394                                 if (!rctx->md)
395                                         rctx->md = EVP_sha1();
396                                 }
397                         if (p1 == RSA_PKCS1_OAEP_PADDING) 
398                                 {
399                                 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
400                                         goto bad_pad;
401                                 if (!rctx->md)
402                                         rctx->md = EVP_sha1();
403                                 }
404                         rctx->pad_mode = p1;
405                         return 1;
406                         }
407                 bad_pad:
408                 RSAerr(RSA_F_PKEY_RSA_CTRL,
409                                 RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
410                 return -2;
411
412                 case EVP_PKEY_CTRL_GET_RSA_PADDING:
413                 *(int *)p2 = rctx->pad_mode;
414                 return 1;
415
416                 case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
417                 case EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN:
418                 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
419                         {
420                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
421                         return -2;
422                         }
423                 if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN)
424                         *(int *)p2 = rctx->saltlen;
425                 else
426                         {
427                         if (p1 < -2)
428                                 return -2;
429                         rctx->saltlen = p1;
430                         }
431                 return 1;
432
433                 case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
434                 if (p1 < 256)
435                         {
436                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_KEYBITS);
437                         return -2;
438                         }
439                 rctx->nbits = p1;
440                 return 1;
441
442                 case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
443                 if (!p2)
444                         return -2;
445                 rctx->pub_exp = p2;
446                 return 1;
447
448                 case EVP_PKEY_CTRL_MD:
449                 if (!check_padding_md(p2, rctx->pad_mode))
450                         return 0;
451                 rctx->md = p2;
452                 return 1;
453
454                 case EVP_PKEY_CTRL_RSA_MGF1_MD:
455                 case EVP_PKEY_CTRL_GET_RSA_MGF1_MD:
456                 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
457                         {
458                         RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_MGF1_MD);
459                         return -2;
460                         }
461                 if (type == EVP_PKEY_CTRL_GET_RSA_MGF1_MD)
462                         {
463                         if (rctx->mgf1md)
464                                 *(const EVP_MD **)p2 = rctx->mgf1md;
465                         else
466                                 *(const EVP_MD **)p2 = rctx->md;
467                         }
468                 else
469                         rctx->mgf1md = p2;
470                 return 1;
471
472                 case EVP_PKEY_CTRL_DIGESTINIT:
473                 case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
474                 case EVP_PKEY_CTRL_PKCS7_DECRYPT:
475                 case EVP_PKEY_CTRL_PKCS7_SIGN:
476 #ifndef OPENSSL_NO_CMS
477                 case EVP_PKEY_CTRL_CMS_ENCRYPT:
478                 case EVP_PKEY_CTRL_CMS_DECRYPT:
479                 case EVP_PKEY_CTRL_CMS_SIGN:
480 #endif
481                 return 1;
482                 case EVP_PKEY_CTRL_PEER_KEY:
483                         RSAerr(RSA_F_PKEY_RSA_CTRL,
484                         RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
485                         return -2;      
486
487                 default:
488                 return -2;
489
490                 }
491         }
492                         
493 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
494                         const char *type, const char *value)
495         {
496         if (!value)
497                 {
498                 RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
499                 return 0;
500                 }
501         if (!strcmp(type, "rsa_padding_mode"))
502                 {
503                 int pm;
504                 if (!strcmp(value, "pkcs1"))
505                         pm = RSA_PKCS1_PADDING;
506                 else if (!strcmp(value, "sslv23"))
507                         pm = RSA_SSLV23_PADDING;
508                 else if (!strcmp(value, "none"))
509                         pm = RSA_NO_PADDING;
510                 else if (!strcmp(value, "oeap"))
511                         pm = RSA_PKCS1_OAEP_PADDING;
512                 else if (!strcmp(value, "x931"))
513                         pm = RSA_X931_PADDING;
514                 else if (!strcmp(value, "pss"))
515                         pm = RSA_PKCS1_PSS_PADDING;
516                 else
517                         {
518                         RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
519                                                 RSA_R_UNKNOWN_PADDING_TYPE);
520                         return -2;
521                         }
522                 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
523                 }
524
525         if (!strcmp(type, "rsa_pss_saltlen"))
526                 {
527                 int saltlen;
528                 saltlen = atoi(value);
529                 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
530                 }
531
532         if (!strcmp(type, "rsa_keygen_bits"))
533                 {
534                 int nbits;
535                 nbits = atoi(value);
536                 return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
537                 }
538
539         if (!strcmp(type, "rsa_keygen_pubexp"))
540                 {
541                 int ret;
542                 BIGNUM *pubexp = NULL;
543                 if (!BN_asc2bn(&pubexp, value))
544                         return 0;
545                 ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
546                 if (ret <= 0)
547                         BN_free(pubexp);
548                 return ret;
549                 }
550
551         return -2;
552         }
553
554 static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
555         {
556         RSA *rsa = NULL;
557         RSA_PKEY_CTX *rctx = ctx->data;
558         BN_GENCB *pcb, cb;
559         int ret;
560         if (!rctx->pub_exp)
561                 {
562                 rctx->pub_exp = BN_new();
563                 if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
564                         return 0;
565                 }
566         rsa = RSA_new();
567         if (!rsa)
568                 return 0;
569         if (ctx->pkey_gencb)
570                 {
571                 pcb = &cb;
572                 evp_pkey_set_cb_translate(pcb, ctx);
573                 }
574         else
575                 pcb = NULL;
576         ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
577         if (ret > 0)
578                 EVP_PKEY_assign_RSA(pkey, rsa);
579         else
580                 RSA_free(rsa);
581         return ret;
582         }
583
584 const EVP_PKEY_METHOD rsa_pkey_meth = 
585         {
586         EVP_PKEY_RSA,
587         EVP_PKEY_FLAG_AUTOARGLEN,
588         pkey_rsa_init,
589         pkey_rsa_copy,
590         pkey_rsa_cleanup,
591
592         0,0,
593
594         0,
595         pkey_rsa_keygen,
596
597         0,
598         pkey_rsa_sign,
599
600         0,
601         pkey_rsa_verify,
602
603         0,
604         pkey_rsa_verifyrecover,
605
606
607         0,0,0,0,
608
609         0,
610         pkey_rsa_encrypt,
611
612         0,
613         pkey_rsa_decrypt,
614
615         0,0,
616
617         pkey_rsa_ctrl,
618         pkey_rsa_ctrl_str
619
620
621         };