56759b5285c5460d9c172a3c5af5e24e129c58f5
[openssl.git] / crypto / rsa / rsa_pmeth.c
1 /* Written by Dr Stephen N Henson (shenson@bigfoot.com) for the OpenSSL
2  * project 2006.
3  */
4 /* ====================================================================
5  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer. 
13  *
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in
16  *    the documentation and/or other materials provided with the
17  *    distribution.
18  *
19  * 3. All advertising materials mentioning features or use of this
20  *    software must display the following acknowledgment:
21  *    "This product includes software developed by the OpenSSL Project
22  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
23  *
24  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
25  *    endorse or promote products derived from this software without
26  *    prior written permission. For written permission, please contact
27  *    licensing@OpenSSL.org.
28  *
29  * 5. Products derived from this software may not be called "OpenSSL"
30  *    nor may "OpenSSL" appear in their names without prior written
31  *    permission of the OpenSSL Project.
32  *
33  * 6. Redistributions of any form whatsoever must retain the following
34  *    acknowledgment:
35  *    "This product includes software developed by the OpenSSL Project
36  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
37  *
38  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
39  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
40  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
41  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
42  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
43  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
44  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
45  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
46  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
47  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
48  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
49  * OF THE POSSIBILITY OF SUCH DAMAGE.
50  * ====================================================================
51  *
52  * This product includes cryptographic software written by Eric Young
53  * (eay@cryptsoft.com).  This product includes software written by Tim
54  * Hudson (tjh@cryptsoft.com).
55  *
56  */
57
58 #include <stdio.h>
59 #include "cryptlib.h"
60 #include <openssl/asn1t.h>
61 #include <openssl/x509.h>
62 #include <openssl/rsa.h>
63 #include <openssl/evp.h>
64 #include "evp_locl.h"
65
66 extern int int_rsa_verify(int dtype, const unsigned char *m, unsigned int m_len,
67                 unsigned char *rm, unsigned int *prm_len,
68                 const unsigned char *sigbuf, unsigned int siglen,
69                 RSA *rsa);
70
71 /* RSA pkey context structure */
72
73 typedef struct
74         {
75         /* Key gen parameters */
76         int nbits;
77         BIGNUM *pub_exp;
78         /* Keygen callback info */
79         int gentmp[2];
80         /* RSA padding mode */
81         int pad_mode;
82         /* message digest */
83         const EVP_MD *md;
84         /* PSS/OAEP salt length */
85         int saltlen;
86         /* Temp buffer */
87         unsigned char *tbuf;
88         } RSA_PKEY_CTX;
89
90 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
91         {
92         RSA_PKEY_CTX *rctx;
93         rctx = OPENSSL_malloc(sizeof(RSA_PKEY_CTX));
94         if (!rctx)
95                 return 0;
96         rctx->nbits = 1024;
97         rctx->pub_exp = NULL;
98         rctx->pad_mode = RSA_PKCS1_PADDING;
99         rctx->md = NULL;
100         rctx->tbuf = NULL;
101
102         rctx->saltlen = -2;
103
104         ctx->data = rctx;
105         ctx->keygen_info = rctx->gentmp;
106         ctx->keygen_info_count = 2;
107         
108         return 1;
109         }
110
111 static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
112         {
113         if (ctx->tbuf)
114                 return 1;
115         ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));
116         if (!ctx->tbuf)
117                 return 0;
118         return 1;
119         }
120
121 static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
122         {
123         RSA_PKEY_CTX *rctx = ctx->data;
124         if (rctx)
125                 {
126                 if (rctx->pub_exp)
127                         BN_free(rctx->pub_exp);
128                 if (rctx->tbuf)
129                         OPENSSL_free(rctx->tbuf);
130                 }
131         OPENSSL_free(rctx);
132         }
133
134 static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, int *siglen,
135                                         const unsigned char *tbs, int tbslen)
136         {
137         int ret;
138         RSA_PKEY_CTX *rctx = ctx->data;
139         RSA *rsa = ctx->pkey->pkey.rsa;
140
141         if (rctx->md)
142                 {
143                 if (tbslen != EVP_MD_size(rctx->md))
144                         {
145                         RSAerr(RSA_F_PKEY_RSA_SIGN,
146                                         RSA_R_INVALID_DIGEST_LENGTH);
147                         return -1;
148                         }
149                 if (rctx->pad_mode == RSA_X931_PADDING)
150                         {
151                         if (!setup_tbuf(rctx, ctx))
152                                 return -1;
153                         memcpy(rctx->tbuf, tbs, tbslen);
154                         rctx->tbuf[tbslen] =
155                                 RSA_X931_hash_id(EVP_MD_type(rctx->md));
156                         ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
157                                                 sig, rsa, RSA_X931_PADDING);
158                         }
159                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
160                         {
161                         unsigned int sltmp;
162                         ret = RSA_sign(EVP_MD_type(rctx->md),
163                                                 tbs, tbslen, sig, &sltmp, rsa);
164                         if (ret <= 0)
165                                 return ret;
166                         ret = sltmp;
167                         }
168                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
169                         {
170                         if (!setup_tbuf(rctx, ctx))
171                                 return -1;
172                         if (!RSA_padding_add_PKCS1_PSS(rsa, rctx->tbuf, tbs,
173                                                 rctx->md, rctx->saltlen))
174                                 return -1;
175                         ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
176                                                 sig, rsa, RSA_NO_PADDING);
177                         }
178                 else
179                         return -1;
180                 }
181         else
182                 ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
183                                                         rctx->pad_mode);
184         if (ret < 0)
185                 return ret;
186         *siglen = ret;
187         return 1;
188         }
189
190
191 static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
192                                         unsigned char *rout, int *routlen,
193                                         const unsigned char *sig, int siglen)
194         {
195         int ret;
196         RSA_PKEY_CTX *rctx = ctx->data;
197
198         if (rctx->md)
199                 {
200                 if (rctx->pad_mode == RSA_X931_PADDING)
201                         {
202                         if (!setup_tbuf(rctx, ctx))
203                                 return -1;
204                         ret = RSA_public_decrypt(siglen, sig,
205                                                 rctx->tbuf, ctx->pkey->pkey.rsa,
206                                                 RSA_X931_PADDING);
207                         if (ret < 1)
208                                 return 0;
209                         ret--;
210                         if (rctx->tbuf[ret] !=
211                                 RSA_X931_hash_id(EVP_MD_type(rctx->md)))
212                                 {
213                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
214                                                 RSA_R_ALGORITHM_MISMATCH);
215                                 return 0;
216                                 }
217                         if (ret != EVP_MD_size(rctx->md))
218                                 {
219                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
220                                         RSA_R_INVALID_DIGEST_LENGTH);
221                                 return 0;
222                                 }
223                         if (rout)
224                                 memcpy(rout, rctx->tbuf, ret);
225                         }
226                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
227                         {
228                         unsigned int sltmp;
229                         ret = int_rsa_verify(EVP_MD_type(rctx->md),
230                                                 NULL, 0, rout, &sltmp,
231                                         sig, siglen, ctx->pkey->pkey.rsa);
232                         ret = sltmp;
233                         }
234                 else
235                         return -1;
236                 }
237         else
238                 ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
239                                                         rctx->pad_mode);
240         if (ret < 0)
241                 return ret;
242         *routlen = ret;
243         return 1;
244         }
245
246 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
247                                         const unsigned char *sig, int siglen,
248                                         const unsigned char *tbs, int tbslen)
249         {
250         RSA_PKEY_CTX *rctx = ctx->data;
251         RSA *rsa = ctx->pkey->pkey.rsa;
252         int rslen;
253         if (rctx->md)
254                 {
255                 if (rctx->pad_mode == RSA_PKCS1_PADDING)
256                         return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
257                                         sig, siglen, rsa);
258                 if (rctx->pad_mode == RSA_X931_PADDING)
259                         {
260                         if (pkey_rsa_verifyrecover(ctx, NULL, &rslen,
261                                         sig, siglen) <= 0)
262                                 return 0;
263                         }
264                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
265                         {
266                         int ret;
267                         if (!setup_tbuf(rctx, ctx))
268                                 return -1;
269                         ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
270                                                         rsa, RSA_NO_PADDING);
271                         if (ret <= 0)
272                                 return 0;
273                         ret = RSA_verify_PKCS1_PSS(rsa, tbs, rctx->md,
274                                                 rctx->tbuf, rctx->saltlen);
275                         if (ret <= 0)
276                                 return 0;
277                         return 1;
278                         }
279                 else
280                         return -1;
281                 }
282         else
283                 {
284                 if (!setup_tbuf(rctx, ctx))
285                         return -1;
286                 rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
287                                                 rsa, rctx->pad_mode);
288                 if (rslen <= 0)
289                         return 0;
290                 }
291
292         if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))
293                 return 0;
294
295         return 1;
296                         
297         }
298         
299
300 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
301                                         const unsigned char *in, int inlen)
302         {
303         int ret;
304         RSA_PKEY_CTX *rctx = ctx->data;
305         ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
306                                                         rctx->pad_mode);
307         if (ret < 0)
308                 return ret;
309         *outlen = ret;
310         return 1;
311         }
312
313 static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
314                                         const unsigned char *in, int inlen)
315         {
316         int ret;
317         RSA_PKEY_CTX *rctx = ctx->data;
318         ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
319                                                         rctx->pad_mode);
320         if (ret < 0)
321                 return ret;
322         *outlen = ret;
323         return 1;
324         }
325
326 static int check_padding_md(const EVP_MD *md, int padding)
327         {
328         if (!md)
329                 return 1;
330
331         if (padding == RSA_NO_PADDING)
332                 {
333                 RSAerr(RSA_F_CHECK_PADDING_NID, RSA_R_INVALID_PADDING_MODE);
334                 return 0;
335                 }
336
337         if (padding == RSA_X931_PADDING)
338                 {
339                 if (RSA_X931_hash_id(EVP_MD_type(md)) == -1)
340                         {
341                         RSAerr(RSA_F_CHECK_PADDING_NID,
342                                                 RSA_R_INVALID_X931_DIGEST);
343                         return 0;
344                         }
345                 return 1;
346                 }
347
348         return 1;
349         }
350                         
351
352 static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
353         {
354         RSA_PKEY_CTX *rctx = ctx->data;
355         switch (type)
356                 {
357                 case EVP_PKEY_CTRL_RSA_PADDING:
358                 if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING))
359                         {
360                         if (!check_padding_md(rctx->md, p1))
361                                 return 0;
362                         if (p1 == RSA_PKCS1_PSS_PADDING) 
363                                 {
364                                 if (ctx->operation == EVP_PKEY_OP_VERIFYRECOVER)
365                                         return -2;
366                                 if (!rctx->md)
367                                         rctx->md = EVP_sha1();
368                                 }
369                         if (p1 == RSA_PKCS1_OAEP_PADDING) 
370                                 {
371                                 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
372                                         return -2;
373                                 if (!rctx->md)
374                                         rctx->md = EVP_sha1();
375                                 }
376                         rctx->pad_mode = p1;
377                         return 1;
378                         }
379                 return -2;
380
381                 case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
382                 if (p1 < -2)
383                         return -2;
384                 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
385                         return -2;
386                 rctx->saltlen = p1;
387                 return 1;
388
389                 case EVP_PKEY_CTRL_MD:
390                 if (!check_padding_md(p2, rctx->pad_mode))
391                         return 0;
392                 rctx->md = p2;
393                 return 1;
394
395                 default:
396                 return -2;
397
398                 }
399         }
400                         
401 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
402                         const char *type, const char *value)
403         {
404         if (!strcmp(type, "rsa_padding_mode"))
405                 {
406                 int pm;
407                 if (!value)
408                         return 0;
409                 if (!strcmp(value, "pkcs1"))
410                         pm = RSA_PKCS1_PADDING;
411                 else if (!strcmp(value, "sslv23"))
412                         pm = RSA_SSLV23_PADDING;
413                 else if (!strcmp(value, "none"))
414                         pm = RSA_NO_PADDING;
415                 else if (!strcmp(value, "oeap"))
416                         pm = RSA_PKCS1_OAEP_PADDING;
417                 else if (!strcmp(value, "x931"))
418                         pm = RSA_X931_PADDING;
419                 else if (!strcmp(value, "pss"))
420                         pm = RSA_PKCS1_PSS_PADDING;
421                 else
422                         return -2;
423                 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
424                 }
425         if (!strcmp(type, "rsa_pss_saltlen"))
426                 {
427                 int saltlen;
428                 saltlen = atoi(value);
429                 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
430                 }
431         return -2;
432         }
433
434 static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
435         {
436         RSA *rsa = NULL;
437         RSA_PKEY_CTX *rctx = ctx->data;
438         BN_GENCB *pcb, cb;
439         int ret;
440         if (!rctx->pub_exp)
441                 {
442                 rctx->pub_exp = BN_new();
443                 if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
444                         return 0;
445                 }
446         rsa = RSA_new();
447         if (!rsa)
448                 return 0;
449         if (ctx->pkey_gencb)
450                 {
451                 pcb = &cb;
452                 evp_pkey_set_cb_translate(pcb, ctx);
453                 }
454         else
455                 pcb = NULL;
456         ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
457         if (ret > 0)
458                 EVP_PKEY_assign_RSA(pkey, rsa);
459         else
460                 RSA_free(rsa);
461         return ret;
462         }
463
464 const EVP_PKEY_METHOD rsa_pkey_meth = 
465         {
466         EVP_PKEY_RSA,
467         0,
468         pkey_rsa_init,
469         pkey_rsa_cleanup,
470
471         0,0,
472
473         0,
474         pkey_rsa_keygen,
475
476         0,
477         pkey_rsa_sign,
478
479         0,
480         pkey_rsa_verify,
481
482         0,
483         pkey_rsa_verifyrecover,
484
485
486         0,0,0,0,
487
488         0,
489         pkey_rsa_encrypt,
490
491         0,
492         pkey_rsa_decrypt,
493
494         pkey_rsa_ctrl,
495         pkey_rsa_ctrl_str
496
497
498         };