378bfe4b4ef16b25829b4d7e065ddd88ba7a6b96
[openssl.git] / crypto / rsa / rsa_pmeth.c
1 /* Written by Dr Stephen N Henson (shenson@bigfoot.com) for the OpenSSL
2  * project 2006.
3  */
4 /* ====================================================================
5  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer. 
13  *
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in
16  *    the documentation and/or other materials provided with the
17  *    distribution.
18  *
19  * 3. All advertising materials mentioning features or use of this
20  *    software must display the following acknowledgment:
21  *    "This product includes software developed by the OpenSSL Project
22  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
23  *
24  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
25  *    endorse or promote products derived from this software without
26  *    prior written permission. For written permission, please contact
27  *    licensing@OpenSSL.org.
28  *
29  * 5. Products derived from this software may not be called "OpenSSL"
30  *    nor may "OpenSSL" appear in their names without prior written
31  *    permission of the OpenSSL Project.
32  *
33  * 6. Redistributions of any form whatsoever must retain the following
34  *    acknowledgment:
35  *    "This product includes software developed by the OpenSSL Project
36  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
37  *
38  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
39  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
40  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
41  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
42  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
43  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
44  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
45  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
46  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
47  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
48  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
49  * OF THE POSSIBILITY OF SUCH DAMAGE.
50  * ====================================================================
51  *
52  * This product includes cryptographic software written by Eric Young
53  * (eay@cryptsoft.com).  This product includes software written by Tim
54  * Hudson (tjh@cryptsoft.com).
55  *
56  */
57
58 #include <stdio.h>
59 #include "cryptlib.h"
60 #include <openssl/asn1t.h>
61 #include <openssl/x509.h>
62 #include <openssl/rsa.h>
63 #include <openssl/evp.h>
64 #include "evp_locl.h"
65
66 extern int int_rsa_verify(int dtype, const unsigned char *m, unsigned int m_len,
67                 unsigned char *rm, unsigned int *prm_len,
68                 const unsigned char *sigbuf, unsigned int siglen,
69                 RSA *rsa);
70
71 /* RSA pkey context structure */
72
73 typedef struct
74         {
75         /* Key gen parameters */
76         int nbits;
77         BIGNUM *pub_exp;
78         /* RSA padding mode */
79         int pad_mode;
80         /* message digest */
81         const EVP_MD *md;
82         /* PSS/OAEP salt length */
83         int saltlen;
84         /* Temp buffer */
85         unsigned char *tbuf;
86         } RSA_PKEY_CTX;
87
88 static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
89         {
90         RSA_PKEY_CTX *rctx;
91         rctx = OPENSSL_malloc(sizeof(RSA_PKEY_CTX));
92         if (!rctx)
93                 return 0;
94         rctx->nbits = 1024;
95         rctx->pub_exp = NULL;
96         rctx->pad_mode = RSA_PKCS1_PADDING;
97         rctx->md = NULL;
98         rctx->tbuf = NULL;
99
100         rctx->saltlen = -2;
101
102         ctx->data = rctx;
103         
104         return 1;
105         }
106
107 static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
108         {
109         if (ctx->tbuf)
110                 return 1;
111         ctx->tbuf = OPENSSL_malloc(EVP_PKEY_size(pk->pkey));
112         if (!ctx->tbuf)
113                 return 0;
114         return 1;
115         }
116
117 static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
118         {
119         RSA_PKEY_CTX *rctx = ctx->data;
120         if (rctx)
121                 {
122                 if (rctx->pub_exp)
123                         BN_free(rctx->pub_exp);
124                 if (rctx->tbuf)
125                         OPENSSL_free(rctx->tbuf);
126                 }
127         OPENSSL_free(rctx);
128         }
129
130 static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, int *siglen,
131                                         const unsigned char *tbs, int tbslen)
132         {
133         int ret;
134         RSA_PKEY_CTX *rctx = ctx->data;
135         RSA *rsa = ctx->pkey->pkey.rsa;
136
137         if (rctx->md)
138                 {
139                 if (tbslen != EVP_MD_size(rctx->md))
140                         {
141                         RSAerr(RSA_F_PKEY_RSA_SIGN,
142                                         RSA_R_INVALID_DIGEST_LENGTH);
143                         return -1;
144                         }
145                 if (rctx->pad_mode == RSA_X931_PADDING)
146                         {
147                         if (!setup_tbuf(rctx, ctx))
148                                 return -1;
149                         memcpy(rctx->tbuf, tbs, tbslen);
150                         rctx->tbuf[tbslen] =
151                                 RSA_X931_hash_id(EVP_MD_type(rctx->md));
152                         ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf,
153                                                 sig, rsa, RSA_X931_PADDING);
154                         }
155                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
156                         {
157                         unsigned int sltmp;
158                         ret = RSA_sign(EVP_MD_type(rctx->md),
159                                                 tbs, tbslen, sig, &sltmp, rsa);
160                         if (ret <= 0)
161                                 return ret;
162                         ret = sltmp;
163                         }
164                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
165                         {
166                         if (!setup_tbuf(rctx, ctx))
167                                 return -1;
168                         if (!RSA_padding_add_PKCS1_PSS(rsa, rctx->tbuf, tbs,
169                                                 rctx->md, rctx->saltlen))
170                                 return -1;
171                         ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
172                                                 sig, rsa, RSA_NO_PADDING);
173                         }
174                 else
175                         return -1;
176                 }
177         else
178                 ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
179                                                         rctx->pad_mode);
180         if (ret < 0)
181                 return ret;
182         *siglen = ret;
183         return 1;
184         }
185
186
187 static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
188                                         unsigned char *rout, int *routlen,
189                                         const unsigned char *sig, int siglen)
190         {
191         int ret;
192         RSA_PKEY_CTX *rctx = ctx->data;
193
194         if (rctx->md)
195                 {
196                 if (rctx->pad_mode == RSA_X931_PADDING)
197                         {
198                         if (!setup_tbuf(rctx, ctx))
199                                 return -1;
200                         ret = RSA_public_decrypt(siglen, sig,
201                                                 rctx->tbuf, ctx->pkey->pkey.rsa,
202                                                 RSA_X931_PADDING);
203                         if (ret < 1)
204                                 return 0;
205                         ret--;
206                         if (rctx->tbuf[ret] !=
207                                 RSA_X931_hash_id(EVP_MD_type(rctx->md)))
208                                 {
209                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
210                                                 RSA_R_ALGORITHM_MISMATCH);
211                                 return 0;
212                                 }
213                         if (ret != EVP_MD_size(rctx->md))
214                                 {
215                                 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
216                                         RSA_R_INVALID_DIGEST_LENGTH);
217                                 return 0;
218                                 }
219                         if (rout)
220                                 memcpy(rout, rctx->tbuf, ret);
221                         }
222                 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
223                         {
224                         unsigned int sltmp;
225                         ret = int_rsa_verify(EVP_MD_type(rctx->md),
226                                                 NULL, 0, rout, &sltmp,
227                                         sig, siglen, ctx->pkey->pkey.rsa);
228                         ret = sltmp;
229                         }
230                 else
231                         return -1;
232                 }
233         else
234                 ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
235                                                         rctx->pad_mode);
236         if (ret < 0)
237                 return ret;
238         *routlen = ret;
239         return 1;
240         }
241
242 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
243                                         const unsigned char *sig, int siglen,
244                                         const unsigned char *tbs, int tbslen)
245         {
246         RSA_PKEY_CTX *rctx = ctx->data;
247         RSA *rsa = ctx->pkey->pkey.rsa;
248         int rslen;
249         if (rctx->md)
250                 {
251                 if (rctx->pad_mode == RSA_PKCS1_PADDING)
252                         return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
253                                         sig, siglen, rsa);
254                 if (rctx->pad_mode == RSA_X931_PADDING)
255                         {
256                         if (pkey_rsa_verifyrecover(ctx, NULL, &rslen,
257                                         sig, siglen) <= 0)
258                                 return 0;
259                         }
260                 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
261                         {
262                         int ret;
263                         if (!setup_tbuf(rctx, ctx))
264                                 return -1;
265                         ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
266                                                         rsa, RSA_NO_PADDING);
267                         if (ret <= 0)
268                                 return 0;
269                         ret = RSA_verify_PKCS1_PSS(rsa, tbs, rctx->md,
270                                                 rctx->tbuf, rctx->saltlen);
271                         if (ret <= 0)
272                                 return 0;
273                         return 1;
274                         }
275                 else
276                         return -1;
277                 }
278         else
279                 {
280                 if (!setup_tbuf(rctx, ctx))
281                         return -1;
282                 rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
283                                                 rsa, rctx->pad_mode);
284                 if (rslen <= 0)
285                         return 0;
286                 }
287
288         if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen))
289                 return 0;
290
291         return 1;
292                         
293         }
294         
295
296 static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
297                                         const unsigned char *in, int inlen)
298         {
299         int ret;
300         RSA_PKEY_CTX *rctx = ctx->data;
301         ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
302                                                         rctx->pad_mode);
303         if (ret < 0)
304                 return ret;
305         *outlen = ret;
306         return 1;
307         }
308
309 static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
310                                         const unsigned char *in, int inlen)
311         {
312         int ret;
313         RSA_PKEY_CTX *rctx = ctx->data;
314         ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
315                                                         rctx->pad_mode);
316         if (ret < 0)
317                 return ret;
318         *outlen = ret;
319         return 1;
320         }
321
322 static int check_padding_md(const EVP_MD *md, int padding)
323         {
324         if (!md)
325                 return 1;
326
327         if (padding == RSA_NO_PADDING)
328                 {
329                 RSAerr(RSA_F_CHECK_PADDING_NID, RSA_R_INVALID_PADDING_MODE);
330                 return 0;
331                 }
332
333         if (padding == RSA_X931_PADDING)
334                 {
335                 if (RSA_X931_hash_id(EVP_MD_type(md)) == -1)
336                         {
337                         RSAerr(RSA_F_CHECK_PADDING_NID,
338                                                 RSA_R_INVALID_X931_DIGEST);
339                         return 0;
340                         }
341                 return 1;
342                 }
343
344         return 1;
345         }
346                         
347
348 static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
349         {
350         RSA_PKEY_CTX *rctx = ctx->data;
351         switch (type)
352                 {
353                 case EVP_PKEY_CTRL_RSA_PADDING:
354                 if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING))
355                         {
356                         if (!check_padding_md(rctx->md, p1))
357                                 return 0;
358                         if (p1 == RSA_PKCS1_PSS_PADDING) 
359                                 {
360                                 if (ctx->operation == EVP_PKEY_OP_VERIFYRECOVER)
361                                         return -2;
362                                 if (!rctx->md)
363                                         rctx->md = EVP_sha1();
364                                 }
365                         if (p1 == RSA_PKCS1_OAEP_PADDING) 
366                                 {
367                                 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
368                                         return -2;
369                                 if (!rctx->md)
370                                         rctx->md = EVP_sha1();
371                                 }
372                         rctx->pad_mode = p1;
373                         return 1;
374                         }
375                 return -2;
376
377                 case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
378                 if (p1 < -2)
379                         return -2;
380                 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
381                         return -2;
382                 rctx->saltlen = p1;
383                 return 1;
384
385                 case EVP_PKEY_CTRL_MD:
386                 if (!check_padding_md(p2, rctx->pad_mode))
387                         return 0;
388                 rctx->md = p2;
389                 return 1;
390
391                 default:
392                 return -2;
393
394                 }
395         }
396                         
397 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
398                         const char *type, const char *value)
399         {
400         if (!strcmp(type, "rsa_padding_mode"))
401                 {
402                 int pm;
403                 if (!value)
404                         return 0;
405                 if (!strcmp(value, "pkcs1"))
406                         pm = RSA_PKCS1_PADDING;
407                 else if (!strcmp(value, "sslv23"))
408                         pm = RSA_SSLV23_PADDING;
409                 else if (!strcmp(value, "none"))
410                         pm = RSA_NO_PADDING;
411                 else if (!strcmp(value, "oeap"))
412                         pm = RSA_PKCS1_OAEP_PADDING;
413                 else if (!strcmp(value, "x931"))
414                         pm = RSA_X931_PADDING;
415                 else if (!strcmp(value, "pss"))
416                         pm = RSA_PKCS1_PSS_PADDING;
417                 else
418                         return -2;
419                 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
420                 }
421         if (!strcmp(type, "rsa_pss_saltlen"))
422                 {
423                 int saltlen;
424                 saltlen = atoi(value);
425                 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
426                 }
427         return -2;
428         }
429
430 const EVP_PKEY_METHOD rsa_pkey_meth = 
431         {
432         EVP_PKEY_RSA,
433         0,
434         pkey_rsa_init,
435         pkey_rsa_cleanup,
436
437         0,0,
438
439         0,0,
440
441         0,
442         pkey_rsa_sign,
443
444         0,
445         pkey_rsa_verify,
446
447         0,
448         pkey_rsa_verifyrecover,
449
450
451         0,0,0,0,
452
453         0,
454         pkey_rsa_encrypt,
455
456         0,
457         pkey_rsa_decrypt,
458
459         pkey_rsa_ctrl,
460         pkey_rsa_ctrl_str
461
462
463         };