Make sure a bad parameter to RSA_verify_PKCS1_PSS() doesn't lead to a crash.
[openssl.git] / crypto / rsa / rsa_ameth.c
1 /* crypto/rsa/rsa_ameth.c */
2 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
3  * project 2006.
4  */
5 /* ====================================================================
6  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer. 
14  *
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in
17  *    the documentation and/or other materials provided with the
18  *    distribution.
19  *
20  * 3. All advertising materials mentioning features or use of this
21  *    software must display the following acknowledgment:
22  *    "This product includes software developed by the OpenSSL Project
23  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
24  *
25  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
26  *    endorse or promote products derived from this software without
27  *    prior written permission. For written permission, please contact
28  *    licensing@OpenSSL.org.
29  *
30  * 5. Products derived from this software may not be called "OpenSSL"
31  *    nor may "OpenSSL" appear in their names without prior written
32  *    permission of the OpenSSL Project.
33  *
34  * 6. Redistributions of any form whatsoever must retain the following
35  *    acknowledgment:
36  *    "This product includes software developed by the OpenSSL Project
37  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
38  *
39  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
40  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
41  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
42  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
43  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
44  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
45  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
46  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
48  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
49  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
50  * OF THE POSSIBILITY OF SUCH DAMAGE.
51  * ====================================================================
52  *
53  * This product includes cryptographic software written by Eric Young
54  * (eay@cryptsoft.com).  This product includes software written by Tim
55  * Hudson (tjh@cryptsoft.com).
56  *
57  */
58
59 #include <stdio.h>
60 #include "cryptlib.h"
61 #include <openssl/asn1t.h>
62 #include <openssl/x509.h>
63 #include <openssl/rsa.h>
64 #include <openssl/bn.h>
65 #ifndef OPENSSL_NO_CMS
66 #include <openssl/cms.h>
67 #endif
68 #include "asn1_locl.h"
69
70 static int rsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
71         {
72         unsigned char *penc = NULL;
73         int penclen;
74         penclen = i2d_RSAPublicKey(pkey->pkey.rsa, &penc);
75         if (penclen <= 0)
76                 return 0;
77         if (X509_PUBKEY_set0_param(pk, OBJ_nid2obj(EVP_PKEY_RSA),
78                                 V_ASN1_NULL, NULL, penc, penclen))
79                 return 1;
80
81         OPENSSL_free(penc);
82         return 0;
83         }
84
85 static int rsa_pub_decode(EVP_PKEY *pkey, X509_PUBKEY *pubkey)
86         {
87         const unsigned char *p;
88         int pklen;
89         RSA *rsa = NULL;
90         if (!X509_PUBKEY_get0_param(NULL, &p, &pklen, NULL, pubkey))
91                 return 0;
92         if (!(rsa = d2i_RSAPublicKey(NULL, &p, pklen)))
93                 {
94                 RSAerr(RSA_F_RSA_PUB_DECODE, ERR_R_RSA_LIB);
95                 return 0;
96                 }
97         EVP_PKEY_assign_RSA (pkey, rsa);
98         return 1;
99         }
100
101 static int rsa_pub_cmp(const EVP_PKEY *a, const EVP_PKEY *b)
102         {
103         if (BN_cmp(b->pkey.rsa->n,a->pkey.rsa->n) != 0
104                 || BN_cmp(b->pkey.rsa->e,a->pkey.rsa->e) != 0)
105                         return 0;
106         return 1;
107         }
108
109 static int old_rsa_priv_decode(EVP_PKEY *pkey,
110                                         const unsigned char **pder, int derlen)
111         {
112         RSA *rsa;
113         if (!(rsa = d2i_RSAPrivateKey (NULL, pder, derlen)))
114                 {
115                 RSAerr(RSA_F_OLD_RSA_PRIV_DECODE, ERR_R_RSA_LIB);
116                 return 0;
117                 }
118         EVP_PKEY_assign_RSA(pkey, rsa);
119         return 1;
120         }
121
122 static int old_rsa_priv_encode(const EVP_PKEY *pkey, unsigned char **pder)
123         {
124         return i2d_RSAPrivateKey(pkey->pkey.rsa, pder);
125         }
126
127 static int rsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey)
128         {
129         unsigned char *rk = NULL;
130         int rklen;
131         rklen = i2d_RSAPrivateKey(pkey->pkey.rsa, &rk);
132
133         if (rklen <= 0)
134                 {
135                 RSAerr(RSA_F_RSA_PRIV_ENCODE,ERR_R_MALLOC_FAILURE);
136                 return 0;
137                 }
138
139         if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(NID_rsaEncryption), 0,
140                                 V_ASN1_NULL, NULL, rk, rklen))
141                 {
142                 RSAerr(RSA_F_RSA_PRIV_ENCODE,ERR_R_MALLOC_FAILURE);
143                 return 0;
144                 }
145
146         return 1;
147         }
148
149 static int rsa_priv_decode(EVP_PKEY *pkey, PKCS8_PRIV_KEY_INFO *p8)
150         {
151         const unsigned char *p;
152         int pklen;
153         if (!PKCS8_pkey_get0(NULL, &p, &pklen, NULL, p8))
154                 return 0;
155         return old_rsa_priv_decode(pkey, &p, pklen);
156         }
157
158 static int int_rsa_size(const EVP_PKEY *pkey)
159         {
160         return RSA_size(pkey->pkey.rsa);
161         }
162
163 static int rsa_bits(const EVP_PKEY *pkey)
164         {
165         return BN_num_bits(pkey->pkey.rsa->n);
166         }
167
168 static void int_rsa_free(EVP_PKEY *pkey)
169         {
170         RSA_free(pkey->pkey.rsa);
171         }
172
173
174 static void update_buflen(const BIGNUM *b, size_t *pbuflen)
175         {
176         size_t i;
177         if (!b)
178                 return;
179         if (*pbuflen < (i = (size_t)BN_num_bytes(b)))
180                         *pbuflen = i;
181         }
182
183 static int do_rsa_print(BIO *bp, const RSA *x, int off, int priv)
184         {
185         char *str;
186         const char *s;
187         unsigned char *m=NULL;
188         int ret=0, mod_len = 0;
189         size_t buf_len=0;
190
191         update_buflen(x->n, &buf_len);
192         update_buflen(x->e, &buf_len);
193
194         if (priv)
195                 {
196                 update_buflen(x->d, &buf_len);
197                 update_buflen(x->p, &buf_len);
198                 update_buflen(x->q, &buf_len);
199                 update_buflen(x->dmp1, &buf_len);
200                 update_buflen(x->dmq1, &buf_len);
201                 update_buflen(x->iqmp, &buf_len);
202                 }
203
204         m=(unsigned char *)OPENSSL_malloc(buf_len+10);
205         if (m == NULL)
206                 {
207                 RSAerr(RSA_F_DO_RSA_PRINT,ERR_R_MALLOC_FAILURE);
208                 goto err;
209                 }
210
211         if (x->n != NULL)
212                 mod_len = BN_num_bits(x->n);
213
214         if(!BIO_indent(bp,off,128))
215                 goto err;
216
217         if (priv && x->d)
218                 {
219                 if (BIO_printf(bp,"Private-Key: (%d bit)\n", mod_len)
220                         <= 0) goto err;
221                 str = "modulus:";
222                 s = "publicExponent:";
223                 }
224         else
225                 {
226                 if (BIO_printf(bp,"Public-Key: (%d bit)\n", mod_len)
227                         <= 0) goto err;
228                 str = "Modulus:";
229                 s= "Exponent:";
230                 }
231         if (!ASN1_bn_print(bp,str,x->n,m,off)) goto err;
232         if (!ASN1_bn_print(bp,s,x->e,m,off))
233                 goto err;
234         if (priv)
235                 {
236                 if (!ASN1_bn_print(bp,"privateExponent:",x->d,m,off))
237                         goto err;
238                 if (!ASN1_bn_print(bp,"prime1:",x->p,m,off))
239                         goto err;
240                 if (!ASN1_bn_print(bp,"prime2:",x->q,m,off))
241                         goto err;
242                 if (!ASN1_bn_print(bp,"exponent1:",x->dmp1,m,off))
243                         goto err;
244                 if (!ASN1_bn_print(bp,"exponent2:",x->dmq1,m,off))
245                         goto err;
246                 if (!ASN1_bn_print(bp,"coefficient:",x->iqmp,m,off))
247                         goto err;
248                 }
249         ret=1;
250 err:
251         if (m != NULL) OPENSSL_free(m);
252         return(ret);
253         }
254
255 static int rsa_pub_print(BIO *bp, const EVP_PKEY *pkey, int indent,
256                                                         ASN1_PCTX *ctx)
257         {
258         return do_rsa_print(bp, pkey->pkey.rsa, indent, 0);
259         }
260
261
262 static int rsa_priv_print(BIO *bp, const EVP_PKEY *pkey, int indent,
263                                                         ASN1_PCTX *ctx)
264         {
265         return do_rsa_print(bp, pkey->pkey.rsa, indent, 1);
266         }
267
268
269 static int rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
270         {
271         X509_ALGOR *alg = NULL;
272         switch (op)
273                 {
274
275                 case ASN1_PKEY_CTRL_PKCS7_SIGN:
276                 if (arg1 == 0)
277                         PKCS7_SIGNER_INFO_get0_algs(arg2, NULL, NULL, &alg);
278                 break;
279
280                 case ASN1_PKEY_CTRL_PKCS7_ENCRYPT:
281                 if (arg1 == 0)
282                         PKCS7_RECIP_INFO_get0_alg(arg2, &alg);
283                 break;
284 #ifndef OPENSSL_NO_CMS
285                 case ASN1_PKEY_CTRL_CMS_SIGN:
286                 if (arg1 == 0)
287                         CMS_SignerInfo_get0_algs(arg2, NULL, NULL, NULL, &alg);
288                 break;
289
290                 case ASN1_PKEY_CTRL_CMS_ENVELOPE:
291                 if (arg1 == 0)
292                         CMS_RecipientInfo_ktri_get0_algs(arg2, NULL, NULL, &alg);
293                 break;
294 #endif
295
296                 case ASN1_PKEY_CTRL_DEFAULT_MD_NID:
297                 *(int *)arg2 = NID_sha1;
298                 return 1;
299
300                 default:
301                 return -2;
302
303                 }
304
305         if (alg)
306                 X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption),
307                                                         V_ASN1_NULL, 0);
308
309         return 1;
310
311         }
312
313
314 const EVP_PKEY_ASN1_METHOD rsa_asn1_meths[] = 
315         {
316                 {
317                 EVP_PKEY_RSA,
318                 EVP_PKEY_RSA,
319                 ASN1_PKEY_SIGPARAM_NULL,
320
321                 "RSA",
322                 "OpenSSL RSA method",
323
324                 rsa_pub_decode,
325                 rsa_pub_encode,
326                 rsa_pub_cmp,
327                 rsa_pub_print,
328
329                 rsa_priv_decode,
330                 rsa_priv_encode,
331                 rsa_priv_print,
332
333                 int_rsa_size,
334                 rsa_bits,
335
336                 0,0,0,0,0,0,
337
338                 int_rsa_free,
339                 rsa_pkey_ctrl,
340                 old_rsa_priv_decode,
341                 old_rsa_priv_encode
342                 },
343
344                 {
345                 EVP_PKEY_RSA2,
346                 EVP_PKEY_RSA,
347                 ASN1_PKEY_ALIAS
348                 }
349         };