Make sure a bad parameter to RSA_verify_PKCS1_PSS() doesn't lead to a crash.

[openssl.git] / crypto / rsa / rsa.h

diff --git a/crypto/rsa/rsa.h b/crypto/rsa/rsa.h
index e9225a53c6cc4e675276e6c73d11e36f593833d0..91cd4198c7a1f9eb17d04c87c141b96dd13c2e57 100644 (file)
--- a/crypto/rsa/rsa.h
+++ b/crypto/rsa/rsa.h
@@ -160,6 +160,17 @@ struct rsa_st
        BN_BLINDING *mt_blinding;
        };
 
+#ifndef OPENSSL_RSA_MAX_MODULUS_BITS
+# define OPENSSL_RSA_MAX_MODULUS_BITS  16384
+#endif
+
+#ifndef OPENSSL_RSA_SMALL_MODULUS_BITS
+# define OPENSSL_RSA_SMALL_MODULUS_BITS        3072
+#endif
+#ifndef OPENSSL_RSA_MAX_PUBEXP_BITS
+# define OPENSSL_RSA_MAX_PUBEXP_BITS   64 /* exponent limit enforced for "large" modulus only */
+#endif
+
 #define RSA_3  0x3L
 #define RSA_F4 0x10001L
 
@@ -185,13 +196,27 @@ struct rsa_st
                                                 * default (ignoring RSA_FLAG_BLINDING),
                                                 * but other engines might not need it
                                                 */
-#define RSA_FLAG_NO_EXP_CONSTTIME      0x0100 /* new with 0.9.7h; the built-in RSA
+#define RSA_FLAG_NO_CONSTTIME          0x0100 /* new with 0.9.8f; the built-in RSA
+                                               * implementation now uses constant time
+                                               * operations by default in private key operations,
+                                               * e.g., constant time modular exponentiation, 
+                                                * modular inverse without leaking branches, 
+                                                * division without leaking branches. This 
+                                                * flag disables these constant time 
+                                                * operations and results in faster RSA 
+                                                * private key operations.
+                                                */ 
+#ifndef OPENSSL_NO_DEPRECATED
+#define RSA_FLAG_NO_EXP_CONSTTIME RSA_FLAG_NO_CONSTTIME /* deprecated name for the flag*/
+                                                /* new with 0.9.7h; the built-in RSA
                                                 * implementation now uses constant time
                                                 * modular exponentiation for secret exponents
                                                 * by default. This flag causes the
                                                 * faster variable sliding window method to
                                                 * be used for all exponents.
                                                 */
+#endif
+
 
 #define EVP_PKEY_CTX_set_rsa_padding(ctx, pad) \
        EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, -1, EVP_PKEY_CTRL_RSA_PADDING, \
@@ -203,9 +228,20 @@ struct rsa_st
                                EVP_PKEY_CTRL_RSA_PSS_SALTLEN, \
                                len, NULL)
 
+#define EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, bits) \
+       EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_KEYGEN, \
+                               EVP_PKEY_CTRL_RSA_KEYGEN_BITS, bits, NULL)
+
+#define EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp) \
+       EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_KEYGEN, \
+                               EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP, 0, pubexp)
+
 #define EVP_PKEY_CTRL_RSA_PADDING      (EVP_PKEY_ALG_CTRL + 1)
 #define EVP_PKEY_CTRL_RSA_PSS_SALTLEN  (EVP_PKEY_ALG_CTRL + 2)
 
+#define EVP_PKEY_CTRL_RSA_KEYGEN_BITS  (EVP_PKEY_ALG_CTRL + 3)
+#define EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP        (EVP_PKEY_ALG_CTRL + 4)
+
 #define RSA_PKCS1_PADDING      1
 #define RSA_SSLV23_PADDING     2
 #define RSA_NO_PADDING         3
@@ -272,6 +308,7 @@ int RSA_print_fp(FILE *fp, const RSA *r,int offset);
 int    RSA_print(BIO *bp, const RSA *r,int offset);
 #endif
 
+#ifndef OPENSSL_NO_RC4
 int i2d_RSA_NET(const RSA *a, unsigned char **pp,
                int (*cb)(char *buf, int len, const char *prompt, int verify),
                int sgckey);
@@ -285,6 +322,7 @@ int i2d_Netscape_RSA(const RSA *a, unsigned char **pp,
 RSA *d2i_Netscape_RSA(RSA **a, const unsigned char **pp, long length,
                      int (*cb)(char *buf, int len, const char *prompt,
                                int verify));
+#endif
 
 /* The following 2 functions sign and verify a X509_SIG ASN1 object
  * inside PKCS#1 padded RSA encryption */
@@ -359,8 +397,13 @@ void ERR_load_RSA_strings(void);
 /* Error codes for the RSA functions. */
 
 /* Function codes. */
-#define RSA_F_CHECK_PADDING_NID                                 140
+#define RSA_F_CHECK_PADDING_MD                          140
+#define RSA_F_DO_RSA_PRINT                              146
+#define RSA_F_INT_RSA_VERIFY                            145
 #define RSA_F_MEMORY_LOCK                               100
+#define RSA_F_OLD_RSA_PRIV_DECODE                       147
+#define RSA_F_PKEY_RSA_CTRL                             143
+#define RSA_F_PKEY_RSA_CTRL_STR                                 144
 #define RSA_F_PKEY_RSA_SIGN                             142
 #define RSA_F_PKEY_RSA_VERIFYRECOVER                    141
 #define RSA_F_RSA_BUILTIN_KEYGEN                        129
@@ -405,6 +448,7 @@ void ERR_load_RSA_strings(void);
 
 /* Reason codes. */
 #define RSA_R_ALGORITHM_MISMATCH                        100
+#define RSA_R_BAD_ARGUMENT                              149
 #define RSA_R_BAD_E_VALUE                               101
 #define RSA_R_BAD_FIXED_HEADER_DECRYPT                  102
 #define RSA_R_BAD_PAD_BYTE_COUNT                        103
@@ -422,21 +466,25 @@ void ERR_load_RSA_strings(void);
 #define RSA_R_DMQ1_NOT_CONGRUENT_TO_D                   125
 #define RSA_R_D_E_NOT_CONGRUENT_TO_1                    123
 #define RSA_R_FIRST_OCTET_INVALID                       133
-#define RSA_R_INVALID_DIGEST                            105
+#define RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE       144
 #define RSA_R_INVALID_DIGEST_LENGTH                     143
 #define RSA_R_INVALID_HEADER                            137
+#define RSA_R_INVALID_KEYBITS                           145
 #define RSA_R_INVALID_MESSAGE_LENGTH                    131
 #define RSA_R_INVALID_PADDING                           138
 #define RSA_R_INVALID_PADDING_MODE                      141
+#define RSA_R_INVALID_PSS_SALTLEN                       146
 #define RSA_R_INVALID_TRAILER                           139
 #define RSA_R_INVALID_X931_DIGEST                       142
 #define RSA_R_IQMP_NOT_INVERSE_OF_Q                     126
 #define RSA_R_KEY_SIZE_TOO_SMALL                        120
 #define RSA_R_LAST_OCTET_INVALID                        134
+#define RSA_R_MODULUS_TOO_LARGE                                 105
 #define RSA_R_NO_PUBLIC_EXPONENT                        140
 #define RSA_R_NULL_BEFORE_BLOCK_MISSING                         113
 #define RSA_R_N_DOES_NOT_EQUAL_P_Q                      127
 #define RSA_R_OAEP_DECODING_ERROR                       121
+#define RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE  148
 #define RSA_R_PADDING_CHECK_FAILED                      114
 #define RSA_R_P_NOT_PRIME                               128
 #define RSA_R_Q_NOT_PRIME                               129
@@ -447,6 +495,7 @@ void ERR_load_RSA_strings(void);
 #define RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD 116
 #define RSA_R_UNKNOWN_ALGORITHM_TYPE                    117
 #define RSA_R_UNKNOWN_PADDING_TYPE                      118
+#define RSA_R_VALUE_MISSING                             147
 #define RSA_R_WRONG_SIGNATURE_LENGTH                    119
 
 #ifdef  __cplusplus