Circumvent an exploitable buffer overrun error in RSA Security's RSAREF

[openssl.git] / rsaref / rsaref.c

diff --git a/rsaref/rsaref.c b/rsaref/rsaref.c
index 0b4fb772326a41179bb74667e992bd5094f4a9b9..1a4c0f378b94bc6567f636a03114ee8498a850b9 100644 (file)
--- a/rsaref/rsaref.c
+++ b/rsaref/rsaref.c
@@ -56,6 +56,7 @@
  * [including the GNU Public Licence.]
  */
 
+#ifndef NO_RSA
 #include <stdio.h>
 #include "cryptlib.h"
 #include <openssl/bn.h>
@@ -77,7 +78,8 @@ int RSA_ref_public_encrypt(int len, unsigned char *from,
        unsigned char *to, RSA *rsa, int padding);
 int RSA_ref_public_decrypt(int len, unsigned char *from,
        unsigned char *to, RSA *rsa, int padding);
-static int BN_ref_mod_exp(BIGNUM *r,BIGNUM *a,BIGNUM *p,BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx);
+static int BN_ref_mod_exp(BIGNUM *r,BIGNUM *a,const BIGNUM *p,const BIGNUM *m,
+                         BN_CTX *ctx, BN_MONT_CTX *m_ctx);
 static int RSA_ref_mod_exp(BIGNUM *r0, BIGNUM *I, RSA *rsa);
 static RSA_METHOD rsa_pkcs1_ref_meth={
        "RSAref PKCS#1 RSA",
@@ -104,8 +106,8 @@ static int RSA_ref_mod_exp(BIGNUM *r0, BIGNUM *I, RSA *rsa)
        return(0);
        }
 
-static int BN_ref_mod_exp(BIGNUM *r, BIGNUM *a, BIGNUM *p, BIGNUM *m,
-            BN_CTX *ctx, BN_MONT_CTX *m_ctx)
+static int BN_ref_mod_exp(BIGNUM *r, BIGNUM *a, const BIGNUM *p,
+                         const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx)
        {
        RSAREFerr(RSAREF_F_BN_REF_MOD_EXP,ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED);
        return(0);
@@ -207,6 +209,11 @@ int RSA_ref_private_decrypt(int len, unsigned char *from, unsigned char *to,
 
        if (!RSAref_Private_eay2ref(rsa,&RSAkey))
                goto err;
+       if (len > RSAref_MAX_LEN)
+               {
+               RSAREFerr(RSAREF_F_RSA_REF_PRIVATE_DECRYPT,RSAREF_R_LEN);
+               goto err;
+               }
        if ((i=RSAPrivateDecrypt(to,&outlen,from,len,&RSAkey)) != 0)
                {
                RSAREFerr(RSAREF_F_RSA_REF_PRIVATE_DECRYPT,i);
@@ -230,6 +237,11 @@ int RSA_ref_private_encrypt(int len, unsigned char *from, unsigned char *to,
        }
        if (!RSAref_Private_eay2ref(rsa,&RSAkey))
                goto err;
+       if (len + 3 > RSAref_MAX_LEN)
+               {
+               RSAREFerr(RSAREF_F_RSA_REF_PRIVATE_ENCRYPT,RSAREF_R_LEN);
+               goto err;
+               }
        if ((i=RSAPrivateEncrypt(to,&outlen,from,len,&RSAkey)) != 0)
                {
                RSAREFerr(RSAREF_F_RSA_REF_PRIVATE_ENCRYPT,i);
@@ -248,6 +260,12 @@ int RSA_ref_public_decrypt(int len, unsigned char *from, unsigned char *to,
 
        if (!RSAref_Public_eay2ref(rsa,&RSAkey))
                goto err;
+       if (len > RSAref_MAX_LEN)
+               {
+               RSAREFerr(RSAREF_F_RSA_REF_PUBLIC_DECRYPT,RSAREF_R_LEN);
+               goto err;
+               }
+               goto err;
        if ((i=RSAPublicDecrypt(to,&outlen,from,len,&RSAkey)) != 0)
                {
                RSAREFerr(RSAREF_F_RSA_REF_PUBLIC_DECRYPT,i);
@@ -284,6 +302,11 @@ int RSA_ref_public_encrypt(int len, unsigned char *from, unsigned char *to,
 
        if (!RSAref_Public_eay2ref(rsa,&RSAkey))
                goto err;
+       if (len + 3 > RSAref_MAX_LEN)
+               {
+               RSAREFerr(RSAREF_F_RSA_REF_PUBLIC_ENCRYPT,RSAREF_R_LEN);
+               goto err;
+               }
        if ((i=RSAPublicEncrypt(to,&outlen,from,len,&RSAkey,&rnd)) != 0)
                {
                RSAREFerr(RSAREF_F_RSA_REF_PUBLIC_ENCRYPT,i);
@@ -296,4 +319,4 @@ err:
        memset(&rnd,0,sizeof(rnd));
        return(outlen);
        }
-
+#endif