Change operation values so they can be used as a mask.

[openssl.git] / crypto / rsa / rsa_pk1.c

diff --git a/crypto/rsa/rsa_pk1.c b/crypto/rsa/rsa_pk1.c
index f0ae51f234ebf9650d51a4a3fc9e322cba88bb2e..8560755f1d12d8745f8272f2cd7441c225556677 100644 (file)
--- a/crypto/rsa/rsa_pk1.c
+++ b/crypto/rsa/rsa_pk1.c
@@ -63,12 +63,12 @@
 #include <openssl/rand.h>
 
 int RSA_padding_add_PKCS1_type_1(unsigned char *to, int tlen,
-            unsigned char *from, int flen)
+            const unsigned char *from, int flen)
        {
        int j;
        unsigned char *p;
 
-       if (flen > (tlen-11))
+       if (flen > (tlen-RSA_PKCS1_PADDING_SIZE))
                {
                RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_TYPE_1,RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
                return(0);
@@ -79,7 +79,7 @@ int RSA_padding_add_PKCS1_type_1(unsigned char *to, int tlen,
        *(p++)=0;
        *(p++)=1; /* Private Key BT (Block Type) */
 
-       /* padd out with 0xff data */
+       /* pad out with 0xff data */
        j=tlen-3-flen;
        memset(p,0xff,j);
        p+=j;
@@ -89,10 +89,10 @@ int RSA_padding_add_PKCS1_type_1(unsigned char *to, int tlen,
        }
 
 int RSA_padding_check_PKCS1_type_1(unsigned char *to, int tlen,
-            unsigned char *from, int flen, int num)
+            const unsigned char *from, int flen, int num)
        {
        int i,j;
-       unsigned char *p;
+       const unsigned char *p;
 
        p=from;
        if ((num != (flen+1)) || (*(p++) != 01))
@@ -130,13 +130,18 @@ int RSA_padding_check_PKCS1_type_1(unsigned char *to, int tlen,
                }
        i++; /* Skip over the '\0' */
        j-=i;
+       if (j > tlen)
+               {
+               RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_TYPE_1,RSA_R_DATA_TOO_LARGE);
+               return(-1);
+               }
        memcpy(to,p,(unsigned int)j);
 
        return(j);
        }
 
 int RSA_padding_add_PKCS1_type_2(unsigned char *to, int tlen,
-            unsigned char *from, int flen)
+            const unsigned char *from, int flen)
        {
        int i,j;
        unsigned char *p;
@@ -155,12 +160,14 @@ int RSA_padding_add_PKCS1_type_2(unsigned char *to, int tlen,
        /* pad out with non-zero random data */
        j=tlen-3-flen;
 
-       RAND_bytes(p,j);
+       if (RAND_bytes(p,j) <= 0)
+               return(0);
        for (i=0; i<j; i++)
                {
                if (*p == '\0')
                        do      {
-                               RAND_bytes(p,1);
+                               if (RAND_bytes(p,1) <= 0)
+                                       return(0);
                                } while (*p == '\0');
                p++;
                }
@@ -172,10 +179,10 @@ int RSA_padding_add_PKCS1_type_2(unsigned char *to, int tlen,
        }
 
 int RSA_padding_check_PKCS1_type_2(unsigned char *to, int tlen,
-            unsigned char *from, int flen, int num)
+            const unsigned char *from, int flen, int num)
        {
        int i,j;
-       unsigned char *p;
+       const unsigned char *p;
 
        p=from;
        if ((num != (flen+1)) || (*(p++) != 02))
@@ -205,6 +212,11 @@ int RSA_padding_check_PKCS1_type_2(unsigned char *to, int tlen,
                }
        i++; /* Skip over the '\0' */
        j-=i;
+       if (j > tlen)
+               {
+               RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_TYPE_2,RSA_R_DATA_TOO_LARGE);
+               return(-1);
+               }
        memcpy(to,p,(unsigned int)j);
 
        return(j);