Change operation values so they can be used as a mask.

[openssl.git] / crypto / rsa / rsa_pk1.c

diff --git a/crypto/rsa/rsa_pk1.c b/crypto/rsa/rsa_pk1.c
index 4638187970b18e19df900ad101225ff3f11dbed6..8560755f1d12d8745f8272f2cd7441c225556677 100644 (file)
--- a/crypto/rsa/rsa_pk1.c
+++ b/crypto/rsa/rsa_pk1.c
@@ -58,32 +58,17 @@
 
 #include <stdio.h>
 #include "cryptlib.h"
-#include "bn.h"
-#include "rsa.h"
-#include "rand.h"
-
-#ifndef NOPROTO
-int RSA_padding_add_PKCS1_type_1();
-int RSA_padding_check_PKCS1_type_1();
-int RSA_padding_add_PKCS1_type_2();
-int RSA_padding_check_PKCS1_type_2();
-int RSA_padding_add_SSLv23();
-int RSA_padding_check_SSLv23();
-int RSA_padding_add_none();
-int RSA_padding_check_none();
+#include <openssl/bn.h>
+#include <openssl/rsa.h>
+#include <openssl/rand.h>
 
-#endif
-
-int RSA_padding_add_PKCS1_type_1(to,tlen,from,flen)
-unsigned char *to;
-int tlen;
-unsigned char *from;
-int flen;
+int RSA_padding_add_PKCS1_type_1(unsigned char *to, int tlen,
+            const unsigned char *from, int flen)
        {
        int j;
        unsigned char *p;
 
-       if (flen > (tlen-11))
+       if (flen > (tlen-RSA_PKCS1_PADDING_SIZE))
                {
                RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_TYPE_1,RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
                return(0);
@@ -94,7 +79,7 @@ int flen;
        *(p++)=0;
        *(p++)=1; /* Private Key BT (Block Type) */
 
-       /* padd out with 0xff data */
+       /* pad out with 0xff data */
        j=tlen-3-flen;
        memset(p,0xff,j);
        p+=j;
@@ -103,15 +88,11 @@ int flen;
        return(1);
        }
 
-int RSA_padding_check_PKCS1_type_1(to,tlen,from,flen,num)
-unsigned char *to;
-int tlen;
-unsigned char *from;
-int flen;
-int num;
+int RSA_padding_check_PKCS1_type_1(unsigned char *to, int tlen,
+            const unsigned char *from, int flen, int num)
        {
        int i,j;
-       unsigned char *p;
+       const unsigned char *p;
 
        p=from;
        if ((num != (flen+1)) || (*(p++) != 01))
@@ -149,16 +130,18 @@ int num;
                }
        i++; /* Skip over the '\0' */
        j-=i;
+       if (j > tlen)
+               {
+               RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_TYPE_1,RSA_R_DATA_TOO_LARGE);
+               return(-1);
+               }
        memcpy(to,p,(unsigned int)j);
 
        return(j);
        }
 
-int RSA_padding_add_PKCS1_type_2(to,tlen,from,flen)
-unsigned char *to;
-int tlen;
-unsigned char *from;
-int flen;
+int RSA_padding_add_PKCS1_type_2(unsigned char *to, int tlen,
+            const unsigned char *from, int flen)
        {
        int i,j;
        unsigned char *p;
@@ -177,12 +160,14 @@ int flen;
        /* pad out with non-zero random data */
        j=tlen-3-flen;
 
-       RAND_bytes(p,j);
+       if (RAND_bytes(p,j) <= 0)
+               return(0);
        for (i=0; i<j; i++)
                {
                if (*p == '\0')
                        do      {
-                               RAND_bytes(p,1);
+                               if (RAND_bytes(p,1) <= 0)
+                                       return(0);
                                } while (*p == '\0');
                p++;
                }
@@ -193,15 +178,11 @@ int flen;
        return(1);
        }
 
-int RSA_padding_check_PKCS1_type_2(to,tlen,from,flen,num)
-unsigned char *to;
-int tlen;
-unsigned char *from;
-int flen;
-int num;
+int RSA_padding_check_PKCS1_type_2(unsigned char *to, int tlen,
+            const unsigned char *from, int flen, int num)
        {
        int i,j;
-       unsigned char *p;
+       const unsigned char *p;
 
        p=from;
        if ((num != (flen+1)) || (*(p++) != 02))
@@ -231,6 +212,11 @@ int num;
                }
        i++; /* Skip over the '\0' */
        j-=i;
+       if (j > tlen)
+               {
+               RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_TYPE_2,RSA_R_DATA_TOO_LARGE);
+               return(-1);
+               }
        memcpy(to,p,(unsigned int)j);
 
        return(j);