A few more constifications of some RSA routines that I forgot
[openssl.git] / crypto / rsa / rsa_eay.c
index 7623189e41a3afbe52699085df5fe44153f853e0..37baaacfc7d95264f811962c0e9329266888f5b4 100644 (file)
 
 #include <stdio.h>
 #include "cryptlib.h"
-#include "bn.h"
-#include "rsa.h"
-#include "rand.h"
+#include <openssl/bn.h>
+#include <openssl/rsa.h>
+#include <openssl/rand.h>
+#include <openssl/engine.h>
 
-#ifndef NOPROTO
-static int RSA_eay_public_encrypt(int flen, unsigned char *from,
+#ifndef RSA_NULL
+
+static int RSA_eay_public_encrypt(int flen, const unsigned char *from,
                unsigned char *to, RSA *rsa,int padding);
-static int RSA_eay_private_encrypt(int flen, unsigned char *from,
+static int RSA_eay_private_encrypt(int flen, const unsigned char *from,
                unsigned char *to, RSA *rsa,int padding);
-static int RSA_eay_public_decrypt(int flen, unsigned char *from,
+static int RSA_eay_public_decrypt(int flen, const unsigned char *from,
                unsigned char *to, RSA *rsa,int padding);
-static int RSA_eay_private_decrypt(int flen, unsigned char *from,
+static int RSA_eay_private_decrypt(int flen, const unsigned char *from,
                unsigned char *to, RSA *rsa,int padding);
-static int RSA_eay_mod_exp(BIGNUM *r0, BIGNUM *i, RSA *rsa);
+static int RSA_eay_mod_exp(BIGNUM *r0, const BIGNUM *i, RSA *rsa);
 static int RSA_eay_init(RSA *rsa);
 static int RSA_eay_finish(RSA *rsa);
-#else
-static int RSA_eay_public_encrypt();
-static int RSA_eay_private_encrypt();
-static int RSA_eay_public_decrypt();
-static int RSA_eay_private_decrypt();
-static int RSA_eay_mod_exp();
-static int RSA_eay_init();
-static int RSA_eay_finish();
-#endif
-
 static RSA_METHOD rsa_pkcs1_eay_meth={
        "Eric Young's PKCS#1 RSA",
        RSA_eay_public_encrypt,
@@ -98,28 +90,26 @@ static RSA_METHOD rsa_pkcs1_eay_meth={
        NULL,
        };
 
-RSA_METHOD *RSA_PKCS1_SSLeay()
+const RSA_METHOD *RSA_PKCS1_SSLeay(void)
        {
        return(&rsa_pkcs1_eay_meth);
        }
 
-static int RSA_eay_public_encrypt(flen, from, to, rsa, padding)
-int flen;
-unsigned char *from;
-unsigned char *to;
-RSA *rsa;
-int padding;
+static int RSA_eay_public_encrypt(int flen, const unsigned char *from,
+            unsigned char *to, RSA *rsa, int padding)
        {
+       const RSA_METHOD *meth;
        BIGNUM f,ret;
        int i,j,k,num=0,r= -1;
        unsigned char *buf=NULL;
        BN_CTX *ctx=NULL;
 
+       meth = ENGINE_get_RSA(rsa->engine);
        BN_init(&f);
        BN_init(&ret);
        if ((ctx=BN_CTX_new()) == NULL) goto err;
        num=BN_num_bytes(rsa->n);
-       if ((buf=(unsigned char *)Malloc(num)) == NULL)
+       if ((buf=(unsigned char *)OPENSSL_malloc(num)) == NULL)
                {
                RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,ERR_R_MALLOC_FAILURE);
                goto err;
@@ -130,6 +120,11 @@ int padding;
        case RSA_PKCS1_PADDING:
                i=RSA_padding_add_PKCS1_type_2(buf,num,from,flen);
                break;
+#ifndef NO_SHA
+       case RSA_PKCS1_OAEP_PADDING:
+               i=RSA_padding_add_PKCS1_OAEP(buf,num,from,flen,NULL,0);
+               break;
+#endif
        case RSA_SSLV23_PADDING:
                i=RSA_padding_add_SSLv23(buf,num,from,flen);
                break;
@@ -144,15 +139,15 @@ int padding;
 
        if (BN_bin2bn(buf,num,&f) == NULL) goto err;
        
-       if ((rsa->method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))
+       if ((rsa->_method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))
                {
-               if ((rsa->method_mod_n=(char *)BN_MONT_CTX_new()) != NULL)
-                       if (!BN_MONT_CTX_set((BN_MONT_CTX *)rsa->method_mod_n,
-                               rsa->n,ctx)) goto err;
+               if ((rsa->_method_mod_n=BN_MONT_CTX_new()) != NULL)
+                       if (!BN_MONT_CTX_set(rsa->_method_mod_n,rsa->n,ctx))
+                           goto err;
                }
 
-       if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
-               rsa->method_mod_n)) goto err;
+       if (!meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
+               rsa->_method_mod_n)) goto err;
 
        /* put in leading 0 bytes if the number is less than the
         * length of the modulus */
@@ -169,29 +164,27 @@ err:
        if (buf != NULL) 
                {
                memset(buf,0,num);
-               Free(buf);
+               OPENSSL_free(buf);
                }
        return(r);
        }
 
-static int RSA_eay_private_encrypt(flen, from, to, rsa, padding)
-int flen;
-unsigned char *from;
-unsigned char *to;
-RSA *rsa;
-int padding;
+static int RSA_eay_private_encrypt(int flen, const unsigned char *from,
+            unsigned char *to, RSA *rsa, int padding)
        {
+       const RSA_METHOD *meth;
        BIGNUM f,ret;
        int i,j,k,num=0,r= -1;
        unsigned char *buf=NULL;
        BN_CTX *ctx=NULL;
 
+       meth = ENGINE_get_RSA(rsa->engine);
        BN_init(&f);
        BN_init(&ret);
 
        if ((ctx=BN_CTX_new()) == NULL) goto err;
        num=BN_num_bytes(rsa->n);
-       if ((buf=(unsigned char *)Malloc(num)) == NULL)
+       if ((buf=(unsigned char *)OPENSSL_malloc(num)) == NULL)
                {
                RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,ERR_R_MALLOC_FAILURE);
                goto err;
@@ -219,15 +212,16 @@ int padding;
        if (rsa->flags & RSA_FLAG_BLINDING)
                if (!BN_BLINDING_convert(&f,rsa->blinding,ctx)) goto err;
 
-       if (    (rsa->p != NULL) &&
+       if ( (rsa->flags & RSA_FLAG_EXT_PKEY) ||
+               ((rsa->p != NULL) &&
                (rsa->q != NULL) &&
                (rsa->dmp1 != NULL) &&
                (rsa->dmq1 != NULL) &&
-               (rsa->iqmp != NULL))
-               { if (!rsa->meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
+               (rsa->iqmp != NULL)) )
+               { if (!meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
        else
                {
-               if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL)) goto err;
+               if (!meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL)) goto err;
                }
 
        if (rsa->flags & RSA_FLAG_BLINDING)
@@ -248,24 +242,22 @@ err:
        if (buf != NULL)
                {
                memset(buf,0,num);
-               Free(buf);
+               OPENSSL_free(buf);
                }
        return(r);
        }
 
-static int RSA_eay_private_decrypt(flen, from, to, rsa,padding)
-int flen;
-unsigned char *from;
-unsigned char *to;
-RSA *rsa;
-int padding;
+static int RSA_eay_private_decrypt(int flen, const unsigned char *from,
+            unsigned char *to, RSA *rsa, int padding)
        {
+       const RSA_METHOD *meth;
        BIGNUM f,ret;
        int j,num=0,r= -1;
        unsigned char *p;
        unsigned char *buf=NULL;
        BN_CTX *ctx=NULL;
 
+       meth = ENGINE_get_RSA(rsa->engine);
        BN_init(&f);
        BN_init(&ret);
        ctx=BN_CTX_new();
@@ -273,13 +265,13 @@ int padding;
 
        num=BN_num_bytes(rsa->n);
 
-       if ((buf=(unsigned char *)Malloc(num)) == NULL)
+       if ((buf=(unsigned char *)OPENSSL_malloc(num)) == NULL)
                {
                RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,ERR_R_MALLOC_FAILURE);
                goto err;
                }
 
-       /* This check was for equallity but PGP does evil things
+       /* This check was for equality but PGP does evil things
         * and chops off the top '0' bytes */
        if (flen > num)
                {
@@ -296,15 +288,16 @@ int padding;
                if (!BN_BLINDING_convert(&f,rsa->blinding,ctx)) goto err;
 
        /* do the decrypt */
-       if (    (rsa->p != NULL) &&
+       if ( (rsa->flags & RSA_FLAG_EXT_PKEY) ||
+               ((rsa->p != NULL) &&
                (rsa->q != NULL) &&
                (rsa->dmp1 != NULL) &&
                (rsa->dmq1 != NULL) &&
-               (rsa->iqmp != NULL))
-               { if (!rsa->meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
+               (rsa->iqmp != NULL)) )
+               { if (!meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
        else
                {
-               if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL))
+               if (!meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL))
                        goto err;
                }
 
@@ -319,7 +312,12 @@ int padding;
        case RSA_PKCS1_PADDING:
                r=RSA_padding_check_PKCS1_type_2(to,num,buf,j,num);
                break;
-       case RSA_SSLV23_PADDING:
+#ifndef NO_SHA
+        case RSA_PKCS1_OAEP_PADDING:
+               r=RSA_padding_check_PKCS1_OAEP(to,num,buf,j,num,NULL,0);
+                break;
+#endif
+       case RSA_SSLV23_PADDING:
                r=RSA_padding_check_SSLv23(to,num,buf,j,num);
                break;
        case RSA_NO_PADDING:
@@ -339,38 +337,36 @@ err:
        if (buf != NULL)
                {
                memset(buf,0,num);
-               Free(buf);
+               OPENSSL_free(buf);
                }
        return(r);
        }
 
-static int RSA_eay_public_decrypt(flen, from, to, rsa, padding)
-int flen;
-unsigned char *from;
-unsigned char *to;
-RSA *rsa;
-int padding;
+static int RSA_eay_public_decrypt(int flen, const unsigned char *from,
+            unsigned char *to, RSA *rsa, int padding)
        {
+       const RSA_METHOD *meth;
        BIGNUM f,ret;
        int i,num=0,r= -1;
        unsigned char *p;
        unsigned char *buf=NULL;
        BN_CTX *ctx=NULL;
 
+       meth = ENGINE_get_RSA(rsa->engine);
        BN_init(&f);
        BN_init(&ret);
        ctx=BN_CTX_new();
        if (ctx == NULL) goto err;
 
        num=BN_num_bytes(rsa->n);
-       buf=(unsigned char *)Malloc(num);
+       buf=(unsigned char *)OPENSSL_malloc(num);
        if (buf == NULL)
                {
                RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,ERR_R_MALLOC_FAILURE);
                goto err;
                }
 
-       /* This check was for equallity but PGP does evil things
+       /* This check was for equality but PGP does evil things
         * and chops off the top '0' bytes */
        if (flen > num)
                {
@@ -380,15 +376,15 @@ int padding;
 
        if (BN_bin2bn(from,flen,&f) == NULL) goto err;
        /* do the decrypt */
-       if ((rsa->method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))
+       if ((rsa->_method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))
                {
-               if ((rsa->method_mod_n=(char *)BN_MONT_CTX_new()) != NULL)
-                       if (!BN_MONT_CTX_set((BN_MONT_CTX *)rsa->method_mod_n,
-                               rsa->n,ctx)) goto err;
+               if ((rsa->_method_mod_n=BN_MONT_CTX_new()) != NULL)
+                       if (!BN_MONT_CTX_set(rsa->_method_mod_n,rsa->n,ctx))
+                           goto err;
                }
 
-       if (!rsa->meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
-               rsa->method_mod_n)) goto err;
+       if (!meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
+               rsa->_method_mod_n)) goto err;
 
        p=buf;
        i=BN_bn2bin(&ret,p);
@@ -415,51 +411,48 @@ err:
        if (buf != NULL)
                {
                memset(buf,0,num);
-               Free(buf);
+               OPENSSL_free(buf);
                }
        return(r);
        }
 
-static int RSA_eay_mod_exp(r0, I, rsa)
-BIGNUM *r0;
-BIGNUM *I;
-RSA *rsa;
+static int RSA_eay_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa)
        {
+       const RSA_METHOD *meth;
        BIGNUM r1,m1;
        int ret=0;
        BN_CTX *ctx;
 
+       meth = ENGINE_get_RSA(rsa->engine);
        if ((ctx=BN_CTX_new()) == NULL) goto err;
        BN_init(&m1);
        BN_init(&r1);
 
        if (rsa->flags & RSA_FLAG_CACHE_PRIVATE)
                {
-               if (rsa->method_mod_p == NULL)
+               if (rsa->_method_mod_p == NULL)
                        {
-                       if ((rsa->method_mod_p=(char *)
-                               BN_MONT_CTX_new()) != NULL)
-                               if (!BN_MONT_CTX_set((BN_MONT_CTX *)
-                                       rsa->method_mod_p,rsa->p,ctx))
+                       if ((rsa->_method_mod_p=BN_MONT_CTX_new()) != NULL)
+                               if (!BN_MONT_CTX_set(rsa->_method_mod_p,rsa->p,
+                                                    ctx))
                                        goto err;
                        }
-               if (rsa->method_mod_q == NULL)
+               if (rsa->_method_mod_q == NULL)
                        {
-                       if ((rsa->method_mod_q=(char *)
-                               BN_MONT_CTX_new()) != NULL)
-                               if (!BN_MONT_CTX_set((BN_MONT_CTX *)
-                                       rsa->method_mod_q,rsa->q,ctx))
+                       if ((rsa->_method_mod_q=BN_MONT_CTX_new()) != NULL)
+                               if (!BN_MONT_CTX_set(rsa->_method_mod_q,rsa->q,
+                                                    ctx))
                                        goto err;
                        }
                }
 
        if (!BN_mod(&r1,I,rsa->q,ctx)) goto err;
-       if (!rsa->meth->bn_mod_exp(&m1,&r1,rsa->dmq1,rsa->q,ctx,
-               rsa->method_mod_q)) goto err;
+       if (!meth->bn_mod_exp(&m1,&r1,rsa->dmq1,rsa->q,ctx,
+               rsa->_method_mod_q)) goto err;
 
        if (!BN_mod(&r1,I,rsa->p,ctx)) goto err;
-       if (!rsa->meth->bn_mod_exp(r0,&r1,rsa->dmp1,rsa->p,ctx,
-               rsa->method_mod_p)) goto err;
+       if (!meth->bn_mod_exp(r0,&r1,rsa->dmp1,rsa->p,ctx,
+               rsa->_method_mod_p)) goto err;
 
        if (!BN_sub(r0,r0,&m1)) goto err;
        /* This will help stop the size of r0 increasing, which does
@@ -469,6 +462,15 @@ RSA *rsa;
 
        if (!BN_mul(&r1,r0,rsa->iqmp,ctx)) goto err;
        if (!BN_mod(r0,&r1,rsa->p,ctx)) goto err;
+       /* If p < q it is occasionally possible for the correction of
+         * adding 'p' if r0 is negative above to leave the result still
+        * negative. This can break the private key operations: the following
+        * second correction should *always* correct this rare occurrence.
+        * This will *never* happen with OpenSSL generated keys because
+         * they ensure p > q [steve]
+         */
+       if (r0->neg)
+               if (!BN_add(r0,r0,rsa->p)) goto err;
        if (!BN_mul(&r1,r0,rsa->q,ctx)) goto err;
        if (!BN_add(r0,&r1,&m1)) goto err;
 
@@ -480,23 +482,21 @@ err:
        return(ret);
        }
 
-static int RSA_eay_init(rsa)
-RSA *rsa;
+static int RSA_eay_init(RSA *rsa)
        {
        rsa->flags|=RSA_FLAG_CACHE_PUBLIC|RSA_FLAG_CACHE_PRIVATE;
        return(1);
        }
 
-static int RSA_eay_finish(rsa)
-RSA *rsa;
+static int RSA_eay_finish(RSA *rsa)
        {
-       if (rsa->method_mod_n != NULL)
-               BN_MONT_CTX_free((BN_MONT_CTX *)rsa->method_mod_n);
-       if (rsa->method_mod_p != NULL)
-               BN_MONT_CTX_free((BN_MONT_CTX *)rsa->method_mod_p);
-       if (rsa->method_mod_q != NULL)
-               BN_MONT_CTX_free((BN_MONT_CTX *)rsa->method_mod_q);
+       if (rsa->_method_mod_n != NULL)
+               BN_MONT_CTX_free(rsa->_method_mod_n);
+       if (rsa->_method_mod_p != NULL)
+               BN_MONT_CTX_free(rsa->_method_mod_p);
+       if (rsa->_method_mod_q != NULL)
+               BN_MONT_CTX_free(rsa->_method_mod_q);
        return(1);
        }
 
-
+#endif