Use a signed value to check return value of do_cipher().

[openssl.git] / crypto / evp / p_open.c

diff --git a/crypto/evp/p_open.c b/crypto/evp/p_open.c
index 85d5097644a51a1d0f5360c00c58fecf8975d953..c748fbea877250031f8b40b60b36ef2d4b5f85c9 100644 (file)
--- a/crypto/evp/p_open.c
+++ b/crypto/evp/p_open.c
@@ -56,52 +56,56 @@
  * [including the GNU Public Licence.]
  */
 
-#ifndef NO_RSA
 #include <stdio.h>
 #include "cryptlib.h"
+
+#ifndef OPENSSL_NO_RSA
+
 #include <openssl/evp.h>
 #include <openssl/objects.h>
 #include <openssl/x509.h>
+#include <openssl/rsa.h>
 
-int EVP_OpenInit(EVP_CIPHER_CTX *ctx, EVP_CIPHER *type, unsigned char *ek,
-            int ekl, unsigned char *iv, EVP_PKEY *priv)
+int EVP_OpenInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
+       const unsigned char *ek, int ekl, const unsigned char *iv,
+       EVP_PKEY *priv)
        {
        unsigned char *key=NULL;
        int i,size=0,ret=0;
-       
+
+       if(type) {      
+               EVP_CIPHER_CTX_init(ctx);
+               if(!EVP_DecryptInit_ex(ctx,type,NULL, NULL,NULL)) return 0;
+       }
+
+       if(!priv) return 1;
+
        if (priv->type != EVP_PKEY_RSA)
                {
                EVPerr(EVP_F_EVP_OPENINIT,EVP_R_PUBLIC_KEY_NOT_RSA);
-               ret= -1;
                goto err;
                 }
 
-       if(type) {      
-               EVP_CIPHER_CTX_init(ctx);
-               EVP_DecryptInit(ctx,type,NULL,NULL);
-       }
-
        size=RSA_size(priv->pkey.rsa);
        key=(unsigned char *)OPENSSL_malloc(size+2);
        if (key == NULL)
                {
                /* ERROR */
                EVPerr(EVP_F_EVP_OPENINIT,ERR_R_MALLOC_FAILURE);
-               ret= -1;
                goto err;
                }
 
-       i=EVP_PKEY_decrypt(key,ek,ekl,priv);
+       i=EVP_PKEY_decrypt_old(key,ek,ekl,priv);
        if ((i <= 0) || !EVP_CIPHER_CTX_set_key_length(ctx, i))
                {
                /* ERROR */
                goto err;
                }
-       if(!EVP_DecryptInit(ctx,NULL,key,iv)) goto err;
+       if(!EVP_DecryptInit_ex(ctx,NULL,NULL,key,iv)) goto err;
 
        ret=1;
 err:
-       if (key != NULL) memset(key,0,size);
+       if (key != NULL) OPENSSL_cleanse(key,size);
        OPENSSL_free(key);
        return(ret);
        }
@@ -110,11 +114,12 @@ int EVP_OpenFinal(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
        {
        int i;
 
-       i=EVP_DecryptFinal(ctx,out,outl);
-       EVP_DecryptInit(ctx,NULL,NULL,NULL);
+       i=EVP_DecryptFinal_ex(ctx,out,outl);
+       if (i)
+               i = EVP_DecryptInit_ex(ctx,NULL,NULL,NULL,NULL);
        return(i);
        }
-#else /* !NO_RSA */
+#else /* !OPENSSL_NO_RSA */
 
 # ifdef PEDANTIC
 static void *dummy=&dummy;