when checking OAEP, signal just a single kind of 'decoding error'

[openssl.git] / crypto / rsa / rsa_sign.c

diff --git a/crypto/rsa/rsa_sign.c b/crypto/rsa/rsa_sign.c
index 05bb7fb74afe9f2cb5a212e847cb49a26c62a5e7..009fba199e3dea6e4a0f94baea4045c7609813fa 100644 (file)
--- a/crypto/rsa/rsa_sign.c
+++ b/crypto/rsa/rsa_sign.c
@@ -62,21 +62,24 @@
 #include <openssl/rsa.h>
 #include <openssl/objects.h>
 #include <openssl/x509.h>
+#include <openssl/engine.h>
 
 /* Size of an SSL signature: MD5+SHA1 */
 #define SSL_SIG_LENGTH 36
 
-int RSA_sign(int type, unsigned char *m, unsigned int m_len,
+int RSA_sign(int type, const unsigned char *m, unsigned int m_len,
             unsigned char *sigret, unsigned int *siglen, RSA *rsa)
        {
        X509_SIG sig;
        ASN1_TYPE parameter;
        int i,j,ret=1;
-       unsigned char *p,*s = NULL;
+       unsigned char *p, *tmps = NULL;
+       const unsigned char *s = NULL;
        X509_ALGOR algor;
        ASN1_OCTET_STRING digest;
        if(rsa->flags & RSA_FLAG_SIGN_VER)
-             return rsa->meth->rsa_sign(type, m, m_len, sigret, siglen, rsa);
+             return ENGINE_get_RSA(rsa->engine)->rsa_sign(type,
+                       m, m_len, sigret, siglen, rsa);
        /* Special case: SSL signature, just check the length */
        if(type == NID_md5_sha1) {
                if(m_len != SSL_SIG_LENGTH) {
@@ -103,7 +106,7 @@ int RSA_sign(int type, unsigned char *m, unsigned int m_len,
                sig.algor->parameter= &parameter;
 
                sig.digest= &digest;
-               sig.digest->data=m;
+               sig.digest->data=(unsigned char *)m; /* TMP UGLY CAST */
                sig.digest->length=m_len;
 
                i=i2d_X509_SIG(&sig,NULL);
@@ -115,14 +118,15 @@ int RSA_sign(int type, unsigned char *m, unsigned int m_len,
                return(0);
                }
        if(type != NID_md5_sha1) {
-               s=(unsigned char *)Malloc((unsigned int)j+1);
-               if (s == NULL)
+               tmps=(unsigned char *)OPENSSL_malloc((unsigned int)j+1);
+               if (tmps == NULL)
                        {
                        RSAerr(RSA_F_RSA_SIGN,ERR_R_MALLOC_FAILURE);
                        return(0);
                        }
-               p=s;
+               p=tmps;
                i2d_X509_SIG(&sig,&p);
+               s=tmps;
        }
        i=RSA_private_encrypt(i,s,sigret,rsa,RSA_PKCS1_PADDING);
        if (i <= 0)
@@ -131,13 +135,13 @@ int RSA_sign(int type, unsigned char *m, unsigned int m_len,
                *siglen=i;
 
        if(type != NID_md5_sha1) {
-               memset(s,0,(unsigned int)j+1);
-               Free(s);
+               memset(tmps,0,(unsigned int)j+1);
+               OPENSSL_free(tmps);
        }
        return(ret);
        }
 
-int RSA_verify(int dtype, unsigned char *m, unsigned int m_len,
+int RSA_verify(int dtype, const unsigned char *m, unsigned int m_len,
             unsigned char *sigbuf, unsigned int siglen, RSA *rsa)
        {
        int i,ret=0,sigtype;
@@ -151,9 +155,10 @@ int RSA_verify(int dtype, unsigned char *m, unsigned int m_len,
                }
 
        if(rsa->flags & RSA_FLAG_SIGN_VER)
-           return rsa->meth->rsa_verify(dtype, m, m_len, sigbuf, siglen, rsa);
+           return ENGINE_get_RSA(rsa->engine)->rsa_verify(dtype,
+                       m, m_len, sigbuf, siglen, rsa);
 
-       s=(unsigned char *)Malloc((unsigned int)siglen);
+       s=(unsigned char *)OPENSSL_malloc((unsigned int)siglen);
        if (s == NULL)
                {
                RSAerr(RSA_F_RSA_VERIFY,ERR_R_MALLOC_FAILURE);
@@ -193,9 +198,9 @@ int RSA_verify(int dtype, unsigned char *m, unsigned int m_len,
                                (sigtype == NID_md2WithRSAEncryption)))
                                {
                                /* ok, we will let it through */
-       #if !defined(NO_STDIO) && !defined(WIN16)
+#if !defined(OPENSSL_NO_STDIO) && !defined(OPENSSL_SYS_WIN16)
                                fprintf(stderr,"signature has problems, re-make with post SSLeay045\n");
-       #endif
+#endif
                                }
                        else
                                {
@@ -215,7 +220,7 @@ int RSA_verify(int dtype, unsigned char *m, unsigned int m_len,
 err:
        if (sig != NULL) X509_SIG_free(sig);
        memset(s,0,(unsigned int)siglen);
-       Free(s);
+       OPENSSL_free(s);
        return(ret);
        }