fix function codes for error
[openssl.git] / crypto / rsa / rsa_sign.c
index 15c6d1a84c3366d66a298d223c5e0bde84401a9a..0be4ec7fb01f08b04e2e6558fc5a9df80d4c8a7a 100644 (file)
 #include <openssl/rsa.h>
 #include <openssl/objects.h>
 #include <openssl/x509.h>
+#include "rsa_locl.h"
 
 /* Size of an SSL signature: MD5+SHA1 */
 #define SSL_SIG_LENGTH 36
 
-int RSA_sign(int type, unsigned char *m, unsigned int m_len,
+int RSA_sign(int type, const unsigned char *m, unsigned int m_len,
             unsigned char *sigret, unsigned int *siglen, RSA *rsa)
        {
        X509_SIG sig;
        ASN1_TYPE parameter;
        int i,j,ret=1;
-       unsigned char *p,*s = NULL;
+       unsigned char *p, *tmps = NULL;
+       const unsigned char *s = NULL;
        X509_ALGOR algor;
        ASN1_OCTET_STRING digest;
-       if(rsa->flags & RSA_FLAG_SIGN_VER)
-             return rsa->meth->rsa_sign(type, m, m_len, sigret, siglen, rsa);
+       if((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_sign)
+               {
+               return rsa->meth->rsa_sign(type, m, m_len,
+                       sigret, siglen, rsa);
+               }
        /* Special case: SSL signature, just check the length */
        if(type == NID_md5_sha1) {
                if(m_len != SSL_SIG_LENGTH) {
@@ -103,26 +108,27 @@ int RSA_sign(int type, unsigned char *m, unsigned int m_len,
                sig.algor->parameter= &parameter;
 
                sig.digest= &digest;
-               sig.digest->data=m;
+               sig.digest->data=(unsigned char *)m; /* TMP UGLY CAST */
                sig.digest->length=m_len;
 
                i=i2d_X509_SIG(&sig,NULL);
        }
        j=RSA_size(rsa);
-       if ((i-RSA_PKCS1_PADDING) > j)
+       if (i > (j-RSA_PKCS1_PADDING_SIZE))
                {
                RSAerr(RSA_F_RSA_SIGN,RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
                return(0);
                }
        if(type != NID_md5_sha1) {
-               s=(unsigned char *)Malloc((unsigned int)j+1);
-               if (s == NULL)
+               tmps=(unsigned char *)OPENSSL_malloc((unsigned int)j+1);
+               if (tmps == NULL)
                        {
                        RSAerr(RSA_F_RSA_SIGN,ERR_R_MALLOC_FAILURE);
                        return(0);
                        }
-               p=s;
+               p=tmps;
                i2d_X509_SIG(&sig,&p);
+               s=tmps;
        }
        i=RSA_private_encrypt(i,s,sigret,rsa,RSA_PKCS1_PADDING);
        if (i <= 0)
@@ -131,37 +137,47 @@ int RSA_sign(int type, unsigned char *m, unsigned int m_len,
                *siglen=i;
 
        if(type != NID_md5_sha1) {
-               memset(s,0,(unsigned int)j+1);
-               Free(s);
+               OPENSSL_cleanse(tmps,(unsigned int)j+1);
+               OPENSSL_free(tmps);
        }
        return(ret);
        }
 
-int RSA_verify(int dtype, unsigned char *m, unsigned int m_len,
-            unsigned char *sigbuf, unsigned int siglen, RSA *rsa)
+int int_rsa_verify(int dtype, const unsigned char *m,
+                         unsigned int m_len,
+                         unsigned char *rm, size_t *prm_len,
+                         const unsigned char *sigbuf, size_t siglen,
+                         RSA *rsa)
        {
        int i,ret=0,sigtype;
-       unsigned char *p,*s;
+       unsigned char *s;
        X509_SIG *sig=NULL;
 
        if (siglen != (unsigned int)RSA_size(rsa))
                {
-               RSAerr(RSA_F_RSA_VERIFY,RSA_R_WRONG_SIGNATURE_LENGTH);
+               RSAerr(RSA_F_INT_RSA_VERIFY,RSA_R_WRONG_SIGNATURE_LENGTH);
                return(0);
                }
 
-       if(rsa->flags & RSA_FLAG_SIGN_VER)
-           return rsa->meth->rsa_verify(dtype, m, m_len, sigbuf, siglen, rsa);
+       if((dtype == NID_md5_sha1) && rm)
+               {
+               i = RSA_public_decrypt((int)siglen,
+                                       sigbuf,rm,rsa,RSA_PKCS1_PADDING);
+               if (i <= 0)
+                       return 0;
+               *prm_len = i;
+               return 1;
+               }
 
-       s=(unsigned char *)Malloc((unsigned int)siglen);
+       s=(unsigned char *)OPENSSL_malloc((unsigned int)siglen);
        if (s == NULL)
                {
-               RSAerr(RSA_F_RSA_VERIFY,ERR_R_MALLOC_FAILURE);
+               RSAerr(RSA_F_INT_RSA_VERIFY,ERR_R_MALLOC_FAILURE);
                goto err;
                }
        if((dtype == NID_md5_sha1) && (m_len != SSL_SIG_LENGTH) ) {
-                       RSAerr(RSA_F_RSA_VERIFY,RSA_R_INVALID_MESSAGE_LENGTH);
-                       return(0);
+                       RSAerr(RSA_F_INT_RSA_VERIFY,RSA_R_INVALID_MESSAGE_LENGTH);
+                       goto err;
        }
        i=RSA_public_decrypt((int)siglen,sigbuf,s,rsa,RSA_PKCS1_PADDING);
 
@@ -170,18 +186,35 @@ int RSA_verify(int dtype, unsigned char *m, unsigned int m_len,
        /* Special case: SSL signature */
        if(dtype == NID_md5_sha1) {
                if((i != SSL_SIG_LENGTH) || memcmp(s, m, SSL_SIG_LENGTH))
-                               RSAerr(RSA_F_RSA_VERIFY,RSA_R_BAD_SIGNATURE);
+                               RSAerr(RSA_F_INT_RSA_VERIFY,RSA_R_BAD_SIGNATURE);
                else ret = 1;
        } else {
-               p=s;
+               const unsigned char *p=s;
                sig=d2i_X509_SIG(NULL,&p,(long)i);
 
                if (sig == NULL) goto err;
+
+               /* Excess data can be used to create forgeries */
+               if(p != s+i)
+                       {
+                       RSAerr(RSA_F_INT_RSA_VERIFY,RSA_R_BAD_SIGNATURE);
+                       goto err;
+                       }
+
+               /* Parameters to the signature algorithm can also be used to
+                  create forgeries */
+               if(sig->algor->parameter
+                  && ASN1_TYPE_get(sig->algor->parameter) != V_ASN1_NULL)
+                       {
+                       RSAerr(RSA_F_INT_RSA_VERIFY,RSA_R_BAD_SIGNATURE);
+                       goto err;
+                       }
+
                sigtype=OBJ_obj2nid(sig->algor->algorithm);
 
 
        #ifdef RSA_DEBUG
-               /* put a backward compatability flag in EAY */
+               /* put a backward compatibility flag in EAY */
                fprintf(stderr,"in(%s) expect(%s)\n",OBJ_nid2ln(sigtype),
                        OBJ_nid2ln(dtype));
        #endif
@@ -193,29 +226,60 @@ int RSA_verify(int dtype, unsigned char *m, unsigned int m_len,
                                (sigtype == NID_md2WithRSAEncryption)))
                                {
                                /* ok, we will let it through */
-       #if !defined(NO_STDIO) && !defined(WIN16)
+#if !defined(OPENSSL_NO_STDIO) && !defined(OPENSSL_SYS_WIN16)
                                fprintf(stderr,"signature has problems, re-make with post SSLeay045\n");
-       #endif
+#endif
                                }
                        else
                                {
-                               RSAerr(RSA_F_RSA_VERIFY,
+                               RSAerr(RSA_F_INT_RSA_VERIFY,
                                                RSA_R_ALGORITHM_MISMATCH);
                                goto err;
                                }
                        }
-               if (    ((unsigned int)sig->digest->length != m_len) ||
+               if (rm)
+                       {
+                       const EVP_MD *md;
+                       md = EVP_get_digestbynid(dtype);
+                       if (md && (EVP_MD_size(md) != sig->digest->length))
+                               RSAerr(RSA_F_INT_RSA_VERIFY,
+                                               RSA_R_INVALID_DIGEST_LENGTH);
+                       else
+                               {
+                               memcpy(rm, sig->digest->data,
+                                                       sig->digest->length);
+                               *prm_len = sig->digest->length;
+                               ret = 1;
+                               }
+                       }
+               else if (((unsigned int)sig->digest->length != m_len) ||
                        (memcmp(m,sig->digest->data,m_len) != 0))
                        {
-                       RSAerr(RSA_F_RSA_VERIFY,RSA_R_BAD_SIGNATURE);
+                       RSAerr(RSA_F_INT_RSA_VERIFY,RSA_R_BAD_SIGNATURE);
                        }
                else
                        ret=1;
        }
 err:
        if (sig != NULL) X509_SIG_free(sig);
-       memset(s,0,(unsigned int)siglen);
-       Free(s);
+       if (s != NULL)
+               {
+               OPENSSL_cleanse(s,(unsigned int)siglen);
+               OPENSSL_free(s);
+               }
        return(ret);
        }
 
+int RSA_verify(int dtype, const unsigned char *m, unsigned int m_len,
+               const unsigned char *sigbuf, unsigned int siglen,
+               RSA *rsa)
+       {
+
+       if((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_verify)
+               {
+               return rsa->meth->rsa_verify(dtype, m, m_len,
+                       sigbuf, siglen, rsa);
+               }
+
+       return int_rsa_verify(dtype, m, m_len, NULL, NULL, sigbuf, siglen, rsa);
+       }