Use correct pointer types for various functions.

[openssl.git] / crypto / rsa / rsa_pmeth.c

diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index 49323bfceea44dcf52d4152bace325d5e0d5f937..5887c4f233bd44f488f98e09d84233a8b5864c7b 100644 (file)
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -1,3 +1,4 @@
+/* crypto/rsa/rsa_pmeth.c */
 /* Written by Dr Stephen N Henson (shenson@bigfoot.com) for the OpenSSL
  * project 2006.
  */
@@ -63,9 +64,9 @@
 #include <openssl/evp.h>
 #include "evp_locl.h"
 
-extern int int_rsa_verify(int dtype, const unsigned char *m, unsigned int m_len,
-               unsigned char *rm, unsigned int *prm_len,
-               const unsigned char *sigbuf, unsigned int siglen,
+extern int int_rsa_verify(int dtype, const unsigned char *m, size_t m_len,
+               unsigned char *rm, size_t *prm_len,
+               const unsigned char *sigbuf, size_t siglen,
                RSA *rsa);
 
 /* RSA pkey context structure */
@@ -108,6 +109,25 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
        return 1;
        }
 
+static int pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
+       {
+       RSA_PKEY_CTX *dctx, *sctx;
+       if (!pkey_rsa_init(dst))
+               return 0;
+               sctx = src->data;
+       dctx = dst->data;
+       dctx->nbits = sctx->nbits;
+       if (sctx->pub_exp)
+               {
+               dctx->pub_exp = BN_dup(sctx->pub_exp);
+               if (!dctx->pub_exp)
+                       return 0;
+               }
+       dctx->pad_mode = sctx->pad_mode;
+       dctx->md = sctx->md;
+       return 1;
+       }
+
 static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
        {
        if (ctx->tbuf)
@@ -127,12 +147,12 @@ static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
                        BN_free(rctx->pub_exp);
                if (rctx->tbuf)
                        OPENSSL_free(rctx->tbuf);
+               OPENSSL_free(rctx);
                }
-       OPENSSL_free(rctx);
        }
 
-static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, int *siglen,
-                                       const unsigned char *tbs, int tbslen)
+static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
+                                       const unsigned char *tbs, size_t tbslen)
        {
        int ret;
        RSA_PKEY_CTX *rctx = ctx->data;
@@ -140,7 +160,7 @@ static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, int *siglen,
 
        if (rctx->md)
                {
-               if (tbslen != EVP_MD_size(rctx->md))
+               if (tbslen != (size_t)EVP_MD_size(rctx->md))
                        {
                        RSAerr(RSA_F_PKEY_RSA_SIGN,
                                        RSA_R_INVALID_DIGEST_LENGTH);
@@ -189,8 +209,8 @@ static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, int *siglen,
 
 
 static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
-                                       unsigned char *rout, int *routlen,
-                                        const unsigned char *sig, int siglen)
+                                       unsigned char *rout, size_t *routlen,
+                                       const unsigned char *sig, size_t siglen)
        {
        int ret;
        RSA_PKEY_CTX *rctx = ctx->data;
@@ -225,7 +245,7 @@ static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
                        }
                else if (rctx->pad_mode == RSA_PKCS1_PADDING)
                        {
-                       unsigned int sltmp;
+                       size_t sltmp;
                        ret = int_rsa_verify(EVP_MD_type(rctx->md),
                                                NULL, 0, rout, &sltmp,
                                        sig, siglen, ctx->pkey->pkey.rsa);
@@ -244,12 +264,12 @@ static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
        }
 
 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
-                                       const unsigned char *sig, int siglen,
-                                        const unsigned char *tbs, int tbslen)
+                                       const unsigned char *sig, size_t siglen,
+                                       const unsigned char *tbs, size_t tbslen)
        {
        RSA_PKEY_CTX *rctx = ctx->data;
        RSA *rsa = ctx->pkey->pkey.rsa;
-       int rslen;
+       size_t rslen;
        if (rctx->md)
                {
                if (rctx->pad_mode == RSA_PKCS1_PADDING)
@@ -285,7 +305,7 @@ static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
                        return -1;
                rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
                                                rsa, rctx->pad_mode);
-               if (rslen <= 0)
+               if (rslen == 0)
                        return 0;
                }
 
@@ -297,8 +317,9 @@ static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
        }
        
 
-static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
-                                        const unsigned char *in, int inlen)
+static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
+                                       unsigned char *out, size_t *outlen,
+                                       const unsigned char *in, size_t inlen)
        {
        int ret;
        RSA_PKEY_CTX *rctx = ctx->data;
@@ -310,8 +331,9 @@ static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
        return 1;
        }
 
-static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
-                                        const unsigned char *in, int inlen)
+static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
+                                       unsigned char *out, size_t *outlen,
+                                       const unsigned char *in, size_t inlen)
        {
        int ret;
        RSA_PKEY_CTX *rctx = ctx->data;
@@ -330,7 +352,7 @@ static int check_padding_md(const EVP_MD *md, int padding)
 
        if (padding == RSA_NO_PADDING)
                {
-               RSAerr(RSA_F_CHECK_PADDING_NID, RSA_R_INVALID_PADDING_MODE);
+               RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
                return 0;
                }
 
@@ -338,7 +360,7 @@ static int check_padding_md(const EVP_MD *md, int padding)
                {
                if (RSA_X931_hash_id(EVP_MD_type(md)) == -1)
                        {
-                       RSAerr(RSA_F_CHECK_PADDING_NID,
+                       RSAerr(RSA_F_CHECK_PADDING_MD,
                                                RSA_R_INVALID_X931_DIGEST);
                        return 0;
                        }
@@ -361,34 +383,44 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
                                return 0;
                        if (p1 == RSA_PKCS1_PSS_PADDING) 
                                {
-                               if (ctx->operation == EVP_PKEY_OP_VERIFYRECOVER)
-                                       return -2;
+                               if (!(ctx->operation &
+                                    (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
+                                       goto bad_pad;
                                if (!rctx->md)
                                        rctx->md = EVP_sha1();
                                }
                        if (p1 == RSA_PKCS1_OAEP_PADDING) 
                                {
                                if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
-                                       return -2;
+                                       goto bad_pad;
                                if (!rctx->md)
                                        rctx->md = EVP_sha1();
                                }
                        rctx->pad_mode = p1;
                        return 1;
                        }
+               bad_pad:
+               RSAerr(RSA_F_PKEY_RSA_CTRL,
+                               RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
                return -2;
 
                case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
                if (p1 < -2)
                        return -2;
                if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
+                       {
+                       RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
                        return -2;
+                       }
                rctx->saltlen = p1;
                return 1;
 
                case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
                if (p1 < 256)
+                       {
+                       RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_KEYBITS);
                        return -2;
+                       }
                rctx->nbits = p1;
                return 1;
 
@@ -404,6 +436,11 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
                rctx->md = p2;
                return 1;
 
+               case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
+               case EVP_PKEY_CTRL_PKCS7_DECRYPT:
+               case EVP_PKEY_CTRL_PKCS7_SIGN:
+               return 1;
+
                default:
                return -2;
 
@@ -413,11 +450,14 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
                        const char *type, const char *value)
        {
+       if (!value)
+               {
+               RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
+               return 0;
+               }
        if (!strcmp(type, "rsa_padding_mode"))
                {
                int pm;
-               if (!value)
-                       return 0;
                if (!strcmp(value, "pkcs1"))
                        pm = RSA_PKCS1_PADDING;
                else if (!strcmp(value, "sslv23"))
@@ -431,7 +471,11 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
                else if (!strcmp(value, "pss"))
                        pm = RSA_PKCS1_PSS_PADDING;
                else
+                       {
+                       RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
+                                               RSA_R_UNKNOWN_PADDING_TYPE);
                        return -2;
+                       }
                return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
                }
 
@@ -497,8 +541,9 @@ static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
 const EVP_PKEY_METHOD rsa_pkey_meth = 
        {
        EVP_PKEY_RSA,
-       0,
+       EVP_PKEY_FLAG_AUTOARGLEN,
        pkey_rsa_init,
+       pkey_rsa_copy,
        pkey_rsa_cleanup,
 
        0,0,
@@ -524,6 +569,8 @@ const EVP_PKEY_METHOD rsa_pkey_meth =
        0,
        pkey_rsa_decrypt,
 
+       0,0,
+
        pkey_rsa_ctrl,
        pkey_rsa_ctrl_str