Indicate support for digest init ctrl.
[openssl.git] / crypto / rsa / rsa_pmeth.c
index e11ae23b16a24c3feabb3bddd52b54263c659ce0..c6710191e4c75ae6ac7ddb276163bc7ebb38770b 100644 (file)
 #include <openssl/asn1t.h>
 #include <openssl/x509.h>
 #include <openssl/rsa.h>
+#include <openssl/bn.h>
 #include <openssl/evp.h>
 #include "evp_locl.h"
-
-extern int int_rsa_verify(int dtype, const unsigned char *m, unsigned int m_len,
-               unsigned char *rm, unsigned int *prm_len,
-               const unsigned char *sigbuf, unsigned int siglen,
-               RSA *rsa);
+#include "rsa_locl.h"
 
 /* RSA pkey context structure */
 
@@ -109,6 +106,25 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
        return 1;
        }
 
+static int pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
+       {
+       RSA_PKEY_CTX *dctx, *sctx;
+       if (!pkey_rsa_init(dst))
+               return 0;
+               sctx = src->data;
+       dctx = dst->data;
+       dctx->nbits = sctx->nbits;
+       if (sctx->pub_exp)
+               {
+               dctx->pub_exp = BN_dup(sctx->pub_exp);
+               if (!dctx->pub_exp)
+                       return 0;
+               }
+       dctx->pad_mode = sctx->pad_mode;
+       dctx->md = sctx->md;
+       return 1;
+       }
+
 static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
        {
        if (ctx->tbuf)
@@ -132,8 +148,8 @@ static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
                }
        }
 
-static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, int *siglen,
-                                       const unsigned char *tbs, int tbslen)
+static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
+                                       const unsigned char *tbs, size_t tbslen)
        {
        int ret;
        RSA_PKEY_CTX *rctx = ctx->data;
@@ -141,7 +157,7 @@ static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, int *siglen,
 
        if (rctx->md)
                {
-               if (tbslen != EVP_MD_size(rctx->md))
+               if (tbslen != (size_t)EVP_MD_size(rctx->md))
                        {
                        RSAerr(RSA_F_PKEY_RSA_SIGN,
                                        RSA_R_INVALID_DIGEST_LENGTH);
@@ -190,8 +206,8 @@ static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, int *siglen,
 
 
 static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
-                                       unsigned char *rout, int *routlen,
-                                        const unsigned char *sig, int siglen)
+                                       unsigned char *rout, size_t *routlen,
+                                       const unsigned char *sig, size_t siglen)
        {
        int ret;
        RSA_PKEY_CTX *rctx = ctx->data;
@@ -226,7 +242,7 @@ static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
                        }
                else if (rctx->pad_mode == RSA_PKCS1_PADDING)
                        {
-                       unsigned int sltmp;
+                       size_t sltmp;
                        ret = int_rsa_verify(EVP_MD_type(rctx->md),
                                                NULL, 0, rout, &sltmp,
                                        sig, siglen, ctx->pkey->pkey.rsa);
@@ -245,12 +261,12 @@ static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
        }
 
 static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
-                                       const unsigned char *sig, int siglen,
-                                        const unsigned char *tbs, int tbslen)
+                                       const unsigned char *sig, size_t siglen,
+                                       const unsigned char *tbs, size_t tbslen)
        {
        RSA_PKEY_CTX *rctx = ctx->data;
        RSA *rsa = ctx->pkey->pkey.rsa;
-       int rslen;
+       size_t rslen;
        if (rctx->md)
                {
                if (rctx->pad_mode == RSA_PKCS1_PADDING)
@@ -286,7 +302,7 @@ static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
                        return -1;
                rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
                                                rsa, rctx->pad_mode);
-               if (rslen <= 0)
+               if (rslen == 0)
                        return 0;
                }
 
@@ -298,8 +314,9 @@ static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
        }
        
 
-static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
-                                        const unsigned char *in, int inlen)
+static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx,
+                                       unsigned char *out, size_t *outlen,
+                                       const unsigned char *in, size_t inlen)
        {
        int ret;
        RSA_PKEY_CTX *rctx = ctx->data;
@@ -311,8 +328,9 @@ static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
        return 1;
        }
 
-static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, int *outlen,
-                                        const unsigned char *in, int inlen)
+static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx,
+                                       unsigned char *out, size_t *outlen,
+                                       const unsigned char *in, size_t inlen)
        {
        int ret;
        RSA_PKEY_CTX *rctx = ctx->data;
@@ -362,34 +380,44 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
                                return 0;
                        if (p1 == RSA_PKCS1_PSS_PADDING) 
                                {
-                               if (ctx->operation == EVP_PKEY_OP_VERIFYRECOVER)
-                                       return -2;
+                               if (!(ctx->operation &
+                                    (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
+                                       goto bad_pad;
                                if (!rctx->md)
                                        rctx->md = EVP_sha1();
                                }
                        if (p1 == RSA_PKCS1_OAEP_PADDING) 
                                {
                                if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
-                                       return -2;
+                                       goto bad_pad;
                                if (!rctx->md)
                                        rctx->md = EVP_sha1();
                                }
                        rctx->pad_mode = p1;
                        return 1;
                        }
+               bad_pad:
+               RSAerr(RSA_F_PKEY_RSA_CTRL,
+                               RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
                return -2;
 
                case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
                if (p1 < -2)
                        return -2;
                if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING)
+                       {
+                       RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
                        return -2;
+                       }
                rctx->saltlen = p1;
                return 1;
 
                case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
                if (p1 < 256)
+                       {
+                       RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_KEYBITS);
                        return -2;
+                       }
                rctx->nbits = p1;
                return 1;
 
@@ -405,6 +433,21 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
                rctx->md = p2;
                return 1;
 
+               case EVP_PKEY_CTRL_DIGESTINIT:
+               case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
+               case EVP_PKEY_CTRL_PKCS7_DECRYPT:
+               case EVP_PKEY_CTRL_PKCS7_SIGN:
+#ifndef OPENSSL_NO_CMS
+               case EVP_PKEY_CTRL_CMS_ENCRYPT:
+               case EVP_PKEY_CTRL_CMS_DECRYPT:
+               case EVP_PKEY_CTRL_CMS_SIGN:
+#endif
+               return 1;
+               case EVP_PKEY_CTRL_PEER_KEY:
+                       RSAerr(RSA_F_PKEY_RSA_CTRL,
+                       RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+                       return -2;      
+
                default:
                return -2;
 
@@ -414,11 +457,14 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
 static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
                        const char *type, const char *value)
        {
+       if (!value)
+               {
+               RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
+               return 0;
+               }
        if (!strcmp(type, "rsa_padding_mode"))
                {
                int pm;
-               if (!value)
-                       return 0;
                if (!strcmp(value, "pkcs1"))
                        pm = RSA_PKCS1_PADDING;
                else if (!strcmp(value, "sslv23"))
@@ -432,7 +478,11 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
                else if (!strcmp(value, "pss"))
                        pm = RSA_PKCS1_PSS_PADDING;
                else
+                       {
+                       RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
+                                               RSA_R_UNKNOWN_PADDING_TYPE);
                        return -2;
+                       }
                return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
                }
 
@@ -498,8 +548,9 @@ static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
 const EVP_PKEY_METHOD rsa_pkey_meth = 
        {
        EVP_PKEY_RSA,
-       0,
+       EVP_PKEY_FLAG_AUTOARGLEN,
        pkey_rsa_init,
+       pkey_rsa_copy,
        pkey_rsa_cleanup,
 
        0,0,