Workaround for some CMS signature formats.
[openssl.git] / apps / pkeyutl.c
index a3e55f5a430fee256d178c763b8b113aaba13920..7eb3f5c544ddd0a3c0b59ee8c7be54559376c70c 100644 (file)
@@ -1,4 +1,4 @@
-/* Written by Dr Stephen N Henson (shenson@bigfoot.com) for the OpenSSL
+/* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
 /* ====================================================================
@@ -79,6 +79,10 @@ static EVP_PKEY_CTX *init_ctx(int *pkeysize,
 static int setup_peer(BIO *err, EVP_PKEY_CTX *ctx, int peerform,
                                                        const char *file);
 
+static int do_keyop(EVP_PKEY_CTX *ctx, int pkey_op,
+               unsigned char *out, size_t *poutlen,
+               unsigned char *in, size_t inlen);
+
 int MAIN(int argc, char **);
 
 int MAIN(int argc, char **argv)
@@ -95,7 +99,8 @@ int MAIN(int argc, char **argv)
        int keysize = -1;
 
        unsigned char *buf_in = NULL, *buf_out = NULL, *sig = NULL;
-       int buf_inlen, buf_outlen, siglen = -1;
+       size_t buf_outlen;
+       int buf_inlen = 0, siglen = -1;
 
        int ret = 1, rv = -1;
 
@@ -114,17 +119,17 @@ int MAIN(int argc, char **argv)
                if (!strcmp(*argv,"-in"))
                        {
                        if (--argc < 1) badarg = 1;
-                        infile= *(++argv);
+                        else infile= *(++argv);
                        }
                else if (!strcmp(*argv,"-out"))
                        {
                        if (--argc < 1) badarg = 1;
-                       outfile= *(++argv);
+                       else outfile= *(++argv);
                        }
                else if (!strcmp(*argv,"-sigfile"))
                        {
                        if (--argc < 1) badarg = 1;
-                       sigfile= *(++argv);
+                       else sigfile= *(++argv);
                        }
                else if(!strcmp(*argv, "-inkey"))
                        {
@@ -154,17 +159,17 @@ int MAIN(int argc, char **argv)
                else if (!strcmp(*argv,"-passin"))
                        {
                        if (--argc < 1) badarg = 1;
-                       passargin= *(++argv);
+                       else passargin= *(++argv);
                        }
                else if (strcmp(*argv,"-peerform") == 0)
                        {
                        if (--argc < 1) badarg = 1;
-                       peerform=str2fmt(*(++argv));
+                       else peerform=str2fmt(*(++argv));
                        }
                else if (strcmp(*argv,"-keyform") == 0)
                        {
                        if (--argc < 1) badarg = 1;
-                       keyform=str2fmt(*(++argv));
+                       else keyform=str2fmt(*(++argv));
                        }
 #ifndef OPENSSL_NO_ENGINE
                else if(!strcmp(*argv, "-engine"))
@@ -299,8 +304,6 @@ int MAIN(int argc, char **argv)
                        }
                }
        
-       buf_out = OPENSSL_malloc(keysize);
-
        if (in)
                {
                /* Read the input data */
@@ -312,53 +315,43 @@ int MAIN(int argc, char **argv)
                        }
                if(rev)
                        {
-                       int i;
+                       size_t i;
                        unsigned char ctmp;
-                       for(i = 0; i < buf_inlen/2; i++)
+                       size_t l = (size_t)buf_inlen;
+                       for(i = 0; i < l/2; i++)
                                {
                                ctmp = buf_in[i];
-                               buf_in[i] = buf_in[buf_inlen - 1 - i];
-                               buf_in[buf_inlen - 1 - i] = ctmp;
+                               buf_in[i] = buf_in[l - 1 - i];
+                               buf_in[l - 1 - i] = ctmp;
                                }
                        }
                }
 
-       switch(pkey_op)
+       if(pkey_op == EVP_PKEY_OP_VERIFY)
                {
-               case EVP_PKEY_OP_VERIFYRECOVER:
-               rv  = EVP_PKEY_verify_recover(ctx, buf_out, &buf_outlen,
-                                                       buf_in, buf_inlen);
-               break;
-
-               case EVP_PKEY_OP_SIGN:
-               rv  = EVP_PKEY_sign(ctx, buf_out, &buf_outlen,
-                                                       buf_in, buf_inlen);
-               break;
-
-               case EVP_PKEY_OP_ENCRYPT:
-               rv  = EVP_PKEY_encrypt(ctx, buf_out, &buf_outlen,
-                                                       buf_in, buf_inlen);
-               break;
-
-               case EVP_PKEY_OP_DECRYPT:
-               rv  = EVP_PKEY_decrypt(ctx, buf_out, &buf_outlen,
-                                                       buf_in, buf_inlen);
-               break; 
-
-               case EVP_PKEY_OP_VERIFY:
-               rv  = EVP_PKEY_verify(ctx, sig, siglen, buf_in, buf_inlen);
+               rv  = EVP_PKEY_verify(ctx, sig, (size_t)siglen,
+                                     buf_in, (size_t)buf_inlen);
                if (rv == 0)
                        BIO_puts(out, "Signature Verification Failure\n");
                else if (rv == 1)
                        BIO_puts(out, "Signature Verified Successfully\n");
                if (rv >= 0)
                        goto end;
-               break; 
-
-               case EVP_PKEY_OP_DERIVE:
-               rv  = EVP_PKEY_derive(ctx, buf_out, &buf_outlen);
-               break;
-
+               }
+       else
+               {       
+               rv = do_keyop(ctx, pkey_op, NULL, (size_t *)&buf_outlen,
+                             buf_in, (size_t)buf_inlen);
+               if (rv > 0)
+                       {
+                       buf_out = OPENSSL_malloc(buf_outlen);
+                       if (!buf_out)
+                               rv = -1;
+                       else
+                               rv = do_keyop(ctx, pkey_op,
+                                               buf_out, (size_t *)&buf_outlen,
+                                               buf_in, (size_t)buf_inlen);
+                       }
                }
 
        if(rv <= 0)
@@ -397,20 +390,23 @@ static void usage()
        BIO_printf(bio_err, "Usage: pkeyutl [options]\n");
        BIO_printf(bio_err, "-in file        input file\n");
        BIO_printf(bio_err, "-out file       output file\n");
+       BIO_printf(bio_err, "-sigfile file signature file (verify operation only)\n");
        BIO_printf(bio_err, "-inkey file     input key\n");
        BIO_printf(bio_err, "-keyform arg    private key format - default PEM\n");
-       BIO_printf(bio_err, "-pubin          input is an RSA public\n");
-       BIO_printf(bio_err, "-certin         input is a certificate carrying an RSA public key\n");
-       BIO_printf(bio_err, "-ctrl X:Y       control parameters\n");
+       BIO_printf(bio_err, "-pubin          input is a public key\n");
+       BIO_printf(bio_err, "-certin         input is a certificate carrying a public key\n");
+       BIO_printf(bio_err, "-pkeyopt X:Y    public key options\n");
        BIO_printf(bio_err, "-sign           sign with private key\n");
        BIO_printf(bio_err, "-verify         verify with public key\n");
+       BIO_printf(bio_err, "-verifyrecover  verify with public key, recover original data\n");
        BIO_printf(bio_err, "-encrypt        encrypt with public key\n");
        BIO_printf(bio_err, "-decrypt        decrypt with private key\n");
+       BIO_printf(bio_err, "-derive         derive shared secret\n");
        BIO_printf(bio_err, "-hexdump        hex dump output\n");
 #ifndef OPENSSL_NO_ENGINE
        BIO_printf(bio_err, "-engine e       use engine e, possibly a hardware device.\n");
-       BIO_printf(bio_err, "-passin arg     pass phrase source\n");
 #endif
+       BIO_printf(bio_err, "-passin arg     pass phrase source\n");
 
 }
 
@@ -541,4 +537,34 @@ static int setup_peer(BIO *err, EVP_PKEY_CTX *ctx, int peerform,
                ERR_print_errors(err);
        return ret;
        }
-                       
+
+static int do_keyop(EVP_PKEY_CTX *ctx, int pkey_op,
+               unsigned char *out, size_t *poutlen,
+               unsigned char *in, size_t inlen)
+       {
+       int rv = 0;
+       switch(pkey_op)
+               {
+               case EVP_PKEY_OP_VERIFYRECOVER:
+               rv  = EVP_PKEY_verify_recover(ctx, out, poutlen, in, inlen);
+               break;
+
+               case EVP_PKEY_OP_SIGN:
+               rv  = EVP_PKEY_sign(ctx, out, poutlen, in, inlen);
+               break;
+
+               case EVP_PKEY_OP_ENCRYPT:
+               rv  = EVP_PKEY_encrypt(ctx, out, poutlen, in, inlen);
+               break;
+
+               case EVP_PKEY_OP_DECRYPT:
+               rv  = EVP_PKEY_decrypt(ctx, out, poutlen, in, inlen);
+               break; 
+
+               case EVP_PKEY_OP_DERIVE:
+               rv  = EVP_PKEY_derive(ctx, out, poutlen);
+               break;
+
+               }
+       return rv;
+       }