Fix pkeyutl/rsautl empty encrypt-input/decrypt-output handling

[openssl.git] / apps / rsautl.c

diff --git a/apps/rsautl.c b/apps/rsautl.c
index 0ef61050cb818c839f94d19057bb5289127fd636..b576ca0b76df2484a257cf7fdf4a22fc0a400fea 100644 (file)
--- a/apps/rsautl.c
+++ b/apps/rsautl.c
@@ -87,7 +87,7 @@ OPTIONS rsautl_options[] = {
     {"in", OPT_IN, '<', "Input file"},
     {"out", OPT_OUT, '>', "Output file"},
     {"inkey", OPT_INKEY, '<', "Input key"},
-    {"keyform", OPT_KEYFORM, 'F', "Private key format - default PEM"},
+    {"keyform", OPT_KEYFORM, 'E', "Private key format - default PEM"},
     {"pubin", OPT_PUBIN, '-', "Input is an RSA public"},
     {"certin", OPT_CERTIN, '-', "Input is a cert carrying an RSA public key"},
     {"ssl", OPT_SSL, '-', "Use SSL v2 padding"},
@@ -137,7 +137,7 @@ int rsautl_main(int argc, char **argv)
             ret = 0;
             goto end;
         case OPT_KEYFORM:
-            if (!opt_format(opt_arg(), OPT_FMT_PEMDER, &keyformat))
+            if (!opt_format(opt_arg(), OPT_FMT_PDE, &keyformat))
                 goto opthelp;
             break;
         case OPT_IN:
@@ -248,10 +248,10 @@ int rsautl_main(int argc, char **argv)
         goto end;
     }
 
-    in = bio_open_default(infile, "rb");
+    in = bio_open_default(infile, 'r', FORMAT_BINARY);
     if (in == NULL)
         goto end;
-    out = bio_open_default(outfile, "wb");
+    out = bio_open_default(outfile, 'w', FORMAT_BINARY);
     if (out == NULL)
         goto end;
 
@@ -262,7 +262,7 @@ int rsautl_main(int argc, char **argv)
 
     /* Read the input data */
     rsa_inlen = BIO_read(in, rsa_in, keysize * 2);
-    if (rsa_inlen <= 0) {
+    if (rsa_inlen < 0) {
         BIO_printf(bio_err, "Error reading input Data\n");
         goto end;
     }
@@ -294,10 +294,9 @@ int rsautl_main(int argc, char **argv)
         rsa_outlen =
             RSA_private_decrypt(rsa_inlen, rsa_in, rsa_out, rsa, pad);
         break;
-
     }
 
-    if (rsa_outlen <= 0) {
+    if (rsa_outlen < 0) {
         BIO_printf(bio_err, "RSA operation error\n");
         ERR_print_errors(bio_err);
         goto end;