Fix custom EVP_PKEY_METHOD implementations where no engine is present

[openssl.git] / apps / rsa.c

diff --git a/apps/rsa.c b/apps/rsa.c
index 1a75681c708bea946312f3fe052254de7413a18a..3e9d320ea38df3026f2269a3fa5476803a8347b6 100644 (file)
--- a/apps/rsa.c
+++ b/apps/rsa.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -25,7 +25,7 @@
 #include <openssl/encoder.h>
 
 /*
- * TODO: This include is to get OSSL_KEYMGMT_SELECT_*, which feels a bit
+ * This include is to get OSSL_KEYMGMT_SELECT_*, which feels a bit
  * much just for those macros...  they might serve better as EVP macros.
  */
 #include <openssl/core_dispatch.h>
@@ -37,7 +37,7 @@
 #endif
 
 typedef enum OPTION_choice {
-    OPT_ERR = -1, OPT_EOF = 0, OPT_HELP,
+    OPT_COMMON,
     OPT_INFORM, OPT_OUTFORM, OPT_ENGINE, OPT_IN, OPT_OUT,
     OPT_PUBIN, OPT_PUBOUT, OPT_PASSOUT, OPT_PASSIN,
     OPT_RSAPUBKEY_IN, OPT_RSAPUBKEY_OUT,
@@ -92,11 +92,11 @@ int rsa_main(int argc, char **argv)
     BIO *out = NULL;
     EVP_PKEY *pkey = NULL;
     EVP_PKEY_CTX *pctx;
-    const EVP_CIPHER *enc = NULL;
+    EVP_CIPHER *enc = NULL;
     char *infile = NULL, *outfile = NULL, *ciphername = NULL, *prog;
     char *passin = NULL, *passout = NULL, *passinarg = NULL, *passoutarg = NULL;
     int private = 0;
-    int informat = FORMAT_PEM, outformat = FORMAT_PEM, text = 0, check = 0;
+    int informat = FORMAT_UNDEF, outformat = FORMAT_PEM, text = 0, check = 0;
     int noout = 0, modulus = 0, pubin = 0, pubout = 0, ret = 1;
     int pvk_encr = DEFAULT_PVK_ENCR_STRENGTH;
     OPTION_CHOICE o;
@@ -204,7 +204,7 @@ int rsa_main(int argc, char **argv)
     }
 
     if (pubin) {
-        int tmpformat = -1;
+        int tmpformat = FORMAT_UNDEF;
 
         if (pubin == 2) {
             if (informat == FORMAT_PEM)
@@ -259,7 +259,7 @@ int rsa_main(int argc, char **argv)
 
         pctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL);
         if (pctx == NULL) {
-            BIO_printf(out, "RSA unable to create PKEY context\n");
+            BIO_printf(bio_err, "RSA unable to create PKEY context\n");
             ERR_print_errors(bio_err);
             goto end;
         }
@@ -269,15 +269,8 @@ int rsa_main(int argc, char **argv)
         if (r == 1) {
             BIO_printf(out, "RSA key ok\n");
         } else if (r == 0) {
-            unsigned long err;
-
-            while ((err = ERR_peek_error()) != 0 &&
-                   ERR_GET_LIB(err) == ERR_LIB_RSA &&
-                   ERR_GET_REASON(err) != ERR_R_MALLOC_FAILURE) {
-                BIO_printf(out, "RSA key error: %s\n",
-                           ERR_reason_error_string(err));
-                ERR_get_error(); /* remove err from error stack */
-            }
+            BIO_printf(bio_err, "RSA key not ok\n");
+            ERR_print_errors(bio_err);
         } else if (r == -1) {
             ERR_print_errors(bio_err);
             goto end;
@@ -329,19 +322,33 @@ int rsa_main(int argc, char **argv)
             if (traditional)
                 output_structure = "pkcs1"; /* "type-specific" would work too */
             else
-                output_structure = "pkcs8";
+                output_structure = "PrivateKeyInfo";
         }
     }
 
     /* Now, perform the encoding */
-    ectx = OSSL_ENCODER_CTX_new_by_EVP_PKEY(pkey, selection,
-                                            output_type, output_structure,
-                                            NULL);
+    ectx = OSSL_ENCODER_CTX_new_for_pkey(pkey, selection,
+                                         output_type, output_structure,
+                                         NULL);
     if (OSSL_ENCODER_CTX_get_num_encoders(ectx) == 0) {
         BIO_printf(bio_err, "%s format not supported\n", output_type);
         goto end;
     }
 
+    /* Passphrase setup */
+    if (enc != NULL)
+        OSSL_ENCODER_CTX_set_cipher(ectx, EVP_CIPHER_get0_name(enc), NULL);
+
+    /* Default passphrase prompter */
+    if (enc != NULL || outformat == FORMAT_PVK) {
+        OSSL_ENCODER_CTX_set_passphrase_ui(ectx, get_ui_method(), NULL);
+        if (passout != NULL)
+            /* When passout given, override the passphrase prompter */
+            OSSL_ENCODER_CTX_set_passphrase(ectx,
+                                            (const unsigned char *)passout,
+                                            strlen(passout));
+    }
+
     /* PVK is a bit special... */
     if (outformat == FORMAT_PVK) {
         OSSL_PARAM params[2] = { OSSL_PARAM_END, OSSL_PARAM_END };
@@ -364,6 +371,7 @@ int rsa_main(int argc, char **argv)
     release_engine(e);
     BIO_free_all(out);
     EVP_PKEY_free(pkey);
+    EVP_CIPHER_free(enc);
     OPENSSL_free(passin);
     OPENSSL_free(passout);
     return ret;