Fix custom EVP_PKEY_METHOD implementations where no engine is present

[openssl.git] / apps / rsa.c

diff --git a/apps/rsa.c b/apps/rsa.c
index da1342b4c0b134ba05f873b12b386830db52d61b..3e9d320ea38df3026f2269a3fa5476803a8347b6 100644 (file)
--- a/apps/rsa.c
+++ b/apps/rsa.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -25,13 +25,19 @@
 #include <openssl/encoder.h>
 
 /*
- * TODO: This include is to get OSSL_KEYMGMT_SELECT_*, which feels a bit
+ * This include is to get OSSL_KEYMGMT_SELECT_*, which feels a bit
  * much just for those macros...  they might serve better as EVP macros.
  */
 #include <openssl/core_dispatch.h>
 
+#ifndef OPENSSL_NO_RC4
+# define DEFAULT_PVK_ENCR_STRENGTH      2
+#else
+# define DEFAULT_PVK_ENCR_STRENGTH      0
+#endif
+
 typedef enum OPTION_choice {
-    OPT_ERR = -1, OPT_EOF = 0, OPT_HELP,
+    OPT_COMMON,
     OPT_INFORM, OPT_OUTFORM, OPT_ENGINE, OPT_IN, OPT_OUT,
     OPT_PUBIN, OPT_PUBOUT, OPT_PASSOUT, OPT_PASSIN,
     OPT_RSAPUBKEY_IN, OPT_RSAPUBKEY_OUT,
@@ -69,10 +75,12 @@ const OPTIONS rsa_options[] = {
     {"traditional", OPT_TRADITIONAL, '-',
      "Use traditional format for private keys"},
 
+#ifndef OPENSSL_NO_RC4
     OPT_SECTION("PVK"),
     {"pvk-strong", OPT_PVK_STRONG, '-', "Enable 'Strong' PVK encoding level (default)"},
     {"pvk-weak", OPT_PVK_WEAK, '-', "Enable 'Weak' PVK encoding level"},
     {"pvk-none", OPT_PVK_NONE, '-', "Don't enforce PVK encoding"},
+#endif
 
     OPT_PROV_OPTIONS,
     {NULL}
@@ -84,13 +92,13 @@ int rsa_main(int argc, char **argv)
     BIO *out = NULL;
     EVP_PKEY *pkey = NULL;
     EVP_PKEY_CTX *pctx;
-    const EVP_CIPHER *enc = NULL;
-    char *infile = NULL, *outfile = NULL, *prog;
+    EVP_CIPHER *enc = NULL;
+    char *infile = NULL, *outfile = NULL, *ciphername = NULL, *prog;
     char *passin = NULL, *passout = NULL, *passinarg = NULL, *passoutarg = NULL;
     int private = 0;
-    int informat = FORMAT_PEM, outformat = FORMAT_PEM, text = 0, check = 0;
+    int informat = FORMAT_UNDEF, outformat = FORMAT_PEM, text = 0, check = 0;
     int noout = 0, modulus = 0, pubin = 0, pubout = 0, ret = 1;
-    int pvk_encr = 2;
+    int pvk_encr = DEFAULT_PVK_ENCR_STRENGTH;
     OPTION_CHOICE o;
     int traditional = 0;
     const char *output_type = NULL;
@@ -163,8 +171,7 @@ int rsa_main(int argc, char **argv)
             check = 1;
             break;
         case OPT_CIPHER:
-            if (!opt_cipher(opt_unknown(), &enc))
-                goto opthelp;
+            ciphername = opt_unknown();
             break;
         case OPT_PROV_CASES:
             if (!opt_provider(o))
@@ -175,10 +182,16 @@ int rsa_main(int argc, char **argv)
             break;
         }
     }
+
+    /* No extra arguments. */
     argc = opt_num_rest();
     if (argc != 0)
         goto opthelp;
 
+    if (ciphername != NULL) {
+        if (!opt_cipher(ciphername, &enc))
+            goto opthelp;
+    }
     private = (text && !pubin) || (!pubout && !noout) ? 1 : 0;
 
     if (!app_passwd(passinarg, passoutarg, &passin, &passout)) {
@@ -191,7 +204,7 @@ int rsa_main(int argc, char **argv)
     }
 
     if (pubin) {
-        int tmpformat = -1;
+        int tmpformat = FORMAT_UNDEF;
 
         if (pubin == 2) {
             if (informat == FORMAT_PEM)
@@ -246,7 +259,7 @@ int rsa_main(int argc, char **argv)
 
         pctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL);
         if (pctx == NULL) {
-            BIO_printf(out, "RSA unable to create PKEY context\n");
+            BIO_printf(bio_err, "RSA unable to create PKEY context\n");
             ERR_print_errors(bio_err);
             goto end;
         }
@@ -256,15 +269,8 @@ int rsa_main(int argc, char **argv)
         if (r == 1) {
             BIO_printf(out, "RSA key ok\n");
         } else if (r == 0) {
-            unsigned long err;
-
-            while ((err = ERR_peek_error()) != 0 &&
-                   ERR_GET_LIB(err) == ERR_LIB_RSA &&
-                   ERR_GET_REASON(err) != ERR_R_MALLOC_FAILURE) {
-                BIO_printf(out, "RSA key error: %s\n",
-                           ERR_reason_error_string(err));
-                ERR_get_error(); /* remove err from error stack */
-            }
+            BIO_printf(bio_err, "RSA key not ok\n");
+            ERR_print_errors(bio_err);
         } else if (r == -1) {
             ERR_print_errors(bio_err);
             goto end;
@@ -308,27 +314,41 @@ int rsa_main(int argc, char **argv)
     if (outformat == FORMAT_ASN1 || outformat == FORMAT_PEM) {
         if (pubout || pubin) {
             if (pubout == 2)
-                output_structure = "SubjectPublicKeyInfo";
-            else
                 output_structure = "pkcs1"; /* "type-specific" would work too */
+            else
+                output_structure = "SubjectPublicKeyInfo";
         } else {
             assert(private);
             if (traditional)
                 output_structure = "pkcs1"; /* "type-specific" would work too */
             else
-                output_structure = "pkcs8";
+                output_structure = "PrivateKeyInfo";
         }
     }
 
     /* Now, perform the encoding */
-    ectx = OSSL_ENCODER_CTX_new_by_EVP_PKEY(pkey, selection,
-                                            output_type, output_structure,
-                                            NULL, NULL);
+    ectx = OSSL_ENCODER_CTX_new_for_pkey(pkey, selection,
+                                         output_type, output_structure,
+                                         NULL);
     if (OSSL_ENCODER_CTX_get_num_encoders(ectx) == 0) {
         BIO_printf(bio_err, "%s format not supported\n", output_type);
         goto end;
     }
 
+    /* Passphrase setup */
+    if (enc != NULL)
+        OSSL_ENCODER_CTX_set_cipher(ectx, EVP_CIPHER_get0_name(enc), NULL);
+
+    /* Default passphrase prompter */
+    if (enc != NULL || outformat == FORMAT_PVK) {
+        OSSL_ENCODER_CTX_set_passphrase_ui(ectx, get_ui_method(), NULL);
+        if (passout != NULL)
+            /* When passout given, override the passphrase prompter */
+            OSSL_ENCODER_CTX_set_passphrase(ectx,
+                                            (const unsigned char *)passout,
+                                            strlen(passout));
+    }
+
     /* PVK is a bit special... */
     if (outformat == FORMAT_PVK) {
         OSSL_PARAM params[2] = { OSSL_PARAM_END, OSSL_PARAM_END };
@@ -351,6 +371,7 @@ int rsa_main(int argc, char **argv)
     release_engine(e);
     BIO_free_all(out);
     EVP_PKEY_free(pkey);
+    EVP_CIPHER_free(enc);
     OPENSSL_free(passin);
     OPENSSL_free(passout);
     return ret;