Make 'openssl prime ""' not segfault

Fixes RT#4699

Reviewed-by: Tim Hudson <tjh@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/1668)

diff --git a/apps/prime.c b/apps/prime.c
index 1fb1c8d8455aa342f63a8c7acd21c5a8edd3d44e..133167f2d4d1e5ed73b8e3b458231afaa17a3d4d 100644 (file)
--- a/apps/prime.c
+++ b/apps/prime.c
@@ -128,16 +128,24 @@ int MAIN(int argc, char **argv)
         BIO_printf(bio_out, "%s\n", s);
         OPENSSL_free(s);
     } else {
+        int r;
+
         if (hex)
-            BN_hex2bn(&bn, argv[0]);
+            r = BN_hex2bn(&bn, argv[0]);
         else
-            BN_dec2bn(&bn, argv[0]);
+            r = BN_dec2bn(&bn, argv[0]);
+
+        if(!r) {
+            BIO_printf(bio_err, "Failed to process value (%s)\n", argv[0]);
+            goto end;
+        }
 
         BN_print(bio_out, bn);
         BIO_printf(bio_out, " is %sprime\n",
                    BN_is_prime_ex(bn, checks, NULL, NULL) ? "" : "not ");
     }
 
+ end:
     BN_free(bn);
     BIO_free_all(bio_out);