Ensure dhparams can handle X9.42 params in DER

dhparams correctly handles X9.42 params in PEM format. However it failed
to correctly processes them when reading/writing DER format.

Fixes #3102

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3111)

diff --git a/apps/dhparam.c b/apps/dhparam.c
index 5fca25e17a4c265836e401321ec3e3862c99f48c..6cd059fd162b7840e500414189cfd9a8cc827b71 100644 (file)
--- a/apps/dhparam.c
+++ b/apps/dhparam.c
@@ -242,10 +242,19 @@ int dhparam_main(int argc, char **argv)
         } else
 # endif
         {
-            if (informat == FORMAT_ASN1)
+            if (informat == FORMAT_ASN1) {
+                /*
+                 * We have no PEM header to determine what type of DH params it
+                 * is. We'll just try both.
+                 */
                 dh = d2i_DHparams_bio(in, NULL);
-            else                /* informat == FORMAT_PEM */
+                /* BIO_reset() returns 0 for success for file BIOs only!!! */
+                if (dh == NULL && BIO_reset(in) == 0)
+                    dh = d2i_DHxparams_bio(in, NULL);
+            } else {
+                /* informat == FORMAT_PEM */
                 dh = PEM_read_bio_DHparams(in, NULL, NULL, NULL);
+            }
 
             if (dh == NULL) {
                 BIO_printf(bio_err, "unable to load DH parameters\n");
@@ -340,9 +349,12 @@ int dhparam_main(int argc, char **argv)
     if (!noout) {
         const BIGNUM *q;
         DH_get0_pqg(dh, NULL, &q, NULL);
-        if (outformat == FORMAT_ASN1)
-            i = i2d_DHparams_bio(out, dh);
-        else if (q != NULL)
+        if (outformat == FORMAT_ASN1) {
+            if (q != NULL)
+                i = i2d_DHxparams_bio(out, dh);
+            else
+                i = i2d_DHparams_bio(out, dh);
+        } else if (q != NULL)
             i = PEM_write_bio_DHxparams(out, dh);
         else
             i = PEM_write_bio_DHparams(out, dh);