change salt handling, way 1

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/4486)

diff --git a/apps/enc.c b/apps/enc.c
index 4339ba4114aea5480cf51e53f9068bcbf135d245..32ed08d943001cf7a84ad79dbe4027892510eae5 100644 (file)
--- a/apps/enc.c
+++ b/apps/enc.c
@@ -429,14 +429,11 @@ int enc_main(int argc, char **argv)
     }
 
     if (cipher != NULL) {
-        /*
-         * Note that str is NULL if a key was passed on the command line, so
-         * we get no salt in that case. Is this a bug?
-         */
-        if (str != NULL) {
+        if (str != NULL) { /* a passphrase is available */
             /*
-             * Salt handling: if encrypting generate a salt and write to
-             * output BIO. If decrypting read salt from input BIO.
+             * Salt handling: if encrypting generate a salt if not supplied,
+             * and write to output BIO. If decrypting use salt from input BIO
+             * if not given with args
              */
             unsigned char *sptr;
             size_t str_len = strlen(str);
@@ -444,36 +441,47 @@ int enc_main(int argc, char **argv)
             if (nosalt) {
                 sptr = NULL;
             } else {
-                if (enc) {
-                    if (hsalt) {
-                        if (!set_hex(hsalt, salt, sizeof(salt))) {
-                            BIO_printf(bio_err, "invalid hex salt value\n");
+                if (hsalt != NULL && !set_hex(hsalt, salt, sizeof(salt))) {
+                    BIO_printf(bio_err, "invalid hex salt value\n");
+                    goto end;
+                }
+                if (enc) {  /* encryption */
+                    if (hsalt == NULL) {
+                        if (RAND_bytes(salt, sizeof(salt)) <= 0) {
+                            BIO_printf(bio_err, "RAND_bytes failed\n");
+                            goto end;
+                        }
+                        /*
+                         * If -P option then don't bother writing.
+                         * If salt is given, shouldn't either ?
+                         */
+                        if ((printkey != 2)
+                            && (BIO_write(wbio, magic,
+                                          sizeof(magic) - 1) != sizeof(magic) - 1
+                                || BIO_write(wbio,
+                                             (char *)salt,
+                                             sizeof(salt)) != sizeof(salt))) {
+                            BIO_printf(bio_err, "error writing output file\n");
                             goto end;
                         }
-                    } else if (RAND_bytes(salt, sizeof(salt)) <= 0) {
-                        goto end;
                     }
-                    /*
-                     * If -P option then don't bother writing
-                     */
-                    if ((printkey != 2)
-                        && (BIO_write(wbio, magic,
-                                      sizeof(magic) - 1) != sizeof(magic) - 1
-                            || BIO_write(wbio,
-                                         (char *)salt,
-                                         sizeof(salt)) != sizeof(salt))) {
-                        BIO_printf(bio_err, "error writing output file\n");
-                        goto end;
+                } else {    /* decryption */
+                    if (hsalt == NULL) {
+                        if (BIO_read(rbio, mbuf, sizeof(mbuf)) != sizeof(mbuf)) {
+                            BIO_printf(bio_err, "error reading input file\n");
+                            goto end;
+                        }
+                        if (memcmp(mbuf, magic, sizeof(mbuf)) == 0) { /* file IS salted */
+                            if (BIO_read(rbio, salt,
+                                         sizeof(salt)) != sizeof(salt)) {
+                                BIO_printf(bio_err, "error reading input file\n");
+                                goto end;
+                            }
+                        } else { /* file is NOT salted, NO salt available */
+                            BIO_printf(bio_err, "bad magic number\n");
+                            goto end;
+                        }
                     }
-                } else if (BIO_read(rbio, mbuf, sizeof(mbuf)) != sizeof(mbuf)
-                           || BIO_read(rbio,
-                                       (unsigned char *)salt,
-                                       sizeof(salt)) != sizeof(salt)) {
-                    BIO_printf(bio_err, "error reading input file\n");
-                    goto end;
-                } else if (memcmp(mbuf, magic, sizeof(magic) - 1)) {
-                    BIO_printf(bio_err, "bad magic number\n");
-                    goto end;
                 }
                 sptr = salt;
             }