EVP_DigestInit_ex(): drop previous context engine earlier

If a EVP_MD_CTX holds a reference to a previously given engine, and
the type of its digest isn't the same as the one given in the new
call, drop that engine reference, allowing providers or other engines
to provide the new algorithm on an equal basis.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/9077)

diff --git a/crypto/evp/digest.c b/crypto/evp/digest.c
index 89cd5c1d006759a05e52a9e7e60d368c5d1a42cc..faa6ccf0de4745bfa4da5d509c9800bc5b100632 100644 (file)
--- a/crypto/evp/digest.c
+++ b/crypto/evp/digest.c
@@ -129,6 +129,16 @@ int EVP_DigestInit_ex(EVP_MD_CTX *ctx, const EVP_MD *type, ENGINE *impl)
         (type == NULL || (type->type == ctx->digest->type)))
         goto skip_to_init;
 
+    if (type != NULL) {
+        /*
+         * Ensure an ENGINE left lying around from last time is cleared (the
+         * previous check attempted to avoid this if the same ENGINE and
+         * EVP_MD could be used).
+         */
+        ENGINE_finish(ctx->engine);
+        ctx->engine = NULL;
+    }
+
     if (type != NULL && impl == NULL)
         tmpimpl = ENGINE_get_digest_engine(type->type);
 #endif
@@ -202,12 +212,6 @@ int EVP_DigestInit_ex(EVP_MD_CTX *ctx, const EVP_MD *type, ENGINE *impl)
 
 #if !defined(OPENSSL_NO_ENGINE) && !defined(FIPS_MODE)
     if (type) {
-        /*
-         * Ensure an ENGINE left lying around from last time is cleared (the
-         * previous check attempted to avoid this if the same ENGINE and
-         * EVP_MD could be used).
-         */
-        ENGINE_finish(ctx->engine);
         if (impl != NULL) {
             if (!ENGINE_init(impl)) {
                 EVPerr(EVP_F_EVP_DIGESTINIT_EX, EVP_R_INITIALIZATION_ERROR);