macs/kmac_prov.c: Add checks for the EVP_MD_get_size()

Add checks for the EVP_MD_get_size() to avoid integer overflow and then explicitly cast from int to size_t.

Fixes: 6e624a6453 ("KMAC implementation using EVP_MAC")
Signed-off-by: Jiasheng Jiang <jiasheng@purdue.edu>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Paul Dale <ppzgs1@gmail.com>
Reviewed-by: Neil Horman <nhorman@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/23946)

diff --git a/providers/implementations/macs/kmac_prov.c b/providers/implementations/macs/kmac_prov.c
index ddaab4ba86dc6d784f52a0ce28289f58c5567369..82cbb4c3879f095ab388d72fdc7dac295ab2fdc7 100644 (file)
--- a/providers/implementations/macs/kmac_prov.c
+++ b/providers/implementations/macs/kmac_prov.c
@@ -178,6 +178,7 @@ static struct kmac_data_st *kmac_new(void *provctx)
 static void *kmac_fetch_new(void *provctx, const OSSL_PARAM *params)
 {
     struct kmac_data_st *kctx = kmac_new(provctx);
+    int md_size;
 
     if (kctx == NULL)
         return 0;
@@ -187,7 +188,12 @@ static void *kmac_fetch_new(void *provctx, const OSSL_PARAM *params)
         return 0;
     }
 
-    kctx->out_len = EVP_MD_get_size(ossl_prov_digest_md(&kctx->digest));
+    md_size = EVP_MD_get_size(ossl_prov_digest_md(&kctx->digest));
+    if (md_size <= 0) {
+        kmac_free(kctx);
+        return 0;
+    }
+    kctx->out_len = (size_t)md_size;
     return kctx;
 }