Add the check before cast from int to unsigned to avoid integer overflow since EVP_MD_get_size() may return negative numbers.
Fixes: 919ba00942 ("DANE support structures, constructructors and accessors")
Signed-off-by: Jiasheng Jiang <jiasheng@purdue.edu>
Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
(Merged from https://github.com/openssl/openssl/pull/23940)
int ilen = (int)dlen;
int i;
int num;
+ int mdsize;
if (dane->trecs == NULL) {
ERR_raise(ERR_LIB_SSL, SSL_R_DANE_NOT_ENABLED);
}
}
- if (md != NULL && dlen != (size_t)EVP_MD_get_size(md)) {
- ERR_raise(ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_DIGEST_LENGTH);
- return 0;
+ if (md != NULL) {
+ mdsize = EVP_MD_get_size(md);
+ if (mdsize < 0 || dlen != (size_t)mdsize) {
+ ERR_raise(ERR_LIB_SSL, SSL_R_DANE_TLSA_BAD_DIGEST_LENGTH);
+ return 0;
+ }
}
if (!data) {
ERR_raise(ERR_LIB_SSL, SSL_R_DANE_TLSA_NULL_DATA);