Fix bug in err_string_data_cmp

Unsigned overflow.  Thanks to Brian Carpenter for reporting this.

Reviewed-by: Tim Hudson <tjh@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3887)

diff --git a/crypto/err/err.c b/crypto/err/err.c
index e50c6d6f9d4e17ffdcfc21ea1efbe4706020c726..8d0ed6faf322ed24651be608a869c2b67967753a 100644 (file)
--- a/crypto/err/err.c
+++ b/crypto/err/err.c
@@ -162,7 +162,9 @@ static unsigned long err_string_data_hash(const ERR_STRING_DATA *a)
 static int err_string_data_cmp(const ERR_STRING_DATA *a,
                                const ERR_STRING_DATA *b)
 {
-    return (int)(a->error - b->error);
+    if (a->error == b->error)
+        return 0;
+    return a->error > b->error ? 1 : -1;
 }
 
 static ERR_STRING_DATA *int_err_get_item(const ERR_STRING_DATA *d)