use '__builtin_expect' to improve EVP_EncryptUpdate performance for gcc/clang.

Signed-off-by: Liu-ErMeng <liuermeng2@huawei.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/21425)

diff --git a/crypto/evp/evp_enc.c b/crypto/evp/evp_enc.c
index 6b6d65163f9cc1e53baee4adf756dcdde41cc9ba..8dddcc0bb51ef8cbeb062224203d2d725fff71c2 100644 (file)
--- a/crypto/evp/evp_enc.c
+++ b/crypto/evp/evp_enc.c
@@ -621,7 +621,7 @@ int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
     size_t soutl, inl_ = (size_t)inl;
     int blocksize;
 
-    if (outl != NULL) {
+    if (likely(outl != NULL)) {
         *outl = 0;
     } else {
         ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER);
@@ -629,22 +629,22 @@ int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
     }
 
     /* Prevent accidental use of decryption context when encrypting */
-    if (!ctx->encrypt) {
+    if (unlikely(!ctx->encrypt)) {
         ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_OPERATION);
         return 0;
     }
 
-    if (ctx->cipher == NULL) {
+    if (unlikely(ctx->cipher == NULL)) {
         ERR_raise(ERR_LIB_EVP, EVP_R_NO_CIPHER_SET);
         return 0;
     }
 
-    if (ctx->cipher->prov == NULL)
+    if (unlikely(ctx->cipher->prov == NULL))
         goto legacy;
 
     blocksize = ctx->cipher->block_size;
 
-    if (ctx->cipher->cupdate == NULL  || blocksize < 1) {
+    if (unlikely(ctx->cipher->cupdate == NULL || blocksize < 1)) {
         ERR_raise(ERR_LIB_EVP, EVP_R_UPDATE_ERROR);
         return 0;
     }
@@ -653,7 +653,7 @@ int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
                                inl_ + (size_t)(blocksize == 1 ? 0 : blocksize),
                                in, inl_);
 
-    if (ret) {
+    if (likely(ret)) {
         if (soutl > INT_MAX) {
             ERR_raise(ERR_LIB_EVP, EVP_R_UPDATE_ERROR);
             return 0;
@@ -770,7 +770,7 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
     size_t soutl, inl_ = (size_t)inl;
     int blocksize;
 
-    if (outl != NULL) {
+    if (likely(outl != NULL)) {
         *outl = 0;
     } else {
         ERR_raise(ERR_LIB_EVP, ERR_R_PASSED_NULL_PARAMETER);
@@ -778,21 +778,21 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
     }
 
     /* Prevent accidental use of encryption context when decrypting */
-    if (ctx->encrypt) {
+    if (unlikely(ctx->encrypt)) {
         ERR_raise(ERR_LIB_EVP, EVP_R_INVALID_OPERATION);
         return 0;
     }
 
-    if (ctx->cipher == NULL) {
+    if (unlikely(ctx->cipher == NULL)) {
         ERR_raise(ERR_LIB_EVP, EVP_R_NO_CIPHER_SET);
         return 0;
     }
-    if (ctx->cipher->prov == NULL)
+    if (unlikely(ctx->cipher->prov == NULL))
         goto legacy;
 
     blocksize = EVP_CIPHER_CTX_get_block_size(ctx);
 
-    if (ctx->cipher->cupdate == NULL || blocksize < 1) {
+    if (unlikely(ctx->cipher->cupdate == NULL || blocksize < 1)) {
         ERR_raise(ERR_LIB_EVP, EVP_R_UPDATE_ERROR);
         return 0;
     }
@@ -800,7 +800,7 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
                                inl_ + (size_t)(blocksize == 1 ? 0 : blocksize),
                                in, inl_);
 
-    if (ret) {
+    if (likely(ret)) {
         if (soutl > INT_MAX) {
             ERR_raise(ERR_LIB_EVP, EVP_R_UPDATE_ERROR);
             return 0;

diff --git a/include/internal/common.h b/include/internal/common.h
index f941eddbe1781c54339d8f911ce0b6d5e8991757..204e7c3eecd376d12c1be11573dd288ee6a791ad 100644 (file)
--- a/include/internal/common.h
+++ b/include/internal/common.h
@@ -18,6 +18,14 @@
 # include "internal/e_os.h" /* ossl_inline in many files */
 # include "internal/nelem.h"
 
+#if defined(__GNUC__) || defined(__clang__)
+    #define likely(x) __builtin_expect(!!(x), 1)
+    #define unlikely(x) __builtin_expect(!!(x), 0)
+#else
+    #define likely(x) x
+    #define unlikely(x) x
+#endif
+
 #ifdef NDEBUG
 # define ossl_assert(x) ((x) != 0)
 #else