Add clmul-based gmult for riscv64 with Zbb, Zbc

[openssl.git] / crypto / modes / gcm128.c

diff --git a/crypto/modes/gcm128.c b/crypto/modes/gcm128.c
index e7e719fc0e262c96cf7559677926e052f886fbe3..8838ffccb9bb5846fb6a50519cfcbbff0b774d00 100644 (file)
--- a/crypto/modes/gcm128.c
+++ b/crypto/modes/gcm128.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2021 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2010-2022 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -27,6 +27,11 @@ typedef size_t size_t_aX;
 # define PUTU32(p,v)     *(u32 *)(p) = BSWAP4(v)
 #endif
 
+/* RISC-V uses C implementation of gmult as a fallback. */
+#if defined(__riscv)
+# define INCLUDE_C_GMULT_4BIT
+#endif
+
 #define PACK(s)         ((size_t)(s)<<(sizeof(size_t)*8-16))
 #define REDUCE1BIT(V)   do { \
         if (sizeof(size_t)==8) { \
@@ -289,7 +294,7 @@ static void gcm_init_4bit(u128 Htable[16], u64 H[2])
 # endif
 }
 
-# ifndef GHASH_ASM
+# if !defined(GHASH_ASM) || defined(INCLUDE_C_GMULT_4BIT)
 static const size_t rem_4bit[16] = {
     PACK(0x0000), PACK(0x1C20), PACK(0x3840), PACK(0x2460),
     PACK(0x7080), PACK(0x6CA0), PACK(0x48C0), PACK(0x54E0),
@@ -364,6 +369,9 @@ static void gcm_gmult_4bit(u64 Xi[2], const u128 Htable[16])
     }
 }
 
+# endif
+
+# if !defined(GHASH_ASM)
 #  if !defined(OPENSSL_SMALL_FOOTPRINT)
 /*
  * Streamed gcm_mult_4bit, see CRYPTO_gcm128_[en|de]crypt for
@@ -689,6 +697,13 @@ void gcm_init_p8(u128 Htable[16], const u64 Xi[2]);
 void gcm_gmult_p8(u64 Xi[2], const u128 Htable[16]);
 void gcm_ghash_p8(u64 Xi[2], const u128 Htable[16], const u8 *inp,
                   size_t len);
+# elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 64
+#  include "crypto/riscv_arch.h"
+#  define GHASH_ASM_RISCV
+#  define GCM_FUNCREF_4BIT
+#  undef  GHASH
+void gcm_init_clmul_rv64i_zbb_zbc(u128 Htable[16], const u64 Xi[2]);
+void gcm_gmult_clmul_rv64i_zbb_zbc(u64 Xi[2], const u128 Htable[16]);
 # endif
 #endif
 
@@ -805,6 +820,14 @@ void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx, void *key, block128_f block)
         ctx->gmult = gcm_gmult_4bit;
         CTX__GHASH(gcm_ghash_4bit);
     }
+# elif  defined(GHASH_ASM_RISCV) && __riscv_xlen == 64
+    if (RISCV_HAS_ZBB() && RISCV_HAS_ZBC()) {
+        gcm_init_clmul_rv64i_zbb_zbc(ctx->Htable, ctx->H.u);
+        ctx->gmult = gcm_gmult_clmul_rv64i_zbb_zbc;
+    } else {
+        gcm_init_4bit(ctx->Htable, ctx->H.u);
+        ctx->gmult = gcm_gmult_4bit;
+    }
 # else
     gcm_init_4bit(ctx->Htable, ctx->H.u);
 # endif