/*
- * Copyright 2019-2021 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2019-2024 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
# endif /* AES_XTS_ASM */
# if defined(OPENSSL_CPUID_OBJ)
-# if (defined(__powerpc__) || defined(__ppc__) || defined(_ARCH_PPC))
+# if (defined(__powerpc__) || defined(__POWERPC__) || defined(_ARCH_PPC))
# include "crypto/ppc_arch.h"
# ifdef VPAES_ASM
# define VPAES_CAPABLE (OPENSSL_ppccap_P & PPC_ALTIVEC)
# endif
-# define HWAES_CAPABLE (OPENSSL_ppccap_P & PPC_CRYPTO207)
-# define HWAES_set_encrypt_key aes_p8_set_encrypt_key
-# define HWAES_set_decrypt_key aes_p8_set_decrypt_key
-# define HWAES_encrypt aes_p8_encrypt
-# define HWAES_decrypt aes_p8_decrypt
-# define HWAES_cbc_encrypt aes_p8_cbc_encrypt
-# define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks
-# define HWAES_xts_encrypt aes_p8_xts_encrypt
-# define HWAES_xts_decrypt aes_p8_xts_decrypt
-# define PPC_AES_GCM_CAPABLE (OPENSSL_ppccap_P & PPC_MADD300)
-# define AES_GCM_ENC_BYTES 128
-# define AES_GCM_DEC_BYTES 128
-size_t ppc_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len,
- const void *key, unsigned char ivec[16], u64 *Xi);
-size_t ppc_aes_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len,
- const void *key, unsigned char ivec[16], u64 *Xi);
+# if !defined(OPENSSL_SYS_AIX) && !defined(OPENSSL_SYS_MACOSX)
+# define HWAES_CAPABLE (OPENSSL_ppccap_P & PPC_CRYPTO207)
+# define HWAES_set_encrypt_key aes_p8_set_encrypt_key
+# define HWAES_set_decrypt_key aes_p8_set_decrypt_key
+# define HWAES_encrypt aes_p8_encrypt
+# define HWAES_decrypt aes_p8_decrypt
+# define HWAES_cbc_encrypt aes_p8_cbc_encrypt
+# define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks
+# define HWAES_xts_encrypt aes_p8_xts_encrypt
+# define HWAES_xts_decrypt aes_p8_xts_decrypt
+# define PPC_AES_GCM_CAPABLE (OPENSSL_ppccap_P & PPC_MADD300)
+# define AES_GCM_ENC_BYTES 128
+# define AES_GCM_DEC_BYTES 128
+size_t ppc_aes_gcm_encrypt(const unsigned char *in, unsigned char *out,
+ size_t len, const void *key, unsigned char ivec[16],
+ u64 *Xi);
+size_t ppc_aes_gcm_decrypt(const unsigned char *in, unsigned char *out,
+ size_t len, const void *key, unsigned char ivec[16],
+ u64 *Xi);
+# define AES_GCM_ASM_PPC(gctx) ((gctx)->ctr==aes_p8_ctr32_encrypt_blocks && \
+ (gctx)->gcm.funcs.ghash==gcm_ghash_p8)
void gcm_ghash_p8(u64 Xi[2],const u128 Htable[16],const u8 *inp, size_t len);
-# define AES_gcm_encrypt ppc_aes_gcm_encrypt
-# define AES_gcm_decrypt ppc_aes_gcm_decrypt
-# define AES_GCM_ASM(gctx) ((gctx)->ctr==aes_p8_ctr32_encrypt_blocks && \
- (gctx)->gcm.ghash==gcm_ghash_p8)
+# endif /* OPENSSL_SYS_AIX || OPENSSL_SYS_MACOSX */
# endif /* PPC */
-# if (defined(__arm__) || defined(__arm) || defined(__aarch64__))
+# if (defined(__arm__) || defined(__arm) || defined(__aarch64__) || defined(_M_ARM64))
# include "arm_arch.h"
# if __ARM_MAX_ARCH__>=7
# if defined(BSAES_ASM)
# define HWAES_decrypt aes_v8_decrypt
# define HWAES_cbc_encrypt aes_v8_cbc_encrypt
# define HWAES_ecb_encrypt aes_v8_ecb_encrypt
-# if __ARM_MAX_ARCH__>=8 && defined(__aarch64__)
+# if __ARM_MAX_ARCH__>=8 && (defined(__aarch64__) || defined(_M_ARM64))
+# define ARMv8_HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES)
# define HWAES_xts_encrypt aes_v8_xts_encrypt
# define HWAES_xts_decrypt aes_v8_xts_decrypt
# endif
# define HWAES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks
+# define HWAES_ctr32_encrypt_blocks_unroll12_eor3 aes_v8_ctr32_encrypt_blocks_unroll12_eor3
# define AES_PMULL_CAPABLE ((OPENSSL_armcap_P & ARMV8_PMULL) && (OPENSSL_armcap_P & ARMV8_AES))
+# define AES_UNROLL12_EOR3_CAPABLE (OPENSSL_armcap_P & ARMV8_UNROLL12_EOR3)
# define AES_GCM_ENC_BYTES 512
# define AES_GCM_DEC_BYTES 512
-# if __ARM_MAX_ARCH__>=8 && defined(__aarch64__)
+# if __ARM_MAX_ARCH__>=8 && (defined(__aarch64__) || defined(_M_ARM64))
# define AES_gcm_encrypt armv8_aes_gcm_encrypt
# define AES_gcm_decrypt armv8_aes_gcm_decrypt
-# define AES_GCM_ASM(gctx) ((gctx)->ctr==aes_v8_ctr32_encrypt_blocks && \
- (gctx)->gcm.ghash==gcm_ghash_v8)
-size_t aes_gcm_enc_128_kernel(const uint8_t * plaintext, uint64_t plaintext_length, uint8_t * ciphertext,
+# define AES_GCM_ASM(gctx) (((gctx)->ctr==aes_v8_ctr32_encrypt_blocks_unroll12_eor3 || \
+ (gctx)->ctr==aes_v8_ctr32_encrypt_blocks) && \
+ (gctx)->gcm.funcs.ghash==gcm_ghash_v8)
+/* The [unroll8_eor3_]aes_gcm_(enc|dec)_(128|192|256)_kernel() functions
+ * take input length in BITS and return number of BYTES processed */
+size_t aes_gcm_enc_128_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext,
+ uint64_t *Xi, unsigned char ivec[16], const void *key);
+size_t aes_gcm_enc_192_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext,
+ uint64_t *Xi, unsigned char ivec[16], const void *key);
+size_t aes_gcm_enc_256_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext,
+ uint64_t *Xi, unsigned char ivec[16], const void *key);
+size_t aes_gcm_dec_128_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext,
+ uint64_t *Xi, unsigned char ivec[16], const void *key);
+size_t aes_gcm_dec_192_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext,
uint64_t *Xi, unsigned char ivec[16], const void *key);
-size_t aes_gcm_enc_192_kernel(const uint8_t * plaintext, uint64_t plaintext_length, uint8_t * ciphertext,
+size_t aes_gcm_dec_256_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext,
uint64_t *Xi, unsigned char ivec[16], const void *key);
-size_t aes_gcm_enc_256_kernel(const uint8_t * plaintext, uint64_t plaintext_length, uint8_t * ciphertext,
+size_t unroll8_eor3_aes_gcm_enc_128_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext,
uint64_t *Xi, unsigned char ivec[16], const void *key);
-size_t aes_gcm_dec_128_kernel(const uint8_t * ciphertext, uint64_t plaintext_length, uint8_t * plaintext,
+size_t unroll8_eor3_aes_gcm_enc_192_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext,
uint64_t *Xi, unsigned char ivec[16], const void *key);
-size_t aes_gcm_dec_192_kernel(const uint8_t * ciphertext, uint64_t plaintext_length, uint8_t * plaintext,
+size_t unroll8_eor3_aes_gcm_enc_256_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext,
uint64_t *Xi, unsigned char ivec[16], const void *key);
-size_t aes_gcm_dec_256_kernel(const uint8_t * ciphertext, uint64_t plaintext_length, uint8_t * plaintext,
+size_t unroll8_eor3_aes_gcm_dec_128_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext,
+ uint64_t *Xi, unsigned char ivec[16], const void *key);
+size_t unroll8_eor3_aes_gcm_dec_192_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext,
+ uint64_t *Xi, unsigned char ivec[16], const void *key);
+size_t unroll8_eor3_aes_gcm_dec_256_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext,
uint64_t *Xi, unsigned char ivec[16], const void *key);
size_t armv8_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len, const void *key,
unsigned char ivec[16], u64 *Xi);
# define AESNI_CBC_HMAC_SHA_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32)))
# endif
+# if defined(__loongarch__) || defined(__loongarch64)
+# include "loongarch_arch.h"
+# if defined(VPAES_ASM)
+# define VPAES_CAPABLE (OPENSSL_loongarch_hwcap_P & LOONGARCH_HWCAP_LSX)
+# endif
+# endif
+
# if defined(AES_ASM) && !defined(I386_ONLY) && ( \
((defined(__i386) || defined(__i386__) || \
defined(_M_IX86)) && defined(OPENSSL_IA32_SSE2))|| \
# define AES_gcm_encrypt aesni_gcm_encrypt
# define AES_gcm_decrypt aesni_gcm_decrypt
# define AES_GCM_ASM(ctx) (ctx->ctr == aesni_ctr32_encrypt_blocks && \
- ctx->gcm.ghash == gcm_ghash_avx)
+ ctx->gcm.funcs.ghash == gcm_ghash_avx)
# endif
/* Convert key size to function code: [16,24,32] -> [18,19,20]. */
# define S390X_AES_FC(keylen) (S390X_AES_128 + ((((keylen) << 3) - 128) >> 6))
+# elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 64
+/* RISC-V 64 support */
+# include "riscv_arch.h"
+
+/* Zkne and Zknd extensions (scalar crypto AES). */
+int rv64i_zkne_set_encrypt_key(const unsigned char *userKey, const int bits,
+ AES_KEY *key);
+int rv64i_zknd_set_decrypt_key(const unsigned char *userKey, const int bits,
+ AES_KEY *key);
+void rv64i_zkne_encrypt(const unsigned char *in, unsigned char *out,
+ const AES_KEY *key);
+void rv64i_zknd_decrypt(const unsigned char *in, unsigned char *out,
+ const AES_KEY *key);
+/* Zvkned extension (vector crypto AES). */
+int rv64i_zvkned_set_encrypt_key(const unsigned char *userKey, const int bits,
+ AES_KEY *key);
+int rv64i_zvkned_set_decrypt_key(const unsigned char *userKey, const int bits,
+ AES_KEY *key);
+void rv64i_zvkned_encrypt(const unsigned char *in, unsigned char *out,
+ const AES_KEY *key);
+void rv64i_zvkned_decrypt(const unsigned char *in, unsigned char *out,
+ const AES_KEY *key);
+
+void rv64i_zvkned_cbc_encrypt(const unsigned char *in, unsigned char *out,
+ size_t length, const AES_KEY *key,
+ unsigned char *ivec, const int enc);
+
+void rv64i_zvkned_cbc_decrypt(const unsigned char *in, unsigned char *out,
+ size_t length, const AES_KEY *key,
+ unsigned char *ivec, const int enc);
+
+void rv64i_zvkned_ecb_encrypt(const unsigned char *in, unsigned char *out,
+ size_t length, const AES_KEY *key,
+ const int enc);
+
+void rv64i_zvkned_ecb_decrypt(const unsigned char *in, unsigned char *out,
+ size_t length, const AES_KEY *key,
+ const int enc);
+
+void rv64i_zvkb_zvkned_ctr32_encrypt_blocks(const unsigned char *in,
+ unsigned char *out, size_t blocks,
+ const void *key,
+ const unsigned char ivec[16]);
+
+size_t rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt(const unsigned char *in,
+ unsigned char *out, size_t len,
+ const void *key,
+ unsigned char ivec[16], u64 *Xi);
+
+size_t rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt(const unsigned char *in,
+ unsigned char *out, size_t len,
+ const void *key,
+ unsigned char ivec[16], u64 *Xi);
+
+void rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt(const unsigned char *in,
+ unsigned char *out, size_t length,
+ const AES_KEY *key1,
+ const AES_KEY *key2,
+ const unsigned char iv[16]);
+
+void rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt(const unsigned char *in,
+ unsigned char *out, size_t length,
+ const AES_KEY *key1,
+ const AES_KEY *key2,
+ const unsigned char iv[16]);
+
+void gcm_ghash_rv64i_zvkg(u64 Xi[2], const u128 Htable[16], const u8 *inp,
+ size_t len);
+
+#define AES_GCM_ENC_BYTES 64
+#define AES_GCM_DEC_BYTES 64
+#define AES_gcm_encrypt rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt
+#define AES_gcm_decrypt rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt
+#define AES_GCM_ASM(ctx) \
+ (ctx->ctr == rv64i_zvkb_zvkned_ctr32_encrypt_blocks && \
+ ctx->gcm.funcs.ghash == gcm_ghash_rv64i_zvkg)
+
+# elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 32
+/* RISC-V 32 support */
+# include "riscv_arch.h"
+
+int rv32i_zkne_set_encrypt_key(const unsigned char *userKey, const int bits,
+ AES_KEY *key);
+/* set_decrypt_key needs both zknd and zkne */
+int rv32i_zknd_zkne_set_decrypt_key(const unsigned char *userKey, const int bits,
+ AES_KEY *key);
+int rv32i_zbkb_zkne_set_encrypt_key(const unsigned char *userKey, const int bits,
+ AES_KEY *key);
+int rv32i_zbkb_zknd_zkne_set_decrypt_key(const unsigned char *userKey, const int bits,
+ AES_KEY *key);
+void rv32i_zkne_encrypt(const unsigned char *in, unsigned char *out,
+ const AES_KEY *key);
+void rv32i_zknd_decrypt(const unsigned char *in, unsigned char *out,
+ const AES_KEY *key);
# endif
# if defined(HWAES_CAPABLE)
void HWAES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
size_t len, const void *key,
const unsigned char ivec[16]);
+# if defined(AES_UNROLL12_EOR3_CAPABLE)
+void HWAES_ctr32_encrypt_blocks_unroll12_eor3(const unsigned char *in, unsigned char *out,
+ size_t len, const void *key,
+ const unsigned char ivec[16]);
+# endif
void HWAES_xts_encrypt(const unsigned char *inp, unsigned char *out,
size_t len, const AES_KEY *key1,
const AES_KEY *key2, const unsigned char iv[16]);
projects / openssl.git / blobdiff