include/crypto/aes_platform.h

   1 /*
   2  * Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved.
   3  *
   4  * Licensed under the Apache License 2.0 (the "License").  You may not use
   5  * this file except in compliance with the License.  You can obtain a copy
   6  * in the file LICENSE in the source distribution or at
   7  * https://www.openssl.org/source/license.html
   8  */
   9
  10 #ifndef OSSL_AES_PLATFORM_H
  11 # define OSSL_AES_PLATFORM_H
  12 # pragma once
  13
  14 # include <openssl/aes.h>
  15
  16 # ifdef VPAES_ASM
  17 int vpaes_set_encrypt_key(const unsigned char *userKey, int bits,
  18                           AES_KEY *key);
  19 int vpaes_set_decrypt_key(const unsigned char *userKey, int bits,
  20                           AES_KEY *key);
  21 void vpaes_encrypt(const unsigned char *in, unsigned char *out,
  22                    const AES_KEY *key);
  23 void vpaes_decrypt(const unsigned char *in, unsigned char *out,
  24                    const AES_KEY *key);
  25 void vpaes_cbc_encrypt(const unsigned char *in,
  26                        unsigned char *out,
  27                        size_t length,
  28                        const AES_KEY *key, unsigned char *ivec, int enc);
  29 # endif /* VPAES_ASM */
  30
  31 # ifdef BSAES_ASM
  32 void ossl_bsaes_cbc_encrypt(const unsigned char *in, unsigned char *out,
  33                             size_t length, const AES_KEY *key,
  34                             unsigned char ivec[16], int enc);
  35 void ossl_bsaes_ctr32_encrypt_blocks(const unsigned char *in,
  36                                      unsigned char *out, size_t len,
  37                                      const AES_KEY *key,
  38                                      const unsigned char ivec[16]);
  39 void ossl_bsaes_xts_encrypt(const unsigned char *inp, unsigned char *out,
  40                             size_t len, const AES_KEY *key1,
  41                             const AES_KEY *key2, const unsigned char iv[16]);
  42 void ossl_bsaes_xts_decrypt(const unsigned char *inp, unsigned char *out,
  43                             size_t len, const AES_KEY *key1,
  44                             const AES_KEY *key2, const unsigned char iv[16]);
  45 # endif /* BSAES_ASM */
  46
  47 # ifdef AES_CTR_ASM
  48 void AES_ctr32_encrypt(const unsigned char *in, unsigned char *out,
  49                        size_t blocks, const AES_KEY *key,
  50                        const unsigned char ivec[AES_BLOCK_SIZE]);
  51 # endif /*  AES_CTR_ASM */
  52
  53 # ifdef AES_XTS_ASM
  54 void AES_xts_encrypt(const unsigned char *inp, unsigned char *out, size_t len,
  55                      const AES_KEY *key1, const AES_KEY *key2,
  56                      const unsigned char iv[16]);
  57 void AES_xts_decrypt(const unsigned char *inp, unsigned char *out, size_t len,
  58                      const AES_KEY *key1, const AES_KEY *key2,
  59                      const unsigned char iv[16]);
  60 # endif /* AES_XTS_ASM */
  61
  62 # if defined(OPENSSL_CPUID_OBJ)
  63 #  if (defined(__powerpc__) || defined(__POWERPC__) || defined(_ARCH_PPC))
  64 #   include "crypto/ppc_arch.h"
  65 #   ifdef VPAES_ASM
  66 #    define VPAES_CAPABLE (OPENSSL_ppccap_P & PPC_ALTIVEC)
  67 #   endif
  68 #   if !defined(OPENSSL_SYS_AIX) && !defined(OPENSSL_SYS_MACOSX)
  69 #    define HWAES_CAPABLE  (OPENSSL_ppccap_P & PPC_CRYPTO207)
  70 #    define HWAES_set_encrypt_key aes_p8_set_encrypt_key
  71 #    define HWAES_set_decrypt_key aes_p8_set_decrypt_key
  72 #    define HWAES_encrypt aes_p8_encrypt
  73 #    define HWAES_decrypt aes_p8_decrypt
  74 #    define HWAES_cbc_encrypt aes_p8_cbc_encrypt
  75 #    define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks
  76 #    define HWAES_xts_encrypt aes_p8_xts_encrypt
  77 #    define HWAES_xts_decrypt aes_p8_xts_decrypt
  78 #    define PPC_AES_GCM_CAPABLE (OPENSSL_ppccap_P & PPC_MADD300)
  79 #    define AES_GCM_ENC_BYTES 128
  80 #    define AES_GCM_DEC_BYTES 128
  81 size_t ppc_aes_gcm_encrypt(const unsigned char *in, unsigned char *out,
  82                            size_t len, const void *key, unsigned char ivec[16],
  83                            u64 *Xi);
  84 size_t ppc_aes_gcm_decrypt(const unsigned char *in, unsigned char *out,
  85                            size_t len, const void *key, unsigned char ivec[16],
  86                            u64 *Xi);
  87 #    define AES_GCM_ASM_PPC(gctx) ((gctx)->ctr==aes_p8_ctr32_encrypt_blocks && \
  88                                    (gctx)->gcm.funcs.ghash==gcm_ghash_p8)
  89 void gcm_ghash_p8(u64 Xi[2],const u128 Htable[16],const u8 *inp, size_t len);
  90 #   endif /* OPENSSL_SYS_AIX || OPENSSL_SYS_MACOSX */
  91 #  endif /* PPC */
  92
  93 #  if (defined(__arm__) || defined(__arm) || defined(__aarch64__) || defined(_M_ARM64))
  94 #   include "arm_arch.h"
  95 #   if __ARM_MAX_ARCH__>=7
  96 #    if defined(BSAES_ASM)
  97 #     define BSAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
  98 #    endif
  99 #    if defined(VPAES_ASM)
 100 #     define VPAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
 101 #    endif
 102 #    define HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES)
 103 #    define HWAES_set_encrypt_key aes_v8_set_encrypt_key
 104 #    define HWAES_set_decrypt_key aes_v8_set_decrypt_key
 105 #    define HWAES_encrypt aes_v8_encrypt
 106 #    define HWAES_decrypt aes_v8_decrypt
 107 #    define HWAES_cbc_encrypt aes_v8_cbc_encrypt
 108 #    define HWAES_ecb_encrypt aes_v8_ecb_encrypt
 109 #    if __ARM_MAX_ARCH__>=8 && (defined(__aarch64__) || defined(_M_ARM64))
 110 #     define ARMv8_HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES)
 111 #     define HWAES_xts_encrypt aes_v8_xts_encrypt
 112 #     define HWAES_xts_decrypt aes_v8_xts_decrypt
 113 #    endif
 114 #    define HWAES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks
 115 #    define HWAES_ctr32_encrypt_blocks_unroll12_eor3 aes_v8_ctr32_encrypt_blocks_unroll12_eor3
 116 #    define AES_PMULL_CAPABLE ((OPENSSL_armcap_P & ARMV8_PMULL) && (OPENSSL_armcap_P & ARMV8_AES))
 117 #    define AES_UNROLL12_EOR3_CAPABLE (OPENSSL_armcap_P & ARMV8_UNROLL12_EOR3)
 118 #    define AES_GCM_ENC_BYTES 512
 119 #    define AES_GCM_DEC_BYTES 512
 120 #    if __ARM_MAX_ARCH__>=8 && (defined(__aarch64__) || defined(_M_ARM64))
 121 #     define AES_gcm_encrypt armv8_aes_gcm_encrypt
 122 #     define AES_gcm_decrypt armv8_aes_gcm_decrypt
 123 #     define AES_GCM_ASM(gctx) (((gctx)->ctr==aes_v8_ctr32_encrypt_blocks_unroll12_eor3 || \
 124                                  (gctx)->ctr==aes_v8_ctr32_encrypt_blocks) && \
 125                                 (gctx)->gcm.funcs.ghash==gcm_ghash_v8)
 126 /* The [unroll8_eor3_]aes_gcm_(enc|dec)_(128|192|256)_kernel() functions
 127  * take input length in BITS and return number of BYTES processed */
 128 size_t aes_gcm_enc_128_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext,
 129                               uint64_t *Xi, unsigned char ivec[16], const void *key);
 130 size_t aes_gcm_enc_192_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext,
 131                               uint64_t *Xi, unsigned char ivec[16], const void *key);
 132 size_t aes_gcm_enc_256_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext,
 133                               uint64_t *Xi, unsigned char ivec[16], const void *key);
 134 size_t aes_gcm_dec_128_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext,
 135                               uint64_t *Xi, unsigned char ivec[16], const void *key);
 136 size_t aes_gcm_dec_192_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext,
 137                               uint64_t *Xi, unsigned char ivec[16], const void *key);
 138 size_t aes_gcm_dec_256_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext,
 139                               uint64_t *Xi, unsigned char ivec[16], const void *key);
 140 size_t unroll8_eor3_aes_gcm_enc_128_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext,
 141                               uint64_t *Xi, unsigned char ivec[16], const void *key);
 142 size_t unroll8_eor3_aes_gcm_enc_192_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext,
 143                               uint64_t *Xi, unsigned char ivec[16], const void *key);
 144 size_t unroll8_eor3_aes_gcm_enc_256_kernel(const uint8_t *plaintext, uint64_t plaintext_length, uint8_t *ciphertext,
 145                               uint64_t *Xi, unsigned char ivec[16], const void *key);
 146 size_t unroll8_eor3_aes_gcm_dec_128_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext,
 147                               uint64_t *Xi, unsigned char ivec[16], const void *key);
 148 size_t unroll8_eor3_aes_gcm_dec_192_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext,
 149                               uint64_t *Xi, unsigned char ivec[16], const void *key);
 150 size_t unroll8_eor3_aes_gcm_dec_256_kernel(const uint8_t *ciphertext, uint64_t plaintext_length, uint8_t *plaintext,
 151                               uint64_t *Xi, unsigned char ivec[16], const void *key);
 152 size_t armv8_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len, const void *key,
 153                              unsigned char ivec[16], u64 *Xi);
 154 size_t armv8_aes_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len, const void *key,
 155                              unsigned char ivec[16], u64 *Xi);
 156 void gcm_ghash_v8(u64 Xi[2],const u128 Htable[16],const u8 *inp, size_t len);
 157 #    endif
 158 #   endif
 159 #  endif
 160 # endif /* OPENSSL_CPUID_OBJ */
 161
 162 # if     defined(AES_ASM) &&     ( \
 163          defined(__x86_64)       || defined(__x86_64__)  || \
 164          defined(_M_AMD64)       || defined(_M_X64)      )
 165 #  define AES_CBC_HMAC_SHA_CAPABLE 1
 166 #  define AESNI_CBC_HMAC_SHA_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32)))
 167 # endif
 168
 169 # if defined(__loongarch__) || defined(__loongarch64)
 170 #  include "loongarch_arch.h"
 171 #  if defined(VPAES_ASM)
 172 #   define VPAES_CAPABLE  (OPENSSL_loongarch_hwcap_P & LOONGARCH_HWCAP_LSX)
 173 #  endif
 174 # endif
 175
 176 # if     defined(AES_ASM) && !defined(I386_ONLY) &&      (  \
 177          ((defined(__i386)       || defined(__i386__)    || \
 178            defined(_M_IX86)) && defined(OPENSSL_IA32_SSE2))|| \
 179          defined(__x86_64)       || defined(__x86_64__)  || \
 180          defined(_M_AMD64)       || defined(_M_X64)      )
 181
 182 /* AES-NI section */
 183
 184 #  define AESNI_CAPABLE   (OPENSSL_ia32cap_P[1]&(1<<(57-32)))
 185 #  ifdef VPAES_ASM
 186 #   define VPAES_CAPABLE   (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
 187 #  endif
 188 #  ifdef BSAES_ASM
 189 #   define BSAES_CAPABLE   (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
 190 #  endif
 191
 192 #  define AES_GCM_ENC_BYTES 32
 193 #  define AES_GCM_DEC_BYTES 16
 194
 195 int aesni_set_encrypt_key(const unsigned char *userKey, int bits,
 196                           AES_KEY *key);
 197 int aesni_set_decrypt_key(const unsigned char *userKey, int bits,
 198                           AES_KEY *key);
 199
 200 void aesni_encrypt(const unsigned char *in, unsigned char *out,
 201                    const AES_KEY *key);
 202 void aesni_decrypt(const unsigned char *in, unsigned char *out,
 203                    const AES_KEY *key);
 204
 205 void aesni_ecb_encrypt(const unsigned char *in,
 206                        unsigned char *out,
 207                        size_t length, const AES_KEY *key, int enc);
 208 void aesni_cbc_encrypt(const unsigned char *in,
 209                        unsigned char *out,
 210                        size_t length,
 211                        const AES_KEY *key, unsigned char *ivec, int enc);
 212 #  ifndef OPENSSL_NO_OCB
 213 void aesni_ocb_encrypt(const unsigned char *in, unsigned char *out,
 214                        size_t blocks, const void *key,
 215                        size_t start_block_num,
 216                        unsigned char offset_i[16],
 217                        const unsigned char L_[][16],
 218                        unsigned char checksum[16]);
 219 void aesni_ocb_decrypt(const unsigned char *in, unsigned char *out,
 220                        size_t blocks, const void *key,
 221                        size_t start_block_num,
 222                        unsigned char offset_i[16],
 223                        const unsigned char L_[][16],
 224                        unsigned char checksum[16]);
 225 #  endif /* OPENSSL_NO_OCB */
 226
 227 void aesni_ctr32_encrypt_blocks(const unsigned char *in,
 228                                 unsigned char *out,
 229                                 size_t blocks,
 230                                 const void *key, const unsigned char *ivec);
 231
 232 void aesni_xts_encrypt(const unsigned char *in,
 233                        unsigned char *out,
 234                        size_t length,
 235                        const AES_KEY *key1, const AES_KEY *key2,
 236                        const unsigned char iv[16]);
 237
 238 void aesni_xts_decrypt(const unsigned char *in,
 239                        unsigned char *out,
 240                        size_t length,
 241                        const AES_KEY *key1, const AES_KEY *key2,
 242                        const unsigned char iv[16]);
 243
 244 void aesni_ccm64_encrypt_blocks(const unsigned char *in,
 245                                 unsigned char *out,
 246                                 size_t blocks,
 247                                 const void *key,
 248                                 const unsigned char ivec[16],
 249                                 unsigned char cmac[16]);
 250
 251 void aesni_ccm64_decrypt_blocks(const unsigned char *in,
 252                                 unsigned char *out,
 253                                 size_t blocks,
 254                                 const void *key,
 255                                 const unsigned char ivec[16],
 256                                 unsigned char cmac[16]);
 257
 258 #  if defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
 259 size_t aesni_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len,
 260                          const void *key, unsigned char ivec[16], u64 *Xi);
 261 size_t aesni_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len,
 262                          const void *key, unsigned char ivec[16], u64 *Xi);
 263 void gcm_ghash_avx(u64 Xi[2], const u128 Htable[16], const u8 *in, size_t len);
 264
 265 #   define AES_gcm_encrypt aesni_gcm_encrypt
 266 #   define AES_gcm_decrypt aesni_gcm_decrypt
 267 #   define AES_GCM_ASM(ctx)    (ctx->ctr == aesni_ctr32_encrypt_blocks && \
 268                                 ctx->gcm.funcs.ghash == gcm_ghash_avx)
 269 #  endif
 270
 271
 272 # elif defined(AES_ASM) && (defined(__sparc) || defined(__sparc__))
 273
 274 /* Fujitsu SPARC64 X support */
 275 #  include "crypto/sparc_arch.h"
 276
 277 #  define SPARC_AES_CAPABLE       (OPENSSL_sparcv9cap_P[1] & CFR_AES)
 278 #  define HWAES_CAPABLE           (OPENSSL_sparcv9cap_P[0] & SPARCV9_FJAESX)
 279 #  define HWAES_set_encrypt_key aes_fx_set_encrypt_key
 280 #  define HWAES_set_decrypt_key aes_fx_set_decrypt_key
 281 #  define HWAES_encrypt aes_fx_encrypt
 282 #  define HWAES_decrypt aes_fx_decrypt
 283 #  define HWAES_cbc_encrypt aes_fx_cbc_encrypt
 284 #  define HWAES_ctr32_encrypt_blocks aes_fx_ctr32_encrypt_blocks
 285
 286 void aes_t4_set_encrypt_key(const unsigned char *key, int bits, AES_KEY *ks);
 287 void aes_t4_set_decrypt_key(const unsigned char *key, int bits, AES_KEY *ks);
 288 void aes_t4_encrypt(const unsigned char *in, unsigned char *out,
 289                     const AES_KEY *key);
 290 void aes_t4_decrypt(const unsigned char *in, unsigned char *out,
 291                     const AES_KEY *key);
 292 /*
 293  * Key-length specific subroutines were chosen for following reason.
 294  * Each SPARC T4 core can execute up to 8 threads which share core's
 295  * resources. Loading as much key material to registers allows to
 296  * minimize references to shared memory interface, as well as amount
 297  * of instructions in inner loops [much needed on T4]. But then having
 298  * non-key-length specific routines would require conditional branches
 299  * either in inner loops or on subroutines' entries. Former is hardly
 300  * acceptable, while latter means code size increase to size occupied
 301  * by multiple key-length specific subroutines, so why fight?
 302  */
 303 void aes128_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
 304                            size_t len, const AES_KEY *key,
 305                            unsigned char *ivec, int /*unused*/);
 306 void aes128_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
 307                            size_t len, const AES_KEY *key,
 308                            unsigned char *ivec, int /*unused*/);
 309 void aes192_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
 310                            size_t len, const AES_KEY *key,
 311                            unsigned char *ivec, int /*unused*/);
 312 void aes192_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
 313                            size_t len, const AES_KEY *key,
 314                            unsigned char *ivec, int /*unused*/);
 315 void aes256_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
 316                            size_t len, const AES_KEY *key,
 317                            unsigned char *ivec, int /*unused*/);
 318 void aes256_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
 319                            size_t len, const AES_KEY *key,
 320                            unsigned char *ivec, int /*unused*/);
 321 void aes128_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
 322                              size_t blocks, const AES_KEY *key,
 323                              unsigned char *ivec);
 324 void aes192_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
 325                              size_t blocks, const AES_KEY *key,
 326                              unsigned char *ivec);
 327 void aes256_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
 328                              size_t blocks, const AES_KEY *key,
 329                              unsigned char *ivec);
 330 void aes128_t4_xts_encrypt(const unsigned char *in, unsigned char *out,
 331                            size_t blocks, const AES_KEY *key1,
 332                            const AES_KEY *key2, const unsigned char *ivec);
 333 void aes128_t4_xts_decrypt(const unsigned char *in, unsigned char *out,
 334                            size_t blocks, const AES_KEY *key1,
 335                            const AES_KEY *key2, const unsigned char *ivec);
 336 void aes256_t4_xts_encrypt(const unsigned char *in, unsigned char *out,
 337                            size_t blocks, const AES_KEY *key1,
 338                            const AES_KEY *key2, const unsigned char *ivec);
 339 void aes256_t4_xts_decrypt(const unsigned char *in, unsigned char *out,
 340                            size_t blocks, const AES_KEY *key1,
 341                            const AES_KEY *key2, const unsigned char *ivec);
 342
 343 # elif defined(OPENSSL_CPUID_OBJ) && defined(__s390__)
 344 /* IBM S390X support */
 345 #  include "s390x_arch.h"
 346
 347
 348 /* Convert key size to function code: [16,24,32] -> [18,19,20]. */
 349 #  define S390X_AES_FC(keylen)  (S390X_AES_128 + ((((keylen) << 3) - 128) >> 6))
 350
 351 /* Most modes of operation need km for partial block processing. */
 352 #  define S390X_aes_128_CAPABLE (OPENSSL_s390xcap_P.km[0] &  \
 353                                 S390X_CAPBIT(S390X_AES_128))
 354 #  define S390X_aes_192_CAPABLE (OPENSSL_s390xcap_P.km[0] &  \
 355                                 S390X_CAPBIT(S390X_AES_192))
 356 #  define S390X_aes_256_CAPABLE (OPENSSL_s390xcap_P.km[0] &  \
 357                                 S390X_CAPBIT(S390X_AES_256))
 358
 359 #  define S390X_aes_128_cbc_CAPABLE     1       /* checked by callee */
 360 #  define S390X_aes_192_cbc_CAPABLE     1
 361 #  define S390X_aes_256_cbc_CAPABLE     1
 362
 363 #  define S390X_aes_128_ecb_CAPABLE     S390X_aes_128_CAPABLE
 364 #  define S390X_aes_192_ecb_CAPABLE     S390X_aes_192_CAPABLE
 365 #  define S390X_aes_256_ecb_CAPABLE     S390X_aes_256_CAPABLE
 366
 367 #  define S390X_aes_128_ofb_CAPABLE (S390X_aes_128_CAPABLE &&           \
 368                                     (OPENSSL_s390xcap_P.kmo[0] &        \
 369                                      S390X_CAPBIT(S390X_AES_128)))
 370 #  define S390X_aes_192_ofb_CAPABLE (S390X_aes_192_CAPABLE &&           \
 371                                     (OPENSSL_s390xcap_P.kmo[0] &        \
 372                                      S390X_CAPBIT(S390X_AES_192)))
 373 #  define S390X_aes_256_ofb_CAPABLE (S390X_aes_256_CAPABLE &&           \
 374                                     (OPENSSL_s390xcap_P.kmo[0] &        \
 375                                      S390X_CAPBIT(S390X_AES_256)))
 376
 377 #  define S390X_aes_128_cfb_CAPABLE (S390X_aes_128_CAPABLE &&           \
 378                                     (OPENSSL_s390xcap_P.kmf[0] &        \
 379                                      S390X_CAPBIT(S390X_AES_128)))
 380 #  define S390X_aes_192_cfb_CAPABLE (S390X_aes_192_CAPABLE &&           \
 381                                     (OPENSSL_s390xcap_P.kmf[0] &        \
 382                                      S390X_CAPBIT(S390X_AES_192)))
 383 #  define S390X_aes_256_cfb_CAPABLE (S390X_aes_256_CAPABLE &&           \
 384                                     (OPENSSL_s390xcap_P.kmf[0] &        \
 385                                      S390X_CAPBIT(S390X_AES_256)))
 386 #  define S390X_aes_128_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] &       \
 387                                      S390X_CAPBIT(S390X_AES_128))
 388 #  define S390X_aes_192_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] &       \
 389                                      S390X_CAPBIT(S390X_AES_192))
 390 #  define S390X_aes_256_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] &       \
 391                                      S390X_CAPBIT(S390X_AES_256))
 392 #  define S390X_aes_128_cfb1_CAPABLE    0
 393 #  define S390X_aes_192_cfb1_CAPABLE    0
 394 #  define S390X_aes_256_cfb1_CAPABLE    0
 395
 396 #  define S390X_aes_128_ctr_CAPABLE     1       /* checked by callee */
 397 #  define S390X_aes_192_ctr_CAPABLE     1
 398 #  define S390X_aes_256_ctr_CAPABLE     1
 399
 400 #  define S390X_aes_128_xts_CAPABLE     1       /* checked by callee */
 401 #  define S390X_aes_256_xts_CAPABLE     1
 402
 403 # define S390X_aes_128_gcm_CAPABLE (S390X_aes_128_CAPABLE &&        \
 404                                     (OPENSSL_s390xcap_P.kma[0] &    \
 405                                      S390X_CAPBIT(S390X_AES_128)))
 406 # define S390X_aes_192_gcm_CAPABLE (S390X_aes_192_CAPABLE &&        \
 407                                     (OPENSSL_s390xcap_P.kma[0] &    \
 408                                      S390X_CAPBIT(S390X_AES_192)))
 409 # define S390X_aes_256_gcm_CAPABLE (S390X_aes_256_CAPABLE &&        \
 410                                     (OPENSSL_s390xcap_P.kma[0] &    \
 411                                      S390X_CAPBIT(S390X_AES_256)))
 412
 413 #  define S390X_aes_128_ccm_CAPABLE (S390X_aes_128_CAPABLE &&       \
 414                                     (OPENSSL_s390xcap_P.kmac[0] &   \
 415                                      S390X_CAPBIT(S390X_AES_128)))
 416 #  define S390X_aes_192_ccm_CAPABLE (S390X_aes_192_CAPABLE &&       \
 417                                     (OPENSSL_s390xcap_P.kmac[0] &   \
 418                                      S390X_CAPBIT(S390X_AES_192)))
 419 #  define S390X_aes_256_ccm_CAPABLE (S390X_aes_256_CAPABLE &&       \
 420                                     (OPENSSL_s390xcap_P.kmac[0] &   \
 421                                      S390X_CAPBIT(S390X_AES_256)))
 422 #  define S390X_CCM_AAD_FLAG    0x40
 423
 424 #  ifndef OPENSSL_NO_OCB
 425 #   define S390X_aes_128_ocb_CAPABLE    0
 426 #   define S390X_aes_192_ocb_CAPABLE    0
 427 #   define S390X_aes_256_ocb_CAPABLE    0
 428 #  endif /* OPENSSL_NO_OCB */
 429
 430 #  ifndef OPENSSL_NO_SIV
 431 #   define S390X_aes_128_siv_CAPABLE    0
 432 #   define S390X_aes_192_siv_CAPABLE    0
 433 #   define S390X_aes_256_siv_CAPABLE    0
 434 #  endif /* OPENSSL_NO_SIV */
 435
 436 /* Convert key size to function code: [16,24,32] -> [18,19,20]. */
 437 #  define S390X_AES_FC(keylen)  (S390X_AES_128 + ((((keylen) << 3) - 128) >> 6))
 438 # elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 64
 439 /* RISC-V 64 support */
 440 #  include "riscv_arch.h"
 441
 442 /* Zkne and Zknd extensions (scalar crypto AES). */
 443 int rv64i_zkne_set_encrypt_key(const unsigned char *userKey, const int bits,
 444                                AES_KEY *key);
 445 int rv64i_zknd_set_decrypt_key(const unsigned char *userKey, const int bits,
 446                                AES_KEY *key);
 447 void rv64i_zkne_encrypt(const unsigned char *in, unsigned char *out,
 448                         const AES_KEY *key);
 449 void rv64i_zknd_decrypt(const unsigned char *in, unsigned char *out,
 450                         const AES_KEY *key);
 451 /* Zvkned extension (vector crypto AES). */
 452 int rv64i_zvkned_set_encrypt_key(const unsigned char *userKey, const int bits,
 453                                  AES_KEY *key);
 454 int rv64i_zvkned_set_decrypt_key(const unsigned char *userKey, const int bits,
 455                                  AES_KEY *key);
 456 void rv64i_zvkned_encrypt(const unsigned char *in, unsigned char *out,
 457                           const AES_KEY *key);
 458 void rv64i_zvkned_decrypt(const unsigned char *in, unsigned char *out,
 459                           const AES_KEY *key);
 460
 461 void rv64i_zvkned_cbc_encrypt(const unsigned char *in, unsigned char *out,
 462                               size_t length, const AES_KEY *key,
 463                               unsigned char *ivec, const int enc);
 464
 465 void rv64i_zvkned_cbc_decrypt(const unsigned char *in, unsigned char *out,
 466                               size_t length, const AES_KEY *key,
 467                               unsigned char *ivec, const int enc);
 468
 469 void rv64i_zvkned_ecb_encrypt(const unsigned char *in, unsigned char *out,
 470                               size_t length, const AES_KEY *key,
 471                               const int enc);
 472
 473 void rv64i_zvkned_ecb_decrypt(const unsigned char *in, unsigned char *out,
 474                               size_t length, const AES_KEY *key,
 475                               const int enc);
 476
 477 void rv64i_zvkb_zvkned_ctr32_encrypt_blocks(const unsigned char *in,
 478                                             unsigned char *out, size_t blocks,
 479                                             const void *key,
 480                                             const unsigned char ivec[16]);
 481
 482 size_t rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt(const unsigned char *in,
 483                                               unsigned char *out, size_t len,
 484                                               const void *key,
 485                                               unsigned char ivec[16], u64 *Xi);
 486
 487 size_t rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt(const unsigned char *in,
 488                                               unsigned char *out, size_t len,
 489                                               const void *key,
 490                                               unsigned char ivec[16], u64 *Xi);
 491
 492 void rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt(const unsigned char *in,
 493                                             unsigned char *out, size_t length,
 494                                             const AES_KEY *key1,
 495                                             const AES_KEY *key2,
 496                                             const unsigned char iv[16]);
 497
 498 void rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt(const unsigned char *in,
 499                                             unsigned char *out, size_t length,
 500                                             const AES_KEY *key1,
 501                                             const AES_KEY *key2,
 502                                             const unsigned char iv[16]);
 503
 504 void gcm_ghash_rv64i_zvkg(u64 Xi[2], const u128 Htable[16], const u8 *inp,
 505                           size_t len);
 506
 507 #define AES_GCM_ENC_BYTES 64
 508 #define AES_GCM_DEC_BYTES 64
 509 #define AES_gcm_encrypt rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt
 510 #define AES_gcm_decrypt rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt
 511 #define AES_GCM_ASM(ctx)                                                       \
 512     (ctx->ctr == rv64i_zvkb_zvkned_ctr32_encrypt_blocks &&                     \
 513      ctx->gcm.funcs.ghash == gcm_ghash_rv64i_zvkg)
 514
 515 # elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 32
 516 /* RISC-V 32 support */
 517 #  include "riscv_arch.h"
 518
 519 int rv32i_zkne_set_encrypt_key(const unsigned char *userKey, const int bits,
 520                                AES_KEY *key);
 521 /* set_decrypt_key needs both zknd and zkne */
 522 int rv32i_zknd_zkne_set_decrypt_key(const unsigned char *userKey, const int bits,
 523                                     AES_KEY *key);
 524 int rv32i_zbkb_zkne_set_encrypt_key(const unsigned char *userKey, const int bits,
 525                                     AES_KEY *key);
 526 int rv32i_zbkb_zknd_zkne_set_decrypt_key(const unsigned char *userKey, const int bits,
 527                                          AES_KEY *key);
 528 void rv32i_zkne_encrypt(const unsigned char *in, unsigned char *out,
 529                         const AES_KEY *key);
 530 void rv32i_zknd_decrypt(const unsigned char *in, unsigned char *out,
 531                         const AES_KEY *key);
 532 # endif
 533
 534 # if defined(HWAES_CAPABLE)
 535 int HWAES_set_encrypt_key(const unsigned char *userKey, const int bits,
 536                           AES_KEY *key);
 537 int HWAES_set_decrypt_key(const unsigned char *userKey, const int bits,
 538                           AES_KEY *key);
 539 void HWAES_encrypt(const unsigned char *in, unsigned char *out,
 540                    const AES_KEY *key);
 541 void HWAES_decrypt(const unsigned char *in, unsigned char *out,
 542                    const AES_KEY *key);
 543 void HWAES_cbc_encrypt(const unsigned char *in, unsigned char *out,
 544                        size_t length, const AES_KEY *key,
 545                        unsigned char *ivec, const int enc);
 546 void HWAES_ecb_encrypt(const unsigned char *in, unsigned char *out,
 547                        size_t length, const AES_KEY *key,
 548                        const int enc);
 549 void HWAES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
 550                                 size_t len, const void *key,
 551                                 const unsigned char ivec[16]);
 552 #  if defined(AES_UNROLL12_EOR3_CAPABLE)
 553 void HWAES_ctr32_encrypt_blocks_unroll12_eor3(const unsigned char *in, unsigned char *out,
 554                                               size_t len, const void *key,
 555                                               const unsigned char ivec[16]);
 556 #  endif
 557 void HWAES_xts_encrypt(const unsigned char *inp, unsigned char *out,
 558                        size_t len, const AES_KEY *key1,
 559                        const AES_KEY *key2, const unsigned char iv[16]);
 560 void HWAES_xts_decrypt(const unsigned char *inp, unsigned char *out,
 561                        size_t len, const AES_KEY *key1,
 562                        const AES_KEY *key2, const unsigned char iv[16]);
 563 #  ifndef OPENSSL_NO_OCB
 564 #   ifdef HWAES_ocb_encrypt
 565 void HWAES_ocb_encrypt(const unsigned char *in, unsigned char *out,
 566                        size_t blocks, const void *key,
 567                        size_t start_block_num,
 568                        unsigned char offset_i[16],
 569                        const unsigned char L_[][16],
 570                        unsigned char checksum[16]);
 571 #   else
 572 #     define HWAES_ocb_encrypt ((ocb128_f)NULL)
 573 #   endif
 574 #   ifdef HWAES_ocb_decrypt
 575 void HWAES_ocb_decrypt(const unsigned char *in, unsigned char *out,
 576                        size_t blocks, const void *key,
 577                        size_t start_block_num,
 578                        unsigned char offset_i[16],
 579                        const unsigned char L_[][16],
 580                        unsigned char checksum[16]);
 581 #   else
 582 #     define HWAES_ocb_decrypt ((ocb128_f)NULL)
 583 #   endif
 584 #  endif /* OPENSSL_NO_OCB */
 585
 586 # endif /* HWAES_CAPABLE */
 587
 588 #endif /* OSSL_AES_PLATFORM_H */