remove end of line spaces
[openssl.git] / crypto / evp / e_aes.c
index 0add393276bccaa4d62a99372d061f2329dd72ef..d687a2cac31eacb6da2269a2aac5857c74743c9f 100644 (file)
@@ -1,27 +1,31 @@
 /*
- * Copyright 2001-2018 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2001-2019 The OpenSSL Project Authors. All Rights Reserved.
  *
- * Licensed under the OpenSSL license (the "License").  You may not use
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
  * in the file LICENSE in the source distribution or at
  * https://www.openssl.org/source/license.html
  */
 
+#include <string.h>
+#include <assert.h>
 #include <openssl/opensslconf.h>
 #include <openssl/crypto.h>
 #include <openssl/evp.h>
 #include <openssl/err.h>
-#include <string.h>
-#include <assert.h>
 #include <openssl/aes.h>
-#include "internal/evp_int.h"
-#include "modes_lcl.h"
 #include <openssl/rand.h>
+#include <openssl/cmac.h>
+#include "internal/evp_int.h"
+#include "internal/cryptlib.h"
+#include "internal/modes_int.h"
+#include "internal/siv_int.h"
+#include "internal/aes_platform.h"
 #include "evp_locl.h"
 
 typedef struct {
     union {
-        double align;
+        OSSL_UNION_ALIGN;
         AES_KEY ks;
     } ks;
     block128_f block;
@@ -33,7 +37,7 @@ typedef struct {
 
 typedef struct {
     union {
-        double align;
+        OSSL_UNION_ALIGN;
         AES_KEY ks;
     } ks;                       /* AES key schedule to use */
     int key_set;                /* Set if key initialised */
@@ -43,13 +47,15 @@ typedef struct {
     int ivlen;                  /* IV length */
     int taglen;
     int iv_gen;                 /* It is OK to generate IVs */
+    int iv_gen_rand;            /* No IV was specified, so generate a rand IV */
     int tls_aad_len;            /* TLS AAD length */
+    uint64_t tls_enc_records;   /* Number of TLS records encrypted */
     ctr128_f ctr;
 } EVP_AES_GCM_CTX;
 
 typedef struct {
     union {
-        double align;
+        OSSL_UNION_ALIGN;
         AES_KEY ks;
     } ks1, ks2;                 /* AES key schedules to use */
     XTS128_CONTEXT xts;
@@ -59,9 +65,15 @@ typedef struct {
                     const unsigned char iv[16]);
 } EVP_AES_XTS_CTX;
 
+#ifdef FIPS_MODE
+static const int allow_insecure_decrypt = 0;
+#else
+static const int allow_insecure_decrypt = 1;
+#endif
+
 typedef struct {
     union {
-        double align;
+        OSSL_UNION_ALIGN;
         AES_KEY ks;
     } ks;                       /* AES key schedule to use */
     int key_set;                /* Set if key initialised */
@@ -77,11 +89,11 @@ typedef struct {
 #ifndef OPENSSL_NO_OCB
 typedef struct {
     union {
-        double align;
+        OSSL_UNION_ALIGN;
         AES_KEY ks;
     } ksenc;                    /* AES key schedule to use for encryption */
     union {
-        double align;
+        OSSL_UNION_ALIGN;
         AES_KEY ks;
     } ksdec;                    /* AES key schedule to use for decryption */
     int key_set;                /* Set if key initialised */
@@ -100,50 +112,6 @@ typedef struct {
 
 #define MAXBITCHUNK     ((size_t)1<<(sizeof(size_t)*8-4))
 
-#ifdef VPAES_ASM
-int vpaes_set_encrypt_key(const unsigned char *userKey, int bits,
-                          AES_KEY *key);
-int vpaes_set_decrypt_key(const unsigned char *userKey, int bits,
-                          AES_KEY *key);
-
-void vpaes_encrypt(const unsigned char *in, unsigned char *out,
-                   const AES_KEY *key);
-void vpaes_decrypt(const unsigned char *in, unsigned char *out,
-                   const AES_KEY *key);
-
-void vpaes_cbc_encrypt(const unsigned char *in,
-                       unsigned char *out,
-                       size_t length,
-                       const AES_KEY *key, unsigned char *ivec, int enc);
-#endif
-#ifdef BSAES_ASM
-void bsaes_cbc_encrypt(const unsigned char *in, unsigned char *out,
-                       size_t length, const AES_KEY *key,
-                       unsigned char ivec[16], int enc);
-void bsaes_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
-                                size_t len, const AES_KEY *key,
-                                const unsigned char ivec[16]);
-void bsaes_xts_encrypt(const unsigned char *inp, unsigned char *out,
-                       size_t len, const AES_KEY *key1,
-                       const AES_KEY *key2, const unsigned char iv[16]);
-void bsaes_xts_decrypt(const unsigned char *inp, unsigned char *out,
-                       size_t len, const AES_KEY *key1,
-                       const AES_KEY *key2, const unsigned char iv[16]);
-#endif
-#ifdef AES_CTR_ASM
-void AES_ctr32_encrypt(const unsigned char *in, unsigned char *out,
-                       size_t blocks, const AES_KEY *key,
-                       const unsigned char ivec[AES_BLOCK_SIZE]);
-#endif
-#ifdef AES_XTS_ASM
-void AES_xts_encrypt(const unsigned char *inp, unsigned char *out, size_t len,
-                     const AES_KEY *key1, const AES_KEY *key2,
-                     const unsigned char iv[16]);
-void AES_xts_decrypt(const unsigned char *inp, unsigned char *out, size_t len,
-                     const AES_KEY *key1, const AES_KEY *key2,
-                     const unsigned char iv[16]);
-#endif
-
 /* increment counter (64-bit int) by 1 */
 static void ctr64_inc(unsigned char *counter)
 {
@@ -160,105 +128,10 @@ static void ctr64_inc(unsigned char *counter)
     } while (n);
 }
 
-#if defined(OPENSSL_CPUID_OBJ) && (defined(__powerpc__) || defined(__ppc__) || defined(_ARCH_PPC))
-# include "ppc_arch.h"
-# ifdef VPAES_ASM
-#  define VPAES_CAPABLE (OPENSSL_ppccap_P & PPC_ALTIVEC)
-# endif
-# define HWAES_CAPABLE  (OPENSSL_ppccap_P & PPC_CRYPTO207)
-# define HWAES_set_encrypt_key aes_p8_set_encrypt_key
-# define HWAES_set_decrypt_key aes_p8_set_decrypt_key
-# define HWAES_encrypt aes_p8_encrypt
-# define HWAES_decrypt aes_p8_decrypt
-# define HWAES_cbc_encrypt aes_p8_cbc_encrypt
-# define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks
-# define HWAES_xts_encrypt aes_p8_xts_encrypt
-# define HWAES_xts_decrypt aes_p8_xts_decrypt
-#endif
-
-#if     defined(AES_ASM) && !defined(I386_ONLY) &&      (  \
-        ((defined(__i386)       || defined(__i386__)    || \
-          defined(_M_IX86)) && defined(OPENSSL_IA32_SSE2))|| \
-        defined(__x86_64)       || defined(__x86_64__)  || \
-        defined(_M_AMD64)       || defined(_M_X64)      )
-
-extern unsigned int OPENSSL_ia32cap_P[];
-
-# ifdef VPAES_ASM
-#  define VPAES_CAPABLE   (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
-# endif
-# ifdef BSAES_ASM
-#  define BSAES_CAPABLE   (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
-# endif
-/*
- * AES-NI section
- */
-# define AESNI_CAPABLE   (OPENSSL_ia32cap_P[1]&(1<<(57-32)))
-
-int aesni_set_encrypt_key(const unsigned char *userKey, int bits,
-                          AES_KEY *key);
-int aesni_set_decrypt_key(const unsigned char *userKey, int bits,
-                          AES_KEY *key);
-
-void aesni_encrypt(const unsigned char *in, unsigned char *out,
-                   const AES_KEY *key);
-void aesni_decrypt(const unsigned char *in, unsigned char *out,
-                   const AES_KEY *key);
-
-void aesni_ecb_encrypt(const unsigned char *in,
-                       unsigned char *out,
-                       size_t length, const AES_KEY *key, int enc);
-void aesni_cbc_encrypt(const unsigned char *in,
-                       unsigned char *out,
-                       size_t length,
-                       const AES_KEY *key, unsigned char *ivec, int enc);
-
-void aesni_ctr32_encrypt_blocks(const unsigned char *in,
-                                unsigned char *out,
-                                size_t blocks,
-                                const void *key, const unsigned char *ivec);
-
-void aesni_xts_encrypt(const unsigned char *in,
-                       unsigned char *out,
-                       size_t length,
-                       const AES_KEY *key1, const AES_KEY *key2,
-                       const unsigned char iv[16]);
-
-void aesni_xts_decrypt(const unsigned char *in,
-                       unsigned char *out,
-                       size_t length,
-                       const AES_KEY *key1, const AES_KEY *key2,
-                       const unsigned char iv[16]);
-
-void aesni_ccm64_encrypt_blocks(const unsigned char *in,
-                                unsigned char *out,
-                                size_t blocks,
-                                const void *key,
-                                const unsigned char ivec[16],
-                                unsigned char cmac[16]);
-
-void aesni_ccm64_decrypt_blocks(const unsigned char *in,
-                                unsigned char *out,
-                                size_t blocks,
-                                const void *key,
-                                const unsigned char ivec[16],
-                                unsigned char cmac[16]);
-
+#if defined(AESNI_CAPABLE)
 # if defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
-size_t aesni_gcm_encrypt(const unsigned char *in,
-                         unsigned char *out,
-                         size_t len,
-                         const void *key, unsigned char ivec[16], u64 *Xi);
 #  define AES_gcm_encrypt aesni_gcm_encrypt
-size_t aesni_gcm_decrypt(const unsigned char *in,
-                         unsigned char *out,
-                         size_t len,
-                         const void *key, unsigned char ivec[16], u64 *Xi);
 #  define AES_gcm_decrypt aesni_gcm_decrypt
-void gcm_ghash_avx(u64 Xi[2], const u128 Htable[16], const u8 *in,
-                   size_t len);
-#  define AES_GCM_ASM(gctx)       (gctx->ctr==aesni_ctr32_encrypt_blocks && \
-                                 gctx->gcm.ghash==gcm_ghash_avx)
 #  define AES_GCM_ASM2(gctx)      (gctx->gcm.block==(block128_f)aesni_encrypt && \
                                  gctx->gcm.ghash==gcm_ghash_avx)
 #  undef AES_GCM_ASM2          /* minor size optimization */
@@ -383,26 +256,39 @@ static int aesni_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
                               const unsigned char *iv, int enc)
 {
     EVP_AES_XTS_CTX *xctx = EVP_C_DATA(EVP_AES_XTS_CTX,ctx);
+
     if (!iv && !key)
         return 1;
 
     if (key) {
+        /* The key is two half length keys in reality */
+        const int bytes = EVP_CIPHER_CTX_key_length(ctx) / 2;
+        const int bits = bytes * 8;
+
+        /*
+         * Verify that the two keys are different.
+         *
+         * This addresses Rogaway's vulnerability.
+         * See comment in aes_xts_init_key() below.
+         */
+        if ((!allow_insecure_decrypt || enc)
+                && CRYPTO_memcmp(key, key + bytes, bytes) == 0) {
+            EVPerr(EVP_F_AESNI_XTS_INIT_KEY, EVP_R_XTS_DUPLICATED_KEYS);
+            return 0;
+        }
+
         /* key_len is two AES keys */
         if (enc) {
-            aesni_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 4,
-                                  &xctx->ks1.ks);
+            aesni_set_encrypt_key(key, bits, &xctx->ks1.ks);
             xctx->xts.block1 = (block128_f) aesni_encrypt;
             xctx->stream = aesni_xts_encrypt;
         } else {
-            aesni_set_decrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 4,
-                                  &xctx->ks1.ks);
+            aesni_set_decrypt_key(key, bits, &xctx->ks1.ks);
             xctx->xts.block1 = (block128_f) aesni_decrypt;
             xctx->stream = aesni_xts_decrypt;
         }
 
-        aesni_set_encrypt_key(key + EVP_CIPHER_CTX_key_length(ctx) / 2,
-                              EVP_CIPHER_CTX_key_length(ctx) * 4,
-                              &xctx->ks2.ks);
+        aesni_set_encrypt_key(key + bytes, bits, &xctx->ks2.ks);
         xctx->xts.block2 = (block128_f) aesni_encrypt;
 
         xctx->xts.key1 = &xctx->ks1;
@@ -447,19 +333,6 @@ static int aesni_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
                             const unsigned char *in, size_t len);
 
 # ifndef OPENSSL_NO_OCB
-void aesni_ocb_encrypt(const unsigned char *in, unsigned char *out,
-                       size_t blocks, const void *key,
-                       size_t start_block_num,
-                       unsigned char offset_i[16],
-                       const unsigned char L_[][16],
-                       unsigned char checksum[16]);
-void aesni_ocb_decrypt(const unsigned char *in, unsigned char *out,
-                       size_t blocks, const void *key,
-                       size_t start_block_num,
-                       unsigned char offset_i[16],
-                       const unsigned char L_[][16],
-                       unsigned char checksum[16]);
-
 static int aesni_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
                               const unsigned char *iv, int enc)
 {
@@ -539,7 +412,8 @@ const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
 # define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \
 static const EVP_CIPHER aesni_##keylen##_##mode = { \
         nid##_##keylen##_##mode,blocksize, \
-        (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \
+        (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE||EVP_CIPH_##MODE##_MODE==EVP_CIPH_SIV_MODE?2:1)*keylen/8, \
+        ivlen,                          \
         flags|EVP_CIPH_##MODE##_MODE,   \
         aesni_##mode##_init_key,        \
         aesni_##mode##_cipher,          \
@@ -548,7 +422,8 @@ static const EVP_CIPHER aesni_##keylen##_##mode = { \
         NULL,NULL,aes_##mode##_ctrl,NULL }; \
 static const EVP_CIPHER aes_##keylen##_##mode = { \
         nid##_##keylen##_##mode,blocksize, \
-        (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \
+        (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE||EVP_CIPH_##MODE##_MODE==EVP_CIPH_SIV_MODE?2:1)*keylen/8, \
+        ivlen,                          \
         flags|EVP_CIPH_##MODE##_MODE,   \
         aes_##mode##_init_key,          \
         aes_##mode##_cipher,            \
@@ -558,81 +433,7 @@ static const EVP_CIPHER aes_##keylen##_##mode = { \
 const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
 { return AESNI_CAPABLE?&aesni_##keylen##_##mode:&aes_##keylen##_##mode; }
 
-#elif   defined(AES_ASM) && (defined(__sparc) || defined(__sparc__))
-
-# include "sparc_arch.h"
-
-extern unsigned int OPENSSL_sparcv9cap_P[];
-
-/*
- * Initial Fujitsu SPARC64 X support
- */
-# define HWAES_CAPABLE           (OPENSSL_sparcv9cap_P[0] & SPARCV9_FJAESX)
-# define HWAES_set_encrypt_key aes_fx_set_encrypt_key
-# define HWAES_set_decrypt_key aes_fx_set_decrypt_key
-# define HWAES_encrypt aes_fx_encrypt
-# define HWAES_decrypt aes_fx_decrypt
-# define HWAES_cbc_encrypt aes_fx_cbc_encrypt
-# define HWAES_ctr32_encrypt_blocks aes_fx_ctr32_encrypt_blocks
-
-# define SPARC_AES_CAPABLE       (OPENSSL_sparcv9cap_P[1] & CFR_AES)
-
-void aes_t4_set_encrypt_key(const unsigned char *key, int bits, AES_KEY *ks);
-void aes_t4_set_decrypt_key(const unsigned char *key, int bits, AES_KEY *ks);
-void aes_t4_encrypt(const unsigned char *in, unsigned char *out,
-                    const AES_KEY *key);
-void aes_t4_decrypt(const unsigned char *in, unsigned char *out,
-                    const AES_KEY *key);
-/*
- * Key-length specific subroutines were chosen for following reason.
- * Each SPARC T4 core can execute up to 8 threads which share core's
- * resources. Loading as much key material to registers allows to
- * minimize references to shared memory interface, as well as amount
- * of instructions in inner loops [much needed on T4]. But then having
- * non-key-length specific routines would require conditional branches
- * either in inner loops or on subroutines' entries. Former is hardly
- * acceptable, while latter means code size increase to size occupied
- * by multiple key-length specific subroutines, so why fight?
- */
-void aes128_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
-                           size_t len, const AES_KEY *key,
-                           unsigned char *ivec);
-void aes128_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
-                           size_t len, const AES_KEY *key,
-                           unsigned char *ivec);
-void aes192_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
-                           size_t len, const AES_KEY *key,
-                           unsigned char *ivec);
-void aes192_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
-                           size_t len, const AES_KEY *key,
-                           unsigned char *ivec);
-void aes256_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
-                           size_t len, const AES_KEY *key,
-                           unsigned char *ivec);
-void aes256_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
-                           size_t len, const AES_KEY *key,
-                           unsigned char *ivec);
-void aes128_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
-                             size_t blocks, const AES_KEY *key,
-                             unsigned char *ivec);
-void aes192_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
-                             size_t blocks, const AES_KEY *key,
-                             unsigned char *ivec);
-void aes256_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
-                             size_t blocks, const AES_KEY *key,
-                             unsigned char *ivec);
-void aes128_t4_xts_encrypt(const unsigned char *in, unsigned char *out,
-                           size_t blocks, const AES_KEY *key1,
-                           const AES_KEY *key2, const unsigned char *ivec);
-void aes128_t4_xts_decrypt(const unsigned char *in, unsigned char *out,
-                           size_t blocks, const AES_KEY *key1,
-                           const AES_KEY *key2, const unsigned char *ivec);
-void aes256_t4_xts_encrypt(const unsigned char *in, unsigned char *out,
-                           size_t blocks, const AES_KEY *key1,
-                           const AES_KEY *key2, const unsigned char *ivec);
-void aes256_t4_xts_decrypt(const unsigned char *in, unsigned char *out,
-                           size_t blocks, const AES_KEY *key1,
-                           const AES_KEY *key2, const unsigned char *ivec);
+#elif defined(SPARC_AES_CAPABLE)
 
 static int aes_t4_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
                            const unsigned char *iv, int enc)
@@ -787,11 +588,27 @@ static int aes_t4_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
                                const unsigned char *iv, int enc)
 {
     EVP_AES_XTS_CTX *xctx = EVP_C_DATA(EVP_AES_XTS_CTX,ctx);
+
     if (!iv && !key)
         return 1;
 
     if (key) {
-        int bits = EVP_CIPHER_CTX_key_length(ctx) * 4;
+        /* The key is two half length keys in reality */
+        const int bytes = EVP_CIPHER_CTX_key_length(ctx) / 2;
+        const int bits = bytes * 8;
+
+        /*
+         * Verify that the two keys are different.
+         *
+         * This addresses Rogaway's vulnerability.
+         * See comment in aes_xts_init_key() below.
+         */
+        if ((!allow_insecure_decrypt || enc)
+                && CRYPTO_memcmp(key, key + bytes, bytes) == 0) {
+            EVPerr(EVP_F_AES_T4_XTS_INIT_KEY, EVP_R_XTS_DUPLICATED_KEYS);
+            return 0;
+        }
+
         xctx->stream = NULL;
         /* key_len is two AES keys */
         if (enc) {
@@ -808,8 +625,7 @@ static int aes_t4_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
                 return 0;
             }
         } else {
-            aes_t4_set_decrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 4,
-                                   &xctx->ks1.ks);
+            aes_t4_set_decrypt_key(key, bits, &xctx->ks1.ks);
             xctx->xts.block1 = (block128_f) aes_t4_decrypt;
             switch (bits) {
             case 128:
@@ -823,9 +639,7 @@ static int aes_t4_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
             }
         }
 
-        aes_t4_set_encrypt_key(key + EVP_CIPHER_CTX_key_length(ctx) / 2,
-                               EVP_CIPHER_CTX_key_length(ctx) * 4,
-                               &xctx->ks2.ks);
+        aes_t4_set_encrypt_key(key + bytes, bits, &xctx->ks2.ks);
         xctx->xts.block2 = (block128_f) aes_t4_encrypt;
 
         xctx->xts.key1 = &xctx->ks1;
@@ -923,6 +737,11 @@ static int aes_t4_ocb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
                              const unsigned char *in, size_t len);
 # endif                        /* OPENSSL_NO_OCB */
 
+# ifndef OPENSSL_NO_SIV
+#  define aes_t4_siv_init_key aes_siv_init_key
+#  define aes_t4_siv_cipher aes_siv_cipher
+# endif /* OPENSSL_NO_SIV */
+
 # define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \
 static const EVP_CIPHER aes_t4_##keylen##_##mode = { \
         nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \
@@ -947,7 +766,8 @@ const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
 # define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \
 static const EVP_CIPHER aes_t4_##keylen##_##mode = { \
         nid##_##keylen##_##mode,blocksize, \
-        (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \
+        (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE||EVP_CIPH_##MODE##_MODE==EVP_CIPH_SIV_MODE?2:1)*keylen/8, \
+        ivlen,                          \
         flags|EVP_CIPH_##MODE##_MODE,   \
         aes_t4_##mode##_init_key,       \
         aes_t4_##mode##_cipher,         \
@@ -956,7 +776,8 @@ static const EVP_CIPHER aes_t4_##keylen##_##mode = { \
         NULL,NULL,aes_##mode##_ctrl,NULL }; \
 static const EVP_CIPHER aes_##keylen##_##mode = { \
         nid##_##keylen##_##mode,blocksize, \
-        (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \
+        (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE||EVP_CIPH_##MODE##_MODE==EVP_CIPH_SIV_MODE?2:1)*keylen/8, \
+        ivlen,                          \
         flags|EVP_CIPH_##MODE##_MODE,   \
         aes_##mode##_init_key,          \
         aes_##mode##_cipher,            \
@@ -966,15 +787,11 @@ static const EVP_CIPHER aes_##keylen##_##mode = { \
 const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
 { return SPARC_AES_CAPABLE?&aes_t4_##keylen##_##mode:&aes_##keylen##_##mode; }
 
-#elif defined(OPENSSL_CPUID_OBJ) && defined(__s390__)
-/*
- * IBM S390X support
- */
-# include "s390x_arch.h"
-
+#elif defined(S390X_aes_128_CAPABLE)
+/* IBM S390X support */
 typedef struct {
     union {
-        double align;
+        OSSL_UNION_ALIGN;
         /*-
          * KM-AES parameter block - begin
          * (see z/Architecture Principles of Operation >= SA22-7832-06)
@@ -989,7 +806,7 @@ typedef struct {
 
 typedef struct {
     union {
-        double align;
+        OSSL_UNION_ALIGN;
         /*-
          * KMO-AES parameter block - begin
          * (see z/Architecture Principles of Operation >= SA22-7832-08)
@@ -1007,7 +824,7 @@ typedef struct {
 
 typedef struct {
     union {
-        double align;
+        OSSL_UNION_ALIGN;
         /*-
          * KMF-AES parameter block - begin
          * (see z/Architecture Principles of Operation >= SA22-7832-08)
@@ -1025,7 +842,7 @@ typedef struct {
 
 typedef struct {
     union {
-        double align;
+        OSSL_UNION_ALIGN;
         /*-
          * KMA-GCM-AES parameter block - begin
          * (see z/Architecture Principles of Operation >= SA22-7832-11)
@@ -1069,11 +886,12 @@ typedef struct {
     int kreslen;
 
     int tls_aad_len;
+    uint64_t tls_enc_records;   /* Number of TLS records encrypted */
 } S390X_AES_GCM_CTX;
 
 typedef struct {
     union {
-        double align;
+        OSSL_UNION_ALIGN;
         /*-
          * Padding is chosen so that ccm.kmac_param.k overlaps with key.k and
          * ccm.fc with key.k.rounds. Remember that on s390x, an AES_KEY's
@@ -1123,25 +941,11 @@ typedef struct {
     } aes;
 } S390X_AES_CCM_CTX;
 
-/* Convert key size to function code: [16,24,32] -> [18,19,20]. */
-# define S390X_AES_FC(keylen)  (S390X_AES_128 + ((((keylen) << 3) - 128) >> 6))
-
-/* Most modes of operation need km for partial block processing. */
-# define S390X_aes_128_CAPABLE (OPENSSL_s390xcap_P.km[0] &     \
-                                S390X_CAPBIT(S390X_AES_128))
-# define S390X_aes_192_CAPABLE (OPENSSL_s390xcap_P.km[0] &     \
-                                S390X_CAPBIT(S390X_AES_192))
-# define S390X_aes_256_CAPABLE (OPENSSL_s390xcap_P.km[0] &     \
-                                S390X_CAPBIT(S390X_AES_256))
-
 # define s390x_aes_init_key aes_init_key
 static int s390x_aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
                               const unsigned char *iv, int enc);
 
-# define S390X_aes_128_cbc_CAPABLE     1       /* checked by callee */
-# define S390X_aes_192_cbc_CAPABLE     1
-# define S390X_aes_256_cbc_CAPABLE     1
-# define S390X_AES_CBC_CTX             EVP_AES_KEY
+# define S390X_AES_CBC_CTX              EVP_AES_KEY
 
 # define s390x_aes_cbc_init_key aes_init_key
 
@@ -1149,10 +953,6 @@ static int s390x_aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 static int s390x_aes_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
                                 const unsigned char *in, size_t len);
 
-# define S390X_aes_128_ecb_CAPABLE     S390X_aes_128_CAPABLE
-# define S390X_aes_192_ecb_CAPABLE     S390X_aes_192_CAPABLE
-# define S390X_aes_256_ecb_CAPABLE     S390X_aes_256_CAPABLE
-
 static int s390x_aes_ecb_init_key(EVP_CIPHER_CTX *ctx,
                                   const unsigned char *key,
                                   const unsigned char *iv, int enc)
@@ -1177,16 +977,6 @@ static int s390x_aes_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     return 1;
 }
 
-# define S390X_aes_128_ofb_CAPABLE (S390X_aes_128_CAPABLE &&           \
-                                    (OPENSSL_s390xcap_P.kmo[0] &       \
-                                     S390X_CAPBIT(S390X_AES_128)))
-# define S390X_aes_192_ofb_CAPABLE (S390X_aes_192_CAPABLE &&           \
-                                    (OPENSSL_s390xcap_P.kmo[0] &       \
-                                     S390X_CAPBIT(S390X_AES_192)))
-# define S390X_aes_256_ofb_CAPABLE (S390X_aes_256_CAPABLE &&           \
-                                    (OPENSSL_s390xcap_P.kmo[0] &       \
-                                     S390X_CAPBIT(S390X_AES_256)))
-
 static int s390x_aes_ofb_init_key(EVP_CIPHER_CTX *ctx,
                                   const unsigned char *key,
                                   const unsigned char *ivec, int enc)
@@ -1242,16 +1032,6 @@ static int s390x_aes_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     return 1;
 }
 
-# define S390X_aes_128_cfb_CAPABLE (S390X_aes_128_CAPABLE &&           \
-                                    (OPENSSL_s390xcap_P.kmf[0] &       \
-                                     S390X_CAPBIT(S390X_AES_128)))
-# define S390X_aes_192_cfb_CAPABLE (S390X_aes_192_CAPABLE &&           \
-                                    (OPENSSL_s390xcap_P.kmf[0] &       \
-                                     S390X_CAPBIT(S390X_AES_192)))
-# define S390X_aes_256_cfb_CAPABLE (S390X_aes_256_CAPABLE &&           \
-                                    (OPENSSL_s390xcap_P.kmf[0] &       \
-                                     S390X_CAPBIT(S390X_AES_256)))
-
 static int s390x_aes_cfb_init_key(EVP_CIPHER_CTX *ctx,
                                   const unsigned char *key,
                                   const unsigned char *ivec, int enc)
@@ -1318,13 +1098,6 @@ static int s390x_aes_cfb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     return 1;
 }
 
-# define S390X_aes_128_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] &       \
-                                     S390X_CAPBIT(S390X_AES_128))
-# define S390X_aes_192_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] &       \
-                                     S390X_CAPBIT(S390X_AES_192))
-# define S390X_aes_256_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] &       \
-                                     S390X_CAPBIT(S390X_AES_256))
-
 static int s390x_aes_cfb8_init_key(EVP_CIPHER_CTX *ctx,
                                    const unsigned char *key,
                                    const unsigned char *ivec, int enc)
@@ -1353,20 +1126,13 @@ static int s390x_aes_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     return 1;
 }
 
-# define S390X_aes_128_cfb1_CAPABLE    0
-# define S390X_aes_192_cfb1_CAPABLE    0
-# define S390X_aes_256_cfb1_CAPABLE    0
-
 # define s390x_aes_cfb1_init_key aes_init_key
 
 # define s390x_aes_cfb1_cipher aes_cfb1_cipher
 static int s390x_aes_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
                                  const unsigned char *in, size_t len);
 
-# define S390X_aes_128_ctr_CAPABLE     1       /* checked by callee */
-# define S390X_aes_192_ctr_CAPABLE     1
-# define S390X_aes_256_ctr_CAPABLE     1
-# define S390X_AES_CTR_CTX             EVP_AES_KEY
+# define S390X_AES_CTR_CTX              EVP_AES_KEY
 
 # define s390x_aes_ctr_init_key aes_init_key
 
@@ -1374,18 +1140,8 @@ static int s390x_aes_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 static int s390x_aes_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
                                 const unsigned char *in, size_t len);
 
-# define S390X_aes_128_gcm_CAPABLE (S390X_aes_128_CAPABLE &&           \
-                                    (OPENSSL_s390xcap_P.kma[0] &       \
-                                     S390X_CAPBIT(S390X_AES_128)))
-# define S390X_aes_192_gcm_CAPABLE (S390X_aes_192_CAPABLE &&           \
-                                    (OPENSSL_s390xcap_P.kma[0] &       \
-                                     S390X_CAPBIT(S390X_AES_192)))
-# define S390X_aes_256_gcm_CAPABLE (S390X_aes_256_CAPABLE &&           \
-                                    (OPENSSL_s390xcap_P.kma[0] &       \
-                                     S390X_CAPBIT(S390X_AES_256)))
-
-/* iv + padding length for iv lenghts != 12 */
-# define S390X_gcm_ivpadlen(i) ((((i) + 15) >> 4 << 4) + 16)
+/* iv + padding length for iv lengths != 12 */
+# define S390X_gcm_ivpadlen(i)  ((((i) + 15) >> 4 << 4) + 16)
 
 /*-
  * Process additional authenticated data. Returns 0 on success. Code is
@@ -1692,6 +1448,7 @@ static int s390x_aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
         buf = EVP_CIPHER_CTX_buf_noconst(c);
         memcpy(buf, ptr, arg);
         gctx->tls_aad_len = arg;
+        gctx->tls_enc_records = 0;
 
         len = buf[arg - 2] << 8 | buf[arg - 1];
         /* Correct length for explicit iv. */
@@ -1791,6 +1548,17 @@ static int s390x_aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     if (out != in || len < (EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN))
         return -1;
 
+    /*
+     * Check for too many keys as per FIPS 140-2 IG A.5 "Key/IV Pair Uniqueness
+     * Requirements from SP 800-38D".  The requirements is for one party to the
+     * communication to fail after 2^64 - 1 keys.  We do this on the encrypting
+     * side only.
+     */
+    if (ctx->encrypt && ++gctx->tls_enc_records == 0) {
+        EVPerr(EVP_F_S390X_AES_GCM_TLS_CIPHER, EVP_R_TOO_MANY_RECORDS);
+        goto err;
+    }
+
     if (EVP_CIPHER_CTX_ctrl(ctx, enc ? EVP_CTRL_GCM_IV_GEN
                                      : EVP_CTRL_GCM_SET_IV_INV,
                             EVP_GCM_TLS_EXPLICIT_IV_LEN, out) <= 0)
@@ -1894,9 +1662,7 @@ static int s390x_aes_gcm_cleanup(EVP_CIPHER_CTX *c)
     return 1;
 }
 
-# define S390X_AES_XTS_CTX             EVP_AES_XTS_CTX
-# define S390X_aes_128_xts_CAPABLE     1       /* checked by callee */
-# define S390X_aes_256_xts_CAPABLE     1
+# define S390X_AES_XTS_CTX              EVP_AES_XTS_CTX
 
 # define s390x_aes_xts_init_key aes_xts_init_key
 static int s390x_aes_xts_init_key(EVP_CIPHER_CTX *ctx,
@@ -1909,18 +1675,6 @@ static int s390x_aes_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 static int s390x_aes_xts_ctrl(EVP_CIPHER_CTX *, int type, int arg, void *ptr);
 # define s390x_aes_xts_cleanup aes_xts_cleanup
 
-# define S390X_aes_128_ccm_CAPABLE (S390X_aes_128_CAPABLE &&           \
-                                    (OPENSSL_s390xcap_P.kmac[0] &      \
-                                     S390X_CAPBIT(S390X_AES_128)))
-# define S390X_aes_192_ccm_CAPABLE (S390X_aes_192_CAPABLE &&           \
-                                    (OPENSSL_s390xcap_P.kmac[0] &      \
-                                     S390X_CAPBIT(S390X_AES_192)))
-# define S390X_aes_256_ccm_CAPABLE (S390X_aes_256_CAPABLE &&           \
-                                    (OPENSSL_s390xcap_P.kmac[0] &      \
-                                     S390X_CAPBIT(S390X_AES_256)))
-
-# define S390X_CCM_AAD_FLAG    0x40
-
 /*-
  * Set nonce and length fields. Code is big-endian.
  */
@@ -2033,13 +1787,13 @@ static int s390x_aes_ccm(S390X_AES_CCM_CTX *ctx, const unsigned char *in,
     ctx->aes.ccm.nonce.b[15] = 1;
 
     if (n != len)
-        return -1;             /* length mismatch */
+        return -1;              /* length mismatch */
 
     if (enc) {
         /* Two operations per block plus one for tag encryption */
         ctx->aes.ccm.blocks += (((len + 15) >> 4) << 1) + 1;
         if (ctx->aes.ccm.blocks > (1ULL << 61))
-            return -2;         /* too much data */
+            return -2;          /* too much data */
     }
 
     num = 0;
@@ -2088,7 +1842,7 @@ static int s390x_aes_ccm(S390X_AES_CCM_CTX *ctx, const unsigned char *in,
     ctx->aes.ccm.kmac_param.icv.g[0] ^= ctx->aes.ccm.buf.g[0];
     ctx->aes.ccm.kmac_param.icv.g[1] ^= ctx->aes.ccm.buf.g[1];
 
-    ctx->aes.ccm.nonce.b[0] = flags;   /* restore flags field */
+    ctx->aes.ccm.nonce.b[0] = flags;    /* restore flags field */
     return 0;
 }
 
@@ -2216,9 +1970,6 @@ static int s390x_aes_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     if (!cctx->aes.ccm.iv_set)
         return -1;
 
-    if (!enc && !cctx->aes.ccm.tag_set)
-        return -1;
-
     if (out == NULL) {
         /* Update(): Pass message length. */
         if (in == NULL) {
@@ -2237,11 +1988,15 @@ static int s390x_aes_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
         return len;
     }
 
+    /* The tag must be set before actually decrypting data */
+    if (!enc && !cctx->aes.ccm.tag_set)
+        return -1;
+
     /* Update(): Process message. */
 
     if (!cctx->aes.ccm.len_set) {
         /*-
-         * In case message length was not previously set explicitely via
+         * In case message length was not previously set explicitly via
          * Update(), set it now.
          */
         ivec = EVP_CIPHER_CTX_iv_noconst(ctx);
@@ -2391,10 +2146,7 @@ static int s390x_aes_ccm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
 # define s390x_aes_ccm_cleanup aes_ccm_cleanup
 
 # ifndef OPENSSL_NO_OCB
-#  define S390X_AES_OCB_CTX            EVP_AES_OCB_CTX
-#  define S390X_aes_128_ocb_CAPABLE    0
-#  define S390X_aes_192_ocb_CAPABLE    0
-#  define S390X_aes_256_ocb_CAPABLE    0
+#  define S390X_AES_OCB_CTX             EVP_AES_OCB_CTX
 
 #  define s390x_aes_ocb_init_key aes_ocb_init_key
 static int s390x_aes_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
@@ -2408,77 +2160,86 @@ static int s390x_aes_ocb_cleanup(EVP_CIPHER_CTX *);
 static int s390x_aes_ocb_ctrl(EVP_CIPHER_CTX *, int type, int arg, void *ptr);
 # endif
 
-# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,   \
-                              MODE,flags)                              \
-static const EVP_CIPHER s390x_aes_##keylen##_##mode = {                        \
-    nid##_##keylen##_##nmode,blocksize,                                        \
-    keylen / 8,                                                                \
-    ivlen,                                                             \
-    flags | EVP_CIPH_##MODE##_MODE,                                    \
-    s390x_aes_##mode##_init_key,                                       \
-    s390x_aes_##mode##_cipher,                                         \
-    NULL,                                                              \
-    sizeof(S390X_AES_##MODE##_CTX),                                    \
-    NULL,                                                              \
-    NULL,                                                              \
-    NULL,                                                              \
-    NULL                                                               \
-};                                                                     \
-static const EVP_CIPHER aes_##keylen##_##mode = {                      \
-    nid##_##keylen##_##nmode,                                          \
-    blocksize,                                                         \
-    keylen / 8,                                                                \
-    ivlen,                                                             \
-    flags | EVP_CIPH_##MODE##_MODE,                                    \
-    aes_init_key,                                                      \
-    aes_##mode##_cipher,                                               \
-    NULL,                                                              \
-    sizeof(EVP_AES_KEY),                                               \
-    NULL,                                                              \
-    NULL,                                                              \
-    NULL,                                                              \
-    NULL                                                               \
-};                                                                     \
-const EVP_CIPHER *EVP_aes_##keylen##_##mode(void)                      \
-{                                                                      \
-    return S390X_aes_##keylen##_##mode##_CAPABLE ?                     \
-           &s390x_aes_##keylen##_##mode : &aes_##keylen##_##mode;      \
+# ifndef OPENSSL_NO_SIV
+#  define S390X_AES_SIV_CTX             EVP_AES_SIV_CTX
+
+#  define s390x_aes_siv_init_key aes_siv_init_key
+#  define s390x_aes_siv_cipher aes_siv_cipher
+#  define s390x_aes_siv_cleanup aes_siv_cleanup
+#  define s390x_aes_siv_ctrl aes_siv_ctrl
+# endif
+
+# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,    \
+                              MODE,flags)                               \
+static const EVP_CIPHER s390x_aes_##keylen##_##mode = {                 \
+    nid##_##keylen##_##nmode,blocksize,                                 \
+    keylen / 8,                                                         \
+    ivlen,                                                              \
+    flags | EVP_CIPH_##MODE##_MODE,                                     \
+    s390x_aes_##mode##_init_key,                                        \
+    s390x_aes_##mode##_cipher,                                          \
+    NULL,                                                               \
+    sizeof(S390X_AES_##MODE##_CTX),                                     \
+    NULL,                                                               \
+    NULL,                                                               \
+    NULL,                                                               \
+    NULL                                                                \
+};                                                                      \
+static const EVP_CIPHER aes_##keylen##_##mode = {                       \
+    nid##_##keylen##_##nmode,                                           \
+    blocksize,                                                          \
+    keylen / 8,                                                         \
+    ivlen,                                                              \
+    flags | EVP_CIPH_##MODE##_MODE,                                     \
+    aes_init_key,                                                       \
+    aes_##mode##_cipher,                                                \
+    NULL,                                                               \
+    sizeof(EVP_AES_KEY),                                                \
+    NULL,                                                               \
+    NULL,                                                               \
+    NULL,                                                               \
+    NULL                                                                \
+};                                                                      \
+const EVP_CIPHER *EVP_aes_##keylen##_##mode(void)                       \
+{                                                                       \
+    return S390X_aes_##keylen##_##mode##_CAPABLE ?                      \
+           &s390x_aes_##keylen##_##mode : &aes_##keylen##_##mode;       \
 }
 
 # define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags)\
-static const EVP_CIPHER s390x_aes_##keylen##_##mode = {                        \
-    nid##_##keylen##_##mode,                                           \
-    blocksize,                                                         \
-    (EVP_CIPH_##MODE##_MODE == EVP_CIPH_XTS_MODE ? 2 : 1) * keylen / 8,        \
-    ivlen,                                                             \
-    flags | EVP_CIPH_##MODE##_MODE,                                    \
-    s390x_aes_##mode##_init_key,                                       \
-    s390x_aes_##mode##_cipher,                                         \
-    s390x_aes_##mode##_cleanup,                                                \
-    sizeof(S390X_AES_##MODE##_CTX),                                    \
-    NULL,                                                              \
-    NULL,                                                              \
-    s390x_aes_##mode##_ctrl,                                           \
-    NULL                                                               \
-};                                                                     \
-static const EVP_CIPHER aes_##keylen##_##mode = {                      \
-    nid##_##keylen##_##mode,blocksize,                                 \
-    (EVP_CIPH_##MODE##_MODE == EVP_CIPH_XTS_MODE ? 2 : 1) * keylen / 8,        \
-    ivlen,                                                             \
-    flags | EVP_CIPH_##MODE##_MODE,                                    \
-    aes_##mode##_init_key,                                             \
-    aes_##mode##_cipher,                                               \
-    aes_##mode##_cleanup,                                              \
-    sizeof(EVP_AES_##MODE##_CTX),                                      \
-    NULL,                                                              \
-    NULL,                                                              \
-    aes_##mode##_ctrl,                                                 \
-    NULL                                                               \
-};                                                                     \
-const EVP_CIPHER *EVP_aes_##keylen##_##mode(void)                      \
-{                                                                      \
-    return S390X_aes_##keylen##_##mode##_CAPABLE ?                     \
-           &s390x_aes_##keylen##_##mode : &aes_##keylen##_##mode;      \
+static const EVP_CIPHER s390x_aes_##keylen##_##mode = {                 \
+    nid##_##keylen##_##mode,                                            \
+    blocksize,                                                          \
+    (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE||EVP_CIPH_##MODE##_MODE==EVP_CIPH_SIV_MODE ? 2 : 1) * keylen / 8,        \
+    ivlen,                                                              \
+    flags | EVP_CIPH_##MODE##_MODE,                                     \
+    s390x_aes_##mode##_init_key,                                        \
+    s390x_aes_##mode##_cipher,                                          \
+    s390x_aes_##mode##_cleanup,                                         \
+    sizeof(S390X_AES_##MODE##_CTX),                                     \
+    NULL,                                                               \
+    NULL,                                                               \
+    s390x_aes_##mode##_ctrl,                                            \
+    NULL                                                                \
+};                                                                      \
+static const EVP_CIPHER aes_##keylen##_##mode = {                       \
+    nid##_##keylen##_##mode,blocksize,                                  \
+    (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE||EVP_CIPH_##MODE##_MODE==EVP_CIPH_SIV_MODE ? 2 : 1) * keylen / 8,        \
+    ivlen,                                                              \
+    flags | EVP_CIPH_##MODE##_MODE,                                     \
+    aes_##mode##_init_key,                                              \
+    aes_##mode##_cipher,                                                \
+    aes_##mode##_cleanup,                                               \
+    sizeof(EVP_AES_##MODE##_CTX),                                       \
+    NULL,                                                               \
+    NULL,                                                               \
+    aes_##mode##_ctrl,                                                  \
+    NULL                                                                \
+};                                                                      \
+const EVP_CIPHER *EVP_aes_##keylen##_##mode(void)                       \
+{                                                                       \
+    return S390X_aes_##keylen##_##mode##_CAPABLE ?                      \
+           &s390x_aes_##keylen##_##mode : &aes_##keylen##_##mode;       \
 }
 
 #else
@@ -2498,7 +2259,8 @@ const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
 # define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \
 static const EVP_CIPHER aes_##keylen##_##mode = { \
         nid##_##keylen##_##mode,blocksize, \
-        (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \
+        (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE||EVP_CIPH_##MODE##_MODE==EVP_CIPH_SIV_MODE?2:1)*keylen/8, \
+        ivlen,                          \
         flags|EVP_CIPH_##MODE##_MODE,   \
         aes_##mode##_init_key,          \
         aes_##mode##_cipher,            \
@@ -2510,48 +2272,6 @@ const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
 
 #endif
 
-#if defined(OPENSSL_CPUID_OBJ) && (defined(__arm__) || defined(__arm) || defined(__aarch64__))
-# include "arm_arch.h"
-# if __ARM_MAX_ARCH__>=7
-#  if defined(BSAES_ASM)
-#   define BSAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
-#  endif
-#  if defined(VPAES_ASM)
-#   define VPAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
-#  endif
-#  define HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES)
-#  define HWAES_set_encrypt_key aes_v8_set_encrypt_key
-#  define HWAES_set_decrypt_key aes_v8_set_decrypt_key
-#  define HWAES_encrypt aes_v8_encrypt
-#  define HWAES_decrypt aes_v8_decrypt
-#  define HWAES_cbc_encrypt aes_v8_cbc_encrypt
-#  define HWAES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks
-# endif
-#endif
-
-#if defined(HWAES_CAPABLE)
-int HWAES_set_encrypt_key(const unsigned char *userKey, const int bits,
-                          AES_KEY *key);
-int HWAES_set_decrypt_key(const unsigned char *userKey, const int bits,
-                          AES_KEY *key);
-void HWAES_encrypt(const unsigned char *in, unsigned char *out,
-                   const AES_KEY *key);
-void HWAES_decrypt(const unsigned char *in, unsigned char *out,
-                   const AES_KEY *key);
-void HWAES_cbc_encrypt(const unsigned char *in, unsigned char *out,
-                       size_t length, const AES_KEY *key,
-                       unsigned char *ivec, const int enc);
-void HWAES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
-                                size_t len, const AES_KEY *key,
-                                const unsigned char ivec[16]);
-void HWAES_xts_encrypt(const unsigned char *inp, unsigned char *out,
-                       size_t len, const AES_KEY *key1,
-                       const AES_KEY *key2, const unsigned char iv[16]);
-void HWAES_xts_decrypt(const unsigned char *inp, unsigned char *out,
-                       size_t len, const AES_KEY *key1,
-                       const AES_KEY *key2, const unsigned char iv[16]);
-#endif
-
 #define BLOCK_CIPHER_generic_pack(nid,keylen,flags)             \
         BLOCK_CIPHER_generic(nid,keylen,16,16,cbc,cbc,CBC,flags|EVP_CIPH_FLAG_DEFAULT_ASN1)     \
         BLOCK_CIPHER_generic(nid,keylen,16,0,ecb,ecb,ECB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1)      \
@@ -2852,6 +2572,14 @@ static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
         memcpy(ptr, c->buf, arg);
         return 1;
 
+    case EVP_CTRL_GET_IV:
+        if (gctx->iv_gen != 1 && gctx->iv_gen_rand != 1)
+            return 0;
+        if (gctx->ivlen != arg)
+            return 0;
+        memcpy(ptr, gctx->iv, arg);
+        return 1;
+
     case EVP_CTRL_GCM_SET_IV_FIXED:
         /* Special case: -1 length restores whole IV */
         if (arg == -1) {
@@ -2901,6 +2629,7 @@ static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
             return 0;
         memcpy(c->buf, ptr, arg);
         gctx->tls_aad_len = arg;
+        gctx->tls_enc_records = 0;
         {
             unsigned int len = c->buf[arg - 2] << 8 | c->buf[arg - 1];
             /* Correct length for explicit IV */
@@ -3035,6 +2764,18 @@ static int aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     if (out != in
         || len < (EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN))
         return -1;
+
+    /*
+     * Check for too many keys as per FIPS 140-2 IG A.5 "Key/IV Pair Uniqueness
+     * Requirements from SP 800-38D".  The requirements is for one party to the
+     * communication to fail after 2^64 - 1 keys.  We do this on the encrypting
+     * side only.
+     */
+    if (ctx->encrypt && ++gctx->tls_enc_records == 0) {
+        EVPerr(EVP_F_AES_GCM_TLS_CIPHER, EVP_R_TOO_MANY_RECORDS);
+        goto err;
+    }
+
     /*
      * Set IV from start of buffer or generate IV and write to start of
      * buffer.
@@ -3144,10 +2885,35 @@ static int aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     return rv;
 }
 
+#ifdef FIPS_MODE
+/*
+ * See SP800-38D (GCM) Section 8 "Uniqueness requirement on IVS and keys"
+ *
+ * See also 8.2.2 RBG-based construction.
+ * Random construction consists of a free field (which can be NULL) and a
+ * random field which will use a DRBG that can return at least 96 bits of
+ * entropy strength. (The DRBG must be seeded by the FIPS module).
+ */
+static int aes_gcm_iv_generate(EVP_AES_GCM_CTX *gctx, int offset)
+{
+    int sz = gctx->ivlen - offset;
+
+    /* Must be at least 96 bits */
+    if (sz <= 0 || gctx->ivlen < 12)
+        return 0;
+
+    /* Use DRBG to generate random iv */
+    if (RAND_bytes(gctx->iv + offset, sz) <= 0)
+        return 0;
+    return 1;
+}
+#endif /* FIPS_MODE */
+
 static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
                           const unsigned char *in, size_t len)
 {
     EVP_AES_GCM_CTX *gctx = EVP_C_DATA(EVP_AES_GCM_CTX,ctx);
+
     /* If not set up, return error */
     if (!gctx->key_set)
         return -1;
@@ -3155,8 +2921,25 @@ static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     if (gctx->tls_aad_len >= 0)
         return aes_gcm_tls_cipher(ctx, out, in, len);
 
+#ifdef FIPS_MODE
+    /*
+     * FIPS requires generation of AES-GCM IV's inside the FIPS module.
+     * The IV can still be set externally (the security policy will state that
+     * this is not FIPS compliant). There are some applications
+     * where setting the IV externally is the only option available.
+     */
+    if (!gctx->iv_set) {
+        if (!ctx->encrypt || !aes_gcm_iv_generate(gctx, 0))
+            return -1;
+        CRYPTO_gcm128_setiv(&gctx->gcm, gctx->iv, gctx->ivlen);
+        gctx->iv_set = 1;
+        gctx->iv_gen_rand = 1;
+    }
+#else
     if (!gctx->iv_set)
         return -1;
+#endif /* FIPS_MODE */
+
     if (in) {
         if (out == NULL) {
             if (CRYPTO_gcm128_aad(&gctx->gcm, in, len))
@@ -3283,10 +3066,12 @@ BLOCK_CIPHER_custom(NID_aes, 128, 1, 12, gcm, GCM,
 
 static int aes_xts_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
 {
-    EVP_AES_XTS_CTX *xctx = EVP_C_DATA(EVP_AES_XTS_CTX,c);
+    EVP_AES_XTS_CTX *xctx = EVP_C_DATA(EVP_AES_XTS_CTX, c);
+
     if (type == EVP_CTRL_COPY) {
         EVP_CIPHER_CTX *out = ptr;
         EVP_AES_XTS_CTX *xctx_out = EVP_C_DATA(EVP_AES_XTS_CTX,out);
+
         if (xctx->xts.key1) {
             if (xctx->xts.key1 != &xctx->ks1)
                 return 0;
@@ -3310,11 +3095,38 @@ static int aes_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
                             const unsigned char *iv, int enc)
 {
     EVP_AES_XTS_CTX *xctx = EVP_C_DATA(EVP_AES_XTS_CTX,ctx);
+
     if (!iv && !key)
         return 1;
 
-    if (key)
+    if (key) {
         do {
+            /* The key is two half length keys in reality */
+            const int bytes = EVP_CIPHER_CTX_key_length(ctx) / 2;
+            const int bits = bytes * 8;
+
+            /*
+             * Verify that the two keys are different.
+             *
+             * This addresses the vulnerability described in Rogaway's
+             * September 2004 paper:
+             *
+             *      "Efficient Instantiations of Tweakable Blockciphers and
+             *       Refinements to Modes OCB and PMAC".
+             *      (http://web.cs.ucdavis.edu/~rogaway/papers/offsets.pdf)
+             *
+             * FIPS 140-2 IG A.9 XTS-AES Key Generation Requirements states
+             * that:
+             *      "The check for Key_1 != Key_2 shall be done at any place
+             *       BEFORE using the keys in the XTS-AES algorithm to process
+             *       data with them."
+             */
+            if ((!allow_insecure_decrypt || enc)
+                    && CRYPTO_memcmp(key, key + bytes, bytes) == 0) {
+                EVPerr(EVP_F_AES_XTS_INIT_KEY, EVP_R_XTS_DUPLICATED_KEYS);
+                return 0;
+            }
+
 #ifdef AES_XTS_ASM
             xctx->stream = enc ? AES_xts_encrypt : AES_xts_decrypt;
 #else
@@ -3324,26 +3136,20 @@ static int aes_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 #ifdef HWAES_CAPABLE
             if (HWAES_CAPABLE) {
                 if (enc) {
-                    HWAES_set_encrypt_key(key,
-                                          EVP_CIPHER_CTX_key_length(ctx) * 4,
-                                          &xctx->ks1.ks);
+                    HWAES_set_encrypt_key(key, bits, &xctx->ks1.ks);
                     xctx->xts.block1 = (block128_f) HWAES_encrypt;
 # ifdef HWAES_xts_encrypt
                     xctx->stream = HWAES_xts_encrypt;
 # endif
                 } else {
-                    HWAES_set_decrypt_key(key,
-                                          EVP_CIPHER_CTX_key_length(ctx) * 4,
-                                          &xctx->ks1.ks);
+                    HWAES_set_decrypt_key(key, bits, &xctx->ks1.ks);
                     xctx->xts.block1 = (block128_f) HWAES_decrypt;
 # ifdef HWAES_xts_decrypt
                     xctx->stream = HWAES_xts_decrypt;
 #endif
                 }
 
-                HWAES_set_encrypt_key(key + EVP_CIPHER_CTX_key_length(ctx) / 2,
-                                      EVP_CIPHER_CTX_key_length(ctx) * 4,
-                                      &xctx->ks2.ks);
+                HWAES_set_encrypt_key(key + bytes, bits, &xctx->ks2.ks);
                 xctx->xts.block2 = (block128_f) HWAES_encrypt;
 
                 xctx->xts.key1 = &xctx->ks1;
@@ -3358,20 +3164,14 @@ static int aes_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 #ifdef VPAES_CAPABLE
             if (VPAES_CAPABLE) {
                 if (enc) {
-                    vpaes_set_encrypt_key(key,
-                                          EVP_CIPHER_CTX_key_length(ctx) * 4,
-                                          &xctx->ks1.ks);
+                    vpaes_set_encrypt_key(key, bits, &xctx->ks1.ks);
                     xctx->xts.block1 = (block128_f) vpaes_encrypt;
                 } else {
-                    vpaes_set_decrypt_key(key,
-                                          EVP_CIPHER_CTX_key_length(ctx) * 4,
-                                          &xctx->ks1.ks);
+                    vpaes_set_decrypt_key(key, bits, &xctx->ks1.ks);
                     xctx->xts.block1 = (block128_f) vpaes_decrypt;
                 }
 
-                vpaes_set_encrypt_key(key + EVP_CIPHER_CTX_key_length(ctx) / 2,
-                                      EVP_CIPHER_CTX_key_length(ctx) * 4,
-                                      &xctx->ks2.ks);
+                vpaes_set_encrypt_key(key + bytes, bits, &xctx->ks2.ks);
                 xctx->xts.block2 = (block128_f) vpaes_encrypt;
 
                 xctx->xts.key1 = &xctx->ks1;
@@ -3381,22 +3181,19 @@ static int aes_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
                 (void)0;        /* terminate potentially open 'else' */
 
             if (enc) {
-                AES_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 4,
-                                    &xctx->ks1.ks);
+                AES_set_encrypt_key(key, bits, &xctx->ks1.ks);
                 xctx->xts.block1 = (block128_f) AES_encrypt;
             } else {
-                AES_set_decrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 4,
-                                    &xctx->ks1.ks);
+                AES_set_decrypt_key(key, bits, &xctx->ks1.ks);
                 xctx->xts.block1 = (block128_f) AES_decrypt;
             }
 
-            AES_set_encrypt_key(key + EVP_CIPHER_CTX_key_length(ctx) / 2,
-                                EVP_CIPHER_CTX_key_length(ctx) * 4,
-                                &xctx->ks2.ks);
+            AES_set_encrypt_key(key + bytes, bits, &xctx->ks2.ks);
             xctx->xts.block2 = (block128_f) AES_encrypt;
 
             xctx->xts.key1 = &xctx->ks1;
         } while (0);
+    }
 
     if (iv) {
         xctx->xts.key2 = &xctx->ks2;
@@ -3410,10 +3207,25 @@ static int aes_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
                           const unsigned char *in, size_t len)
 {
     EVP_AES_XTS_CTX *xctx = EVP_C_DATA(EVP_AES_XTS_CTX,ctx);
-    if (!xctx->xts.key1 || !xctx->xts.key2)
+
+    if (xctx->xts.key1 == NULL
+            || xctx->xts.key2 == NULL
+            || out == NULL
+            || in == NULL
+            || len < AES_BLOCK_SIZE)
         return 0;
-    if (!out || !in || len < AES_BLOCK_SIZE)
+
+    /*
+     * Impose a limit of 2^20 blocks per data unit as specifed by
+     * IEEE Std 1619-2018.  The earlier and obsolete IEEE Std 1619-2007
+     * indicated that this was a SHOULD NOT rather than a MUST NOT.
+     * NIST SP 800-38E mandates the same limit.
+     */
+    if (len > XTS_MAX_BLOCKS_PER_DATA_UNIT * AES_BLOCK_SIZE) {
+        EVPerr(EVP_F_AES_XTS_CIPHER, EVP_R_XTS_DATA_UNIT_IS_TOO_LARGE);
         return 0;
+    }
+
     if (xctx->stream)
         (*xctx->stream) (in, out, len,
                          xctx->xts.key1, xctx->xts.key2,
@@ -3643,8 +3455,6 @@ static int aes_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     if (!cctx->iv_set)
         return -1;
 
-    if (!EVP_CIPHER_CTX_encrypting(ctx) && !cctx->tag_set)
-        return -1;
     if (!out) {
         if (!in) {
             if (CRYPTO_ccm128_setiv(ccm, EVP_CIPHER_CTX_iv_noconst(ctx),
@@ -3659,6 +3469,11 @@ static int aes_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
         CRYPTO_ccm128_aad(ccm, in, len);
         return len;
     }
+
+    /* The tag must be set before actually decrypting data */
+    if (!EVP_CIPHER_CTX_encrypting(ctx) && !cctx->tag_set)
+        return -1;
+
     /* If not set length yet do it */
     if (!cctx->len_set) {
         if (CRYPTO_ccm128_setiv(ccm, EVP_CIPHER_CTX_iv_noconst(ctx),
@@ -3705,7 +3520,7 @@ BLOCK_CIPHER_custom(NID_aes, 128, 1, 12, ccm, CCM,
 
 typedef struct {
     union {
-        double align;
+        OSSL_UNION_ALIGN;
         AES_KEY ks;
     } ks;
     /* Indicates if IV has been set */
@@ -3942,29 +3757,6 @@ static int aes_ocb_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
     }
 }
 
-# ifdef HWAES_CAPABLE
-#  ifdef HWAES_ocb_encrypt
-void HWAES_ocb_encrypt(const unsigned char *in, unsigned char *out,
-                       size_t blocks, const void *key,
-                       size_t start_block_num,
-                       unsigned char offset_i[16],
-                       const unsigned char L_[][16],
-                       unsigned char checksum[16]);
-#  else
-#    define HWAES_ocb_encrypt ((ocb128_f)NULL)
-#  endif
-#  ifdef HWAES_ocb_decrypt
-void HWAES_ocb_decrypt(const unsigned char *in, unsigned char *out,
-                       size_t blocks, const void *key,
-                       size_t start_block_num,
-                       unsigned char offset_i[16],
-                       const unsigned char L_[][16],
-                       unsigned char checksum[16]);
-#  else
-#    define HWAES_ocb_decrypt ((ocb128_f)NULL)
-#  endif
-# endif
-
 static int aes_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
                             const unsigned char *iv, int enc)
 {
@@ -4208,3 +4000,114 @@ BLOCK_CIPHER_custom(NID_aes, 192, 16, 12, ocb, OCB,
 BLOCK_CIPHER_custom(NID_aes, 256, 16, 12, ocb, OCB,
                     EVP_CIPH_FLAG_AEAD_CIPHER | CUSTOM_FLAGS)
 #endif                         /* OPENSSL_NO_OCB */
+
+/* AES-SIV mode */
+#ifndef OPENSSL_NO_SIV
+
+typedef SIV128_CONTEXT EVP_AES_SIV_CTX;
+
+#define aesni_siv_init_key aes_siv_init_key
+static int aes_siv_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
+                            const unsigned char *iv, int enc)
+{
+    const EVP_CIPHER *ctr;
+    const EVP_CIPHER *cbc;
+    SIV128_CONTEXT *sctx = EVP_C_DATA(SIV128_CONTEXT, ctx);
+    int klen = EVP_CIPHER_CTX_key_length(ctx) / 2;
+
+    if (key == NULL)
+        return 1;
+
+    switch (klen) {
+    case 16:
+        cbc = EVP_aes_128_cbc();
+        ctr = EVP_aes_128_ctr();
+        break;
+    case 24:
+        cbc = EVP_aes_192_cbc();
+        ctr = EVP_aes_192_ctr();
+        break;
+    case 32:
+        cbc = EVP_aes_256_cbc();
+        ctr = EVP_aes_256_ctr();
+        break;
+    default:
+        return 0;
+    }
+
+    /* klen is the length of the underlying cipher, not the input key,
+       which should be twice as long */
+    return CRYPTO_siv128_init(sctx, key, klen, cbc, ctr);
+}
+
+#define aesni_siv_cipher aes_siv_cipher
+static int aes_siv_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
+                          const unsigned char *in, size_t len)
+{
+    SIV128_CONTEXT *sctx = EVP_C_DATA(SIV128_CONTEXT, ctx);
+
+    /* EncryptFinal or DecryptFinal */
+    if (in == NULL)
+        return CRYPTO_siv128_finish(sctx);
+
+    /* Deal with associated data */
+    if (out == NULL)
+        return CRYPTO_siv128_aad(sctx, in, len);
+
+    if (EVP_CIPHER_CTX_encrypting(ctx))
+        return CRYPTO_siv128_encrypt(sctx, in, out, len);
+
+    return CRYPTO_siv128_decrypt(sctx, in, out, len);
+}
+
+#define aesni_siv_cleanup aes_siv_cleanup
+static int aes_siv_cleanup(EVP_CIPHER_CTX *c)
+{
+    SIV128_CONTEXT *sctx = EVP_C_DATA(SIV128_CONTEXT, c);
+
+    return CRYPTO_siv128_cleanup(sctx);
+}
+
+
+#define aesni_siv_ctrl aes_siv_ctrl
+static int aes_siv_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
+{
+    SIV128_CONTEXT *sctx = EVP_C_DATA(SIV128_CONTEXT, c);
+    SIV128_CONTEXT *sctx_out;
+
+    switch (type) {
+    case EVP_CTRL_INIT:
+        return CRYPTO_siv128_cleanup(sctx);
+
+    case EVP_CTRL_SET_SPEED:
+        return CRYPTO_siv128_speed(sctx, arg);
+
+    case EVP_CTRL_AEAD_SET_TAG:
+        if (!EVP_CIPHER_CTX_encrypting(c))
+            return CRYPTO_siv128_set_tag(sctx, ptr, arg);
+        return 1;
+
+    case EVP_CTRL_AEAD_GET_TAG:
+        if (!EVP_CIPHER_CTX_encrypting(c))
+            return 0;
+        return CRYPTO_siv128_get_tag(sctx, ptr, arg);
+
+    case EVP_CTRL_COPY:
+        sctx_out = EVP_C_DATA(SIV128_CONTEXT, (EVP_CIPHER_CTX*)ptr);
+        return CRYPTO_siv128_copy_ctx(sctx_out, sctx);
+
+    default:
+        return -1;
+
+    }
+}
+
+#define SIV_FLAGS    (EVP_CIPH_FLAG_AEAD_CIPHER | EVP_CIPH_FLAG_DEFAULT_ASN1 \
+                      | EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \
+                      | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_CUSTOM_COPY \
+                      | EVP_CIPH_CTRL_INIT)
+
+BLOCK_CIPHER_custom(NID_aes, 128, 1, 0, siv, SIV, SIV_FLAGS)
+BLOCK_CIPHER_custom(NID_aes, 192, 1, 0, siv, SIV, SIV_FLAGS)
+BLOCK_CIPHER_custom(NID_aes, 256, 1, 0, siv, SIV, SIV_FLAGS)
+#endif