crypto/evp/e_aes.c

   1 /* ====================================================================
   2  * Copyright (c) 2001-2014 The OpenSSL Project.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  *
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in
  13  *    the documentation and/or other materials provided with the
  14  *    distribution.
  15  *
  16  * 3. All advertising materials mentioning features or use of this
  17  *    software must display the following acknowledgment:
  18  *    "This product includes software developed by the OpenSSL Project
  19  *    for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
  20  *
  21  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
  22  *    endorse or promote products derived from this software without
  23  *    prior written permission. For written permission, please contact
  24  *    openssl-core@openssl.org.
  25  *
  26  * 5. Products derived from this software may not be called "OpenSSL"
  27  *    nor may "OpenSSL" appear in their names without prior written
  28  *    permission of the OpenSSL Project.
  29  *
  30  * 6. Redistributions of any form whatsoever must retain the following
  31  *    acknowledgment:
  32  *    "This product includes software developed by the OpenSSL Project
  33  *    for use in the OpenSSL Toolkit (http://www.openssl.org/)"
  34  *
  35  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
  36  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  37  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  38  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
  39  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  41  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  42  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  43  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  44  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  45  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
  46  * OF THE POSSIBILITY OF SUCH DAMAGE.
  47  * ====================================================================
  48  *
  49  */
  50
  51 #define OPENSSL_FIPSAPI
  52
  53 #include <openssl/opensslconf.h>
  54 #ifndef OPENSSL_NO_AES
  55 #include <openssl/evp.h>
  56 #include <openssl/err.h>
  57 #include <string.h>
  58 #include <assert.h>
  59 #include <openssl/aes.h>
  60 #include "evp_locl.h"
  61 #include "modes_lcl.h"
  62 #include <openssl/rand.h>
  63
  64 typedef struct
  65         {
  66         union { double align; AES_KEY ks; } ks;
  67         block128_f block;
  68         union {
  69                 cbc128_f cbc;
  70                 ctr128_f ctr;
  71         } stream;
  72         } EVP_AES_KEY;
  73
  74 typedef struct
  75         {
  76         union { double align; AES_KEY ks; } ks; /* AES key schedule to use */
  77         int key_set;            /* Set if key initialised */
  78         int iv_set;             /* Set if an iv is set */
  79         GCM128_CONTEXT gcm;
  80         unsigned char *iv;      /* Temporary IV store */
  81         int ivlen;              /* IV length */
  82         int taglen;
  83         int iv_gen;             /* It is OK to generate IVs */
  84         int tls_aad_len;        /* TLS AAD length */
  85         ctr128_f ctr;
  86         } EVP_AES_GCM_CTX;
  87
  88 typedef struct
  89         {
  90         union { double align; AES_KEY ks; } ks1, ks2;   /* AES key schedules to use */
  91         XTS128_CONTEXT xts;
  92         void     (*stream)(const unsigned char *in,
  93                         unsigned char *out, size_t length,
  94                         const AES_KEY *key1, const AES_KEY *key2,
  95                         const unsigned char iv[16]);
  96         } EVP_AES_XTS_CTX;
  97
  98 typedef struct
  99         {
 100         union { double align; AES_KEY ks; } ks; /* AES key schedule to use */
 101         int key_set;            /* Set if key initialised */
 102         int iv_set;             /* Set if an iv is set */
 103         int tag_set;            /* Set if tag is valid */
 104         int len_set;            /* Set if message length set */
 105         int L, M;               /* L and M parameters from RFC3610 */
 106         CCM128_CONTEXT ccm;
 107         ccm128_f str;
 108         } EVP_AES_CCM_CTX;
 109
 110
 111 typedef struct
 112         {
 113         AES_KEY ksenc;          /* AES key schedule to use for encryption */
 114         AES_KEY ksdec;          /* AES key schedule to use for decryption */
 115         int key_set;            /* Set if key initialised */
 116         int iv_set;             /* Set if an iv is set */
 117         OCB128_CONTEXT ocb;
 118         unsigned char *iv;      /* Temporary IV store */
 119         unsigned char tag[16];
 120         unsigned char data_buf[16]; /* Store partial data blocks */
 121         unsigned char aad_buf[16]; /* Store partial AAD blocks */
 122         int data_buf_len;
 123         int aad_buf_len;
 124         int ivlen;              /* IV length */
 125         int taglen;
 126         } EVP_AES_OCB_CTX;
 127
 128 #define MAXBITCHUNK     ((size_t)1<<(sizeof(size_t)*8-4))
 129
 130 #ifdef VPAES_ASM
 131 int vpaes_set_encrypt_key(const unsigned char *userKey, int bits,
 132                         AES_KEY *key);
 133 int vpaes_set_decrypt_key(const unsigned char *userKey, int bits,
 134                         AES_KEY *key);
 135
 136 void vpaes_encrypt(const unsigned char *in, unsigned char *out,
 137                         const AES_KEY *key);
 138 void vpaes_decrypt(const unsigned char *in, unsigned char *out,
 139                         const AES_KEY *key);
 140
 141 void vpaes_cbc_encrypt(const unsigned char *in,
 142                         unsigned char *out,
 143                         size_t length,
 144                         const AES_KEY *key,
 145                         unsigned char *ivec, int enc);
 146 #endif
 147 #ifdef BSAES_ASM
 148 void bsaes_cbc_encrypt(const unsigned char *in, unsigned char *out,
 149                         size_t length, const AES_KEY *key,
 150                         unsigned char ivec[16], int enc);
 151 void bsaes_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
 152                         size_t len, const AES_KEY *key,
 153                         const unsigned char ivec[16]);
 154 void bsaes_xts_encrypt(const unsigned char *inp, unsigned char *out,
 155                         size_t len, const AES_KEY *key1,
 156                         const AES_KEY *key2, const unsigned char iv[16]);
 157 void bsaes_xts_decrypt(const unsigned char *inp, unsigned char *out,
 158                         size_t len, const AES_KEY *key1,
 159                         const AES_KEY *key2, const unsigned char iv[16]);
 160 #endif
 161 #ifdef AES_CTR_ASM
 162 void AES_ctr32_encrypt(const unsigned char *in, unsigned char *out,
 163                         size_t blocks, const AES_KEY *key,
 164                         const unsigned char ivec[AES_BLOCK_SIZE]);
 165 #endif
 166 #ifdef AES_XTS_ASM
 167 void AES_xts_encrypt(const char *inp,char *out,size_t len,
 168                         const AES_KEY *key1, const AES_KEY *key2,
 169                         const unsigned char iv[16]);
 170 void AES_xts_decrypt(const char *inp,char *out,size_t len,
 171                         const AES_KEY *key1, const AES_KEY *key2,
 172                         const unsigned char iv[16]);
 173 #endif
 174
 175 #if     defined(OPENSSL_CPUID_OBJ) && (defined(__powerpc__) || defined(__ppc__) || defined(_ARCH_PPC))
 176 # include "ppc_arch.h"
 177 # ifdef VPAES_ASM
 178 #  define VPAES_CAPABLE (OPENSSL_ppccap_P & PPC_ALTIVEC)
 179 # endif
 180 # define HWAES_CAPABLE  (OPENSSL_ppccap_P & PPC_CRYPTO207)
 181 # define HWAES_set_encrypt_key aes_p8_set_encrypt_key
 182 # define HWAES_set_decrypt_key aes_p8_set_decrypt_key
 183 # define HWAES_encrypt aes_p8_encrypt
 184 # define HWAES_decrypt aes_p8_decrypt
 185 # define HWAES_cbc_encrypt aes_p8_cbc_encrypt
 186 # define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks
 187 #endif
 188
 189 #if     defined(AES_ASM) && !defined(I386_ONLY) &&      (  \
 190         ((defined(__i386)       || defined(__i386__)    || \
 191           defined(_M_IX86)) && defined(OPENSSL_IA32_SSE2))|| \
 192         defined(__x86_64)       || defined(__x86_64__)  || \
 193         defined(_M_AMD64)       || defined(_M_X64)      || \
 194         defined(__INTEL__)                              )
 195
 196 extern unsigned int OPENSSL_ia32cap_P[];
 197
 198 #ifdef VPAES_ASM
 199 #define VPAES_CAPABLE   (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
 200 #endif
 201 #ifdef BSAES_ASM
 202 #define BSAES_CAPABLE   (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
 203 #endif
 204 /*
 205  * AES-NI section
 206  */
 207 #define AESNI_CAPABLE   (OPENSSL_ia32cap_P[1]&(1<<(57-32)))
 208
 209 int aesni_set_encrypt_key(const unsigned char *userKey, int bits,
 210                         AES_KEY *key);
 211 int aesni_set_decrypt_key(const unsigned char *userKey, int bits,
 212                         AES_KEY *key);
 213
 214 void aesni_encrypt(const unsigned char *in, unsigned char *out,
 215                         const AES_KEY *key);
 216 void aesni_decrypt(const unsigned char *in, unsigned char *out,
 217                         const AES_KEY *key);
 218
 219 void aesni_ecb_encrypt(const unsigned char *in,
 220                         unsigned char *out,
 221                         size_t length,
 222                         const AES_KEY *key,
 223                         int enc);
 224 void aesni_cbc_encrypt(const unsigned char *in,
 225                         unsigned char *out,
 226                         size_t length,
 227                         const AES_KEY *key,
 228                         unsigned char *ivec, int enc);
 229
 230 void aesni_ctr32_encrypt_blocks(const unsigned char *in,
 231                         unsigned char *out,
 232                         size_t blocks,
 233                         const void *key,
 234                         const unsigned char *ivec);
 235
 236 void aesni_xts_encrypt(const unsigned char *in,
 237                         unsigned char *out,
 238                         size_t length,
 239                         const AES_KEY *key1, const AES_KEY *key2,
 240                         const unsigned char iv[16]);
 241
 242 void aesni_xts_decrypt(const unsigned char *in,
 243                         unsigned char *out,
 244                         size_t length,
 245                         const AES_KEY *key1, const AES_KEY *key2,
 246                         const unsigned char iv[16]);
 247
 248 void aesni_ccm64_encrypt_blocks (const unsigned char *in,
 249                         unsigned char *out,
 250                         size_t blocks,
 251                         const void *key,
 252                         const unsigned char ivec[16],
 253                         unsigned char cmac[16]);
 254
 255 void aesni_ccm64_decrypt_blocks (const unsigned char *in,
 256                         unsigned char *out,
 257                         size_t blocks,
 258                         const void *key,
 259                         const unsigned char ivec[16],
 260                         unsigned char cmac[16]);
 261
 262 #if defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
 263 size_t aesni_gcm_encrypt(const unsigned char *in,
 264                         unsigned char *out,
 265                         size_t len,
 266                         const void *key,
 267                         unsigned char ivec[16],
 268                         u64 *Xi);
 269 #define AES_gcm_encrypt aesni_gcm_encrypt
 270 size_t aesni_gcm_decrypt(const unsigned char *in,
 271                         unsigned char *out,
 272                         size_t len,
 273                         const void *key,
 274                         unsigned char ivec[16],
 275                         u64 *Xi);
 276 #define AES_gcm_decrypt aesni_gcm_decrypt
 277 void gcm_ghash_avx(u64 Xi[2],const u128 Htable[16],const u8 *in,size_t len);
 278 #define AES_GCM_ASM(gctx)       (gctx->ctr==aesni_ctr32_encrypt_blocks && \
 279                                  gctx->gcm.ghash==gcm_ghash_avx)
 280 #define AES_GCM_ASM2(gctx)      (gctx->gcm.block==(block128_f)aesni_encrypt && \
 281                                  gctx->gcm.ghash==gcm_ghash_avx)
 282 #undef AES_GCM_ASM2             /* minor size optimization */
 283 #endif
 284
 285 static int aesni_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 286                    const unsigned char *iv, int enc)
 287         {
 288         int ret, mode;
 289         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
 290
 291         mode = ctx->cipher->flags & EVP_CIPH_MODE;
 292         if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE)
 293             && !enc)
 294                 {
 295                 ret = aesni_set_decrypt_key(key, ctx->key_len*8, ctx->cipher_data);
 296                 dat->block      = (block128_f)aesni_decrypt;
 297                 dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
 298                                         (cbc128_f)aesni_cbc_encrypt :
 299                                         NULL;
 300                 }
 301         else    {
 302                 ret = aesni_set_encrypt_key(key, ctx->key_len*8, ctx->cipher_data);
 303                 dat->block      = (block128_f)aesni_encrypt;
 304                 if (mode==EVP_CIPH_CBC_MODE)
 305                         dat->stream.cbc = (cbc128_f)aesni_cbc_encrypt;
 306                 else if (mode==EVP_CIPH_CTR_MODE)
 307                         dat->stream.ctr = (ctr128_f)aesni_ctr32_encrypt_blocks;
 308                 else
 309                         dat->stream.cbc = NULL;
 310                 }
 311
 312         if(ret < 0)
 313                 {
 314                 EVPerr(EVP_F_AESNI_INIT_KEY,EVP_R_AES_KEY_SETUP_FAILED);
 315                 return 0;
 316                 }
 317
 318         return 1;
 319         }
 320
 321 static int aesni_cbc_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 322         const unsigned char *in, size_t len)
 323 {
 324         aesni_cbc_encrypt(in,out,len,ctx->cipher_data,ctx->iv,ctx->encrypt);
 325
 326         return 1;
 327 }
 328
 329 static int aesni_ecb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 330         const unsigned char *in, size_t len)
 331 {
 332         size_t  bl = ctx->cipher->block_size;
 333
 334         if (len<bl)     return 1;
 335
 336         aesni_ecb_encrypt(in,out,len,ctx->cipher_data,ctx->encrypt);
 337
 338         return 1;
 339 }
 340
 341 #define aesni_ofb_cipher aes_ofb_cipher
 342 static int aesni_ofb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 343         const unsigned char *in,size_t len);
 344
 345 #define aesni_cfb_cipher aes_cfb_cipher
 346 static int aesni_cfb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 347         const unsigned char *in,size_t len);
 348
 349 #define aesni_cfb8_cipher aes_cfb8_cipher
 350 static int aesni_cfb8_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 351         const unsigned char *in,size_t len);
 352
 353 #define aesni_cfb1_cipher aes_cfb1_cipher
 354 static int aesni_cfb1_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 355         const unsigned char *in,size_t len);
 356
 357 #define aesni_ctr_cipher aes_ctr_cipher
 358 static int aesni_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 359                 const unsigned char *in, size_t len);
 360
 361 static int aesni_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 362                         const unsigned char *iv, int enc)
 363         {
 364         EVP_AES_GCM_CTX *gctx = ctx->cipher_data;
 365         if (!iv && !key)
 366                 return 1;
 367         if (key)
 368                 {
 369                 aesni_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks);
 370                 CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks,
 371                                 (block128_f)aesni_encrypt);
 372                 gctx->ctr = (ctr128_f)aesni_ctr32_encrypt_blocks;
 373                 /* If we have an iv can set it directly, otherwise use
 374                  * saved IV.
 375                  */
 376                 if (iv == NULL && gctx->iv_set)
 377                         iv = gctx->iv;
 378                 if (iv)
 379                         {
 380                         CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
 381                         gctx->iv_set = 1;
 382                         }
 383                 gctx->key_set = 1;
 384                 }
 385         else
 386                 {
 387                 /* If key set use IV, otherwise copy */
 388                 if (gctx->key_set)
 389                         CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
 390                 else
 391                         memcpy(gctx->iv, iv, gctx->ivlen);
 392                 gctx->iv_set = 1;
 393                 gctx->iv_gen = 0;
 394                 }
 395         return 1;
 396         }
 397
 398 #define aesni_gcm_cipher aes_gcm_cipher
 399 static int aesni_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 400                 const unsigned char *in, size_t len);
 401
 402 static int aesni_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 403                         const unsigned char *iv, int enc)
 404         {
 405         EVP_AES_XTS_CTX *xctx = ctx->cipher_data;
 406         if (!iv && !key)
 407                 return 1;
 408
 409         if (key)
 410                 {
 411                 /* key_len is two AES keys */
 412                 if (enc)
 413                         {
 414                         aesni_set_encrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
 415                         xctx->xts.block1 = (block128_f)aesni_encrypt;
 416                         xctx->stream = aesni_xts_encrypt;
 417                         }
 418                 else
 419                         {
 420                         aesni_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
 421                         xctx->xts.block1 = (block128_f)aesni_decrypt;
 422                         xctx->stream = aesni_xts_decrypt;
 423                         }
 424
 425                 aesni_set_encrypt_key(key + ctx->key_len/2,
 426                                                 ctx->key_len * 4, &xctx->ks2.ks);
 427                 xctx->xts.block2 = (block128_f)aesni_encrypt;
 428
 429                 xctx->xts.key1 = &xctx->ks1;
 430                 }
 431
 432         if (iv)
 433                 {
 434                 xctx->xts.key2 = &xctx->ks2;
 435                 memcpy(ctx->iv, iv, 16);
 436                 }
 437
 438         return 1;
 439         }
 440
 441 #define aesni_xts_cipher aes_xts_cipher
 442 static int aesni_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 443                 const unsigned char *in, size_t len);
 444
 445 static int aesni_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 446                         const unsigned char *iv, int enc)
 447         {
 448         EVP_AES_CCM_CTX *cctx = ctx->cipher_data;
 449         if (!iv && !key)
 450                 return 1;
 451         if (key)
 452                 {
 453                 aesni_set_encrypt_key(key, ctx->key_len * 8, &cctx->ks.ks);
 454                 CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L,
 455                                         &cctx->ks, (block128_f)aesni_encrypt);
 456                 cctx->str = enc?(ccm128_f)aesni_ccm64_encrypt_blocks :
 457                                 (ccm128_f)aesni_ccm64_decrypt_blocks;
 458                 cctx->key_set = 1;
 459                 }
 460         if (iv)
 461                 {
 462                 memcpy(ctx->iv, iv, 15 - cctx->L);
 463                 cctx->iv_set = 1;
 464                 }
 465         return 1;
 466         }
 467
 468 #define aesni_ccm_cipher aes_ccm_cipher
 469 static int aesni_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 470                 const unsigned char *in, size_t len);
 471
 472 static int aesni_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 473                         const unsigned char *iv, int enc)
 474         {
 475         EVP_AES_OCB_CTX *octx = ctx->cipher_data;
 476         if (!iv && !key)
 477                 return 1;
 478         if (key)
 479                 {
 480                 do
 481                         {
 482                         /* We set both the encrypt and decrypt key here because decrypt
 483                          * needs both. We could possibly optimise to remove setting the
 484                          * decrypt for an encryption operation.
 485                          */
 486                         aesni_set_encrypt_key(key, ctx->key_len * 8, &octx->ksenc);
 487                         aesni_set_decrypt_key(key, ctx->key_len * 8, &octx->ksdec);
 488                         if(!CRYPTO_ocb128_init(&octx->ocb, &octx->ksenc, &octx->ksdec,
 489                                         (block128_f)aesni_encrypt, (block128_f)aesni_decrypt))
 490                                 return 0;
 491                         }
 492                 while (0);
 493
 494                 /* If we have an iv we can set it directly, otherwise use
 495                  * saved IV.
 496                  */
 497                 if (iv == NULL && octx->iv_set)
 498                         iv = octx->iv;
 499                 if (iv)
 500                         {
 501                         if(CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen) != 1)
 502                                 return 0;
 503                         octx->iv_set = 1;
 504                         }
 505                 octx->key_set = 1;
 506                 }
 507         else
 508                 {
 509                 /* If key set use IV, otherwise copy */
 510                 if (octx->key_set)
 511                         CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen);
 512                 else
 513                         memcpy(octx->iv, iv, octx->ivlen);
 514                 octx->iv_set = 1;
 515                 }
 516         return 1;
 517         }
 518
 519 #define aesni_ocb_cipher aes_ocb_cipher
 520 static int aesni_ocb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 521                 const unsigned char *in, size_t len);
 522
 523
 524 #define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \
 525 static const EVP_CIPHER aesni_##keylen##_##mode = { \
 526         nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \
 527         flags|EVP_CIPH_##MODE##_MODE,   \
 528         aesni_init_key,                 \
 529         aesni_##mode##_cipher,          \
 530         NULL,                           \
 531         sizeof(EVP_AES_KEY),            \
 532         NULL,NULL,NULL,NULL }; \
 533 static const EVP_CIPHER aes_##keylen##_##mode = { \
 534         nid##_##keylen##_##nmode,blocksize,     \
 535         keylen/8,ivlen, \
 536         flags|EVP_CIPH_##MODE##_MODE,   \
 537         aes_init_key,                   \
 538         aes_##mode##_cipher,            \
 539         NULL,                           \
 540         sizeof(EVP_AES_KEY),            \
 541         NULL,NULL,NULL,NULL }; \
 542 const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
 543 { return AESNI_CAPABLE?&aesni_##keylen##_##mode:&aes_##keylen##_##mode; }
 544
 545 #define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \
 546 static const EVP_CIPHER aesni_##keylen##_##mode = { \
 547         nid##_##keylen##_##mode,blocksize, \
 548         (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \
 549         flags|EVP_CIPH_##MODE##_MODE,   \
 550         aesni_##mode##_init_key,        \
 551         aesni_##mode##_cipher,          \
 552         aes_##mode##_cleanup,           \
 553         sizeof(EVP_AES_##MODE##_CTX),   \
 554         NULL,NULL,aes_##mode##_ctrl,NULL }; \
 555 static const EVP_CIPHER aes_##keylen##_##mode = { \
 556         nid##_##keylen##_##mode,blocksize, \
 557         (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \
 558         flags|EVP_CIPH_##MODE##_MODE,   \
 559         aes_##mode##_init_key,          \
 560         aes_##mode##_cipher,            \
 561         aes_##mode##_cleanup,           \
 562         sizeof(EVP_AES_##MODE##_CTX),   \
 563         NULL,NULL,aes_##mode##_ctrl,NULL }; \
 564 const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
 565 { return AESNI_CAPABLE?&aesni_##keylen##_##mode:&aes_##keylen##_##mode; }
 566
 567 #elif   defined(AES_ASM) && (defined(__sparc) || defined(__sparc__))
 568
 569 #include "sparc_arch.h"
 570
 571 extern unsigned int OPENSSL_sparcv9cap_P[];
 572
 573 #define SPARC_AES_CAPABLE       (OPENSSL_sparcv9cap_P[1] & CFR_AES)
 574
 575 void    aes_t4_set_encrypt_key (const unsigned char *key, int bits,
 576                                 AES_KEY *ks);
 577 void    aes_t4_set_decrypt_key (const unsigned char *key, int bits,
 578                                 AES_KEY *ks);
 579 void    aes_t4_encrypt (const unsigned char *in, unsigned char *out,
 580                                 const AES_KEY *key);
 581 void    aes_t4_decrypt (const unsigned char *in, unsigned char *out,
 582                                 const AES_KEY *key);
 583 /*
 584  * Key-length specific subroutines were chosen for following reason.
 585  * Each SPARC T4 core can execute up to 8 threads which share core's
 586  * resources. Loading as much key material to registers allows to
 587  * minimize references to shared memory interface, as well as amount
 588  * of instructions in inner loops [much needed on T4]. But then having
 589  * non-key-length specific routines would require conditional branches
 590  * either in inner loops or on subroutines' entries. Former is hardly
 591  * acceptable, while latter means code size increase to size occupied
 592  * by multiple key-length specfic subroutines, so why fight?
 593  */
 594 void    aes128_t4_cbc_encrypt (const unsigned char *in, unsigned char *out,
 595                                 size_t len, const AES_KEY *key,
 596                                 unsigned char *ivec);
 597 void    aes128_t4_cbc_decrypt (const unsigned char *in, unsigned char *out,
 598                                 size_t len, const AES_KEY *key,
 599                                 unsigned char *ivec);
 600 void    aes192_t4_cbc_encrypt (const unsigned char *in, unsigned char *out,
 601                                 size_t len, const AES_KEY *key,
 602                                 unsigned char *ivec);
 603 void    aes192_t4_cbc_decrypt (const unsigned char *in, unsigned char *out,
 604                                 size_t len, const AES_KEY *key,
 605                                 unsigned char *ivec);
 606 void    aes256_t4_cbc_encrypt (const unsigned char *in, unsigned char *out,
 607                                 size_t len, const AES_KEY *key,
 608                                 unsigned char *ivec);
 609 void    aes256_t4_cbc_decrypt (const unsigned char *in, unsigned char *out,
 610                                 size_t len, const AES_KEY *key,
 611                                 unsigned char *ivec);
 612 void    aes128_t4_ctr32_encrypt (const unsigned char *in, unsigned char *out,
 613                                 size_t blocks, const AES_KEY *key,
 614                                 unsigned char *ivec);
 615 void    aes192_t4_ctr32_encrypt (const unsigned char *in, unsigned char *out,
 616                                 size_t blocks, const AES_KEY *key,
 617                                 unsigned char *ivec);
 618 void    aes256_t4_ctr32_encrypt (const unsigned char *in, unsigned char *out,
 619                                 size_t blocks, const AES_KEY *key,
 620                                 unsigned char *ivec);
 621 void    aes128_t4_xts_encrypt (const unsigned char *in, unsigned char *out,
 622                                 size_t blocks, const AES_KEY *key1,
 623                                 const AES_KEY *key2, const unsigned char *ivec);
 624 void    aes128_t4_xts_decrypt (const unsigned char *in, unsigned char *out,
 625                                 size_t blocks, const AES_KEY *key1,
 626                                 const AES_KEY *key2, const unsigned char *ivec);
 627 void    aes256_t4_xts_encrypt (const unsigned char *in, unsigned char *out,
 628                                 size_t blocks, const AES_KEY *key1,
 629                                 const AES_KEY *key2, const unsigned char *ivec);
 630 void    aes256_t4_xts_decrypt (const unsigned char *in, unsigned char *out,
 631                                 size_t blocks, const AES_KEY *key1,
 632                                 const AES_KEY *key2, const unsigned char *ivec);
 633
 634 static int aes_t4_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 635                    const unsigned char *iv, int enc)
 636         {
 637         int ret, mode, bits;
 638         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
 639
 640         mode = ctx->cipher->flags & EVP_CIPH_MODE;
 641         bits = ctx->key_len*8;
 642         if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE)
 643             && !enc)
 644                 {
 645                     ret = 0;
 646                     aes_t4_set_decrypt_key(key, bits, ctx->cipher_data);
 647                     dat->block  = (block128_f)aes_t4_decrypt;
 648                     switch (bits) {
 649                     case 128:
 650                         dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
 651                                                 (cbc128_f)aes128_t4_cbc_decrypt :
 652                                                 NULL;
 653                         break;
 654                     case 192:
 655                         dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
 656                                                 (cbc128_f)aes192_t4_cbc_decrypt :
 657                                                 NULL;
 658                         break;
 659                     case 256:
 660                         dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
 661                                                 (cbc128_f)aes256_t4_cbc_decrypt :
 662                                                 NULL;
 663                         break;
 664                     default:
 665                         ret = -1;
 666                     }
 667                 }
 668         else    {
 669                     ret = 0;
 670                     aes_t4_set_encrypt_key(key, bits, ctx->cipher_data);
 671                     dat->block  = (block128_f)aes_t4_encrypt;
 672                     switch (bits) {
 673                     case 128:
 674                         if (mode==EVP_CIPH_CBC_MODE)
 675                                 dat->stream.cbc = (cbc128_f)aes128_t4_cbc_encrypt;
 676                         else if (mode==EVP_CIPH_CTR_MODE)
 677                                 dat->stream.ctr = (ctr128_f)aes128_t4_ctr32_encrypt;
 678                         else
 679                                 dat->stream.cbc = NULL;
 680                         break;
 681                     case 192:
 682                         if (mode==EVP_CIPH_CBC_MODE)
 683                                 dat->stream.cbc = (cbc128_f)aes192_t4_cbc_encrypt;
 684                         else if (mode==EVP_CIPH_CTR_MODE)
 685                                 dat->stream.ctr = (ctr128_f)aes192_t4_ctr32_encrypt;
 686                         else
 687                                 dat->stream.cbc = NULL;
 688                         break;
 689                     case 256:
 690                         if (mode==EVP_CIPH_CBC_MODE)
 691                                 dat->stream.cbc = (cbc128_f)aes256_t4_cbc_encrypt;
 692                         else if (mode==EVP_CIPH_CTR_MODE)
 693                                 dat->stream.ctr = (ctr128_f)aes256_t4_ctr32_encrypt;
 694                         else
 695                                 dat->stream.cbc = NULL;
 696                         break;
 697                     default:
 698                         ret = -1;
 699                     }
 700                 }
 701
 702         if(ret < 0)
 703                 {
 704                 EVPerr(EVP_F_AES_T4_INIT_KEY,EVP_R_AES_KEY_SETUP_FAILED);
 705                 return 0;
 706                 }
 707
 708         return 1;
 709         }
 710
 711 #define aes_t4_cbc_cipher aes_cbc_cipher
 712 static int aes_t4_cbc_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 713         const unsigned char *in, size_t len);
 714
 715 #define aes_t4_ecb_cipher aes_ecb_cipher
 716 static int aes_t4_ecb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 717         const unsigned char *in, size_t len);
 718
 719 #define aes_t4_ofb_cipher aes_ofb_cipher
 720 static int aes_t4_ofb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 721         const unsigned char *in,size_t len);
 722
 723 #define aes_t4_cfb_cipher aes_cfb_cipher
 724 static int aes_t4_cfb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 725         const unsigned char *in,size_t len);
 726
 727 #define aes_t4_cfb8_cipher aes_cfb8_cipher
 728 static int aes_t4_cfb8_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 729         const unsigned char *in,size_t len);
 730
 731 #define aes_t4_cfb1_cipher aes_cfb1_cipher
 732 static int aes_t4_cfb1_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 733         const unsigned char *in,size_t len);
 734
 735 #define aes_t4_ctr_cipher aes_ctr_cipher
 736 static int aes_t4_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 737                 const unsigned char *in, size_t len);
 738
 739 static int aes_t4_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 740                         const unsigned char *iv, int enc)
 741         {
 742         EVP_AES_GCM_CTX *gctx = ctx->cipher_data;
 743         if (!iv && !key)
 744                 return 1;
 745         if (key)
 746                 {
 747                 int bits = ctx->key_len * 8;
 748                 aes_t4_set_encrypt_key(key, bits, &gctx->ks.ks);
 749                 CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks,
 750                                 (block128_f)aes_t4_encrypt);
 751                 switch (bits) {
 752                     case 128:
 753                         gctx->ctr = (ctr128_f)aes128_t4_ctr32_encrypt;
 754                         break;
 755                     case 192:
 756                         gctx->ctr = (ctr128_f)aes192_t4_ctr32_encrypt;
 757                         break;
 758                     case 256:
 759                         gctx->ctr = (ctr128_f)aes256_t4_ctr32_encrypt;
 760                         break;
 761                     default:
 762                         return 0;
 763                 }
 764                 /* If we have an iv can set it directly, otherwise use
 765                  * saved IV.
 766                  */
 767                 if (iv == NULL && gctx->iv_set)
 768                         iv = gctx->iv;
 769                 if (iv)
 770                         {
 771                         CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
 772                         gctx->iv_set = 1;
 773                         }
 774                 gctx->key_set = 1;
 775                 }
 776         else
 777                 {
 778                 /* If key set use IV, otherwise copy */
 779                 if (gctx->key_set)
 780                         CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
 781                 else
 782                         memcpy(gctx->iv, iv, gctx->ivlen);
 783                 gctx->iv_set = 1;
 784                 gctx->iv_gen = 0;
 785                 }
 786         return 1;
 787         }
 788
 789 #define aes_t4_gcm_cipher aes_gcm_cipher
 790 static int aes_t4_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 791                 const unsigned char *in, size_t len);
 792
 793 static int aes_t4_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 794                         const unsigned char *iv, int enc)
 795         {
 796         EVP_AES_XTS_CTX *xctx = ctx->cipher_data;
 797         if (!iv && !key)
 798                 return 1;
 799
 800         if (key)
 801                 {
 802                 int bits = ctx->key_len * 4;
 803                 xctx->stream = NULL;
 804                 /* key_len is two AES keys */
 805                 if (enc)
 806                         {
 807                         aes_t4_set_encrypt_key(key, bits, &xctx->ks1.ks);
 808                         xctx->xts.block1 = (block128_f)aes_t4_encrypt;
 809                         switch (bits) {
 810                             case 128:
 811                                 xctx->stream = aes128_t4_xts_encrypt;
 812                                 break;
 813 #if 0 /* not yet */
 814                             case 192:
 815                                 xctx->stream = aes192_t4_xts_encrypt;
 816                                 break;
 817 #endif
 818                             case 256:
 819                                 xctx->stream = aes256_t4_xts_encrypt;
 820                                 break;
 821                             default:
 822                                 return 0;
 823                             }
 824                         }
 825                 else
 826                         {
 827                         aes_t4_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
 828                         xctx->xts.block1 = (block128_f)aes_t4_decrypt;
 829                         switch (bits) {
 830                             case 128:
 831                                 xctx->stream = aes128_t4_xts_decrypt;
 832                                 break;
 833 #if 0 /* not yet */
 834                             case 192:
 835                                 xctx->stream = aes192_t4_xts_decrypt;
 836                                 break;
 837 #endif
 838                             case 256:
 839                                 xctx->stream = aes256_t4_xts_decrypt;
 840                                 break;
 841                             default:
 842                                 return 0;
 843                             }
 844                         }
 845
 846                 aes_t4_set_encrypt_key(key + ctx->key_len/2,
 847                                                 ctx->key_len * 4, &xctx->ks2.ks);
 848                 xctx->xts.block2 = (block128_f)aes_t4_encrypt;
 849
 850                 xctx->xts.key1 = &xctx->ks1;
 851                 }
 852
 853         if (iv)
 854                 {
 855                 xctx->xts.key2 = &xctx->ks2;
 856                 memcpy(ctx->iv, iv, 16);
 857                 }
 858
 859         return 1;
 860         }
 861
 862 #define aes_t4_xts_cipher aes_xts_cipher
 863 static int aes_t4_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 864                 const unsigned char *in, size_t len);
 865
 866 static int aes_t4_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 867                         const unsigned char *iv, int enc)
 868         {
 869         EVP_AES_CCM_CTX *cctx = ctx->cipher_data;
 870         if (!iv && !key)
 871                 return 1;
 872         if (key)
 873                 {
 874                 int bits = ctx->key_len * 8;
 875                 aes_t4_set_encrypt_key(key, bits, &cctx->ks.ks);
 876                 CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L,
 877                                         &cctx->ks, (block128_f)aes_t4_encrypt);
 878 #if 0 /* not yet */
 879                 switch (bits) {
 880                     case 128:
 881                         cctx->str = enc?(ccm128_f)aes128_t4_ccm64_encrypt :
 882                                 (ccm128_f)ae128_t4_ccm64_decrypt;
 883                         break;
 884                     case 192:
 885                         cctx->str = enc?(ccm128_f)aes192_t4_ccm64_encrypt :
 886                                 (ccm128_f)ae192_t4_ccm64_decrypt;
 887                         break;
 888                     case 256:
 889                         cctx->str = enc?(ccm128_f)aes256_t4_ccm64_encrypt :
 890                                 (ccm128_f)ae256_t4_ccm64_decrypt;
 891                         break;
 892                     default:
 893                         return 0;
 894                     }
 895 #endif
 896                 cctx->key_set = 1;
 897                 }
 898         if (iv)
 899                 {
 900                 memcpy(ctx->iv, iv, 15 - cctx->L);
 901                 cctx->iv_set = 1;
 902                 }
 903         return 1;
 904         }
 905
 906 #define aes_t4_ccm_cipher aes_ccm_cipher
 907 static int aes_t4_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 908                 const unsigned char *in, size_t len);
 909
 910
 911 static int aes_t4_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 912                         const unsigned char *iv, int enc)
 913         {
 914         EVP_AES_OCB_CTX *octx = ctx->cipher_data;
 915         if (!iv && !key)
 916                 return 1;
 917         if (key)
 918                 {
 919                 do
 920                         {
 921                         /* We set both the encrypt and decrypt key here because decrypt
 922                          * needs both. We could possibly optimise to remove setting the
 923                          * decrypt for an encryption operation.
 924                          */
 925                         aes_t4_set_encrypt_key(key, ctx->key_len * 8, &octx->ksenc);
 926                         aes_t4_set_decrypt_key(key, ctx->key_len * 8, &octx->ksdec);
 927                         if(!CRYPTO_ocb128_init(&octx->ocb, &octx->ksenc, &octx->ksdec,
 928                                         (block128_f)aes_t4_encrypt, (block128_f)aes_t4_decrypt))
 929                                 return 0;
 930                         }
 931                 while (0);
 932
 933                 /* If we have an iv we can set it directly, otherwise use
 934                  * saved IV.
 935                  */
 936                 if (iv == NULL && octx->iv_set)
 937                         iv = octx->iv;
 938                 if (iv)
 939                         {
 940                         if(CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen) != 1)
 941                                 return 0;
 942                         octx->iv_set = 1;
 943                         }
 944                 octx->key_set = 1;
 945                 }
 946         else
 947                 {
 948                 /* If key set use IV, otherwise copy */
 949                 if (octx->key_set)
 950                         CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen);
 951                 else
 952                         memcpy(octx->iv, iv, octx->ivlen);
 953                 octx->iv_set = 1;
 954                 }
 955         return 1;
 956         }
 957
 958 #define aes_t4_ocb_cipher aes_ocb_cipher
 959 static int aes_t4_ocb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 960                 const unsigned char *in, size_t len);
 961
 962
 963 #define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \
 964 static const EVP_CIPHER aes_t4_##keylen##_##mode = { \
 965         nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \
 966         flags|EVP_CIPH_##MODE##_MODE,   \
 967         aes_t4_init_key,                \
 968         aes_t4_##mode##_cipher,         \
 969         NULL,                           \
 970         sizeof(EVP_AES_KEY),            \
 971         NULL,NULL,NULL,NULL }; \
 972 static const EVP_CIPHER aes_##keylen##_##mode = { \
 973         nid##_##keylen##_##nmode,blocksize,     \
 974         keylen/8,ivlen, \
 975         flags|EVP_CIPH_##MODE##_MODE,   \
 976         aes_init_key,                   \
 977         aes_##mode##_cipher,            \
 978         NULL,                           \
 979         sizeof(EVP_AES_KEY),            \
 980         NULL,NULL,NULL,NULL }; \
 981 const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
 982 { return SPARC_AES_CAPABLE?&aes_t4_##keylen##_##mode:&aes_##keylen##_##mode; }
 983
 984 #define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \
 985 static const EVP_CIPHER aes_t4_##keylen##_##mode = { \
 986         nid##_##keylen##_##mode,blocksize, \
 987         (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \
 988         flags|EVP_CIPH_##MODE##_MODE,   \
 989         aes_t4_##mode##_init_key,       \
 990         aes_t4_##mode##_cipher,         \
 991         aes_##mode##_cleanup,           \
 992         sizeof(EVP_AES_##MODE##_CTX),   \
 993         NULL,NULL,aes_##mode##_ctrl,NULL }; \
 994 static const EVP_CIPHER aes_##keylen##_##mode = { \
 995         nid##_##keylen##_##mode,blocksize, \
 996         (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \
 997         flags|EVP_CIPH_##MODE##_MODE,   \
 998         aes_##mode##_init_key,          \
 999         aes_##mode##_cipher,            \
1000         aes_##mode##_cleanup,           \
1001         sizeof(EVP_AES_##MODE##_CTX),   \
1002         NULL,NULL,aes_##mode##_ctrl,NULL }; \
1003 const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
1004 { return SPARC_AES_CAPABLE?&aes_t4_##keylen##_##mode:&aes_##keylen##_##mode; }
1005
1006 #else
1007
1008 #define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \
1009 static const EVP_CIPHER aes_##keylen##_##mode = { \
1010         nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \
1011         flags|EVP_CIPH_##MODE##_MODE,   \
1012         aes_init_key,                   \
1013         aes_##mode##_cipher,            \
1014         NULL,                           \
1015         sizeof(EVP_AES_KEY),            \
1016         NULL,NULL,NULL,NULL }; \
1017 const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
1018 { return &aes_##keylen##_##mode; }
1019
1020 #define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \
1021 static const EVP_CIPHER aes_##keylen##_##mode = { \
1022         nid##_##keylen##_##mode,blocksize, \
1023         (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \
1024         flags|EVP_CIPH_##MODE##_MODE,   \
1025         aes_##mode##_init_key,          \
1026         aes_##mode##_cipher,            \
1027         aes_##mode##_cleanup,           \
1028         sizeof(EVP_AES_##MODE##_CTX),   \
1029         NULL,NULL,aes_##mode##_ctrl,NULL }; \
1030 const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
1031 { return &aes_##keylen##_##mode; }
1032
1033 #endif
1034
1035 #if defined(OPENSSL_CPUID_OBJ) && (defined(__arm__) || defined(__arm) || defined(__aarch64__))
1036 #include "arm_arch.h"
1037 #if __ARM_ARCH__>=7
1038 # if defined(BSAES_ASM)
1039 #  define BSAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
1040 # endif
1041 # define HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES)
1042 # define HWAES_set_encrypt_key aes_v8_set_encrypt_key
1043 # define HWAES_set_decrypt_key aes_v8_set_decrypt_key
1044 # define HWAES_encrypt aes_v8_encrypt
1045 # define HWAES_decrypt aes_v8_decrypt
1046 # define HWAES_cbc_encrypt aes_v8_cbc_encrypt
1047 # define HWAES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks
1048 #endif
1049 #endif
1050
1051 #if defined(HWAES_CAPABLE)
1052 int HWAES_set_encrypt_key(const unsigned char *userKey, const int bits,
1053         AES_KEY *key);
1054 int HWAES_set_decrypt_key(const unsigned char *userKey, const int bits,
1055         AES_KEY *key);
1056 void HWAES_encrypt(const unsigned char *in, unsigned char *out,
1057         const AES_KEY *key);
1058 void HWAES_decrypt(const unsigned char *in, unsigned char *out,
1059         const AES_KEY *key);
1060 void HWAES_cbc_encrypt(const unsigned char *in, unsigned char *out,
1061         size_t length, const AES_KEY *key,
1062         unsigned char *ivec, const int enc);
1063 void HWAES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
1064         size_t len, const AES_KEY *key, const unsigned char ivec[16]);
1065 #endif
1066
1067 #define BLOCK_CIPHER_generic_pack(nid,keylen,flags)             \
1068         BLOCK_CIPHER_generic(nid,keylen,16,16,cbc,cbc,CBC,flags|EVP_CIPH_FLAG_DEFAULT_ASN1)     \
1069         BLOCK_CIPHER_generic(nid,keylen,16,0,ecb,ecb,ECB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1)      \
1070         BLOCK_CIPHER_generic(nid,keylen,1,16,ofb128,ofb,OFB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1)   \
1071         BLOCK_CIPHER_generic(nid,keylen,1,16,cfb128,cfb,CFB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1)   \
1072         BLOCK_CIPHER_generic(nid,keylen,1,16,cfb1,cfb1,CFB,flags)       \
1073         BLOCK_CIPHER_generic(nid,keylen,1,16,cfb8,cfb8,CFB,flags)       \
1074         BLOCK_CIPHER_generic(nid,keylen,1,16,ctr,ctr,CTR,flags)
1075
1076 static int aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
1077                    const unsigned char *iv, int enc)
1078         {
1079         int ret, mode;
1080         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1081
1082         mode = ctx->cipher->flags & EVP_CIPH_MODE;
1083         if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE)
1084             && !enc)
1085 #ifdef HWAES_CAPABLE
1086             if (HWAES_CAPABLE)
1087                 {
1088                 ret = HWAES_set_decrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1089                 dat->block      = (block128_f)HWAES_decrypt;
1090                 dat->stream.cbc = NULL;
1091 #ifdef HWAES_cbc_encrypt
1092                 if (mode==EVP_CIPH_CBC_MODE)
1093                     dat->stream.cbc = (cbc128_f)HWAES_cbc_encrypt;
1094 #endif
1095                 }
1096             else
1097 #endif
1098 #ifdef BSAES_CAPABLE
1099             if (BSAES_CAPABLE && mode==EVP_CIPH_CBC_MODE)
1100                 {
1101                 ret = AES_set_decrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1102                 dat->block      = (block128_f)AES_decrypt;
1103                 dat->stream.cbc = (cbc128_f)bsaes_cbc_encrypt;
1104                 }
1105             else
1106 #endif
1107 #ifdef VPAES_CAPABLE
1108             if (VPAES_CAPABLE)
1109                 {
1110                 ret = vpaes_set_decrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1111                 dat->block      = (block128_f)vpaes_decrypt;
1112                 dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
1113                                         (cbc128_f)vpaes_cbc_encrypt :
1114                                         NULL;
1115                 }
1116             else
1117 #endif
1118                 {
1119                 ret = AES_set_decrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1120                 dat->block      = (block128_f)AES_decrypt;
1121                 dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
1122                                         (cbc128_f)AES_cbc_encrypt :
1123                                         NULL;
1124                 }
1125         else
1126 #ifdef HWAES_CAPABLE
1127             if (HWAES_CAPABLE)
1128                 {
1129                 ret = HWAES_set_encrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1130                 dat->block      = (block128_f)HWAES_encrypt;
1131                 dat->stream.cbc = NULL;
1132 #ifdef HWAES_cbc_encrypt
1133                 if (mode==EVP_CIPH_CBC_MODE)
1134                     dat->stream.cbc = (cbc128_f)HWAES_cbc_encrypt;
1135                 else
1136 #endif
1137 #ifdef HWAES_ctr32_encrypt_blocks
1138                 if (mode==EVP_CIPH_CTR_MODE)
1139                     dat->stream.ctr = (ctr128_f)HWAES_ctr32_encrypt_blocks;
1140                 else
1141 #endif
1142                 (void)0;        /* terminate potentially open 'else' */
1143                 }
1144             else
1145 #endif
1146 #ifdef BSAES_CAPABLE
1147             if (BSAES_CAPABLE && mode==EVP_CIPH_CTR_MODE)
1148                 {
1149                 ret = AES_set_encrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1150                 dat->block      = (block128_f)AES_encrypt;
1151                 dat->stream.ctr = (ctr128_f)bsaes_ctr32_encrypt_blocks;
1152                 }
1153             else
1154 #endif
1155 #ifdef VPAES_CAPABLE
1156             if (VPAES_CAPABLE)
1157                 {
1158                 ret = vpaes_set_encrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1159                 dat->block      = (block128_f)vpaes_encrypt;
1160                 dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
1161                                         (cbc128_f)vpaes_cbc_encrypt :
1162                                         NULL;
1163                 }
1164             else
1165 #endif
1166                 {
1167                 ret = AES_set_encrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1168                 dat->block      = (block128_f)AES_encrypt;
1169                 dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
1170                                         (cbc128_f)AES_cbc_encrypt :
1171                                         NULL;
1172 #ifdef AES_CTR_ASM
1173                 if (mode==EVP_CIPH_CTR_MODE)
1174                         dat->stream.ctr = (ctr128_f)AES_ctr32_encrypt;
1175 #endif
1176                 }
1177
1178         if(ret < 0)
1179                 {
1180                 EVPerr(EVP_F_AES_INIT_KEY,EVP_R_AES_KEY_SETUP_FAILED);
1181                 return 0;
1182                 }
1183
1184         return 1;
1185         }
1186
1187 static int aes_cbc_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
1188         const unsigned char *in, size_t len)
1189 {
1190         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1191
1192         if (dat->stream.cbc)
1193                 (*dat->stream.cbc)(in,out,len,&dat->ks,ctx->iv,ctx->encrypt);
1194         else if (ctx->encrypt)
1195                 CRYPTO_cbc128_encrypt(in,out,len,&dat->ks,ctx->iv,dat->block);
1196         else
1197                 CRYPTO_cbc128_decrypt(in,out,len,&dat->ks,ctx->iv,dat->block);
1198
1199         return 1;
1200 }
1201
1202 static int aes_ecb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
1203         const unsigned char *in, size_t len)
1204 {
1205         size_t  bl = ctx->cipher->block_size;
1206         size_t  i;
1207         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1208
1209         if (len<bl)     return 1;
1210
1211         for (i=0,len-=bl;i<=len;i+=bl)
1212                 (*dat->block)(in+i,out+i,&dat->ks);
1213
1214         return 1;
1215 }
1216
1217 static int aes_ofb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
1218         const unsigned char *in,size_t len)
1219 {
1220         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1221
1222         CRYPTO_ofb128_encrypt(in,out,len,&dat->ks,
1223                         ctx->iv,&ctx->num,dat->block);
1224         return 1;
1225 }
1226
1227 static int aes_cfb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
1228         const unsigned char *in,size_t len)
1229 {
1230         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1231
1232         CRYPTO_cfb128_encrypt(in,out,len,&dat->ks,
1233                         ctx->iv,&ctx->num,ctx->encrypt,dat->block);
1234         return 1;
1235 }
1236
1237 static int aes_cfb8_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
1238         const unsigned char *in,size_t len)
1239 {
1240         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1241
1242         CRYPTO_cfb128_8_encrypt(in,out,len,&dat->ks,
1243                         ctx->iv,&ctx->num,ctx->encrypt,dat->block);
1244         return 1;
1245 }
1246
1247 static int aes_cfb1_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
1248         const unsigned char *in,size_t len)
1249 {
1250         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1251
1252         if (ctx->flags&EVP_CIPH_FLAG_LENGTH_BITS) {
1253                 CRYPTO_cfb128_1_encrypt(in,out,len,&dat->ks,
1254                         ctx->iv,&ctx->num,ctx->encrypt,dat->block);
1255                 return 1;
1256         }
1257
1258         while (len>=MAXBITCHUNK) {
1259                 CRYPTO_cfb128_1_encrypt(in,out,MAXBITCHUNK*8,&dat->ks,
1260                         ctx->iv,&ctx->num,ctx->encrypt,dat->block);
1261                 len-=MAXBITCHUNK;
1262         }
1263         if (len)
1264                 CRYPTO_cfb128_1_encrypt(in,out,len*8,&dat->ks,
1265                         ctx->iv,&ctx->num,ctx->encrypt,dat->block);
1266
1267         return 1;
1268 }
1269
1270 static int aes_ctr_cipher (EVP_CIPHER_CTX *ctx, unsigned char *out,
1271                 const unsigned char *in, size_t len)
1272 {
1273         unsigned int num = ctx->num;
1274         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1275
1276         if (dat->stream.ctr)
1277                 CRYPTO_ctr128_encrypt_ctr32(in,out,len,&dat->ks,
1278                         ctx->iv,ctx->buf,&num,dat->stream.ctr);
1279         else
1280                 CRYPTO_ctr128_encrypt(in,out,len,&dat->ks,
1281                         ctx->iv,ctx->buf,&num,dat->block);
1282         ctx->num = (size_t)num;
1283         return 1;
1284 }
1285
1286 BLOCK_CIPHER_generic_pack(NID_aes,128,EVP_CIPH_FLAG_FIPS)
1287 BLOCK_CIPHER_generic_pack(NID_aes,192,EVP_CIPH_FLAG_FIPS)
1288 BLOCK_CIPHER_generic_pack(NID_aes,256,EVP_CIPH_FLAG_FIPS)
1289
1290 static int aes_gcm_cleanup(EVP_CIPHER_CTX *c)
1291         {
1292         EVP_AES_GCM_CTX *gctx = c->cipher_data;
1293         OPENSSL_cleanse(&gctx->gcm, sizeof(gctx->gcm));
1294         if (gctx->iv != c->iv)
1295                 OPENSSL_free(gctx->iv);
1296         return 1;
1297         }
1298
1299 /* increment counter (64-bit int) by 1 */
1300 static void ctr64_inc(unsigned char *counter) {
1301         int n=8;
1302         unsigned char  c;
1303
1304         do {
1305                 --n;
1306                 c = counter[n];
1307                 ++c;
1308                 counter[n] = c;
1309                 if (c) return;
1310         } while (n);
1311 }
1312
1313 static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
1314         {
1315         EVP_AES_GCM_CTX *gctx = c->cipher_data;
1316         switch (type)
1317                 {
1318         case EVP_CTRL_INIT:
1319                 gctx->key_set = 0;
1320                 gctx->iv_set = 0;
1321                 gctx->ivlen = c->cipher->iv_len;
1322                 gctx->iv = c->iv;
1323                 gctx->taglen = -1;
1324                 gctx->iv_gen = 0;
1325                 gctx->tls_aad_len = -1;
1326                 return 1;
1327
1328         case EVP_CTRL_GCM_SET_IVLEN:
1329                 if (arg <= 0)
1330                         return 0;
1331 #ifdef OPENSSL_FIPS
1332                 if (FIPS_module_mode() && !(c->flags & EVP_CIPH_FLAG_NON_FIPS_ALLOW)
1333                                                  && arg < 12)
1334                         return 0;
1335 #endif
1336                 /* Allocate memory for IV if needed */
1337                 if ((arg > EVP_MAX_IV_LENGTH) && (arg > gctx->ivlen))
1338                         {
1339                         if (gctx->iv != c->iv)
1340                                 OPENSSL_free(gctx->iv);
1341                         gctx->iv = OPENSSL_malloc(arg);
1342                         if (!gctx->iv)
1343                                 return 0;
1344                         }
1345                 gctx->ivlen = arg;
1346                 return 1;
1347
1348         case EVP_CTRL_GCM_SET_TAG:
1349                 if (arg <= 0 || arg > 16 || c->encrypt)
1350                         return 0;
1351                 memcpy(c->buf, ptr, arg);
1352                 gctx->taglen = arg;
1353                 return 1;
1354
1355         case EVP_CTRL_GCM_GET_TAG:
1356                 if (arg <= 0 || arg > 16 || !c->encrypt || gctx->taglen < 0)
1357                         return 0;
1358                 memcpy(ptr, c->buf, arg);
1359                 return 1;
1360
1361         case EVP_CTRL_GCM_SET_IV_FIXED:
1362                 /* Special case: -1 length restores whole IV */
1363                 if (arg == -1)
1364                         {
1365                         memcpy(gctx->iv, ptr, gctx->ivlen);
1366                         gctx->iv_gen = 1;
1367                         return 1;
1368                         }
1369                 /* Fixed field must be at least 4 bytes and invocation field
1370                  * at least 8.
1371                  */
1372                 if ((arg < 4) || (gctx->ivlen - arg) < 8)
1373                         return 0;
1374                 if (arg)
1375                         memcpy(gctx->iv, ptr, arg);
1376                 if (c->encrypt &&
1377                         RAND_bytes(gctx->iv + arg, gctx->ivlen - arg) <= 0)
1378                         return 0;
1379                 gctx->iv_gen = 1;
1380                 return 1;
1381
1382         case EVP_CTRL_GCM_IV_GEN:
1383                 if (gctx->iv_gen == 0 || gctx->key_set == 0)
1384                         return 0;
1385                 CRYPTO_gcm128_setiv(&gctx->gcm, gctx->iv, gctx->ivlen);
1386                 if (arg <= 0 || arg > gctx->ivlen)
1387                         arg = gctx->ivlen;
1388                 memcpy(ptr, gctx->iv + gctx->ivlen - arg, arg);
1389                 /* Invocation field will be at least 8 bytes in size and
1390                  * so no need to check wrap around or increment more than
1391                  * last 8 bytes.
1392                  */
1393                 ctr64_inc(gctx->iv + gctx->ivlen - 8);
1394                 gctx->iv_set = 1;
1395                 return 1;
1396
1397         case EVP_CTRL_GCM_SET_IV_INV:
1398                 if (gctx->iv_gen == 0 || gctx->key_set == 0 || c->encrypt)
1399                         return 0;
1400                 memcpy(gctx->iv + gctx->ivlen - arg, ptr, arg);
1401                 CRYPTO_gcm128_setiv(&gctx->gcm, gctx->iv, gctx->ivlen);
1402                 gctx->iv_set = 1;
1403                 return 1;
1404
1405         case EVP_CTRL_AEAD_TLS1_AAD:
1406                 /* Save the AAD for later use */
1407                 if (arg != 13)
1408                         return 0;
1409                 memcpy(c->buf, ptr, arg);
1410                 gctx->tls_aad_len = arg;
1411                         {
1412                         unsigned int len=c->buf[arg-2]<<8|c->buf[arg-1];
1413                         /* Correct length for explicit IV */
1414                         len -= EVP_GCM_TLS_EXPLICIT_IV_LEN;
1415                         /* If decrypting correct for tag too */
1416                         if (!c->encrypt)
1417                                 len -= EVP_GCM_TLS_TAG_LEN;
1418                         c->buf[arg-2] = len>>8;
1419                         c->buf[arg-1] = len & 0xff;
1420                         }
1421                 /* Extra padding: tag appended to record */
1422                 return EVP_GCM_TLS_TAG_LEN;
1423
1424         case EVP_CTRL_COPY:
1425                 {
1426                         EVP_CIPHER_CTX *out = ptr;
1427                         EVP_AES_GCM_CTX *gctx_out = out->cipher_data;
1428                         if (gctx->gcm.key)
1429                                 {
1430                                 if (gctx->gcm.key != &gctx->ks)
1431                                         return 0;
1432                                 gctx_out->gcm.key = &gctx_out->ks;
1433                                 }
1434                         if (gctx->iv == c->iv)
1435                                 gctx_out->iv = out->iv;
1436                         else
1437                         {
1438                                 gctx_out->iv = OPENSSL_malloc(gctx->ivlen);
1439                                 if (!gctx_out->iv)
1440                                         return 0;
1441                                 memcpy(gctx_out->iv, gctx->iv, gctx->ivlen);
1442                         }
1443                         return 1;
1444                 }
1445
1446         default:
1447                 return -1;
1448
1449                 }
1450         }
1451
1452 static int aes_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
1453                         const unsigned char *iv, int enc)
1454         {
1455         EVP_AES_GCM_CTX *gctx = ctx->cipher_data;
1456         if (!iv && !key)
1457                 return 1;
1458         if (key)
1459                 { do {
1460 #ifdef HWAES_CAPABLE
1461                 if (HWAES_CAPABLE)
1462                         {
1463                         HWAES_set_encrypt_key(key,ctx->key_len*8,&gctx->ks.ks);
1464                         CRYPTO_gcm128_init(&gctx->gcm,&gctx->ks,
1465                                         (block128_f)HWAES_encrypt);
1466 #ifdef HWAES_ctr32_encrypt_blocks
1467                         gctx->ctr = (ctr128_f)HWAES_ctr32_encrypt_blocks;
1468 #else
1469                         gctx->ctr = NULL;
1470 #endif
1471                         break;
1472                         }
1473                 else
1474 #endif
1475 #ifdef BSAES_CAPABLE
1476                 if (BSAES_CAPABLE)
1477                         {
1478                         AES_set_encrypt_key(key,ctx->key_len*8,&gctx->ks.ks);
1479                         CRYPTO_gcm128_init(&gctx->gcm,&gctx->ks,
1480                                         (block128_f)AES_encrypt);
1481                         gctx->ctr = (ctr128_f)bsaes_ctr32_encrypt_blocks;
1482                         break;
1483                         }
1484                 else
1485 #endif
1486 #ifdef VPAES_CAPABLE
1487                 if (VPAES_CAPABLE)
1488                         {
1489                         vpaes_set_encrypt_key(key,ctx->key_len*8,&gctx->ks.ks);
1490                         CRYPTO_gcm128_init(&gctx->gcm,&gctx->ks,
1491                                         (block128_f)vpaes_encrypt);
1492                         gctx->ctr = NULL;
1493                         break;
1494                         }
1495                 else
1496 #endif
1497                 (void)0;        /* terminate potentially open 'else' */
1498
1499                 AES_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks);
1500                 CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f)AES_encrypt);
1501 #ifdef AES_CTR_ASM
1502                 gctx->ctr = (ctr128_f)AES_ctr32_encrypt;
1503 #else
1504                 gctx->ctr = NULL;
1505 #endif
1506                 } while (0);
1507
1508                 /* If we have an iv can set it directly, otherwise use
1509                  * saved IV.
1510                  */
1511                 if (iv == NULL && gctx->iv_set)
1512                         iv = gctx->iv;
1513                 if (iv)
1514                         {
1515                         CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
1516                         gctx->iv_set = 1;
1517                         }
1518                 gctx->key_set = 1;
1519                 }
1520         else
1521                 {
1522                 /* If key set use IV, otherwise copy */
1523                 if (gctx->key_set)
1524                         CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
1525                 else
1526                         memcpy(gctx->iv, iv, gctx->ivlen);
1527                 gctx->iv_set = 1;
1528                 gctx->iv_gen = 0;
1529                 }
1530         return 1;
1531         }
1532
1533 /* Handle TLS GCM packet format. This consists of the last portion of the IV
1534  * followed by the payload and finally the tag. On encrypt generate IV,
1535  * encrypt payload and write the tag. On verify retrieve IV, decrypt payload
1536  * and verify tag.
1537  */
1538
1539 static int aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
1540                 const unsigned char *in, size_t len)
1541         {
1542         EVP_AES_GCM_CTX *gctx = ctx->cipher_data;
1543         int rv = -1;
1544         /* Encrypt/decrypt must be performed in place */
1545         if (out != in || len < (EVP_GCM_TLS_EXPLICIT_IV_LEN+EVP_GCM_TLS_TAG_LEN))
1546                 return -1;
1547         /* Set IV from start of buffer or generate IV and write to start
1548          * of buffer.
1549          */
1550         if (EVP_CIPHER_CTX_ctrl(ctx, ctx->encrypt ?
1551                                 EVP_CTRL_GCM_IV_GEN : EVP_CTRL_GCM_SET_IV_INV,
1552                                 EVP_GCM_TLS_EXPLICIT_IV_LEN, out) <= 0)
1553                 goto err;
1554         /* Use saved AAD */
1555         if (CRYPTO_gcm128_aad(&gctx->gcm, ctx->buf, gctx->tls_aad_len))
1556                 goto err;
1557         /* Fix buffer and length to point to payload */
1558         in += EVP_GCM_TLS_EXPLICIT_IV_LEN;
1559         out += EVP_GCM_TLS_EXPLICIT_IV_LEN;
1560         len -= EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN;
1561         if (ctx->encrypt)
1562                 {
1563                 /* Encrypt payload */
1564                 if (gctx->ctr)
1565                         {
1566                         size_t bulk=0;
1567 #if defined(AES_GCM_ASM)
1568                         if (len>=32 && AES_GCM_ASM(gctx))
1569                                 {
1570                                 if (CRYPTO_gcm128_encrypt(&gctx->gcm,NULL,NULL,0))
1571                                         return -1;
1572
1573                                 bulk = AES_gcm_encrypt(in,out,len,
1574                                                         gctx->gcm.key,
1575                                                         gctx->gcm.Yi.c,
1576                                                         gctx->gcm.Xi.u);
1577                                 gctx->gcm.len.u[1] += bulk;
1578                                 }
1579 #endif
1580                         if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm,
1581                                                         in +bulk,
1582                                                         out+bulk,
1583                                                         len-bulk,
1584                                                         gctx->ctr))
1585                                 goto err;
1586                         }
1587                 else    {
1588                         size_t bulk=0;
1589 #if defined(AES_GCM_ASM2)
1590                         if (len>=32 && AES_GCM_ASM2(gctx))
1591                                 {
1592                                 if (CRYPTO_gcm128_encrypt(&gctx->gcm,NULL,NULL,0))
1593                                         return -1;
1594
1595                                 bulk = AES_gcm_encrypt(in,out,len,
1596                                                         gctx->gcm.key,
1597                                                         gctx->gcm.Yi.c,
1598                                                         gctx->gcm.Xi.u);
1599                                 gctx->gcm.len.u[1] += bulk;
1600                                 }
1601 #endif
1602                         if (CRYPTO_gcm128_encrypt(&gctx->gcm,
1603                                                         in +bulk,
1604                                                         out+bulk,
1605                                                         len-bulk))
1606                                 goto err;
1607                         }
1608                 out += len;
1609                 /* Finally write tag */
1610                 CRYPTO_gcm128_tag(&gctx->gcm, out, EVP_GCM_TLS_TAG_LEN);
1611                 rv = len + EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN;
1612                 }
1613         else
1614                 {
1615                 /* Decrypt */
1616                 if (gctx->ctr)
1617                         {
1618                         size_t bulk=0;
1619 #if defined(AES_GCM_ASM)
1620                         if (len>=16 && AES_GCM_ASM(gctx))
1621                                 {
1622                                 if (CRYPTO_gcm128_decrypt(&gctx->gcm,NULL,NULL,0))
1623                                         return -1;
1624
1625                                 bulk = AES_gcm_decrypt(in,out,len,
1626                                                         gctx->gcm.key,
1627                                                         gctx->gcm.Yi.c,
1628                                                         gctx->gcm.Xi.u);
1629                                 gctx->gcm.len.u[1] += bulk;
1630                                 }
1631 #endif
1632                         if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm,
1633                                                         in +bulk,
1634                                                         out+bulk,
1635                                                         len-bulk,
1636                                                         gctx->ctr))
1637                                 goto err;
1638                         }
1639                 else    {
1640                         size_t bulk=0;
1641 #if defined(AES_GCM_ASM2)
1642                         if (len>=16 && AES_GCM_ASM2(gctx))
1643                                 {
1644                                 if (CRYPTO_gcm128_decrypt(&gctx->gcm,NULL,NULL,0))
1645                                         return -1;
1646
1647                                 bulk = AES_gcm_decrypt(in,out,len,
1648                                                         gctx->gcm.key,
1649                                                         gctx->gcm.Yi.c,
1650                                                         gctx->gcm.Xi.u);
1651                                 gctx->gcm.len.u[1] += bulk;
1652                                 }
1653 #endif
1654                         if (CRYPTO_gcm128_decrypt(&gctx->gcm,
1655                                                         in +bulk,
1656                                                         out+bulk,
1657                                                         len-bulk))
1658                                 goto err;
1659                         }
1660                 /* Retrieve tag */
1661                 CRYPTO_gcm128_tag(&gctx->gcm, ctx->buf,
1662                                         EVP_GCM_TLS_TAG_LEN);
1663                 /* If tag mismatch wipe buffer */
1664                 if (memcmp(ctx->buf, in + len, EVP_GCM_TLS_TAG_LEN))
1665                         {
1666                         OPENSSL_cleanse(out, len);
1667                         goto err;
1668                         }
1669                 rv = len;
1670                 }
1671
1672         err:
1673         gctx->iv_set = 0;
1674         gctx->tls_aad_len = -1;
1675         return rv;
1676         }
1677
1678 static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
1679                 const unsigned char *in, size_t len)
1680         {
1681         EVP_AES_GCM_CTX *gctx = ctx->cipher_data;
1682         /* If not set up, return error */
1683         if (!gctx->key_set)
1684                 return -1;
1685
1686         if (gctx->tls_aad_len >= 0)
1687                 return aes_gcm_tls_cipher(ctx, out, in, len);
1688
1689         if (!gctx->iv_set)
1690                 return -1;
1691         if (in)
1692                 {
1693                 if (out == NULL)
1694                         {
1695                         if (CRYPTO_gcm128_aad(&gctx->gcm, in, len))
1696                                 return -1;
1697                         }
1698                 else if (ctx->encrypt)
1699                         {
1700                         if (gctx->ctr)
1701                                 {
1702                                 size_t bulk=0;
1703 #if defined(AES_GCM_ASM)
1704                                 if (len>=32 && AES_GCM_ASM(gctx))
1705                                         {
1706                                         size_t res = (16-gctx->gcm.mres)%16;
1707
1708                                         if (CRYPTO_gcm128_encrypt(&gctx->gcm,
1709                                                         in,out,res))
1710                                                 return -1;
1711
1712                                         bulk = AES_gcm_encrypt(in+res,
1713                                                         out+res,len-res,                                                                gctx->gcm.key,
1714                                                         gctx->gcm.Yi.c,
1715                                                         gctx->gcm.Xi.u);
1716                                         gctx->gcm.len.u[1] += bulk;
1717                                         bulk += res;
1718                                         }
1719 #endif
1720                                 if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm,
1721                                                         in +bulk,
1722                                                         out+bulk,
1723                                                         len-bulk,
1724                                                         gctx->ctr))
1725                                         return -1;
1726                                 }
1727                         else    {
1728                                 size_t bulk=0;
1729 #if defined(AES_GCM_ASM2)
1730                                 if (len>=32 && AES_GCM_ASM2(gctx))
1731                                         {
1732                                         size_t res = (16-gctx->gcm.mres)%16;
1733
1734                                         if (CRYPTO_gcm128_encrypt(&gctx->gcm,
1735                                                         in,out,res))
1736                                                 return -1;
1737
1738                                         bulk = AES_gcm_encrypt(in+res,
1739                                                         out+res,len-res,                                                                gctx->gcm.key,
1740                                                         gctx->gcm.Yi.c,
1741                                                         gctx->gcm.Xi.u);
1742                                         gctx->gcm.len.u[1] += bulk;
1743                                         bulk += res;
1744                                         }
1745 #endif
1746                                 if (CRYPTO_gcm128_encrypt(&gctx->gcm,
1747                                                         in +bulk,
1748                                                         out+bulk,
1749                                                         len-bulk))
1750                                         return -1;
1751                                 }
1752                         }
1753                 else
1754                         {
1755                         if (gctx->ctr)
1756                                 {
1757                                 size_t bulk=0;
1758 #if defined(AES_GCM_ASM)
1759                                 if (len>=16 && AES_GCM_ASM(gctx))
1760                                         {
1761                                         size_t res = (16-gctx->gcm.mres)%16;
1762
1763                                         if (CRYPTO_gcm128_decrypt(&gctx->gcm,
1764                                                         in,out,res))
1765                                                 return -1;
1766
1767                                         bulk = AES_gcm_decrypt(in+res,
1768                                                         out+res,len-res,
1769                                                         gctx->gcm.key,
1770                                                         gctx->gcm.Yi.c,
1771                                                         gctx->gcm.Xi.u);
1772                                         gctx->gcm.len.u[1] += bulk;
1773                                         bulk += res;
1774                                         }
1775 #endif
1776                                 if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm,
1777                                                         in +bulk,
1778                                                         out+bulk,
1779                                                         len-bulk,
1780                                                         gctx->ctr))
1781                                         return -1;
1782                                 }
1783                         else    {
1784                                 size_t bulk=0;
1785 #if defined(AES_GCM_ASM2)
1786                                 if (len>=16 && AES_GCM_ASM2(gctx))
1787                                         {
1788                                         size_t res = (16-gctx->gcm.mres)%16;
1789
1790                                         if (CRYPTO_gcm128_decrypt(&gctx->gcm,
1791                                                         in,out,res))
1792                                                 return -1;
1793
1794                                         bulk = AES_gcm_decrypt(in+res,
1795                                                         out+res,len-res,
1796                                                         gctx->gcm.key,
1797                                                         gctx->gcm.Yi.c,
1798                                                         gctx->gcm.Xi.u);
1799                                         gctx->gcm.len.u[1] += bulk;
1800                                         bulk += res;
1801                                         }
1802 #endif
1803                                 if (CRYPTO_gcm128_decrypt(&gctx->gcm,
1804                                                         in +bulk,
1805                                                         out+bulk,
1806                                                         len-bulk))
1807                                         return -1;
1808                                 }
1809                         }
1810                 return len;
1811                 }
1812         else
1813                 {
1814                 if (!ctx->encrypt)
1815                         {
1816                         if (gctx->taglen < 0)
1817                                 return -1;
1818                         if (CRYPTO_gcm128_finish(&gctx->gcm,
1819                                         ctx->buf, gctx->taglen) != 0)
1820                                 return -1;
1821                         gctx->iv_set = 0;
1822                         return 0;
1823                         }
1824                 CRYPTO_gcm128_tag(&gctx->gcm, ctx->buf, 16);
1825                 gctx->taglen = 16;
1826                 /* Don't reuse the IV */
1827                 gctx->iv_set = 0;
1828                 return 0;
1829                 }
1830
1831         }
1832
1833 #define CUSTOM_FLAGS    (EVP_CIPH_FLAG_DEFAULT_ASN1 \
1834                 | EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \
1835                 | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_CTRL_INIT \
1836                 | EVP_CIPH_CUSTOM_COPY)
1837
1838 BLOCK_CIPHER_custom(NID_aes,128,1,12,gcm,GCM,
1839                 EVP_CIPH_FLAG_FIPS|EVP_CIPH_FLAG_AEAD_CIPHER|CUSTOM_FLAGS)
1840 BLOCK_CIPHER_custom(NID_aes,192,1,12,gcm,GCM,
1841                 EVP_CIPH_FLAG_FIPS|EVP_CIPH_FLAG_AEAD_CIPHER|CUSTOM_FLAGS)
1842 BLOCK_CIPHER_custom(NID_aes,256,1,12,gcm,GCM,
1843                 EVP_CIPH_FLAG_FIPS|EVP_CIPH_FLAG_AEAD_CIPHER|CUSTOM_FLAGS)
1844
1845 static int aes_xts_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
1846         {
1847         EVP_AES_XTS_CTX *xctx = c->cipher_data;
1848         if (type == EVP_CTRL_COPY)
1849                 {
1850                 EVP_CIPHER_CTX *out = ptr;
1851                 EVP_AES_XTS_CTX *xctx_out = out->cipher_data;
1852                 if (xctx->xts.key1)
1853                         {
1854                         if (xctx->xts.key1 != &xctx->ks1)
1855                                 return 0;
1856                         xctx_out->xts.key1 = &xctx_out->ks1;
1857                         }
1858                 if (xctx->xts.key2)
1859                         {
1860                         if (xctx->xts.key2 != &xctx->ks2)
1861                                 return 0;
1862                         xctx_out->xts.key2 = &xctx_out->ks2;
1863                         }
1864                 return 1;
1865                 }
1866         else if (type != EVP_CTRL_INIT)
1867                 return -1;
1868         /* key1 and key2 are used as an indicator both key and IV are set */
1869         xctx->xts.key1 = NULL;
1870         xctx->xts.key2 = NULL;
1871         return 1;
1872         }
1873
1874 static int aes_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
1875                         const unsigned char *iv, int enc)
1876         {
1877         EVP_AES_XTS_CTX *xctx = ctx->cipher_data;
1878         if (!iv && !key)
1879                 return 1;
1880
1881         if (key) do
1882                 {
1883 #ifdef AES_XTS_ASM
1884                 xctx->stream = enc ? AES_xts_encrypt : AES_xts_decrypt;
1885 #else
1886                 xctx->stream = NULL;
1887 #endif
1888                 /* key_len is two AES keys */
1889 #ifdef HWAES_CAPABLE
1890                 if (HWAES_CAPABLE)
1891                         {
1892                         if (enc)
1893                             {
1894                             HWAES_set_encrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
1895                             xctx->xts.block1 = (block128_f)HWAES_encrypt;
1896                             }
1897                         else
1898                             {
1899                             HWAES_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
1900                             xctx->xts.block1 = (block128_f)HWAES_decrypt;
1901                             }
1902
1903                         HWAES_set_encrypt_key(key + ctx->key_len/2,
1904                                                     ctx->key_len * 4, &xctx->ks2.ks);
1905                         xctx->xts.block2 = (block128_f)HWAES_encrypt;
1906
1907                         xctx->xts.key1 = &xctx->ks1;
1908                         break;
1909                         }
1910                 else
1911 #endif
1912 #ifdef BSAES_CAPABLE
1913                 if (BSAES_CAPABLE)
1914                         xctx->stream = enc ? bsaes_xts_encrypt : bsaes_xts_decrypt;
1915                 else
1916 #endif
1917 #ifdef VPAES_CAPABLE
1918                 if (VPAES_CAPABLE)
1919                     {
1920                     if (enc)
1921                         {
1922                         vpaes_set_encrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
1923                         xctx->xts.block1 = (block128_f)vpaes_encrypt;
1924                         }
1925                     else
1926                         {
1927                         vpaes_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
1928                         xctx->xts.block1 = (block128_f)vpaes_decrypt;
1929                         }
1930
1931                     vpaes_set_encrypt_key(key + ctx->key_len/2,
1932                                                 ctx->key_len * 4, &xctx->ks2.ks);
1933                     xctx->xts.block2 = (block128_f)vpaes_encrypt;
1934
1935                     xctx->xts.key1 = &xctx->ks1;
1936                     break;
1937                     }
1938                 else
1939 #endif
1940                 (void)0;        /* terminate potentially open 'else' */
1941
1942                 if (enc)
1943                         {
1944                         AES_set_encrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
1945                         xctx->xts.block1 = (block128_f)AES_encrypt;
1946                         }
1947                 else
1948                         {
1949                         AES_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
1950                         xctx->xts.block1 = (block128_f)AES_decrypt;
1951                         }
1952
1953                 AES_set_encrypt_key(key + ctx->key_len/2,
1954                                                 ctx->key_len * 4, &xctx->ks2.ks);
1955                 xctx->xts.block2 = (block128_f)AES_encrypt;
1956
1957                 xctx->xts.key1 = &xctx->ks1;
1958                 } while (0);
1959
1960         if (iv)
1961                 {
1962                 xctx->xts.key2 = &xctx->ks2;
1963                 memcpy(ctx->iv, iv, 16);
1964                 }
1965
1966         return 1;
1967         }
1968
1969 static int aes_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
1970                 const unsigned char *in, size_t len)
1971         {
1972         EVP_AES_XTS_CTX *xctx = ctx->cipher_data;
1973         if (!xctx->xts.key1 || !xctx->xts.key2)
1974                 return 0;
1975         if (!out || !in || len<AES_BLOCK_SIZE)
1976                 return 0;
1977 #ifdef OPENSSL_FIPS
1978         /* Requirement of SP800-38E */
1979         if (FIPS_module_mode() && !(ctx->flags & EVP_CIPH_FLAG_NON_FIPS_ALLOW) &&
1980                         (len > (1UL<<20)*16))
1981                 {
1982                 EVPerr(EVP_F_AES_XTS_CIPHER, EVP_R_TOO_LARGE);
1983                 return 0;
1984                 }
1985 #endif
1986         if (xctx->stream)
1987                 (*xctx->stream)(in, out, len,
1988                                 xctx->xts.key1, xctx->xts.key2, ctx->iv);
1989         else if (CRYPTO_xts128_encrypt(&xctx->xts, ctx->iv, in, out, len,
1990                                                                 ctx->encrypt))
1991                 return 0;
1992         return 1;
1993         }
1994
1995 #define aes_xts_cleanup NULL
1996
1997 #define XTS_FLAGS       (EVP_CIPH_FLAG_DEFAULT_ASN1 | EVP_CIPH_CUSTOM_IV \
1998                          | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_CTRL_INIT \
1999                          | EVP_CIPH_CUSTOM_COPY)
2000
2001 BLOCK_CIPHER_custom(NID_aes,128,1,16,xts,XTS,EVP_CIPH_FLAG_FIPS|XTS_FLAGS)
2002 BLOCK_CIPHER_custom(NID_aes,256,1,16,xts,XTS,EVP_CIPH_FLAG_FIPS|XTS_FLAGS)
2003
2004 static int aes_ccm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
2005         {
2006         EVP_AES_CCM_CTX *cctx = c->cipher_data;
2007         switch (type)
2008                 {
2009         case EVP_CTRL_INIT:
2010                 cctx->key_set = 0;
2011                 cctx->iv_set = 0;
2012                 cctx->L = 8;
2013                 cctx->M = 12;
2014                 cctx->tag_set = 0;
2015                 cctx->len_set = 0;
2016                 return 1;
2017
2018         case EVP_CTRL_CCM_SET_IVLEN:
2019                 arg = 15 - arg;
2020         case EVP_CTRL_CCM_SET_L:
2021                 if (arg < 2 || arg > 8)
2022                         return 0;
2023                 cctx->L = arg;
2024                 return 1;
2025
2026         case EVP_CTRL_CCM_SET_TAG:
2027                 if ((arg & 1) || arg < 4 || arg > 16)
2028                         return 0;
2029                 if ((c->encrypt && ptr) || (!c->encrypt && !ptr))
2030                         return 0;
2031                 if (ptr)
2032                         {
2033                         cctx->tag_set = 1;
2034                         memcpy(c->buf, ptr, arg);
2035                         }
2036                 cctx->M = arg;
2037                 return 1;
2038
2039         case EVP_CTRL_CCM_GET_TAG:
2040                 if (!c->encrypt || !cctx->tag_set)
2041                         return 0;
2042                 if(!CRYPTO_ccm128_tag(&cctx->ccm, ptr, (size_t)arg))
2043                         return 0;
2044                 cctx->tag_set = 0;
2045                 cctx->iv_set = 0;
2046                 cctx->len_set = 0;
2047                 return 1;
2048
2049         case EVP_CTRL_COPY:
2050                 {
2051                         EVP_CIPHER_CTX *out = ptr;
2052                         EVP_AES_CCM_CTX *cctx_out = out->cipher_data;
2053                         if (cctx->ccm.key)
2054                                 {
2055                                 if (cctx->ccm.key != &cctx->ks)
2056                                         return 0;
2057                                 cctx_out->ccm.key = &cctx_out->ks;
2058                                 }
2059                         return 1;
2060                 }
2061
2062         default:
2063                 return -1;
2064
2065                 }
2066         }
2067
2068 static int aes_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
2069                         const unsigned char *iv, int enc)
2070         {
2071         EVP_AES_CCM_CTX *cctx = ctx->cipher_data;
2072         if (!iv && !key)
2073                 return 1;
2074         if (key) do
2075                 {
2076 #ifdef HWAES_CAPABLE
2077                 if (HWAES_CAPABLE)
2078                         {
2079                         HWAES_set_encrypt_key(key,ctx->key_len*8,&cctx->ks.ks);
2080
2081                         CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L,
2082                                         &cctx->ks, (block128_f)HWAES_encrypt);
2083                         cctx->str = NULL;
2084                         cctx->key_set = 1;
2085                         break;
2086                         }
2087                 else
2088 #endif
2089 #ifdef VPAES_CAPABLE
2090                 if (VPAES_CAPABLE)
2091                         {
2092                         vpaes_set_encrypt_key(key, ctx->key_len*8, &cctx->ks.ks);
2093                         CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L,
2094                                         &cctx->ks, (block128_f)vpaes_encrypt);
2095                         cctx->str = NULL;
2096                         cctx->key_set = 1;
2097                         break;
2098                         }
2099 #endif
2100                 AES_set_encrypt_key(key, ctx->key_len * 8, &cctx->ks.ks);
2101                 CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L,
2102                                         &cctx->ks, (block128_f)AES_encrypt);
2103                 cctx->str = NULL;
2104                 cctx->key_set = 1;
2105                 } while (0);
2106         if (iv)
2107                 {
2108                 memcpy(ctx->iv, iv, 15 - cctx->L);
2109                 cctx->iv_set = 1;
2110                 }
2111         return 1;
2112         }
2113
2114 static int aes_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
2115                 const unsigned char *in, size_t len)
2116         {
2117         EVP_AES_CCM_CTX *cctx = ctx->cipher_data;
2118         CCM128_CONTEXT *ccm = &cctx->ccm;
2119         /* If not set up, return error */
2120         if (!cctx->iv_set && !cctx->key_set)
2121                 return -1;
2122         if (!ctx->encrypt && !cctx->tag_set)
2123                 return -1;
2124         if (!out)
2125                 {
2126                 if (!in)
2127                         {
2128                         if (CRYPTO_ccm128_setiv(ccm, ctx->iv, 15 - cctx->L,len))
2129                                 return -1;
2130                         cctx->len_set = 1;
2131                         return len;
2132                         }
2133                 /* If have AAD need message length */
2134                 if (!cctx->len_set && len)
2135                         return -1;
2136                 CRYPTO_ccm128_aad(ccm, in, len);
2137                 return len;
2138                 }
2139         /* EVP_*Final() doesn't return any data */
2140         if (!in)
2141                 return 0;
2142         /* If not set length yet do it */
2143         if (!cctx->len_set)
2144                 {
2145                 if (CRYPTO_ccm128_setiv(ccm, ctx->iv, 15 - cctx->L, len))
2146                         return -1;
2147                 cctx->len_set = 1;
2148                 }
2149         if (ctx->encrypt)
2150                 {
2151                 if (cctx->str ? CRYPTO_ccm128_encrypt_ccm64(ccm, in, out, len,
2152                                                 cctx->str) :
2153                                 CRYPTO_ccm128_encrypt(ccm, in, out, len))
2154                         return -1;
2155                 cctx->tag_set = 1;
2156                 return len;
2157                 }
2158         else
2159                 {
2160                 int rv = -1;
2161                 if (cctx->str ? !CRYPTO_ccm128_decrypt_ccm64(ccm, in, out, len,
2162                                                 cctx->str) :
2163                                 !CRYPTO_ccm128_decrypt(ccm, in, out, len))
2164                         {
2165                         unsigned char tag[16];
2166                         if (CRYPTO_ccm128_tag(ccm, tag, cctx->M))
2167                                 {
2168                                 if (!memcmp(tag, ctx->buf, cctx->M))
2169                                         rv = len;
2170                                 }
2171                         }
2172                 if (rv == -1)
2173                         OPENSSL_cleanse(out, len);
2174                 cctx->iv_set = 0;
2175                 cctx->tag_set = 0;
2176                 cctx->len_set = 0;
2177                 return rv;
2178                 }
2179
2180         }
2181
2182 #define aes_ccm_cleanup NULL
2183
2184 BLOCK_CIPHER_custom(NID_aes,128,1,12,ccm,CCM,EVP_CIPH_FLAG_FIPS|CUSTOM_FLAGS)
2185 BLOCK_CIPHER_custom(NID_aes,192,1,12,ccm,CCM,EVP_CIPH_FLAG_FIPS|CUSTOM_FLAGS)
2186 BLOCK_CIPHER_custom(NID_aes,256,1,12,ccm,CCM,EVP_CIPH_FLAG_FIPS|CUSTOM_FLAGS)
2187
2188 typedef struct
2189         {
2190         union { double align; AES_KEY ks; } ks;
2191         /* Indicates if IV has been set */
2192         unsigned char *iv;
2193         } EVP_AES_WRAP_CTX;
2194
2195 static int aes_wrap_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
2196                         const unsigned char *iv, int enc)
2197         {
2198         EVP_AES_WRAP_CTX *wctx = ctx->cipher_data;
2199         if (!iv && !key)
2200                 return 1;
2201         if (key)
2202                 {
2203                 if (ctx->encrypt)
2204                         AES_set_encrypt_key(key, ctx->key_len * 8, &wctx->ks.ks);
2205                 else
2206                         AES_set_decrypt_key(key, ctx->key_len * 8, &wctx->ks.ks);
2207                 if (!iv)
2208                         wctx->iv = NULL;
2209                 }
2210         if (iv)
2211                 {
2212                 memcpy(ctx->iv, iv, EVP_CIPHER_CTX_iv_length(ctx));
2213                 wctx->iv = ctx->iv;
2214                 }
2215         return 1;
2216         }
2217
2218 static int aes_wrap_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
2219                 const unsigned char *in, size_t inlen)
2220         {
2221         EVP_AES_WRAP_CTX *wctx = ctx->cipher_data;
2222         size_t rv;
2223         /* AES wrap with padding has IV length of 4, without padding 8 */
2224         int pad = EVP_CIPHER_CTX_iv_length(ctx) == 4;
2225         /* No final operation so always return zero length */
2226         if (!in)
2227                 return 0;
2228         /* Input length must always be non-zero */
2229         if (!inlen)
2230                 return -1;
2231         /* If decrypting need at least 16 bytes and multiple of 8 */
2232         if (!ctx->encrypt && (inlen < 16 || inlen & 0x7))
2233                 return -1;
2234         /* If not padding input must be multiple of 8 */
2235         if (!pad && inlen & 0x7)
2236                 return -1;
2237         if (!out)
2238                 {
2239                 if (ctx->encrypt)
2240                         {
2241                         /* If padding round up to multiple of 8 */
2242                         if (pad)
2243                                 inlen = (inlen + 7)/8 * 8;
2244                         /* 8 byte prefix */
2245                         return inlen + 8;
2246                         }
2247                 else
2248                         {
2249                         /* If not padding output will be exactly 8 bytes
2250                          * smaller than input. If padding it will be at
2251                          * least 8 bytes smaller but we don't know how
2252                          * much.
2253                          */
2254                         return inlen - 8;
2255                         }
2256                 }
2257         if (pad)
2258                 {
2259                 if (ctx->encrypt)
2260                         rv = CRYPTO_128_wrap_pad(&wctx->ks.ks, wctx->iv,
2261                                                 out, in, inlen,
2262                                                 (block128_f)AES_encrypt);
2263                 else
2264                         rv = CRYPTO_128_unwrap_pad(&wctx->ks.ks, wctx->iv,
2265                                                 out, in, inlen,
2266                                                 (block128_f)AES_decrypt);
2267                 }
2268         else
2269                 {
2270                 if (ctx->encrypt)
2271                         rv = CRYPTO_128_wrap(&wctx->ks.ks, wctx->iv,
2272                                                 out, in, inlen,
2273                                                 (block128_f)AES_encrypt);
2274                 else
2275                         rv = CRYPTO_128_unwrap(&wctx->ks.ks, wctx->iv,
2276                                                 out, in, inlen,
2277                                                 (block128_f)AES_decrypt);
2278                 }
2279         return rv ? (int)rv : -1;
2280         }
2281
2282 #define WRAP_FLAGS      (EVP_CIPH_WRAP_MODE \
2283                 | EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \
2284                 | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_FLAG_DEFAULT_ASN1)
2285
2286 static const EVP_CIPHER aes_128_wrap = {
2287         NID_id_aes128_wrap,
2288         8, 16, 8, WRAP_FLAGS,
2289         aes_wrap_init_key, aes_wrap_cipher,
2290         NULL,
2291         sizeof(EVP_AES_WRAP_CTX),
2292         NULL,NULL,NULL,NULL };
2293
2294 const EVP_CIPHER *EVP_aes_128_wrap(void)
2295         {
2296         return &aes_128_wrap;
2297         }
2298
2299 static const EVP_CIPHER aes_192_wrap = {
2300         NID_id_aes192_wrap,
2301         8, 24, 8, WRAP_FLAGS,
2302         aes_wrap_init_key, aes_wrap_cipher,
2303         NULL,
2304         sizeof(EVP_AES_WRAP_CTX),
2305         NULL,NULL,NULL,NULL };
2306
2307 const EVP_CIPHER *EVP_aes_192_wrap(void)
2308         {
2309         return &aes_192_wrap;
2310         }
2311
2312 static const EVP_CIPHER aes_256_wrap = {
2313         NID_id_aes256_wrap,
2314         8, 32, 8, WRAP_FLAGS,
2315         aes_wrap_init_key, aes_wrap_cipher,
2316         NULL,
2317         sizeof(EVP_AES_WRAP_CTX),
2318         NULL,NULL,NULL,NULL };
2319
2320 const EVP_CIPHER *EVP_aes_256_wrap(void)
2321         {
2322         return &aes_256_wrap;
2323         }
2324
2325 static const EVP_CIPHER aes_128_wrap_pad = {
2326         NID_id_aes128_wrap_pad,
2327         8, 16, 4, WRAP_FLAGS,
2328         aes_wrap_init_key, aes_wrap_cipher,
2329         NULL,
2330         sizeof(EVP_AES_WRAP_CTX),
2331         NULL,NULL,NULL,NULL };
2332
2333 const EVP_CIPHER *EVP_aes_128_wrap_pad(void)
2334         {
2335         return &aes_128_wrap_pad;
2336         }
2337
2338 static const EVP_CIPHER aes_192_wrap_pad = {
2339         NID_id_aes192_wrap_pad,
2340         8, 24, 4, WRAP_FLAGS,
2341         aes_wrap_init_key, aes_wrap_cipher,
2342         NULL,
2343         sizeof(EVP_AES_WRAP_CTX),
2344         NULL,NULL,NULL,NULL };
2345
2346 const EVP_CIPHER *EVP_aes_192_wrap_pad(void)
2347         {
2348         return &aes_192_wrap_pad;
2349         }
2350
2351 static const EVP_CIPHER aes_256_wrap_pad = {
2352         NID_id_aes256_wrap_pad,
2353         8, 32, 4, WRAP_FLAGS,
2354         aes_wrap_init_key, aes_wrap_cipher,
2355         NULL,
2356         sizeof(EVP_AES_WRAP_CTX),
2357         NULL,NULL,NULL,NULL };
2358
2359 const EVP_CIPHER *EVP_aes_256_wrap_pad(void)
2360         {
2361         return &aes_256_wrap_pad;
2362         }
2363
2364 static int aes_ocb_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
2365         {
2366         EVP_AES_OCB_CTX *octx = c->cipher_data;
2367         EVP_CIPHER_CTX *newc;
2368         EVP_AES_OCB_CTX *new_octx;
2369
2370         switch (type)
2371                 {
2372         case EVP_CTRL_INIT:
2373                 octx->key_set = 0;
2374                 octx->iv_set = 0;
2375                 octx->ivlen = c->cipher->iv_len;
2376                 octx->iv = c->iv;
2377                 octx->taglen = 16;
2378                 octx->data_buf_len = 0;
2379                 octx->aad_buf_len = 0;
2380                 return 1;
2381
2382         case EVP_CTRL_SET_IVLEN:
2383                 /* IV len must be 1 to 15 */
2384                 if (arg <= 0 || arg > 15)
2385                         return 0;
2386
2387                 octx->ivlen = arg;
2388                 return 1;
2389
2390         case EVP_CTRL_OCB_SET_TAGLEN:
2391                 /* Tag len must be 0 to 16 */
2392                 if (arg < 0 || arg > 16)
2393                         return 0;
2394
2395                 octx->taglen = arg;
2396                 return 1;
2397
2398         case EVP_CTRL_SET_TAG:
2399                 if (arg != octx->taglen || c->encrypt)
2400                         return 0;
2401                 memcpy(octx->tag, ptr, arg);
2402                 return 1;
2403
2404         case EVP_CTRL_GET_TAG:
2405                 if (arg != octx->taglen || !c->encrypt)
2406                         return 0;
2407
2408                 memcpy(ptr, octx->tag, arg);
2409                 return 1;
2410
2411         case EVP_CTRL_COPY:
2412                 newc = (EVP_CIPHER_CTX *)ptr;
2413                 new_octx = newc->cipher_data;
2414                 return CRYPTO_ocb128_copy_ctx(&new_octx->ocb, &octx->ocb,
2415                         &new_octx->ksenc, &new_octx->ksdec);
2416
2417         default:
2418                 return -1;
2419
2420                 }
2421         }
2422
2423
2424 static int aes_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
2425                 const unsigned char *iv, int enc)
2426         {
2427         EVP_AES_OCB_CTX *octx = ctx->cipher_data;
2428         if (!iv && !key)
2429                 return 1;
2430         if (key)
2431                 {
2432                 do
2433                         {
2434                         /* We set both the encrypt and decrypt key here because decrypt
2435                          * needs both. We could possibly optimise to remove setting the
2436                          * decrypt for an encryption operation.
2437                          */
2438 #ifdef VPAES_CAPABLE
2439                         if (VPAES_CAPABLE)
2440                                 {
2441                                 vpaes_set_encrypt_key(key,ctx->key_len*8,&octx->ksenc);
2442                                 vpaes_set_decrypt_key(key,ctx->key_len*8,&octx->ksdec);
2443                                 if(!CRYPTO_ocb128_init(&octx->ocb,&octx->ksenc,&octx->ksdec,
2444                                                 (block128_f)vpaes_encrypt,(block128_f)vpaes_decrypt))
2445                                         return 0;
2446                                 break;
2447                                 }
2448 #endif
2449                         AES_set_encrypt_key(key, ctx->key_len * 8, &octx->ksenc);
2450                         AES_set_decrypt_key(key, ctx->key_len * 8, &octx->ksdec);
2451                         if(!CRYPTO_ocb128_init(&octx->ocb, &octx->ksenc, &octx->ksdec,
2452                                         (block128_f)AES_encrypt, (block128_f)AES_decrypt))
2453                                 return 0;
2454                         }
2455                 while (0);
2456
2457                 /* If we have an iv we can set it directly, otherwise use
2458                  * saved IV.
2459                  */
2460                 if (iv == NULL && octx->iv_set)
2461                         iv = octx->iv;
2462                 if (iv)
2463                         {
2464                         if(CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen) != 1)
2465                                 return 0;
2466                         octx->iv_set = 1;
2467                         }
2468                 octx->key_set = 1;
2469                 }
2470         else
2471                 {
2472                 /* If key set use IV, otherwise copy */
2473                 if (octx->key_set)
2474                         CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen);
2475                 else
2476                         memcpy(octx->iv, iv, octx->ivlen);
2477                 octx->iv_set = 1;
2478                 }
2479         return 1;
2480         }
2481
2482 static int aes_ocb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
2483                 const unsigned char *in, size_t len)
2484         {
2485         unsigned char *buf;
2486         int *buf_len;
2487         int written_len = 0;
2488         size_t trailing_len;
2489         EVP_AES_OCB_CTX *octx = ctx->cipher_data;
2490
2491         /* If IV or Key not set then return error */
2492         if (!octx->iv_set)
2493                 return -1;
2494
2495         if (!octx->key_set)
2496                 return -1;
2497
2498         if (in)
2499                 {
2500                 /* Need to ensure we are only passing full blocks to low level OCB
2501                  * routines. We do it here rather than in EVP_EncryptUpdate/
2502                  * EVP_DecryptUpdate because we need to pass full blocks of AAD too
2503                  * and those routines don't support that
2504                  */
2505
2506                 /* Are we dealing with AAD or normal data here? */
2507                 if (out == NULL)
2508                         {
2509                         buf = octx->aad_buf;
2510                         buf_len = &(octx->aad_buf_len);
2511                         }
2512                 else
2513                         {
2514                         buf = octx->data_buf;
2515                         buf_len = &(octx->data_buf_len);
2516                         }
2517
2518                 /* If we've got a partially filled buffer from a previous call then use
2519                  * that data first
2520                  */
2521                 if(*buf_len)
2522                         {
2523                         unsigned int remaining;
2524
2525                         remaining = 16 - (*buf_len);
2526                         if(remaining > len)
2527                                 {
2528                                 memcpy(buf+(*buf_len), in, len);
2529                                 *(buf_len)+=len;
2530                                 return 0;
2531                                 }
2532                         memcpy(buf+(*buf_len), in, remaining);
2533
2534                         /* If we get here we've filled the buffer, so process it */
2535                         len -= remaining;
2536                         in += remaining;
2537                         if (out == NULL)
2538                                 {
2539                                 if(!CRYPTO_ocb128_aad(&octx->ocb, buf, 16))
2540                                         return -1;
2541                                 }
2542                         else if (ctx->encrypt)
2543                                 {
2544                                 if(!CRYPTO_ocb128_encrypt(&octx->ocb, buf, out, 16))
2545                                         return -1;
2546                                 }
2547                         else
2548                                 {
2549                                 if(!CRYPTO_ocb128_decrypt(&octx->ocb, buf, out, 16))
2550                                         return -1;
2551                                 }
2552                         written_len = 16;
2553                         *buf_len = 0;
2554                         }
2555
2556                 /* Do we have a partial block to handle at the end? */
2557                 trailing_len = len % 16;
2558
2559                 /* If we've got some full blocks to handle, then process these first */
2560                 if(len != trailing_len)
2561                         {
2562                         if (out == NULL)
2563                                 {
2564                                 if(!CRYPTO_ocb128_aad(&octx->ocb, in, len-trailing_len))
2565                                         return -1;
2566                                 }
2567                         else if (ctx->encrypt)
2568                                 {
2569                                 if(!CRYPTO_ocb128_encrypt(&octx->ocb, in, out, len-trailing_len))
2570                                         return -1;
2571                                 }
2572                         else
2573                                 {
2574                                 if(!CRYPTO_ocb128_decrypt(&octx->ocb, in, out, len-trailing_len))
2575                                         return -1;
2576                                 }
2577                         written_len += len-trailing_len;
2578                         in += len-trailing_len;
2579                         }
2580
2581                 /* Handle any trailing partial block */
2582                 if(trailing_len)
2583                         {
2584                         memcpy(buf, in, trailing_len);
2585                         *buf_len = trailing_len;
2586                         }
2587
2588                 return written_len;
2589                 }
2590         else
2591                 {
2592                 /* First of all empty the buffer of any partial block that we might
2593                  * have been provided - both for data and AAD
2594                  */
2595                 if(octx->data_buf_len)
2596                         {
2597                         if (ctx->encrypt)
2598                                 {
2599                                 if(!CRYPTO_ocb128_encrypt(&octx->ocb, octx->data_buf, out,
2600                                                 octx->data_buf_len))
2601                                         return -1;
2602                                 }
2603                         else
2604                                 {
2605                                 if(!CRYPTO_ocb128_decrypt(&octx->ocb, octx->data_buf, out,
2606                                                 octx->data_buf_len))
2607                                         return -1;
2608                                 }
2609                         written_len = octx->data_buf_len;
2610                         octx->data_buf_len = 0;
2611                         }
2612                 if(octx->aad_buf_len)
2613                         {
2614                         if(!CRYPTO_ocb128_aad(&octx->ocb, octx->aad_buf, octx->aad_buf_len))
2615                                 return -1;
2616                         octx->aad_buf_len = 0;
2617                         }
2618                 /* If decrypting then verify */
2619                 if (!ctx->encrypt)
2620                         {
2621                         if (octx->taglen < 0)
2622                                 return -1;
2623                         if (CRYPTO_ocb128_finish(&octx->ocb,
2624                                         octx->tag, octx->taglen) != 0)
2625                                 return -1;
2626                         octx->iv_set = 0;
2627                         return written_len;
2628                         }
2629                 /* If encrypting then just get the tag */
2630                 if(CRYPTO_ocb128_tag(&octx->ocb, octx->tag, 16) != 1)
2631                         return -1;
2632                 /* Don't reuse the IV */
2633                 octx->iv_set = 0;
2634                 return written_len;
2635                 }
2636         }
2637
2638 static int aes_ocb_cleanup(EVP_CIPHER_CTX *c)
2639         {
2640         EVP_AES_OCB_CTX *octx = c->cipher_data;
2641         CRYPTO_ocb128_cleanup(&octx->ocb);
2642         return 1;
2643         }
2644
2645 BLOCK_CIPHER_custom(NID_aes,128,16,12,ocb,OCB,CUSTOM_FLAGS)
2646 BLOCK_CIPHER_custom(NID_aes,192,16,12,ocb,OCB,CUSTOM_FLAGS)
2647 BLOCK_CIPHER_custom(NID_aes,256,16,12,ocb,OCB,CUSTOM_FLAGS)
2648
2649 #endif