crypto/evp/e_aes.c

   1 /* ====================================================================
   2  * Copyright (c) 2001-2014 The OpenSSL Project.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  *
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in
  13  *    the documentation and/or other materials provided with the
  14  *    distribution.
  15  *
  16  * 3. All advertising materials mentioning features or use of this
  17  *    software must display the following acknowledgment:
  18  *    "This product includes software developed by the OpenSSL Project
  19  *    for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
  20  *
  21  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
  22  *    endorse or promote products derived from this software without
  23  *    prior written permission. For written permission, please contact
  24  *    openssl-core@openssl.org.
  25  *
  26  * 5. Products derived from this software may not be called "OpenSSL"
  27  *    nor may "OpenSSL" appear in their names without prior written
  28  *    permission of the OpenSSL Project.
  29  *
  30  * 6. Redistributions of any form whatsoever must retain the following
  31  *    acknowledgment:
  32  *    "This product includes software developed by the OpenSSL Project
  33  *    for use in the OpenSSL Toolkit (http://www.openssl.org/)"
  34  *
  35  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
  36  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  37  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  38  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
  39  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  41  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  42  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  43  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  44  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  45  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
  46  * OF THE POSSIBILITY OF SUCH DAMAGE.
  47  * ====================================================================
  48  *
  49  */
  50
  51
  52
  53 #include <openssl/opensslconf.h>
  54 #ifndef OPENSSL_NO_AES
  55 #include <openssl/evp.h>
  56 #include <openssl/err.h>
  57 #include <string.h>
  58 #include <assert.h>
  59 #include <openssl/aes.h>
  60 #include "evp_locl.h"
  61 #include "modes_lcl.h"
  62 #include <openssl/rand.h>
  63
  64 typedef struct
  65         {
  66         union { double align; AES_KEY ks; } ks;
  67         block128_f block;
  68         union {
  69                 cbc128_f cbc;
  70                 ctr128_f ctr;
  71         } stream;
  72         } EVP_AES_KEY;
  73
  74 typedef struct
  75         {
  76         union { double align; AES_KEY ks; } ks; /* AES key schedule to use */
  77         int key_set;            /* Set if key initialised */
  78         int iv_set;             /* Set if an iv is set */
  79         GCM128_CONTEXT gcm;
  80         unsigned char *iv;      /* Temporary IV store */
  81         int ivlen;              /* IV length */
  82         int taglen;
  83         int iv_gen;             /* It is OK to generate IVs */
  84         int tls_aad_len;        /* TLS AAD length */
  85         ctr128_f ctr;
  86         } EVP_AES_GCM_CTX;
  87
  88 typedef struct
  89         {
  90         union { double align; AES_KEY ks; } ks1, ks2;   /* AES key schedules to use */
  91         XTS128_CONTEXT xts;
  92         void     (*stream)(const unsigned char *in,
  93                         unsigned char *out, size_t length,
  94                         const AES_KEY *key1, const AES_KEY *key2,
  95                         const unsigned char iv[16]);
  96         } EVP_AES_XTS_CTX;
  97
  98 typedef struct
  99         {
 100         union { double align; AES_KEY ks; } ks; /* AES key schedule to use */
 101         int key_set;            /* Set if key initialised */
 102         int iv_set;             /* Set if an iv is set */
 103         int tag_set;            /* Set if tag is valid */
 104         int len_set;            /* Set if message length set */
 105         int L, M;               /* L and M parameters from RFC3610 */
 106         CCM128_CONTEXT ccm;
 107         ccm128_f str;
 108         } EVP_AES_CCM_CTX;
 109
 110 #ifndef OPENSSL_NO_OCB
 111 typedef struct
 112         {
 113         AES_KEY ksenc;          /* AES key schedule to use for encryption */
 114         AES_KEY ksdec;          /* AES key schedule to use for decryption */
 115         int key_set;            /* Set if key initialised */
 116         int iv_set;             /* Set if an iv is set */
 117         OCB128_CONTEXT ocb;
 118         unsigned char *iv;      /* Temporary IV store */
 119         unsigned char tag[16];
 120         unsigned char data_buf[16]; /* Store partial data blocks */
 121         unsigned char aad_buf[16]; /* Store partial AAD blocks */
 122         int data_buf_len;
 123         int aad_buf_len;
 124         int ivlen;              /* IV length */
 125         int taglen;
 126         } EVP_AES_OCB_CTX;
 127 #endif
 128
 129 #define MAXBITCHUNK     ((size_t)1<<(sizeof(size_t)*8-4))
 130
 131 #ifdef VPAES_ASM
 132 int vpaes_set_encrypt_key(const unsigned char *userKey, int bits,
 133                         AES_KEY *key);
 134 int vpaes_set_decrypt_key(const unsigned char *userKey, int bits,
 135                         AES_KEY *key);
 136
 137 void vpaes_encrypt(const unsigned char *in, unsigned char *out,
 138                         const AES_KEY *key);
 139 void vpaes_decrypt(const unsigned char *in, unsigned char *out,
 140                         const AES_KEY *key);
 141
 142 void vpaes_cbc_encrypt(const unsigned char *in,
 143                         unsigned char *out,
 144                         size_t length,
 145                         const AES_KEY *key,
 146                         unsigned char *ivec, int enc);
 147 #endif
 148 #ifdef BSAES_ASM
 149 void bsaes_cbc_encrypt(const unsigned char *in, unsigned char *out,
 150                         size_t length, const AES_KEY *key,
 151                         unsigned char ivec[16], int enc);
 152 void bsaes_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
 153                         size_t len, const AES_KEY *key,
 154                         const unsigned char ivec[16]);
 155 void bsaes_xts_encrypt(const unsigned char *inp, unsigned char *out,
 156                         size_t len, const AES_KEY *key1,
 157                         const AES_KEY *key2, const unsigned char iv[16]);
 158 void bsaes_xts_decrypt(const unsigned char *inp, unsigned char *out,
 159                         size_t len, const AES_KEY *key1,
 160                         const AES_KEY *key2, const unsigned char iv[16]);
 161 #endif
 162 #ifdef AES_CTR_ASM
 163 void AES_ctr32_encrypt(const unsigned char *in, unsigned char *out,
 164                         size_t blocks, const AES_KEY *key,
 165                         const unsigned char ivec[AES_BLOCK_SIZE]);
 166 #endif
 167 #ifdef AES_XTS_ASM
 168 void AES_xts_encrypt(const char *inp,char *out,size_t len,
 169                         const AES_KEY *key1, const AES_KEY *key2,
 170                         const unsigned char iv[16]);
 171 void AES_xts_decrypt(const char *inp,char *out,size_t len,
 172                         const AES_KEY *key1, const AES_KEY *key2,
 173                         const unsigned char iv[16]);
 174 #endif
 175
 176 #if     defined(OPENSSL_CPUID_OBJ) && (defined(__powerpc__) || defined(__ppc__) || defined(_ARCH_PPC))
 177 # include "ppc_arch.h"
 178 # ifdef VPAES_ASM
 179 #  define VPAES_CAPABLE (OPENSSL_ppccap_P & PPC_ALTIVEC)
 180 # endif
 181 # define HWAES_CAPABLE  (OPENSSL_ppccap_P & PPC_CRYPTO207)
 182 # define HWAES_set_encrypt_key aes_p8_set_encrypt_key
 183 # define HWAES_set_decrypt_key aes_p8_set_decrypt_key
 184 # define HWAES_encrypt aes_p8_encrypt
 185 # define HWAES_decrypt aes_p8_decrypt
 186 # define HWAES_cbc_encrypt aes_p8_cbc_encrypt
 187 # define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks
 188 #endif
 189
 190 #if     defined(AES_ASM) && !defined(I386_ONLY) &&      (  \
 191         ((defined(__i386)       || defined(__i386__)    || \
 192           defined(_M_IX86)) && defined(OPENSSL_IA32_SSE2))|| \
 193         defined(__x86_64)       || defined(__x86_64__)  || \
 194         defined(_M_AMD64)       || defined(_M_X64)      || \
 195         defined(__INTEL__)                              )
 196
 197 extern unsigned int OPENSSL_ia32cap_P[];
 198
 199 #ifdef VPAES_ASM
 200 #define VPAES_CAPABLE   (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
 201 #endif
 202 #ifdef BSAES_ASM
 203 #define BSAES_CAPABLE   (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
 204 #endif
 205 /*
 206  * AES-NI section
 207  */
 208 #define AESNI_CAPABLE   (OPENSSL_ia32cap_P[1]&(1<<(57-32)))
 209
 210 int aesni_set_encrypt_key(const unsigned char *userKey, int bits,
 211                         AES_KEY *key);
 212 int aesni_set_decrypt_key(const unsigned char *userKey, int bits,
 213                         AES_KEY *key);
 214
 215 void aesni_encrypt(const unsigned char *in, unsigned char *out,
 216                         const AES_KEY *key);
 217 void aesni_decrypt(const unsigned char *in, unsigned char *out,
 218                         const AES_KEY *key);
 219
 220 void aesni_ecb_encrypt(const unsigned char *in,
 221                         unsigned char *out,
 222                         size_t length,
 223                         const AES_KEY *key,
 224                         int enc);
 225 void aesni_cbc_encrypt(const unsigned char *in,
 226                         unsigned char *out,
 227                         size_t length,
 228                         const AES_KEY *key,
 229                         unsigned char *ivec, int enc);
 230
 231 void aesni_ctr32_encrypt_blocks(const unsigned char *in,
 232                         unsigned char *out,
 233                         size_t blocks,
 234                         const void *key,
 235                         const unsigned char *ivec);
 236
 237 void aesni_xts_encrypt(const unsigned char *in,
 238                         unsigned char *out,
 239                         size_t length,
 240                         const AES_KEY *key1, const AES_KEY *key2,
 241                         const unsigned char iv[16]);
 242
 243 void aesni_xts_decrypt(const unsigned char *in,
 244                         unsigned char *out,
 245                         size_t length,
 246                         const AES_KEY *key1, const AES_KEY *key2,
 247                         const unsigned char iv[16]);
 248
 249 void aesni_ccm64_encrypt_blocks (const unsigned char *in,
 250                         unsigned char *out,
 251                         size_t blocks,
 252                         const void *key,
 253                         const unsigned char ivec[16],
 254                         unsigned char cmac[16]);
 255
 256 void aesni_ccm64_decrypt_blocks (const unsigned char *in,
 257                         unsigned char *out,
 258                         size_t blocks,
 259                         const void *key,
 260                         const unsigned char ivec[16],
 261                         unsigned char cmac[16]);
 262
 263 #if defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
 264 size_t aesni_gcm_encrypt(const unsigned char *in,
 265                         unsigned char *out,
 266                         size_t len,
 267                         const void *key,
 268                         unsigned char ivec[16],
 269                         u64 *Xi);
 270 #define AES_gcm_encrypt aesni_gcm_encrypt
 271 size_t aesni_gcm_decrypt(const unsigned char *in,
 272                         unsigned char *out,
 273                         size_t len,
 274                         const void *key,
 275                         unsigned char ivec[16],
 276                         u64 *Xi);
 277 #define AES_gcm_decrypt aesni_gcm_decrypt
 278 void gcm_ghash_avx(u64 Xi[2],const u128 Htable[16],const u8 *in,size_t len);
 279 #define AES_GCM_ASM(gctx)       (gctx->ctr==aesni_ctr32_encrypt_blocks && \
 280                                  gctx->gcm.ghash==gcm_ghash_avx)
 281 #define AES_GCM_ASM2(gctx)      (gctx->gcm.block==(block128_f)aesni_encrypt && \
 282                                  gctx->gcm.ghash==gcm_ghash_avx)
 283 #undef AES_GCM_ASM2             /* minor size optimization */
 284 #endif
 285
 286 static int aesni_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 287                    const unsigned char *iv, int enc)
 288         {
 289         int ret, mode;
 290         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
 291
 292         mode = ctx->cipher->flags & EVP_CIPH_MODE;
 293         if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE)
 294             && !enc)
 295                 {
 296                 ret = aesni_set_decrypt_key(key, ctx->key_len*8, ctx->cipher_data);
 297                 dat->block      = (block128_f)aesni_decrypt;
 298                 dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
 299                                         (cbc128_f)aesni_cbc_encrypt :
 300                                         NULL;
 301                 }
 302         else    {
 303                 ret = aesni_set_encrypt_key(key, ctx->key_len*8, ctx->cipher_data);
 304                 dat->block      = (block128_f)aesni_encrypt;
 305                 if (mode==EVP_CIPH_CBC_MODE)
 306                         dat->stream.cbc = (cbc128_f)aesni_cbc_encrypt;
 307                 else if (mode==EVP_CIPH_CTR_MODE)
 308                         dat->stream.ctr = (ctr128_f)aesni_ctr32_encrypt_blocks;
 309                 else
 310                         dat->stream.cbc = NULL;
 311                 }
 312
 313         if(ret < 0)
 314                 {
 315                 EVPerr(EVP_F_AESNI_INIT_KEY,EVP_R_AES_KEY_SETUP_FAILED);
 316                 return 0;
 317                 }
 318
 319         return 1;
 320         }
 321
 322 static int aesni_cbc_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 323         const unsigned char *in, size_t len)
 324 {
 325         aesni_cbc_encrypt(in,out,len,ctx->cipher_data,ctx->iv,ctx->encrypt);
 326
 327         return 1;
 328 }
 329
 330 static int aesni_ecb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 331         const unsigned char *in, size_t len)
 332 {
 333         size_t  bl = ctx->cipher->block_size;
 334
 335         if (len<bl)     return 1;
 336
 337         aesni_ecb_encrypt(in,out,len,ctx->cipher_data,ctx->encrypt);
 338
 339         return 1;
 340 }
 341
 342 #define aesni_ofb_cipher aes_ofb_cipher
 343 static int aesni_ofb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 344         const unsigned char *in,size_t len);
 345
 346 #define aesni_cfb_cipher aes_cfb_cipher
 347 static int aesni_cfb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 348         const unsigned char *in,size_t len);
 349
 350 #define aesni_cfb8_cipher aes_cfb8_cipher
 351 static int aesni_cfb8_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 352         const unsigned char *in,size_t len);
 353
 354 #define aesni_cfb1_cipher aes_cfb1_cipher
 355 static int aesni_cfb1_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 356         const unsigned char *in,size_t len);
 357
 358 #define aesni_ctr_cipher aes_ctr_cipher
 359 static int aesni_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 360                 const unsigned char *in, size_t len);
 361
 362 static int aesni_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 363                         const unsigned char *iv, int enc)
 364         {
 365         EVP_AES_GCM_CTX *gctx = ctx->cipher_data;
 366         if (!iv && !key)
 367                 return 1;
 368         if (key)
 369                 {
 370                 aesni_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks);
 371                 CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks,
 372                                 (block128_f)aesni_encrypt);
 373                 gctx->ctr = (ctr128_f)aesni_ctr32_encrypt_blocks;
 374                 /* If we have an iv can set it directly, otherwise use
 375                  * saved IV.
 376                  */
 377                 if (iv == NULL && gctx->iv_set)
 378                         iv = gctx->iv;
 379                 if (iv)
 380                         {
 381                         CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
 382                         gctx->iv_set = 1;
 383                         }
 384                 gctx->key_set = 1;
 385                 }
 386         else
 387                 {
 388                 /* If key set use IV, otherwise copy */
 389                 if (gctx->key_set)
 390                         CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
 391                 else
 392                         memcpy(gctx->iv, iv, gctx->ivlen);
 393                 gctx->iv_set = 1;
 394                 gctx->iv_gen = 0;
 395                 }
 396         return 1;
 397         }
 398
 399 #define aesni_gcm_cipher aes_gcm_cipher
 400 static int aesni_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 401                 const unsigned char *in, size_t len);
 402
 403 static int aesni_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 404                         const unsigned char *iv, int enc)
 405         {
 406         EVP_AES_XTS_CTX *xctx = ctx->cipher_data;
 407         if (!iv && !key)
 408                 return 1;
 409
 410         if (key)
 411                 {
 412                 /* key_len is two AES keys */
 413                 if (enc)
 414                         {
 415                         aesni_set_encrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
 416                         xctx->xts.block1 = (block128_f)aesni_encrypt;
 417                         xctx->stream = aesni_xts_encrypt;
 418                         }
 419                 else
 420                         {
 421                         aesni_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
 422                         xctx->xts.block1 = (block128_f)aesni_decrypt;
 423                         xctx->stream = aesni_xts_decrypt;
 424                         }
 425
 426                 aesni_set_encrypt_key(key + ctx->key_len/2,
 427                                                 ctx->key_len * 4, &xctx->ks2.ks);
 428                 xctx->xts.block2 = (block128_f)aesni_encrypt;
 429
 430                 xctx->xts.key1 = &xctx->ks1;
 431                 }
 432
 433         if (iv)
 434                 {
 435                 xctx->xts.key2 = &xctx->ks2;
 436                 memcpy(ctx->iv, iv, 16);
 437                 }
 438
 439         return 1;
 440         }
 441
 442 #define aesni_xts_cipher aes_xts_cipher
 443 static int aesni_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 444                 const unsigned char *in, size_t len);
 445
 446 static int aesni_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 447                         const unsigned char *iv, int enc)
 448         {
 449         EVP_AES_CCM_CTX *cctx = ctx->cipher_data;
 450         if (!iv && !key)
 451                 return 1;
 452         if (key)
 453                 {
 454                 aesni_set_encrypt_key(key, ctx->key_len * 8, &cctx->ks.ks);
 455                 CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L,
 456                                         &cctx->ks, (block128_f)aesni_encrypt);
 457                 cctx->str = enc?(ccm128_f)aesni_ccm64_encrypt_blocks :
 458                                 (ccm128_f)aesni_ccm64_decrypt_blocks;
 459                 cctx->key_set = 1;
 460                 }
 461         if (iv)
 462                 {
 463                 memcpy(ctx->iv, iv, 15 - cctx->L);
 464                 cctx->iv_set = 1;
 465                 }
 466         return 1;
 467         }
 468
 469 #define aesni_ccm_cipher aes_ccm_cipher
 470 static int aesni_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 471                 const unsigned char *in, size_t len);
 472
 473 #ifndef OPENSSL_NO_OCB
 474 static int aesni_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 475                         const unsigned char *iv, int enc)
 476         {
 477         EVP_AES_OCB_CTX *octx = ctx->cipher_data;
 478         if (!iv && !key)
 479                 return 1;
 480         if (key)
 481                 {
 482                 do
 483                         {
 484                         /* We set both the encrypt and decrypt key here because decrypt
 485                          * needs both. We could possibly optimise to remove setting the
 486                          * decrypt for an encryption operation.
 487                          */
 488                         aesni_set_encrypt_key(key, ctx->key_len * 8, &octx->ksenc);
 489                         aesni_set_decrypt_key(key, ctx->key_len * 8, &octx->ksdec);
 490                         if(!CRYPTO_ocb128_init(&octx->ocb, &octx->ksenc, &octx->ksdec,
 491                                         (block128_f)aesni_encrypt, (block128_f)aesni_decrypt))
 492                                 return 0;
 493                         }
 494                 while (0);
 495
 496                 /* If we have an iv we can set it directly, otherwise use
 497                  * saved IV.
 498                  */
 499                 if (iv == NULL && octx->iv_set)
 500                         iv = octx->iv;
 501                 if (iv)
 502                         {
 503                         if(CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen) != 1)
 504                                 return 0;
 505                         octx->iv_set = 1;
 506                         }
 507                 octx->key_set = 1;
 508                 }
 509         else
 510                 {
 511                 /* If key set use IV, otherwise copy */
 512                 if (octx->key_set)
 513                         CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen);
 514                 else
 515                         memcpy(octx->iv, iv, octx->ivlen);
 516                 octx->iv_set = 1;
 517                 }
 518         return 1;
 519         }
 520
 521 #define aesni_ocb_cipher aes_ocb_cipher
 522 static int aesni_ocb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 523                 const unsigned char *in, size_t len);
 524 #endif /* OPENSSL_NO_OCB */
 525
 526 #define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \
 527 static const EVP_CIPHER aesni_##keylen##_##mode = { \
 528         nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \
 529         flags|EVP_CIPH_##MODE##_MODE,   \
 530         aesni_init_key,                 \
 531         aesni_##mode##_cipher,          \
 532         NULL,                           \
 533         sizeof(EVP_AES_KEY),            \
 534         NULL,NULL,NULL,NULL }; \
 535 static const EVP_CIPHER aes_##keylen##_##mode = { \
 536         nid##_##keylen##_##nmode,blocksize,     \
 537         keylen/8,ivlen, \
 538         flags|EVP_CIPH_##MODE##_MODE,   \
 539         aes_init_key,                   \
 540         aes_##mode##_cipher,            \
 541         NULL,                           \
 542         sizeof(EVP_AES_KEY),            \
 543         NULL,NULL,NULL,NULL }; \
 544 const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
 545 { return AESNI_CAPABLE?&aesni_##keylen##_##mode:&aes_##keylen##_##mode; }
 546
 547 #define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \
 548 static const EVP_CIPHER aesni_##keylen##_##mode = { \
 549         nid##_##keylen##_##mode,blocksize, \
 550         (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \
 551         flags|EVP_CIPH_##MODE##_MODE,   \
 552         aesni_##mode##_init_key,        \
 553         aesni_##mode##_cipher,          \
 554         aes_##mode##_cleanup,           \
 555         sizeof(EVP_AES_##MODE##_CTX),   \
 556         NULL,NULL,aes_##mode##_ctrl,NULL }; \
 557 static const EVP_CIPHER aes_##keylen##_##mode = { \
 558         nid##_##keylen##_##mode,blocksize, \
 559         (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \
 560         flags|EVP_CIPH_##MODE##_MODE,   \
 561         aes_##mode##_init_key,          \
 562         aes_##mode##_cipher,            \
 563         aes_##mode##_cleanup,           \
 564         sizeof(EVP_AES_##MODE##_CTX),   \
 565         NULL,NULL,aes_##mode##_ctrl,NULL }; \
 566 const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
 567 { return AESNI_CAPABLE?&aesni_##keylen##_##mode:&aes_##keylen##_##mode; }
 568
 569 #elif   defined(AES_ASM) && (defined(__sparc) || defined(__sparc__))
 570
 571 #include "sparc_arch.h"
 572
 573 extern unsigned int OPENSSL_sparcv9cap_P[];
 574
 575 #define SPARC_AES_CAPABLE       (OPENSSL_sparcv9cap_P[1] & CFR_AES)
 576
 577 void    aes_t4_set_encrypt_key (const unsigned char *key, int bits,
 578                                 AES_KEY *ks);
 579 void    aes_t4_set_decrypt_key (const unsigned char *key, int bits,
 580                                 AES_KEY *ks);
 581 void    aes_t4_encrypt (const unsigned char *in, unsigned char *out,
 582                                 const AES_KEY *key);
 583 void    aes_t4_decrypt (const unsigned char *in, unsigned char *out,
 584                                 const AES_KEY *key);
 585 /*
 586  * Key-length specific subroutines were chosen for following reason.
 587  * Each SPARC T4 core can execute up to 8 threads which share core's
 588  * resources. Loading as much key material to registers allows to
 589  * minimize references to shared memory interface, as well as amount
 590  * of instructions in inner loops [much needed on T4]. But then having
 591  * non-key-length specific routines would require conditional branches
 592  * either in inner loops or on subroutines' entries. Former is hardly
 593  * acceptable, while latter means code size increase to size occupied
 594  * by multiple key-length specfic subroutines, so why fight?
 595  */
 596 void    aes128_t4_cbc_encrypt (const unsigned char *in, unsigned char *out,
 597                                 size_t len, const AES_KEY *key,
 598                                 unsigned char *ivec);
 599 void    aes128_t4_cbc_decrypt (const unsigned char *in, unsigned char *out,
 600                                 size_t len, const AES_KEY *key,
 601                                 unsigned char *ivec);
 602 void    aes192_t4_cbc_encrypt (const unsigned char *in, unsigned char *out,
 603                                 size_t len, const AES_KEY *key,
 604                                 unsigned char *ivec);
 605 void    aes192_t4_cbc_decrypt (const unsigned char *in, unsigned char *out,
 606                                 size_t len, const AES_KEY *key,
 607                                 unsigned char *ivec);
 608 void    aes256_t4_cbc_encrypt (const unsigned char *in, unsigned char *out,
 609                                 size_t len, const AES_KEY *key,
 610                                 unsigned char *ivec);
 611 void    aes256_t4_cbc_decrypt (const unsigned char *in, unsigned char *out,
 612                                 size_t len, const AES_KEY *key,
 613                                 unsigned char *ivec);
 614 void    aes128_t4_ctr32_encrypt (const unsigned char *in, unsigned char *out,
 615                                 size_t blocks, const AES_KEY *key,
 616                                 unsigned char *ivec);
 617 void    aes192_t4_ctr32_encrypt (const unsigned char *in, unsigned char *out,
 618                                 size_t blocks, const AES_KEY *key,
 619                                 unsigned char *ivec);
 620 void    aes256_t4_ctr32_encrypt (const unsigned char *in, unsigned char *out,
 621                                 size_t blocks, const AES_KEY *key,
 622                                 unsigned char *ivec);
 623 void    aes128_t4_xts_encrypt (const unsigned char *in, unsigned char *out,
 624                                 size_t blocks, const AES_KEY *key1,
 625                                 const AES_KEY *key2, const unsigned char *ivec);
 626 void    aes128_t4_xts_decrypt (const unsigned char *in, unsigned char *out,
 627                                 size_t blocks, const AES_KEY *key1,
 628                                 const AES_KEY *key2, const unsigned char *ivec);
 629 void    aes256_t4_xts_encrypt (const unsigned char *in, unsigned char *out,
 630                                 size_t blocks, const AES_KEY *key1,
 631                                 const AES_KEY *key2, const unsigned char *ivec);
 632 void    aes256_t4_xts_decrypt (const unsigned char *in, unsigned char *out,
 633                                 size_t blocks, const AES_KEY *key1,
 634                                 const AES_KEY *key2, const unsigned char *ivec);
 635
 636 static int aes_t4_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 637                    const unsigned char *iv, int enc)
 638         {
 639         int ret, mode, bits;
 640         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
 641
 642         mode = ctx->cipher->flags & EVP_CIPH_MODE;
 643         bits = ctx->key_len*8;
 644         if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE)
 645             && !enc)
 646                 {
 647                     ret = 0;
 648                     aes_t4_set_decrypt_key(key, bits, ctx->cipher_data);
 649                     dat->block  = (block128_f)aes_t4_decrypt;
 650                     switch (bits) {
 651                     case 128:
 652                         dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
 653                                                 (cbc128_f)aes128_t4_cbc_decrypt :
 654                                                 NULL;
 655                         break;
 656                     case 192:
 657                         dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
 658                                                 (cbc128_f)aes192_t4_cbc_decrypt :
 659                                                 NULL;
 660                         break;
 661                     case 256:
 662                         dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
 663                                                 (cbc128_f)aes256_t4_cbc_decrypt :
 664                                                 NULL;
 665                         break;
 666                     default:
 667                         ret = -1;
 668                     }
 669                 }
 670         else    {
 671                     ret = 0;
 672                     aes_t4_set_encrypt_key(key, bits, ctx->cipher_data);
 673                     dat->block  = (block128_f)aes_t4_encrypt;
 674                     switch (bits) {
 675                     case 128:
 676                         if (mode==EVP_CIPH_CBC_MODE)
 677                                 dat->stream.cbc = (cbc128_f)aes128_t4_cbc_encrypt;
 678                         else if (mode==EVP_CIPH_CTR_MODE)
 679                                 dat->stream.ctr = (ctr128_f)aes128_t4_ctr32_encrypt;
 680                         else
 681                                 dat->stream.cbc = NULL;
 682                         break;
 683                     case 192:
 684                         if (mode==EVP_CIPH_CBC_MODE)
 685                                 dat->stream.cbc = (cbc128_f)aes192_t4_cbc_encrypt;
 686                         else if (mode==EVP_CIPH_CTR_MODE)
 687                                 dat->stream.ctr = (ctr128_f)aes192_t4_ctr32_encrypt;
 688                         else
 689                                 dat->stream.cbc = NULL;
 690                         break;
 691                     case 256:
 692                         if (mode==EVP_CIPH_CBC_MODE)
 693                                 dat->stream.cbc = (cbc128_f)aes256_t4_cbc_encrypt;
 694                         else if (mode==EVP_CIPH_CTR_MODE)
 695                                 dat->stream.ctr = (ctr128_f)aes256_t4_ctr32_encrypt;
 696                         else
 697                                 dat->stream.cbc = NULL;
 698                         break;
 699                     default:
 700                         ret = -1;
 701                     }
 702                 }
 703
 704         if(ret < 0)
 705                 {
 706                 EVPerr(EVP_F_AES_T4_INIT_KEY,EVP_R_AES_KEY_SETUP_FAILED);
 707                 return 0;
 708                 }
 709
 710         return 1;
 711         }
 712
 713 #define aes_t4_cbc_cipher aes_cbc_cipher
 714 static int aes_t4_cbc_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 715         const unsigned char *in, size_t len);
 716
 717 #define aes_t4_ecb_cipher aes_ecb_cipher
 718 static int aes_t4_ecb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 719         const unsigned char *in, size_t len);
 720
 721 #define aes_t4_ofb_cipher aes_ofb_cipher
 722 static int aes_t4_ofb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 723         const unsigned char *in,size_t len);
 724
 725 #define aes_t4_cfb_cipher aes_cfb_cipher
 726 static int aes_t4_cfb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 727         const unsigned char *in,size_t len);
 728
 729 #define aes_t4_cfb8_cipher aes_cfb8_cipher
 730 static int aes_t4_cfb8_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 731         const unsigned char *in,size_t len);
 732
 733 #define aes_t4_cfb1_cipher aes_cfb1_cipher
 734 static int aes_t4_cfb1_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
 735         const unsigned char *in,size_t len);
 736
 737 #define aes_t4_ctr_cipher aes_ctr_cipher
 738 static int aes_t4_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 739                 const unsigned char *in, size_t len);
 740
 741 static int aes_t4_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 742                         const unsigned char *iv, int enc)
 743         {
 744         EVP_AES_GCM_CTX *gctx = ctx->cipher_data;
 745         if (!iv && !key)
 746                 return 1;
 747         if (key)
 748                 {
 749                 int bits = ctx->key_len * 8;
 750                 aes_t4_set_encrypt_key(key, bits, &gctx->ks.ks);
 751                 CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks,
 752                                 (block128_f)aes_t4_encrypt);
 753                 switch (bits) {
 754                     case 128:
 755                         gctx->ctr = (ctr128_f)aes128_t4_ctr32_encrypt;
 756                         break;
 757                     case 192:
 758                         gctx->ctr = (ctr128_f)aes192_t4_ctr32_encrypt;
 759                         break;
 760                     case 256:
 761                         gctx->ctr = (ctr128_f)aes256_t4_ctr32_encrypt;
 762                         break;
 763                     default:
 764                         return 0;
 765                 }
 766                 /* If we have an iv can set it directly, otherwise use
 767                  * saved IV.
 768                  */
 769                 if (iv == NULL && gctx->iv_set)
 770                         iv = gctx->iv;
 771                 if (iv)
 772                         {
 773                         CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
 774                         gctx->iv_set = 1;
 775                         }
 776                 gctx->key_set = 1;
 777                 }
 778         else
 779                 {
 780                 /* If key set use IV, otherwise copy */
 781                 if (gctx->key_set)
 782                         CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
 783                 else
 784                         memcpy(gctx->iv, iv, gctx->ivlen);
 785                 gctx->iv_set = 1;
 786                 gctx->iv_gen = 0;
 787                 }
 788         return 1;
 789         }
 790
 791 #define aes_t4_gcm_cipher aes_gcm_cipher
 792 static int aes_t4_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 793                 const unsigned char *in, size_t len);
 794
 795 static int aes_t4_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 796                         const unsigned char *iv, int enc)
 797         {
 798         EVP_AES_XTS_CTX *xctx = ctx->cipher_data;
 799         if (!iv && !key)
 800                 return 1;
 801
 802         if (key)
 803                 {
 804                 int bits = ctx->key_len * 4;
 805                 xctx->stream = NULL;
 806                 /* key_len is two AES keys */
 807                 if (enc)
 808                         {
 809                         aes_t4_set_encrypt_key(key, bits, &xctx->ks1.ks);
 810                         xctx->xts.block1 = (block128_f)aes_t4_encrypt;
 811                         switch (bits) {
 812                             case 128:
 813                                 xctx->stream = aes128_t4_xts_encrypt;
 814                                 break;
 815 #if 0 /* not yet */
 816                             case 192:
 817                                 xctx->stream = aes192_t4_xts_encrypt;
 818                                 break;
 819 #endif
 820                             case 256:
 821                                 xctx->stream = aes256_t4_xts_encrypt;
 822                                 break;
 823                             default:
 824                                 return 0;
 825                             }
 826                         }
 827                 else
 828                         {
 829                         aes_t4_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
 830                         xctx->xts.block1 = (block128_f)aes_t4_decrypt;
 831                         switch (bits) {
 832                             case 128:
 833                                 xctx->stream = aes128_t4_xts_decrypt;
 834                                 break;
 835 #if 0 /* not yet */
 836                             case 192:
 837                                 xctx->stream = aes192_t4_xts_decrypt;
 838                                 break;
 839 #endif
 840                             case 256:
 841                                 xctx->stream = aes256_t4_xts_decrypt;
 842                                 break;
 843                             default:
 844                                 return 0;
 845                             }
 846                         }
 847
 848                 aes_t4_set_encrypt_key(key + ctx->key_len/2,
 849                                                 ctx->key_len * 4, &xctx->ks2.ks);
 850                 xctx->xts.block2 = (block128_f)aes_t4_encrypt;
 851
 852                 xctx->xts.key1 = &xctx->ks1;
 853                 }
 854
 855         if (iv)
 856                 {
 857                 xctx->xts.key2 = &xctx->ks2;
 858                 memcpy(ctx->iv, iv, 16);
 859                 }
 860
 861         return 1;
 862         }
 863
 864 #define aes_t4_xts_cipher aes_xts_cipher
 865 static int aes_t4_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 866                 const unsigned char *in, size_t len);
 867
 868 static int aes_t4_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 869                         const unsigned char *iv, int enc)
 870         {
 871         EVP_AES_CCM_CTX *cctx = ctx->cipher_data;
 872         if (!iv && !key)
 873                 return 1;
 874         if (key)
 875                 {
 876                 int bits = ctx->key_len * 8;
 877                 aes_t4_set_encrypt_key(key, bits, &cctx->ks.ks);
 878                 CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L,
 879                                         &cctx->ks, (block128_f)aes_t4_encrypt);
 880 #if 0 /* not yet */
 881                 switch (bits) {
 882                     case 128:
 883                         cctx->str = enc?(ccm128_f)aes128_t4_ccm64_encrypt :
 884                                 (ccm128_f)ae128_t4_ccm64_decrypt;
 885                         break;
 886                     case 192:
 887                         cctx->str = enc?(ccm128_f)aes192_t4_ccm64_encrypt :
 888                                 (ccm128_f)ae192_t4_ccm64_decrypt;
 889                         break;
 890                     case 256:
 891                         cctx->str = enc?(ccm128_f)aes256_t4_ccm64_encrypt :
 892                                 (ccm128_f)ae256_t4_ccm64_decrypt;
 893                         break;
 894                     default:
 895                         return 0;
 896                     }
 897 #endif
 898                 cctx->key_set = 1;
 899                 }
 900         if (iv)
 901                 {
 902                 memcpy(ctx->iv, iv, 15 - cctx->L);
 903                 cctx->iv_set = 1;
 904                 }
 905         return 1;
 906         }
 907
 908 #define aes_t4_ccm_cipher aes_ccm_cipher
 909 static int aes_t4_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 910                 const unsigned char *in, size_t len);
 911
 912 #ifndef OPENSSL_NO_OCB
 913 static int aes_t4_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 914                         const unsigned char *iv, int enc)
 915         {
 916         EVP_AES_OCB_CTX *octx = ctx->cipher_data;
 917         if (!iv && !key)
 918                 return 1;
 919         if (key)
 920                 {
 921                 do
 922                         {
 923                         /* We set both the encrypt and decrypt key here because decrypt
 924                          * needs both. We could possibly optimise to remove setting the
 925                          * decrypt for an encryption operation.
 926                          */
 927                         aes_t4_set_encrypt_key(key, ctx->key_len * 8, &octx->ksenc);
 928                         aes_t4_set_decrypt_key(key, ctx->key_len * 8, &octx->ksdec);
 929                         if(!CRYPTO_ocb128_init(&octx->ocb, &octx->ksenc, &octx->ksdec,
 930                                         (block128_f)aes_t4_encrypt, (block128_f)aes_t4_decrypt))
 931                                 return 0;
 932                         }
 933                 while (0);
 934
 935                 /* If we have an iv we can set it directly, otherwise use
 936                  * saved IV.
 937                  */
 938                 if (iv == NULL && octx->iv_set)
 939                         iv = octx->iv;
 940                 if (iv)
 941                         {
 942                         if(CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen) != 1)
 943                                 return 0;
 944                         octx->iv_set = 1;
 945                         }
 946                 octx->key_set = 1;
 947                 }
 948         else
 949                 {
 950                 /* If key set use IV, otherwise copy */
 951                 if (octx->key_set)
 952                         CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen);
 953                 else
 954                         memcpy(octx->iv, iv, octx->ivlen);
 955                 octx->iv_set = 1;
 956                 }
 957         return 1;
 958         }
 959
 960 #define aes_t4_ocb_cipher aes_ocb_cipher
 961 static int aes_t4_ocb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 962                 const unsigned char *in, size_t len);
 963 #endif /* OPENSSL_NO_OCB */
 964
 965 #define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \
 966 static const EVP_CIPHER aes_t4_##keylen##_##mode = { \
 967         nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \
 968         flags|EVP_CIPH_##MODE##_MODE,   \
 969         aes_t4_init_key,                \
 970         aes_t4_##mode##_cipher,         \
 971         NULL,                           \
 972         sizeof(EVP_AES_KEY),            \
 973         NULL,NULL,NULL,NULL }; \
 974 static const EVP_CIPHER aes_##keylen##_##mode = { \
 975         nid##_##keylen##_##nmode,blocksize,     \
 976         keylen/8,ivlen, \
 977         flags|EVP_CIPH_##MODE##_MODE,   \
 978         aes_init_key,                   \
 979         aes_##mode##_cipher,            \
 980         NULL,                           \
 981         sizeof(EVP_AES_KEY),            \
 982         NULL,NULL,NULL,NULL }; \
 983 const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
 984 { return SPARC_AES_CAPABLE?&aes_t4_##keylen##_##mode:&aes_##keylen##_##mode; }
 985
 986 #define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \
 987 static const EVP_CIPHER aes_t4_##keylen##_##mode = { \
 988         nid##_##keylen##_##mode,blocksize, \
 989         (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \
 990         flags|EVP_CIPH_##MODE##_MODE,   \
 991         aes_t4_##mode##_init_key,       \
 992         aes_t4_##mode##_cipher,         \
 993         aes_##mode##_cleanup,           \
 994         sizeof(EVP_AES_##MODE##_CTX),   \
 995         NULL,NULL,aes_##mode##_ctrl,NULL }; \
 996 static const EVP_CIPHER aes_##keylen##_##mode = { \
 997         nid##_##keylen##_##mode,blocksize, \
 998         (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \
 999         flags|EVP_CIPH_##MODE##_MODE,   \
1000         aes_##mode##_init_key,          \
1001         aes_##mode##_cipher,            \
1002         aes_##mode##_cleanup,           \
1003         sizeof(EVP_AES_##MODE##_CTX),   \
1004         NULL,NULL,aes_##mode##_ctrl,NULL }; \
1005 const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
1006 { return SPARC_AES_CAPABLE?&aes_t4_##keylen##_##mode:&aes_##keylen##_##mode; }
1007
1008 #else
1009
1010 #define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \
1011 static const EVP_CIPHER aes_##keylen##_##mode = { \
1012         nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \
1013         flags|EVP_CIPH_##MODE##_MODE,   \
1014         aes_init_key,                   \
1015         aes_##mode##_cipher,            \
1016         NULL,                           \
1017         sizeof(EVP_AES_KEY),            \
1018         NULL,NULL,NULL,NULL }; \
1019 const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
1020 { return &aes_##keylen##_##mode; }
1021
1022 #define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \
1023 static const EVP_CIPHER aes_##keylen##_##mode = { \
1024         nid##_##keylen##_##mode,blocksize, \
1025         (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE?2:1)*keylen/8, ivlen, \
1026         flags|EVP_CIPH_##MODE##_MODE,   \
1027         aes_##mode##_init_key,          \
1028         aes_##mode##_cipher,            \
1029         aes_##mode##_cleanup,           \
1030         sizeof(EVP_AES_##MODE##_CTX),   \
1031         NULL,NULL,aes_##mode##_ctrl,NULL }; \
1032 const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
1033 { return &aes_##keylen##_##mode; }
1034
1035 #endif
1036
1037 #if defined(OPENSSL_CPUID_OBJ) && (defined(__arm__) || defined(__arm) || defined(__aarch64__))
1038 #include "arm_arch.h"
1039 #if __ARM_ARCH__>=7
1040 # if defined(BSAES_ASM)
1041 #  define BSAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
1042 # endif
1043 # define HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES)
1044 # define HWAES_set_encrypt_key aes_v8_set_encrypt_key
1045 # define HWAES_set_decrypt_key aes_v8_set_decrypt_key
1046 # define HWAES_encrypt aes_v8_encrypt
1047 # define HWAES_decrypt aes_v8_decrypt
1048 # define HWAES_cbc_encrypt aes_v8_cbc_encrypt
1049 # define HWAES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks
1050 #endif
1051 #endif
1052
1053 #if defined(HWAES_CAPABLE)
1054 int HWAES_set_encrypt_key(const unsigned char *userKey, const int bits,
1055         AES_KEY *key);
1056 int HWAES_set_decrypt_key(const unsigned char *userKey, const int bits,
1057         AES_KEY *key);
1058 void HWAES_encrypt(const unsigned char *in, unsigned char *out,
1059         const AES_KEY *key);
1060 void HWAES_decrypt(const unsigned char *in, unsigned char *out,
1061         const AES_KEY *key);
1062 void HWAES_cbc_encrypt(const unsigned char *in, unsigned char *out,
1063         size_t length, const AES_KEY *key,
1064         unsigned char *ivec, const int enc);
1065 void HWAES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
1066         size_t len, const AES_KEY *key, const unsigned char ivec[16]);
1067 #endif
1068
1069 #define BLOCK_CIPHER_generic_pack(nid,keylen,flags)             \
1070         BLOCK_CIPHER_generic(nid,keylen,16,16,cbc,cbc,CBC,flags|EVP_CIPH_FLAG_DEFAULT_ASN1)     \
1071         BLOCK_CIPHER_generic(nid,keylen,16,0,ecb,ecb,ECB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1)      \
1072         BLOCK_CIPHER_generic(nid,keylen,1,16,ofb128,ofb,OFB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1)   \
1073         BLOCK_CIPHER_generic(nid,keylen,1,16,cfb128,cfb,CFB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1)   \
1074         BLOCK_CIPHER_generic(nid,keylen,1,16,cfb1,cfb1,CFB,flags)       \
1075         BLOCK_CIPHER_generic(nid,keylen,1,16,cfb8,cfb8,CFB,flags)       \
1076         BLOCK_CIPHER_generic(nid,keylen,1,16,ctr,ctr,CTR,flags)
1077
1078 static int aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
1079                    const unsigned char *iv, int enc)
1080         {
1081         int ret, mode;
1082         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1083
1084         mode = ctx->cipher->flags & EVP_CIPH_MODE;
1085         if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE)
1086             && !enc)
1087 #ifdef HWAES_CAPABLE
1088             if (HWAES_CAPABLE)
1089                 {
1090                 ret = HWAES_set_decrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1091                 dat->block      = (block128_f)HWAES_decrypt;
1092                 dat->stream.cbc = NULL;
1093 #ifdef HWAES_cbc_encrypt
1094                 if (mode==EVP_CIPH_CBC_MODE)
1095                     dat->stream.cbc = (cbc128_f)HWAES_cbc_encrypt;
1096 #endif
1097                 }
1098             else
1099 #endif
1100 #ifdef BSAES_CAPABLE
1101             if (BSAES_CAPABLE && mode==EVP_CIPH_CBC_MODE)
1102                 {
1103                 ret = AES_set_decrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1104                 dat->block      = (block128_f)AES_decrypt;
1105                 dat->stream.cbc = (cbc128_f)bsaes_cbc_encrypt;
1106                 }
1107             else
1108 #endif
1109 #ifdef VPAES_CAPABLE
1110             if (VPAES_CAPABLE)
1111                 {
1112                 ret = vpaes_set_decrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1113                 dat->block      = (block128_f)vpaes_decrypt;
1114                 dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
1115                                         (cbc128_f)vpaes_cbc_encrypt :
1116                                         NULL;
1117                 }
1118             else
1119 #endif
1120                 {
1121                 ret = AES_set_decrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1122                 dat->block      = (block128_f)AES_decrypt;
1123                 dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
1124                                         (cbc128_f)AES_cbc_encrypt :
1125                                         NULL;
1126                 }
1127         else
1128 #ifdef HWAES_CAPABLE
1129             if (HWAES_CAPABLE)
1130                 {
1131                 ret = HWAES_set_encrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1132                 dat->block      = (block128_f)HWAES_encrypt;
1133                 dat->stream.cbc = NULL;
1134 #ifdef HWAES_cbc_encrypt
1135                 if (mode==EVP_CIPH_CBC_MODE)
1136                     dat->stream.cbc = (cbc128_f)HWAES_cbc_encrypt;
1137                 else
1138 #endif
1139 #ifdef HWAES_ctr32_encrypt_blocks
1140                 if (mode==EVP_CIPH_CTR_MODE)
1141                     dat->stream.ctr = (ctr128_f)HWAES_ctr32_encrypt_blocks;
1142                 else
1143 #endif
1144                 (void)0;        /* terminate potentially open 'else' */
1145                 }
1146             else
1147 #endif
1148 #ifdef BSAES_CAPABLE
1149             if (BSAES_CAPABLE && mode==EVP_CIPH_CTR_MODE)
1150                 {
1151                 ret = AES_set_encrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1152                 dat->block      = (block128_f)AES_encrypt;
1153                 dat->stream.ctr = (ctr128_f)bsaes_ctr32_encrypt_blocks;
1154                 }
1155             else
1156 #endif
1157 #ifdef VPAES_CAPABLE
1158             if (VPAES_CAPABLE)
1159                 {
1160                 ret = vpaes_set_encrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1161                 dat->block      = (block128_f)vpaes_encrypt;
1162                 dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
1163                                         (cbc128_f)vpaes_cbc_encrypt :
1164                                         NULL;
1165                 }
1166             else
1167 #endif
1168                 {
1169                 ret = AES_set_encrypt_key(key,ctx->key_len*8,&dat->ks.ks);
1170                 dat->block      = (block128_f)AES_encrypt;
1171                 dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ?
1172                                         (cbc128_f)AES_cbc_encrypt :
1173                                         NULL;
1174 #ifdef AES_CTR_ASM
1175                 if (mode==EVP_CIPH_CTR_MODE)
1176                         dat->stream.ctr = (ctr128_f)AES_ctr32_encrypt;
1177 #endif
1178                 }
1179
1180         if(ret < 0)
1181                 {
1182                 EVPerr(EVP_F_AES_INIT_KEY,EVP_R_AES_KEY_SETUP_FAILED);
1183                 return 0;
1184                 }
1185
1186         return 1;
1187         }
1188
1189 static int aes_cbc_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
1190         const unsigned char *in, size_t len)
1191 {
1192         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1193
1194         if (dat->stream.cbc)
1195                 (*dat->stream.cbc)(in,out,len,&dat->ks,ctx->iv,ctx->encrypt);
1196         else if (ctx->encrypt)
1197                 CRYPTO_cbc128_encrypt(in,out,len,&dat->ks,ctx->iv,dat->block);
1198         else
1199                 CRYPTO_cbc128_decrypt(in,out,len,&dat->ks,ctx->iv,dat->block);
1200
1201         return 1;
1202 }
1203
1204 static int aes_ecb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
1205         const unsigned char *in, size_t len)
1206 {
1207         size_t  bl = ctx->cipher->block_size;
1208         size_t  i;
1209         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1210
1211         if (len<bl)     return 1;
1212
1213         for (i=0,len-=bl;i<=len;i+=bl)
1214                 (*dat->block)(in+i,out+i,&dat->ks);
1215
1216         return 1;
1217 }
1218
1219 static int aes_ofb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
1220         const unsigned char *in,size_t len)
1221 {
1222         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1223
1224         CRYPTO_ofb128_encrypt(in,out,len,&dat->ks,
1225                         ctx->iv,&ctx->num,dat->block);
1226         return 1;
1227 }
1228
1229 static int aes_cfb_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
1230         const unsigned char *in,size_t len)
1231 {
1232         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1233
1234         CRYPTO_cfb128_encrypt(in,out,len,&dat->ks,
1235                         ctx->iv,&ctx->num,ctx->encrypt,dat->block);
1236         return 1;
1237 }
1238
1239 static int aes_cfb8_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
1240         const unsigned char *in,size_t len)
1241 {
1242         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1243
1244         CRYPTO_cfb128_8_encrypt(in,out,len,&dat->ks,
1245                         ctx->iv,&ctx->num,ctx->encrypt,dat->block);
1246         return 1;
1247 }
1248
1249 static int aes_cfb1_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out,
1250         const unsigned char *in,size_t len)
1251 {
1252         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1253
1254         if (ctx->flags&EVP_CIPH_FLAG_LENGTH_BITS) {
1255                 CRYPTO_cfb128_1_encrypt(in,out,len,&dat->ks,
1256                         ctx->iv,&ctx->num,ctx->encrypt,dat->block);
1257                 return 1;
1258         }
1259
1260         while (len>=MAXBITCHUNK) {
1261                 CRYPTO_cfb128_1_encrypt(in,out,MAXBITCHUNK*8,&dat->ks,
1262                         ctx->iv,&ctx->num,ctx->encrypt,dat->block);
1263                 len-=MAXBITCHUNK;
1264         }
1265         if (len)
1266                 CRYPTO_cfb128_1_encrypt(in,out,len*8,&dat->ks,
1267                         ctx->iv,&ctx->num,ctx->encrypt,dat->block);
1268
1269         return 1;
1270 }
1271
1272 static int aes_ctr_cipher (EVP_CIPHER_CTX *ctx, unsigned char *out,
1273                 const unsigned char *in, size_t len)
1274 {
1275         unsigned int num = ctx->num;
1276         EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;
1277
1278         if (dat->stream.ctr)
1279                 CRYPTO_ctr128_encrypt_ctr32(in,out,len,&dat->ks,
1280                         ctx->iv,ctx->buf,&num,dat->stream.ctr);
1281         else
1282                 CRYPTO_ctr128_encrypt(in,out,len,&dat->ks,
1283                         ctx->iv,ctx->buf,&num,dat->block);
1284         ctx->num = (size_t)num;
1285         return 1;
1286 }
1287
1288 BLOCK_CIPHER_generic_pack(NID_aes,128,0)
1289 BLOCK_CIPHER_generic_pack(NID_aes,192,0)
1290 BLOCK_CIPHER_generic_pack(NID_aes,256,0)
1291
1292 static int aes_gcm_cleanup(EVP_CIPHER_CTX *c)
1293         {
1294         EVP_AES_GCM_CTX *gctx = c->cipher_data;
1295         OPENSSL_cleanse(&gctx->gcm, sizeof(gctx->gcm));
1296         if (gctx->iv != c->iv)
1297                 OPENSSL_free(gctx->iv);
1298         return 1;
1299         }
1300
1301 /* increment counter (64-bit int) by 1 */
1302 static void ctr64_inc(unsigned char *counter) {
1303         int n=8;
1304         unsigned char  c;
1305
1306         do {
1307                 --n;
1308                 c = counter[n];
1309                 ++c;
1310                 counter[n] = c;
1311                 if (c) return;
1312         } while (n);
1313 }
1314
1315 static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
1316         {
1317         EVP_AES_GCM_CTX *gctx = c->cipher_data;
1318         switch (type)
1319                 {
1320         case EVP_CTRL_INIT:
1321                 gctx->key_set = 0;
1322                 gctx->iv_set = 0;
1323                 gctx->ivlen = c->cipher->iv_len;
1324                 gctx->iv = c->iv;
1325                 gctx->taglen = -1;
1326                 gctx->iv_gen = 0;
1327                 gctx->tls_aad_len = -1;
1328                 return 1;
1329
1330         case EVP_CTRL_GCM_SET_IVLEN:
1331                 if (arg <= 0)
1332                         return 0;
1333                 /* Allocate memory for IV if needed */
1334                 if ((arg > EVP_MAX_IV_LENGTH) && (arg > gctx->ivlen))
1335                         {
1336                         if (gctx->iv != c->iv)
1337                                 OPENSSL_free(gctx->iv);
1338                         gctx->iv = OPENSSL_malloc(arg);
1339                         if (!gctx->iv)
1340                                 return 0;
1341                         }
1342                 gctx->ivlen = arg;
1343                 return 1;
1344
1345         case EVP_CTRL_GCM_SET_TAG:
1346                 if (arg <= 0 || arg > 16 || c->encrypt)
1347                         return 0;
1348                 memcpy(c->buf, ptr, arg);
1349                 gctx->taglen = arg;
1350                 return 1;
1351
1352         case EVP_CTRL_GCM_GET_TAG:
1353                 if (arg <= 0 || arg > 16 || !c->encrypt || gctx->taglen < 0)
1354                         return 0;
1355                 memcpy(ptr, c->buf, arg);
1356                 return 1;
1357
1358         case EVP_CTRL_GCM_SET_IV_FIXED:
1359                 /* Special case: -1 length restores whole IV */
1360                 if (arg == -1)
1361                         {
1362                         memcpy(gctx->iv, ptr, gctx->ivlen);
1363                         gctx->iv_gen = 1;
1364                         return 1;
1365                         }
1366                 /* Fixed field must be at least 4 bytes and invocation field
1367                  * at least 8.
1368                  */
1369                 if ((arg < 4) || (gctx->ivlen - arg) < 8)
1370                         return 0;
1371                 if (arg)
1372                         memcpy(gctx->iv, ptr, arg);
1373                 if (c->encrypt &&
1374                         RAND_bytes(gctx->iv + arg, gctx->ivlen - arg) <= 0)
1375                         return 0;
1376                 gctx->iv_gen = 1;
1377                 return 1;
1378
1379         case EVP_CTRL_GCM_IV_GEN:
1380                 if (gctx->iv_gen == 0 || gctx->key_set == 0)
1381                         return 0;
1382                 CRYPTO_gcm128_setiv(&gctx->gcm, gctx->iv, gctx->ivlen);
1383                 if (arg <= 0 || arg > gctx->ivlen)
1384                         arg = gctx->ivlen;
1385                 memcpy(ptr, gctx->iv + gctx->ivlen - arg, arg);
1386                 /* Invocation field will be at least 8 bytes in size and
1387                  * so no need to check wrap around or increment more than
1388                  * last 8 bytes.
1389                  */
1390                 ctr64_inc(gctx->iv + gctx->ivlen - 8);
1391                 gctx->iv_set = 1;
1392                 return 1;
1393
1394         case EVP_CTRL_GCM_SET_IV_INV:
1395                 if (gctx->iv_gen == 0 || gctx->key_set == 0 || c->encrypt)
1396                         return 0;
1397                 memcpy(gctx->iv + gctx->ivlen - arg, ptr, arg);
1398                 CRYPTO_gcm128_setiv(&gctx->gcm, gctx->iv, gctx->ivlen);
1399                 gctx->iv_set = 1;
1400                 return 1;
1401
1402         case EVP_CTRL_AEAD_TLS1_AAD:
1403                 /* Save the AAD for later use */
1404                 if (arg != 13)
1405                         return 0;
1406                 memcpy(c->buf, ptr, arg);
1407                 gctx->tls_aad_len = arg;
1408                         {
1409                         unsigned int len=c->buf[arg-2]<<8|c->buf[arg-1];
1410                         /* Correct length for explicit IV */
1411                         len -= EVP_GCM_TLS_EXPLICIT_IV_LEN;
1412                         /* If decrypting correct for tag too */
1413                         if (!c->encrypt)
1414                                 len -= EVP_GCM_TLS_TAG_LEN;
1415                         c->buf[arg-2] = len>>8;
1416                         c->buf[arg-1] = len & 0xff;
1417                         }
1418                 /* Extra padding: tag appended to record */
1419                 return EVP_GCM_TLS_TAG_LEN;
1420
1421         case EVP_CTRL_COPY:
1422                 {
1423                         EVP_CIPHER_CTX *out = ptr;
1424                         EVP_AES_GCM_CTX *gctx_out = out->cipher_data;
1425                         if (gctx->gcm.key)
1426                                 {
1427                                 if (gctx->gcm.key != &gctx->ks)
1428                                         return 0;
1429                                 gctx_out->gcm.key = &gctx_out->ks;
1430                                 }
1431                         if (gctx->iv == c->iv)
1432                                 gctx_out->iv = out->iv;
1433                         else
1434                         {
1435                                 gctx_out->iv = OPENSSL_malloc(gctx->ivlen);
1436                                 if (!gctx_out->iv)
1437                                         return 0;
1438                                 memcpy(gctx_out->iv, gctx->iv, gctx->ivlen);
1439                         }
1440                         return 1;
1441                 }
1442
1443         default:
1444                 return -1;
1445
1446                 }
1447         }
1448
1449 static int aes_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
1450                         const unsigned char *iv, int enc)
1451         {
1452         EVP_AES_GCM_CTX *gctx = ctx->cipher_data;
1453         if (!iv && !key)
1454                 return 1;
1455         if (key)
1456                 { do {
1457 #ifdef HWAES_CAPABLE
1458                 if (HWAES_CAPABLE)
1459                         {
1460                         HWAES_set_encrypt_key(key,ctx->key_len*8,&gctx->ks.ks);
1461                         CRYPTO_gcm128_init(&gctx->gcm,&gctx->ks,
1462                                         (block128_f)HWAES_encrypt);
1463 #ifdef HWAES_ctr32_encrypt_blocks
1464                         gctx->ctr = (ctr128_f)HWAES_ctr32_encrypt_blocks;
1465 #else
1466                         gctx->ctr = NULL;
1467 #endif
1468                         break;
1469                         }
1470                 else
1471 #endif
1472 #ifdef BSAES_CAPABLE
1473                 if (BSAES_CAPABLE)
1474                         {
1475                         AES_set_encrypt_key(key,ctx->key_len*8,&gctx->ks.ks);
1476                         CRYPTO_gcm128_init(&gctx->gcm,&gctx->ks,
1477                                         (block128_f)AES_encrypt);
1478                         gctx->ctr = (ctr128_f)bsaes_ctr32_encrypt_blocks;
1479                         break;
1480                         }
1481                 else
1482 #endif
1483 #ifdef VPAES_CAPABLE
1484                 if (VPAES_CAPABLE)
1485                         {
1486                         vpaes_set_encrypt_key(key,ctx->key_len*8,&gctx->ks.ks);
1487                         CRYPTO_gcm128_init(&gctx->gcm,&gctx->ks,
1488                                         (block128_f)vpaes_encrypt);
1489                         gctx->ctr = NULL;
1490                         break;
1491                         }
1492                 else
1493 #endif
1494                 (void)0;        /* terminate potentially open 'else' */
1495
1496                 AES_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks);
1497                 CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f)AES_encrypt);
1498 #ifdef AES_CTR_ASM
1499                 gctx->ctr = (ctr128_f)AES_ctr32_encrypt;
1500 #else
1501                 gctx->ctr = NULL;
1502 #endif
1503                 } while (0);
1504
1505                 /* If we have an iv can set it directly, otherwise use
1506                  * saved IV.
1507                  */
1508                 if (iv == NULL && gctx->iv_set)
1509                         iv = gctx->iv;
1510                 if (iv)
1511                         {
1512                         CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
1513                         gctx->iv_set = 1;
1514                         }
1515                 gctx->key_set = 1;
1516                 }
1517         else
1518                 {
1519                 /* If key set use IV, otherwise copy */
1520                 if (gctx->key_set)
1521                         CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
1522                 else
1523                         memcpy(gctx->iv, iv, gctx->ivlen);
1524                 gctx->iv_set = 1;
1525                 gctx->iv_gen = 0;
1526                 }
1527         return 1;
1528         }
1529
1530 /* Handle TLS GCM packet format. This consists of the last portion of the IV
1531  * followed by the payload and finally the tag. On encrypt generate IV,
1532  * encrypt payload and write the tag. On verify retrieve IV, decrypt payload
1533  * and verify tag.
1534  */
1535
1536 static int aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
1537                 const unsigned char *in, size_t len)
1538         {
1539         EVP_AES_GCM_CTX *gctx = ctx->cipher_data;
1540         int rv = -1;
1541         /* Encrypt/decrypt must be performed in place */
1542         if (out != in || len < (EVP_GCM_TLS_EXPLICIT_IV_LEN+EVP_GCM_TLS_TAG_LEN))
1543                 return -1;
1544         /* Set IV from start of buffer or generate IV and write to start
1545          * of buffer.
1546          */
1547         if (EVP_CIPHER_CTX_ctrl(ctx, ctx->encrypt ?
1548                                 EVP_CTRL_GCM_IV_GEN : EVP_CTRL_GCM_SET_IV_INV,
1549                                 EVP_GCM_TLS_EXPLICIT_IV_LEN, out) <= 0)
1550                 goto err;
1551         /* Use saved AAD */
1552         if (CRYPTO_gcm128_aad(&gctx->gcm, ctx->buf, gctx->tls_aad_len))
1553                 goto err;
1554         /* Fix buffer and length to point to payload */
1555         in += EVP_GCM_TLS_EXPLICIT_IV_LEN;
1556         out += EVP_GCM_TLS_EXPLICIT_IV_LEN;
1557         len -= EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN;
1558         if (ctx->encrypt)
1559                 {
1560                 /* Encrypt payload */
1561                 if (gctx->ctr)
1562                         {
1563                         size_t bulk=0;
1564 #if defined(AES_GCM_ASM)
1565                         if (len>=32 && AES_GCM_ASM(gctx))
1566                                 {
1567                                 if (CRYPTO_gcm128_encrypt(&gctx->gcm,NULL,NULL,0))
1568                                         return -1;
1569
1570                                 bulk = AES_gcm_encrypt(in,out,len,
1571                                                         gctx->gcm.key,
1572                                                         gctx->gcm.Yi.c,
1573                                                         gctx->gcm.Xi.u);
1574                                 gctx->gcm.len.u[1] += bulk;
1575                                 }
1576 #endif
1577                         if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm,
1578                                                         in +bulk,
1579                                                         out+bulk,
1580                                                         len-bulk,
1581                                                         gctx->ctr))
1582                                 goto err;
1583                         }
1584                 else    {
1585                         size_t bulk=0;
1586 #if defined(AES_GCM_ASM2)
1587                         if (len>=32 && AES_GCM_ASM2(gctx))
1588                                 {
1589                                 if (CRYPTO_gcm128_encrypt(&gctx->gcm,NULL,NULL,0))
1590                                         return -1;
1591
1592                                 bulk = AES_gcm_encrypt(in,out,len,
1593                                                         gctx->gcm.key,
1594                                                         gctx->gcm.Yi.c,
1595                                                         gctx->gcm.Xi.u);
1596                                 gctx->gcm.len.u[1] += bulk;
1597                                 }
1598 #endif
1599                         if (CRYPTO_gcm128_encrypt(&gctx->gcm,
1600                                                         in +bulk,
1601                                                         out+bulk,
1602                                                         len-bulk))
1603                                 goto err;
1604                         }
1605                 out += len;
1606                 /* Finally write tag */
1607                 CRYPTO_gcm128_tag(&gctx->gcm, out, EVP_GCM_TLS_TAG_LEN);
1608                 rv = len + EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN;
1609                 }
1610         else
1611                 {
1612                 /* Decrypt */
1613                 if (gctx->ctr)
1614                         {
1615                         size_t bulk=0;
1616 #if defined(AES_GCM_ASM)
1617                         if (len>=16 && AES_GCM_ASM(gctx))
1618                                 {
1619                                 if (CRYPTO_gcm128_decrypt(&gctx->gcm,NULL,NULL,0))
1620                                         return -1;
1621
1622                                 bulk = AES_gcm_decrypt(in,out,len,
1623                                                         gctx->gcm.key,
1624                                                         gctx->gcm.Yi.c,
1625                                                         gctx->gcm.Xi.u);
1626                                 gctx->gcm.len.u[1] += bulk;
1627                                 }
1628 #endif
1629                         if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm,
1630                                                         in +bulk,
1631                                                         out+bulk,
1632                                                         len-bulk,
1633                                                         gctx->ctr))
1634                                 goto err;
1635                         }
1636                 else    {
1637                         size_t bulk=0;
1638 #if defined(AES_GCM_ASM2)
1639                         if (len>=16 && AES_GCM_ASM2(gctx))
1640                                 {
1641                                 if (CRYPTO_gcm128_decrypt(&gctx->gcm,NULL,NULL,0))
1642                                         return -1;
1643
1644                                 bulk = AES_gcm_decrypt(in,out,len,
1645                                                         gctx->gcm.key,
1646                                                         gctx->gcm.Yi.c,
1647                                                         gctx->gcm.Xi.u);
1648                                 gctx->gcm.len.u[1] += bulk;
1649                                 }
1650 #endif
1651                         if (CRYPTO_gcm128_decrypt(&gctx->gcm,
1652                                                         in +bulk,
1653                                                         out+bulk,
1654                                                         len-bulk))
1655                                 goto err;
1656                         }
1657                 /* Retrieve tag */
1658                 CRYPTO_gcm128_tag(&gctx->gcm, ctx->buf,
1659                                         EVP_GCM_TLS_TAG_LEN);
1660                 /* If tag mismatch wipe buffer */
1661                 if (memcmp(ctx->buf, in + len, EVP_GCM_TLS_TAG_LEN))
1662                         {
1663                         OPENSSL_cleanse(out, len);
1664                         goto err;
1665                         }
1666                 rv = len;
1667                 }
1668
1669         err:
1670         gctx->iv_set = 0;
1671         gctx->tls_aad_len = -1;
1672         return rv;
1673         }
1674
1675 static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
1676                 const unsigned char *in, size_t len)
1677         {
1678         EVP_AES_GCM_CTX *gctx = ctx->cipher_data;
1679         /* If not set up, return error */
1680         if (!gctx->key_set)
1681                 return -1;
1682
1683         if (gctx->tls_aad_len >= 0)
1684                 return aes_gcm_tls_cipher(ctx, out, in, len);
1685
1686         if (!gctx->iv_set)
1687                 return -1;
1688         if (in)
1689                 {
1690                 if (out == NULL)
1691                         {
1692                         if (CRYPTO_gcm128_aad(&gctx->gcm, in, len))
1693                                 return -1;
1694                         }
1695                 else if (ctx->encrypt)
1696                         {
1697                         if (gctx->ctr)
1698                                 {
1699                                 size_t bulk=0;
1700 #if defined(AES_GCM_ASM)
1701                                 if (len>=32 && AES_GCM_ASM(gctx))
1702                                         {
1703                                         size_t res = (16-gctx->gcm.mres)%16;
1704
1705                                         if (CRYPTO_gcm128_encrypt(&gctx->gcm,
1706                                                         in,out,res))
1707                                                 return -1;
1708
1709                                         bulk = AES_gcm_encrypt(in+res,
1710                                                         out+res,len-res,                                                                gctx->gcm.key,
1711                                                         gctx->gcm.Yi.c,
1712                                                         gctx->gcm.Xi.u);
1713                                         gctx->gcm.len.u[1] += bulk;
1714                                         bulk += res;
1715                                         }
1716 #endif
1717                                 if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm,
1718                                                         in +bulk,
1719                                                         out+bulk,
1720                                                         len-bulk,
1721                                                         gctx->ctr))
1722                                         return -1;
1723                                 }
1724                         else    {
1725                                 size_t bulk=0;
1726 #if defined(AES_GCM_ASM2)
1727                                 if (len>=32 && AES_GCM_ASM2(gctx))
1728                                         {
1729                                         size_t res = (16-gctx->gcm.mres)%16;
1730
1731                                         if (CRYPTO_gcm128_encrypt(&gctx->gcm,
1732                                                         in,out,res))
1733                                                 return -1;
1734
1735                                         bulk = AES_gcm_encrypt(in+res,
1736                                                         out+res,len-res,                                                                gctx->gcm.key,
1737                                                         gctx->gcm.Yi.c,
1738                                                         gctx->gcm.Xi.u);
1739                                         gctx->gcm.len.u[1] += bulk;
1740                                         bulk += res;
1741                                         }
1742 #endif
1743                                 if (CRYPTO_gcm128_encrypt(&gctx->gcm,
1744                                                         in +bulk,
1745                                                         out+bulk,
1746                                                         len-bulk))
1747                                         return -1;
1748                                 }
1749                         }
1750                 else
1751                         {
1752                         if (gctx->ctr)
1753                                 {
1754                                 size_t bulk=0;
1755 #if defined(AES_GCM_ASM)
1756                                 if (len>=16 && AES_GCM_ASM(gctx))
1757                                         {
1758                                         size_t res = (16-gctx->gcm.mres)%16;
1759
1760                                         if (CRYPTO_gcm128_decrypt(&gctx->gcm,
1761                                                         in,out,res))
1762                                                 return -1;
1763
1764                                         bulk = AES_gcm_decrypt(in+res,
1765                                                         out+res,len-res,
1766                                                         gctx->gcm.key,
1767                                                         gctx->gcm.Yi.c,
1768                                                         gctx->gcm.Xi.u);
1769                                         gctx->gcm.len.u[1] += bulk;
1770                                         bulk += res;
1771                                         }
1772 #endif
1773                                 if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm,
1774                                                         in +bulk,
1775                                                         out+bulk,
1776                                                         len-bulk,
1777                                                         gctx->ctr))
1778                                         return -1;
1779                                 }
1780                         else    {
1781                                 size_t bulk=0;
1782 #if defined(AES_GCM_ASM2)
1783                                 if (len>=16 && AES_GCM_ASM2(gctx))
1784                                         {
1785                                         size_t res = (16-gctx->gcm.mres)%16;
1786
1787                                         if (CRYPTO_gcm128_decrypt(&gctx->gcm,
1788                                                         in,out,res))
1789                                                 return -1;
1790
1791                                         bulk = AES_gcm_decrypt(in+res,
1792                                                         out+res,len-res,
1793                                                         gctx->gcm.key,
1794                                                         gctx->gcm.Yi.c,
1795                                                         gctx->gcm.Xi.u);
1796                                         gctx->gcm.len.u[1] += bulk;
1797                                         bulk += res;
1798                                         }
1799 #endif
1800                                 if (CRYPTO_gcm128_decrypt(&gctx->gcm,
1801                                                         in +bulk,
1802                                                         out+bulk,
1803                                                         len-bulk))
1804                                         return -1;
1805                                 }
1806                         }
1807                 return len;
1808                 }
1809         else
1810                 {
1811                 if (!ctx->encrypt)
1812                         {
1813                         if (gctx->taglen < 0)
1814                                 return -1;
1815                         if (CRYPTO_gcm128_finish(&gctx->gcm,
1816                                         ctx->buf, gctx->taglen) != 0)
1817                                 return -1;
1818                         gctx->iv_set = 0;
1819                         return 0;
1820                         }
1821                 CRYPTO_gcm128_tag(&gctx->gcm, ctx->buf, 16);
1822                 gctx->taglen = 16;
1823                 /* Don't reuse the IV */
1824                 gctx->iv_set = 0;
1825                 return 0;
1826                 }
1827
1828         }
1829
1830 #define CUSTOM_FLAGS    (EVP_CIPH_FLAG_DEFAULT_ASN1 \
1831                 | EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \
1832                 | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_CTRL_INIT \
1833                 | EVP_CIPH_CUSTOM_COPY)
1834
1835 BLOCK_CIPHER_custom(NID_aes,128,1,12,gcm,GCM,
1836                 EVP_CIPH_FLAG_AEAD_CIPHER|CUSTOM_FLAGS)
1837 BLOCK_CIPHER_custom(NID_aes,192,1,12,gcm,GCM,
1838                 EVP_CIPH_FLAG_AEAD_CIPHER|CUSTOM_FLAGS)
1839 BLOCK_CIPHER_custom(NID_aes,256,1,12,gcm,GCM,
1840                 EVP_CIPH_FLAG_AEAD_CIPHER|CUSTOM_FLAGS)
1841
1842 static int aes_xts_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
1843         {
1844         EVP_AES_XTS_CTX *xctx = c->cipher_data;
1845         if (type == EVP_CTRL_COPY)
1846                 {
1847                 EVP_CIPHER_CTX *out = ptr;
1848                 EVP_AES_XTS_CTX *xctx_out = out->cipher_data;
1849                 if (xctx->xts.key1)
1850                         {
1851                         if (xctx->xts.key1 != &xctx->ks1)
1852                                 return 0;
1853                         xctx_out->xts.key1 = &xctx_out->ks1;
1854                         }
1855                 if (xctx->xts.key2)
1856                         {
1857                         if (xctx->xts.key2 != &xctx->ks2)
1858                                 return 0;
1859                         xctx_out->xts.key2 = &xctx_out->ks2;
1860                         }
1861                 return 1;
1862                 }
1863         else if (type != EVP_CTRL_INIT)
1864                 return -1;
1865         /* key1 and key2 are used as an indicator both key and IV are set */
1866         xctx->xts.key1 = NULL;
1867         xctx->xts.key2 = NULL;
1868         return 1;
1869         }
1870
1871 static int aes_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
1872                         const unsigned char *iv, int enc)
1873         {
1874         EVP_AES_XTS_CTX *xctx = ctx->cipher_data;
1875         if (!iv && !key)
1876                 return 1;
1877
1878         if (key) do
1879                 {
1880 #ifdef AES_XTS_ASM
1881                 xctx->stream = enc ? AES_xts_encrypt : AES_xts_decrypt;
1882 #else
1883                 xctx->stream = NULL;
1884 #endif
1885                 /* key_len is two AES keys */
1886 #ifdef HWAES_CAPABLE
1887                 if (HWAES_CAPABLE)
1888                         {
1889                         if (enc)
1890                             {
1891                             HWAES_set_encrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
1892                             xctx->xts.block1 = (block128_f)HWAES_encrypt;
1893                             }
1894                         else
1895                             {
1896                             HWAES_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
1897                             xctx->xts.block1 = (block128_f)HWAES_decrypt;
1898                             }
1899
1900                         HWAES_set_encrypt_key(key + ctx->key_len/2,
1901                                                     ctx->key_len * 4, &xctx->ks2.ks);
1902                         xctx->xts.block2 = (block128_f)HWAES_encrypt;
1903
1904                         xctx->xts.key1 = &xctx->ks1;
1905                         break;
1906                         }
1907                 else
1908 #endif
1909 #ifdef BSAES_CAPABLE
1910                 if (BSAES_CAPABLE)
1911                         xctx->stream = enc ? bsaes_xts_encrypt : bsaes_xts_decrypt;
1912                 else
1913 #endif
1914 #ifdef VPAES_CAPABLE
1915                 if (VPAES_CAPABLE)
1916                     {
1917                     if (enc)
1918                         {
1919                         vpaes_set_encrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
1920                         xctx->xts.block1 = (block128_f)vpaes_encrypt;
1921                         }
1922                     else
1923                         {
1924                         vpaes_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
1925                         xctx->xts.block1 = (block128_f)vpaes_decrypt;
1926                         }
1927
1928                     vpaes_set_encrypt_key(key + ctx->key_len/2,
1929                                                 ctx->key_len * 4, &xctx->ks2.ks);
1930                     xctx->xts.block2 = (block128_f)vpaes_encrypt;
1931
1932                     xctx->xts.key1 = &xctx->ks1;
1933                     break;
1934                     }
1935                 else
1936 #endif
1937                 (void)0;        /* terminate potentially open 'else' */
1938
1939                 if (enc)
1940                         {
1941                         AES_set_encrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
1942                         xctx->xts.block1 = (block128_f)AES_encrypt;
1943                         }
1944                 else
1945                         {
1946                         AES_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks);
1947                         xctx->xts.block1 = (block128_f)AES_decrypt;
1948                         }
1949
1950                 AES_set_encrypt_key(key + ctx->key_len/2,
1951                                                 ctx->key_len * 4, &xctx->ks2.ks);
1952                 xctx->xts.block2 = (block128_f)AES_encrypt;
1953
1954                 xctx->xts.key1 = &xctx->ks1;
1955                 } while (0);
1956
1957         if (iv)
1958                 {
1959                 xctx->xts.key2 = &xctx->ks2;
1960                 memcpy(ctx->iv, iv, 16);
1961                 }
1962
1963         return 1;
1964         }
1965
1966 static int aes_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
1967                 const unsigned char *in, size_t len)
1968         {
1969         EVP_AES_XTS_CTX *xctx = ctx->cipher_data;
1970         if (!xctx->xts.key1 || !xctx->xts.key2)
1971                 return 0;
1972         if (!out || !in || len<AES_BLOCK_SIZE)
1973                 return 0;
1974         if (xctx->stream)
1975                 (*xctx->stream)(in, out, len,
1976                                 xctx->xts.key1, xctx->xts.key2, ctx->iv);
1977         else if (CRYPTO_xts128_encrypt(&xctx->xts, ctx->iv, in, out, len,
1978                                                                 ctx->encrypt))
1979                 return 0;
1980         return 1;
1981         }
1982
1983 #define aes_xts_cleanup NULL
1984
1985 #define XTS_FLAGS       (EVP_CIPH_FLAG_DEFAULT_ASN1 | EVP_CIPH_CUSTOM_IV \
1986                          | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_CTRL_INIT \
1987                          | EVP_CIPH_CUSTOM_COPY)
1988
1989 BLOCK_CIPHER_custom(NID_aes,128,1,16,xts,XTS,XTS_FLAGS)
1990 BLOCK_CIPHER_custom(NID_aes,256,1,16,xts,XTS,XTS_FLAGS)
1991
1992 static int aes_ccm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
1993         {
1994         EVP_AES_CCM_CTX *cctx = c->cipher_data;
1995         switch (type)
1996                 {
1997         case EVP_CTRL_INIT:
1998                 cctx->key_set = 0;
1999                 cctx->iv_set = 0;
2000                 cctx->L = 8;
2001                 cctx->M = 12;
2002                 cctx->tag_set = 0;
2003                 cctx->len_set = 0;
2004                 return 1;
2005
2006         case EVP_CTRL_CCM_SET_IVLEN:
2007                 arg = 15 - arg;
2008         case EVP_CTRL_CCM_SET_L:
2009                 if (arg < 2 || arg > 8)
2010                         return 0;
2011                 cctx->L = arg;
2012                 return 1;
2013
2014         case EVP_CTRL_CCM_SET_TAG:
2015                 if ((arg & 1) || arg < 4 || arg > 16)
2016                         return 0;
2017                 if ((c->encrypt && ptr) || (!c->encrypt && !ptr))
2018                         return 0;
2019                 if (ptr)
2020                         {
2021                         cctx->tag_set = 1;
2022                         memcpy(c->buf, ptr, arg);
2023                         }
2024                 cctx->M = arg;
2025                 return 1;
2026
2027         case EVP_CTRL_CCM_GET_TAG:
2028                 if (!c->encrypt || !cctx->tag_set)
2029                         return 0;
2030                 if(!CRYPTO_ccm128_tag(&cctx->ccm, ptr, (size_t)arg))
2031                         return 0;
2032                 cctx->tag_set = 0;
2033                 cctx->iv_set = 0;
2034                 cctx->len_set = 0;
2035                 return 1;
2036
2037         case EVP_CTRL_COPY:
2038                 {
2039                         EVP_CIPHER_CTX *out = ptr;
2040                         EVP_AES_CCM_CTX *cctx_out = out->cipher_data;
2041                         if (cctx->ccm.key)
2042                                 {
2043                                 if (cctx->ccm.key != &cctx->ks)
2044                                         return 0;
2045                                 cctx_out->ccm.key = &cctx_out->ks;
2046                                 }
2047                         return 1;
2048                 }
2049
2050         default:
2051                 return -1;
2052
2053                 }
2054         }
2055
2056 static int aes_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
2057                         const unsigned char *iv, int enc)
2058         {
2059         EVP_AES_CCM_CTX *cctx = ctx->cipher_data;
2060         if (!iv && !key)
2061                 return 1;
2062         if (key) do
2063                 {
2064 #ifdef HWAES_CAPABLE
2065                 if (HWAES_CAPABLE)
2066                         {
2067                         HWAES_set_encrypt_key(key,ctx->key_len*8,&cctx->ks.ks);
2068
2069                         CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L,
2070                                         &cctx->ks, (block128_f)HWAES_encrypt);
2071                         cctx->str = NULL;
2072                         cctx->key_set = 1;
2073                         break;
2074                         }
2075                 else
2076 #endif
2077 #ifdef VPAES_CAPABLE
2078                 if (VPAES_CAPABLE)
2079                         {
2080                         vpaes_set_encrypt_key(key, ctx->key_len*8, &cctx->ks.ks);
2081                         CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L,
2082                                         &cctx->ks, (block128_f)vpaes_encrypt);
2083                         cctx->str = NULL;
2084                         cctx->key_set = 1;
2085                         break;
2086                         }
2087 #endif
2088                 AES_set_encrypt_key(key, ctx->key_len * 8, &cctx->ks.ks);
2089                 CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L,
2090                                         &cctx->ks, (block128_f)AES_encrypt);
2091                 cctx->str = NULL;
2092                 cctx->key_set = 1;
2093                 } while (0);
2094         if (iv)
2095                 {
2096                 memcpy(ctx->iv, iv, 15 - cctx->L);
2097                 cctx->iv_set = 1;
2098                 }
2099         return 1;
2100         }
2101
2102 static int aes_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
2103                 const unsigned char *in, size_t len)
2104         {
2105         EVP_AES_CCM_CTX *cctx = ctx->cipher_data;
2106         CCM128_CONTEXT *ccm = &cctx->ccm;
2107         /* If not set up, return error */
2108         if (!cctx->iv_set && !cctx->key_set)
2109                 return -1;
2110         if (!ctx->encrypt && !cctx->tag_set)
2111                 return -1;
2112         if (!out)
2113                 {
2114                 if (!in)
2115                         {
2116                         if (CRYPTO_ccm128_setiv(ccm, ctx->iv, 15 - cctx->L,len))
2117                                 return -1;
2118                         cctx->len_set = 1;
2119                         return len;
2120                         }
2121                 /* If have AAD need message length */
2122                 if (!cctx->len_set && len)
2123                         return -1;
2124                 CRYPTO_ccm128_aad(ccm, in, len);
2125                 return len;
2126                 }
2127         /* EVP_*Final() doesn't return any data */
2128         if (!in)
2129                 return 0;
2130         /* If not set length yet do it */
2131         if (!cctx->len_set)
2132                 {
2133                 if (CRYPTO_ccm128_setiv(ccm, ctx->iv, 15 - cctx->L, len))
2134                         return -1;
2135                 cctx->len_set = 1;
2136                 }
2137         if (ctx->encrypt)
2138                 {
2139                 if (cctx->str ? CRYPTO_ccm128_encrypt_ccm64(ccm, in, out, len,
2140                                                 cctx->str) :
2141                                 CRYPTO_ccm128_encrypt(ccm, in, out, len))
2142                         return -1;
2143                 cctx->tag_set = 1;
2144                 return len;
2145                 }
2146         else
2147                 {
2148                 int rv = -1;
2149                 if (cctx->str ? !CRYPTO_ccm128_decrypt_ccm64(ccm, in, out, len,
2150                                                 cctx->str) :
2151                                 !CRYPTO_ccm128_decrypt(ccm, in, out, len))
2152                         {
2153                         unsigned char tag[16];
2154                         if (CRYPTO_ccm128_tag(ccm, tag, cctx->M))
2155                                 {
2156                                 if (!memcmp(tag, ctx->buf, cctx->M))
2157                                         rv = len;
2158                                 }
2159                         }
2160                 if (rv == -1)
2161                         OPENSSL_cleanse(out, len);
2162                 cctx->iv_set = 0;
2163                 cctx->tag_set = 0;
2164                 cctx->len_set = 0;
2165                 return rv;
2166                 }
2167
2168         }
2169
2170 #define aes_ccm_cleanup NULL
2171
2172 BLOCK_CIPHER_custom(NID_aes,128,1,12,ccm,CCM,CUSTOM_FLAGS)
2173 BLOCK_CIPHER_custom(NID_aes,192,1,12,ccm,CCM,CUSTOM_FLAGS)
2174 BLOCK_CIPHER_custom(NID_aes,256,1,12,ccm,CCM,CUSTOM_FLAGS)
2175
2176 typedef struct
2177         {
2178         union { double align; AES_KEY ks; } ks;
2179         /* Indicates if IV has been set */
2180         unsigned char *iv;
2181         } EVP_AES_WRAP_CTX;
2182
2183 static int aes_wrap_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
2184                         const unsigned char *iv, int enc)
2185         {
2186         EVP_AES_WRAP_CTX *wctx = ctx->cipher_data;
2187         if (!iv && !key)
2188                 return 1;
2189         if (key)
2190                 {
2191                 if (ctx->encrypt)
2192                         AES_set_encrypt_key(key, ctx->key_len * 8, &wctx->ks.ks);
2193                 else
2194                         AES_set_decrypt_key(key, ctx->key_len * 8, &wctx->ks.ks);
2195                 if (!iv)
2196                         wctx->iv = NULL;
2197                 }
2198         if (iv)
2199                 {
2200                 memcpy(ctx->iv, iv, EVP_CIPHER_CTX_iv_length(ctx));
2201                 wctx->iv = ctx->iv;
2202                 }
2203         return 1;
2204         }
2205
2206 static int aes_wrap_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
2207                 const unsigned char *in, size_t inlen)
2208         {
2209         EVP_AES_WRAP_CTX *wctx = ctx->cipher_data;
2210         size_t rv;
2211         /* AES wrap with padding has IV length of 4, without padding 8 */
2212         int pad = EVP_CIPHER_CTX_iv_length(ctx) == 4;
2213         /* No final operation so always return zero length */
2214         if (!in)
2215                 return 0;
2216         /* Input length must always be non-zero */
2217         if (!inlen)
2218                 return -1;
2219         /* If decrypting need at least 16 bytes and multiple of 8 */
2220         if (!ctx->encrypt && (inlen < 16 || inlen & 0x7))
2221                 return -1;
2222         /* If not padding input must be multiple of 8 */
2223         if (!pad && inlen & 0x7)
2224                 return -1;
2225         if (!out)
2226                 {
2227                 if (ctx->encrypt)
2228                         {
2229                         /* If padding round up to multiple of 8 */
2230                         if (pad)
2231                                 inlen = (inlen + 7)/8 * 8;
2232                         /* 8 byte prefix */
2233                         return inlen + 8;
2234                         }
2235                 else
2236                         {
2237                         /* If not padding output will be exactly 8 bytes
2238                          * smaller than input. If padding it will be at
2239                          * least 8 bytes smaller but we don't know how
2240                          * much.
2241                          */
2242                         return inlen - 8;
2243                         }
2244                 }
2245         if (pad)
2246                 {
2247                 if (ctx->encrypt)
2248                         rv = CRYPTO_128_wrap_pad(&wctx->ks.ks, wctx->iv,
2249                                                 out, in, inlen,
2250                                                 (block128_f)AES_encrypt);
2251                 else
2252                         rv = CRYPTO_128_unwrap_pad(&wctx->ks.ks, wctx->iv,
2253                                                 out, in, inlen,
2254                                                 (block128_f)AES_decrypt);
2255                 }
2256         else
2257                 {
2258                 if (ctx->encrypt)
2259                         rv = CRYPTO_128_wrap(&wctx->ks.ks, wctx->iv,
2260                                                 out, in, inlen,
2261                                                 (block128_f)AES_encrypt);
2262                 else
2263                         rv = CRYPTO_128_unwrap(&wctx->ks.ks, wctx->iv,
2264                                                 out, in, inlen,
2265                                                 (block128_f)AES_decrypt);
2266                 }
2267         return rv ? (int)rv : -1;
2268         }
2269
2270 #define WRAP_FLAGS      (EVP_CIPH_WRAP_MODE \
2271                 | EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \
2272                 | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_FLAG_DEFAULT_ASN1)
2273
2274 static const EVP_CIPHER aes_128_wrap = {
2275         NID_id_aes128_wrap,
2276         8, 16, 8, WRAP_FLAGS,
2277         aes_wrap_init_key, aes_wrap_cipher,
2278         NULL,
2279         sizeof(EVP_AES_WRAP_CTX),
2280         NULL,NULL,NULL,NULL };
2281
2282 const EVP_CIPHER *EVP_aes_128_wrap(void)
2283         {
2284         return &aes_128_wrap;
2285         }
2286
2287 static const EVP_CIPHER aes_192_wrap = {
2288         NID_id_aes192_wrap,
2289         8, 24, 8, WRAP_FLAGS,
2290         aes_wrap_init_key, aes_wrap_cipher,
2291         NULL,
2292         sizeof(EVP_AES_WRAP_CTX),
2293         NULL,NULL,NULL,NULL };
2294
2295 const EVP_CIPHER *EVP_aes_192_wrap(void)
2296         {
2297         return &aes_192_wrap;
2298         }
2299
2300 static const EVP_CIPHER aes_256_wrap = {
2301         NID_id_aes256_wrap,
2302         8, 32, 8, WRAP_FLAGS,
2303         aes_wrap_init_key, aes_wrap_cipher,
2304         NULL,
2305         sizeof(EVP_AES_WRAP_CTX),
2306         NULL,NULL,NULL,NULL };
2307
2308 const EVP_CIPHER *EVP_aes_256_wrap(void)
2309         {
2310         return &aes_256_wrap;
2311         }
2312
2313 static const EVP_CIPHER aes_128_wrap_pad = {
2314         NID_id_aes128_wrap_pad,
2315         8, 16, 4, WRAP_FLAGS,
2316         aes_wrap_init_key, aes_wrap_cipher,
2317         NULL,
2318         sizeof(EVP_AES_WRAP_CTX),
2319         NULL,NULL,NULL,NULL };
2320
2321 const EVP_CIPHER *EVP_aes_128_wrap_pad(void)
2322         {
2323         return &aes_128_wrap_pad;
2324         }
2325
2326 static const EVP_CIPHER aes_192_wrap_pad = {
2327         NID_id_aes192_wrap_pad,
2328         8, 24, 4, WRAP_FLAGS,
2329         aes_wrap_init_key, aes_wrap_cipher,
2330         NULL,
2331         sizeof(EVP_AES_WRAP_CTX),
2332         NULL,NULL,NULL,NULL };
2333
2334 const EVP_CIPHER *EVP_aes_192_wrap_pad(void)
2335         {
2336         return &aes_192_wrap_pad;
2337         }
2338
2339 static const EVP_CIPHER aes_256_wrap_pad = {
2340         NID_id_aes256_wrap_pad,
2341         8, 32, 4, WRAP_FLAGS,
2342         aes_wrap_init_key, aes_wrap_cipher,
2343         NULL,
2344         sizeof(EVP_AES_WRAP_CTX),
2345         NULL,NULL,NULL,NULL };
2346
2347 const EVP_CIPHER *EVP_aes_256_wrap_pad(void)
2348         {
2349         return &aes_256_wrap_pad;
2350         }
2351
2352 #ifndef OPENSSL_NO_OCB
2353 static int aes_ocb_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
2354         {
2355         EVP_AES_OCB_CTX *octx = c->cipher_data;
2356         EVP_CIPHER_CTX *newc;
2357         EVP_AES_OCB_CTX *new_octx;
2358
2359         switch (type)
2360                 {
2361         case EVP_CTRL_INIT:
2362                 octx->key_set = 0;
2363                 octx->iv_set = 0;
2364                 octx->ivlen = c->cipher->iv_len;
2365                 octx->iv = c->iv;
2366                 octx->taglen = 16;
2367                 octx->data_buf_len = 0;
2368                 octx->aad_buf_len = 0;
2369                 return 1;
2370
2371         case EVP_CTRL_SET_IVLEN:
2372                 /* IV len must be 1 to 15 */
2373                 if (arg <= 0 || arg > 15)
2374                         return 0;
2375
2376                 octx->ivlen = arg;
2377                 return 1;
2378
2379         case EVP_CTRL_OCB_SET_TAGLEN:
2380                 /* Tag len must be 0 to 16 */
2381                 if (arg < 0 || arg > 16)
2382                         return 0;
2383
2384                 octx->taglen = arg;
2385                 return 1;
2386
2387         case EVP_CTRL_SET_TAG:
2388                 if (arg != octx->taglen || c->encrypt)
2389                         return 0;
2390                 memcpy(octx->tag, ptr, arg);
2391                 return 1;
2392
2393         case EVP_CTRL_GET_TAG:
2394                 if (arg != octx->taglen || !c->encrypt)
2395                         return 0;
2396
2397                 memcpy(ptr, octx->tag, arg);
2398                 return 1;
2399
2400         case EVP_CTRL_COPY:
2401                 newc = (EVP_CIPHER_CTX *)ptr;
2402                 new_octx = newc->cipher_data;
2403                 return CRYPTO_ocb128_copy_ctx(&new_octx->ocb, &octx->ocb,
2404                         &new_octx->ksenc, &new_octx->ksdec);
2405
2406         default:
2407                 return -1;
2408
2409                 }
2410         }
2411
2412
2413 static int aes_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
2414                 const unsigned char *iv, int enc)
2415         {
2416         EVP_AES_OCB_CTX *octx = ctx->cipher_data;
2417         if (!iv && !key)
2418                 return 1;
2419         if (key)
2420                 {
2421                 do
2422                         {
2423                         /* We set both the encrypt and decrypt key here because decrypt
2424                          * needs both. We could possibly optimise to remove setting the
2425                          * decrypt for an encryption operation.
2426                          */
2427 #ifdef VPAES_CAPABLE
2428                         if (VPAES_CAPABLE)
2429                                 {
2430                                 vpaes_set_encrypt_key(key,ctx->key_len*8,&octx->ksenc);
2431                                 vpaes_set_decrypt_key(key,ctx->key_len*8,&octx->ksdec);
2432                                 if(!CRYPTO_ocb128_init(&octx->ocb,&octx->ksenc,&octx->ksdec,
2433                                                 (block128_f)vpaes_encrypt,(block128_f)vpaes_decrypt))
2434                                         return 0;
2435                                 break;
2436                                 }
2437 #endif
2438                         AES_set_encrypt_key(key, ctx->key_len * 8, &octx->ksenc);
2439                         AES_set_decrypt_key(key, ctx->key_len * 8, &octx->ksdec);
2440                         if(!CRYPTO_ocb128_init(&octx->ocb, &octx->ksenc, &octx->ksdec,
2441                                         (block128_f)AES_encrypt, (block128_f)AES_decrypt))
2442                                 return 0;
2443                         }
2444                 while (0);
2445
2446                 /* If we have an iv we can set it directly, otherwise use
2447                  * saved IV.
2448                  */
2449                 if (iv == NULL && octx->iv_set)
2450                         iv = octx->iv;
2451                 if (iv)
2452                         {
2453                         if(CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen) != 1)
2454                                 return 0;
2455                         octx->iv_set = 1;
2456                         }
2457                 octx->key_set = 1;
2458                 }
2459         else
2460                 {
2461                 /* If key set use IV, otherwise copy */
2462                 if (octx->key_set)
2463                         CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen);
2464                 else
2465                         memcpy(octx->iv, iv, octx->ivlen);
2466                 octx->iv_set = 1;
2467                 }
2468         return 1;
2469         }
2470
2471 static int aes_ocb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
2472                 const unsigned char *in, size_t len)
2473         {
2474         unsigned char *buf;
2475         int *buf_len;
2476         int written_len = 0;
2477         size_t trailing_len;
2478         EVP_AES_OCB_CTX *octx = ctx->cipher_data;
2479
2480         /* If IV or Key not set then return error */
2481         if (!octx->iv_set)
2482                 return -1;
2483
2484         if (!octx->key_set)
2485                 return -1;
2486
2487         if (in)
2488                 {
2489                 /* Need to ensure we are only passing full blocks to low level OCB
2490                  * routines. We do it here rather than in EVP_EncryptUpdate/
2491                  * EVP_DecryptUpdate because we need to pass full blocks of AAD too
2492                  * and those routines don't support that
2493                  */
2494
2495                 /* Are we dealing with AAD or normal data here? */
2496                 if (out == NULL)
2497                         {
2498                         buf = octx->aad_buf;
2499                         buf_len = &(octx->aad_buf_len);
2500                         }
2501                 else
2502                         {
2503                         buf = octx->data_buf;
2504                         buf_len = &(octx->data_buf_len);
2505                         }
2506
2507                 /* If we've got a partially filled buffer from a previous call then use
2508                  * that data first
2509                  */
2510                 if(*buf_len)
2511                         {
2512                         unsigned int remaining;
2513
2514                         remaining = 16 - (*buf_len);
2515                         if(remaining > len)
2516                                 {
2517                                 memcpy(buf+(*buf_len), in, len);
2518                                 *(buf_len)+=len;
2519                                 return 0;
2520                                 }
2521                         memcpy(buf+(*buf_len), in, remaining);
2522
2523                         /* If we get here we've filled the buffer, so process it */
2524                         len -= remaining;
2525                         in += remaining;
2526                         if (out == NULL)
2527                                 {
2528                                 if(!CRYPTO_ocb128_aad(&octx->ocb, buf, 16))
2529                                         return -1;
2530                                 }
2531                         else if (ctx->encrypt)
2532                                 {
2533                                 if(!CRYPTO_ocb128_encrypt(&octx->ocb, buf, out, 16))
2534                                         return -1;
2535                                 }
2536                         else
2537                                 {
2538                                 if(!CRYPTO_ocb128_decrypt(&octx->ocb, buf, out, 16))
2539                                         return -1;
2540                                 }
2541                         written_len = 16;
2542                         *buf_len = 0;
2543                         }
2544
2545                 /* Do we have a partial block to handle at the end? */
2546                 trailing_len = len % 16;
2547
2548                 /* If we've got some full blocks to handle, then process these first */
2549                 if(len != trailing_len)
2550                         {
2551                         if (out == NULL)
2552                                 {
2553                                 if(!CRYPTO_ocb128_aad(&octx->ocb, in, len-trailing_len))
2554                                         return -1;
2555                                 }
2556                         else if (ctx->encrypt)
2557                                 {
2558                                 if(!CRYPTO_ocb128_encrypt(&octx->ocb, in, out, len-trailing_len))
2559                                         return -1;
2560                                 }
2561                         else
2562                                 {
2563                                 if(!CRYPTO_ocb128_decrypt(&octx->ocb, in, out, len-trailing_len))
2564                                         return -1;
2565                                 }
2566                         written_len += len-trailing_len;
2567                         in += len-trailing_len;
2568                         }
2569
2570                 /* Handle any trailing partial block */
2571                 if(trailing_len)
2572                         {
2573                         memcpy(buf, in, trailing_len);
2574                         *buf_len = trailing_len;
2575                         }
2576
2577                 return written_len;
2578                 }
2579         else
2580                 {
2581                 /* First of all empty the buffer of any partial block that we might
2582                  * have been provided - both for data and AAD
2583                  */
2584                 if(octx->data_buf_len)
2585                         {
2586                         if (ctx->encrypt)
2587                                 {
2588                                 if(!CRYPTO_ocb128_encrypt(&octx->ocb, octx->data_buf, out,
2589                                                 octx->data_buf_len))
2590                                         return -1;
2591                                 }
2592                         else
2593                                 {
2594                                 if(!CRYPTO_ocb128_decrypt(&octx->ocb, octx->data_buf, out,
2595                                                 octx->data_buf_len))
2596                                         return -1;
2597                                 }
2598                         written_len = octx->data_buf_len;
2599                         octx->data_buf_len = 0;
2600                         }
2601                 if(octx->aad_buf_len)
2602                         {
2603                         if(!CRYPTO_ocb128_aad(&octx->ocb, octx->aad_buf, octx->aad_buf_len))
2604                                 return -1;
2605                         octx->aad_buf_len = 0;
2606                         }
2607                 /* If decrypting then verify */
2608                 if (!ctx->encrypt)
2609                         {
2610                         if (octx->taglen < 0)
2611                                 return -1;
2612                         if (CRYPTO_ocb128_finish(&octx->ocb,
2613                                         octx->tag, octx->taglen) != 0)
2614                                 return -1;
2615                         octx->iv_set = 0;
2616                         return written_len;
2617                         }
2618                 /* If encrypting then just get the tag */
2619                 if(CRYPTO_ocb128_tag(&octx->ocb, octx->tag, 16) != 1)
2620                         return -1;
2621                 /* Don't reuse the IV */
2622                 octx->iv_set = 0;
2623                 return written_len;
2624                 }
2625         }
2626
2627 static int aes_ocb_cleanup(EVP_CIPHER_CTX *c)
2628         {
2629         EVP_AES_OCB_CTX *octx = c->cipher_data;
2630         CRYPTO_ocb128_cleanup(&octx->ocb);
2631         return 1;
2632         }
2633
2634 BLOCK_CIPHER_custom(NID_aes,128,16,12,ocb,OCB,CUSTOM_FLAGS)
2635 BLOCK_CIPHER_custom(NID_aes,192,16,12,ocb,OCB,CUSTOM_FLAGS)
2636 BLOCK_CIPHER_custom(NID_aes,256,16,12,ocb,OCB,CUSTOM_FLAGS)
2637 #endif /* OPENSSL_NO_OCB */
2638
2639 #endif