crypto/modes/gcm128.c

   1 /* ====================================================================
   2  * Copyright (c) 2010 The OpenSSL Project.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  *
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in
  13  *    the documentation and/or other materials provided with the
  14  *    distribution.
  15  *
  16  * 3. All advertising materials mentioning features or use of this
  17  *    software must display the following acknowledgment:
  18  *    "This product includes software developed by the OpenSSL Project
  19  *    for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
  20  *
  21  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
  22  *    endorse or promote products derived from this software without
  23  *    prior written permission. For written permission, please contact
  24  *    openssl-core@openssl.org.
  25  *
  26  * 5. Products derived from this software may not be called "OpenSSL"
  27  *    nor may "OpenSSL" appear in their names without prior written
  28  *    permission of the OpenSSL Project.
  29  *
  30  * 6. Redistributions of any form whatsoever must retain the following
  31  *    acknowledgment:
  32  *    "This product includes software developed by the OpenSSL Project
  33  *    for use in the OpenSSL Toolkit (http://www.openssl.org/)"
  34  *
  35  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
  36  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  37  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  38  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
  39  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  41  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  42  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  43  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  44  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  45  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
  46  * OF THE POSSIBILITY OF SUCH DAMAGE.
  47  * ====================================================================
  48  */
  49
  50 #include "modes.h"
  51 #include <string.h>
  52
  53 #ifndef MODES_DEBUG
  54 # ifndef NDEBUG
  55 #  define NDEBUG
  56 # endif
  57 #endif
  58 #include <assert.h>
  59
  60 #if (defined(_WIN32) || defined(_WIN64)) && !defined(__MINGW32__)
  61 typedef __int64 i64;
  62 typedef unsigned __int64 u64;
  63 #define U64(C) C##UI64
  64 #elif defined(__arch64__)
  65 typedef long i64;
  66 typedef unsigned long u64;
  67 #define U64(C) C##UL
  68 #else
  69 typedef long long i64;
  70 typedef unsigned long long u64;
  71 #define U64(C) C##ULL
  72 #endif
  73
  74 typedef unsigned int u32;
  75 typedef unsigned char u8;
  76 typedef struct { u64 hi,lo; } u128;
  77
  78 #define STRICT_ALIGNMENT
  79 #if defined(__i386)     || defined(__i386__)    || \
  80     defined(__x86_64)   || defined(__x86_64__)  || \
  81     defined(_M_IX86)    || defined(_M_AMD64)    || defined(_M_X64) || \
  82     defined(__s390__)   || defined(__s390x__)
  83 # undef STRICT_ALIGNMENT
  84 #endif
  85
  86 #if defined(__GNUC__) && __GNUC__>=2
  87 # if defined(__x86_64) || defined(__x86_64__)
  88 #  define BSWAP8(x) ({  u64 ret=(x);                    \
  89                         asm volatile ("bswapq %0"       \
  90                         : "+r"(ret));   ret;            })
  91 #  define BSWAP4(x) ({  u32 ret=(x);                    \
  92                         asm volatile ("bswapl %0"       \
  93                         : "+r"(ret));   ret;            })
  94 # elif defined(__i386) || defined(__i386__)
  95 #  define BSWAP8(x) ({  u32 lo=(u64)(x)>>32,hi=(x);     \
  96                         asm volatile ("bswapl %0; bswapl %1"    \
  97                         : "+r"(hi),"+r"(lo));           \
  98                         (u64)hi<<32|lo;                 })
  99 #  define BSWAP4(x) ({  u32 ret=(x);                    \
 100                         asm volatile ("bswapl %0"       \
 101                         : "+r"(ret));   ret;            })
 102 # endif
 103 #elif defined(_MSC_VER)
 104 # if _MSC_VER>=1300
 105 #  pragma intrinsic(_byteswap_uint64,_byteswap_ulong)
 106 #  define BSWAP8(x)     _byteswap_uint64((u64)(x))
 107 #  define BSWAP4(x)     _byteswap_ulong((u32)(x))
 108 # elif defined(_M_IX86)
 109 # endif
 110 #endif
 111
 112 #ifdef BSWAP4
 113 #define GETU32(p)       BSWAP4(*(const u32 *)(p))
 114 #define PUTU32(p,v)     *(u32 *)(p) = BSWAP4(v)
 115 #else
 116 #define GETU32(p)       ((u32)(p)[0]<<24|(u32)(p)[1]<<16|(u32)(p)[2]<<8|(u32)(p)[3])
 117 #define PUTU32(p,v)     ((p)[0]=(u8)((v)>>24),(p)[1]=(u8)((v)>>16),(p)[2]=(u8)((v)>>8),(p)[3]=(u8)(v))
 118 #endif
 119
 120 #define PACK(s) ((size_t)(s)<<(sizeof(size_t)*8-16))
 121 #ifdef  TABLE_BITS
 122 #undef  TABLE_BITS
 123 #endif
 124 /*
 125  * Even though permitted values for TABLE_BITS are 8, 4 and 1, it should
 126  * never be set to 8. 8 is effectively reserved for testing purposes.
 127  * Under ideal conditions "8-bit" version should be twice as fast as
 128  * "4-bit" one. But world is far from ideal. For gcc-generated x86 code,
 129  * "8-bit" was observed to run only ~50% faster. On x86_64 observed
 130  * improvement was ~75%, much closer to optimal, but the fact of
 131  * deviation means that references to pre-computed tables end up on
 132  * critical path and as tables are pretty big, 4KB per key+1KB shared,
 133  * execution time is sensitive to cache timing. It's not actually
 134  * proven, but 4-bit procedure is believed to provide adequate
 135  * all-round performance...
 136  */
 137 #define TABLE_BITS 4
 138
 139 #if     TABLE_BITS==8
 140
 141 static void gcm_init_8bit(u128 Htable[256], u64 H[2])
 142 {
 143         int  i, j;
 144         u128 V;
 145
 146         Htable[0].hi = 0;
 147         Htable[0].lo = 0;
 148         V.hi = H[0];
 149         V.lo = H[1];
 150
 151         for (Htable[128]=V, i=64; i>0; i>>=1) {
 152                 if (sizeof(size_t)==8) {
 153                         u64 T = U64(0xe100000000000000) & (0-(V.lo&1));
 154                         V.lo  = (V.hi<<63)|(V.lo>>1);
 155                         V.hi  = (V.hi>>1 )^T;
 156                 }
 157                 else {
 158                         u32 T = 0xe1000000U & (0-(u32)(V.lo&1));
 159                         V.lo  = (V.hi<<63)|(V.lo>>1);
 160                         V.hi  = (V.hi>>1 )^((u64)T<<32);
 161                 }
 162                 Htable[i] = V;
 163         }
 164
 165         for (i=2; i<256; i<<=1) {
 166                 u128 *Hi = Htable+i, H0 = *Hi;
 167                 for (j=1; j<i; ++j) {
 168                         Hi[j].hi = H0.hi^Htable[j].hi;
 169                         Hi[j].lo = H0.lo^Htable[j].lo;
 170                 }
 171         }
 172 }
 173
 174 static void gcm_gmult_8bit(u64 Xi[2], u128 Htable[256])
 175 {
 176         u128 Z = { 0, 0};
 177         const u8 *xi = (const u8 *)Xi+15;
 178         size_t rem, n = *xi;
 179         const union { long one; char little; } is_endian = {1};
 180         static const size_t rem_8bit[256] = {
 181                 PACK(0x0000), PACK(0x01C2), PACK(0x0384), PACK(0x0246),
 182                 PACK(0x0708), PACK(0x06CA), PACK(0x048C), PACK(0x054E),
 183                 PACK(0x0E10), PACK(0x0FD2), PACK(0x0D94), PACK(0x0C56),
 184                 PACK(0x0918), PACK(0x08DA), PACK(0x0A9C), PACK(0x0B5E),
 185                 PACK(0x1C20), PACK(0x1DE2), PACK(0x1FA4), PACK(0x1E66),
 186                 PACK(0x1B28), PACK(0x1AEA), PACK(0x18AC), PACK(0x196E),
 187                 PACK(0x1230), PACK(0x13F2), PACK(0x11B4), PACK(0x1076),
 188                 PACK(0x1538), PACK(0x14FA), PACK(0x16BC), PACK(0x177E),
 189                 PACK(0x3840), PACK(0x3982), PACK(0x3BC4), PACK(0x3A06),
 190                 PACK(0x3F48), PACK(0x3E8A), PACK(0x3CCC), PACK(0x3D0E),
 191                 PACK(0x3650), PACK(0x3792), PACK(0x35D4), PACK(0x3416),
 192                 PACK(0x3158), PACK(0x309A), PACK(0x32DC), PACK(0x331E),
 193                 PACK(0x2460), PACK(0x25A2), PACK(0x27E4), PACK(0x2626),
 194                 PACK(0x2368), PACK(0x22AA), PACK(0x20EC), PACK(0x212E),
 195                 PACK(0x2A70), PACK(0x2BB2), PACK(0x29F4), PACK(0x2836),
 196                 PACK(0x2D78), PACK(0x2CBA), PACK(0x2EFC), PACK(0x2F3E),
 197                 PACK(0x7080), PACK(0x7142), PACK(0x7304), PACK(0x72C6),
 198                 PACK(0x7788), PACK(0x764A), PACK(0x740C), PACK(0x75CE),
 199                 PACK(0x7E90), PACK(0x7F52), PACK(0x7D14), PACK(0x7CD6),
 200                 PACK(0x7998), PACK(0x785A), PACK(0x7A1C), PACK(0x7BDE),
 201                 PACK(0x6CA0), PACK(0x6D62), PACK(0x6F24), PACK(0x6EE6),
 202                 PACK(0x6BA8), PACK(0x6A6A), PACK(0x682C), PACK(0x69EE),
 203                 PACK(0x62B0), PACK(0x6372), PACK(0x6134), PACK(0x60F6),
 204                 PACK(0x65B8), PACK(0x647A), PACK(0x663C), PACK(0x67FE),
 205                 PACK(0x48C0), PACK(0x4902), PACK(0x4B44), PACK(0x4A86),
 206                 PACK(0x4FC8), PACK(0x4E0A), PACK(0x4C4C), PACK(0x4D8E),
 207                 PACK(0x46D0), PACK(0x4712), PACK(0x4554), PACK(0x4496),
 208                 PACK(0x41D8), PACK(0x401A), PACK(0x425C), PACK(0x439E),
 209                 PACK(0x54E0), PACK(0x5522), PACK(0x5764), PACK(0x56A6),
 210                 PACK(0x53E8), PACK(0x522A), PACK(0x506C), PACK(0x51AE),
 211                 PACK(0x5AF0), PACK(0x5B32), PACK(0x5974), PACK(0x58B6),
 212                 PACK(0x5DF8), PACK(0x5C3A), PACK(0x5E7C), PACK(0x5FBE),
 213                 PACK(0xE100), PACK(0xE0C2), PACK(0xE284), PACK(0xE346),
 214                 PACK(0xE608), PACK(0xE7CA), PACK(0xE58C), PACK(0xE44E),
 215                 PACK(0xEF10), PACK(0xEED2), PACK(0xEC94), PACK(0xED56),
 216                 PACK(0xE818), PACK(0xE9DA), PACK(0xEB9C), PACK(0xEA5E),
 217                 PACK(0xFD20), PACK(0xFCE2), PACK(0xFEA4), PACK(0xFF66),
 218                 PACK(0xFA28), PACK(0xFBEA), PACK(0xF9AC), PACK(0xF86E),
 219                 PACK(0xF330), PACK(0xF2F2), PACK(0xF0B4), PACK(0xF176),
 220                 PACK(0xF438), PACK(0xF5FA), PACK(0xF7BC), PACK(0xF67E),
 221                 PACK(0xD940), PACK(0xD882), PACK(0xDAC4), PACK(0xDB06),
 222                 PACK(0xDE48), PACK(0xDF8A), PACK(0xDDCC), PACK(0xDC0E),
 223                 PACK(0xD750), PACK(0xD692), PACK(0xD4D4), PACK(0xD516),
 224                 PACK(0xD058), PACK(0xD19A), PACK(0xD3DC), PACK(0xD21E),
 225                 PACK(0xC560), PACK(0xC4A2), PACK(0xC6E4), PACK(0xC726),
 226                 PACK(0xC268), PACK(0xC3AA), PACK(0xC1EC), PACK(0xC02E),
 227                 PACK(0xCB70), PACK(0xCAB2), PACK(0xC8F4), PACK(0xC936),
 228                 PACK(0xCC78), PACK(0xCDBA), PACK(0xCFFC), PACK(0xCE3E),
 229                 PACK(0x9180), PACK(0x9042), PACK(0x9204), PACK(0x93C6),
 230                 PACK(0x9688), PACK(0x974A), PACK(0x950C), PACK(0x94CE),
 231                 PACK(0x9F90), PACK(0x9E52), PACK(0x9C14), PACK(0x9DD6),
 232                 PACK(0x9898), PACK(0x995A), PACK(0x9B1C), PACK(0x9ADE),
 233                 PACK(0x8DA0), PACK(0x8C62), PACK(0x8E24), PACK(0x8FE6),
 234                 PACK(0x8AA8), PACK(0x8B6A), PACK(0x892C), PACK(0x88EE),
 235                 PACK(0x83B0), PACK(0x8272), PACK(0x8034), PACK(0x81F6),
 236                 PACK(0x84B8), PACK(0x857A), PACK(0x873C), PACK(0x86FE),
 237                 PACK(0xA9C0), PACK(0xA802), PACK(0xAA44), PACK(0xAB86),
 238                 PACK(0xAEC8), PACK(0xAF0A), PACK(0xAD4C), PACK(0xAC8E),
 239                 PACK(0xA7D0), PACK(0xA612), PACK(0xA454), PACK(0xA596),
 240                 PACK(0xA0D8), PACK(0xA11A), PACK(0xA35C), PACK(0xA29E),
 241                 PACK(0xB5E0), PACK(0xB422), PACK(0xB664), PACK(0xB7A6),
 242                 PACK(0xB2E8), PACK(0xB32A), PACK(0xB16C), PACK(0xB0AE),
 243                 PACK(0xBBF0), PACK(0xBA32), PACK(0xB874), PACK(0xB9B6),
 244                 PACK(0xBCF8), PACK(0xBD3A), PACK(0xBF7C), PACK(0xBEBE) };
 245
 246         while (1) {
 247                 Z.hi ^= Htable[n].hi;
 248                 Z.lo ^= Htable[n].lo;
 249
 250                 if ((u8 *)Xi==xi)       break;
 251
 252                 n = *(--xi);
 253
 254                 rem  = (size_t)Z.lo&0xff;
 255                 Z.lo = (Z.hi<<56)|(Z.lo>>8);
 256                 Z.hi = (Z.hi>>8);
 257                 if (sizeof(size_t)==8)
 258                         Z.hi ^= rem_8bit[rem];
 259                 else
 260                         Z.hi ^= (u64)rem_8bit[rem]<<32;
 261         }
 262
 263         if (is_endian.little) {
 264 #ifdef BSWAP8
 265                 Xi[0] = BSWAP8(Z.hi);
 266                 Xi[1] = BSWAP8(Z.lo);
 267 #else
 268                 u8 *p = (u8 *)Xi;
 269                 u32 v;
 270                 v = (u32)(Z.hi>>32);    PUTU32(p,v);
 271                 v = (u32)(Z.hi);        PUTU32(p+4,v);
 272                 v = (u32)(Z.lo>>32);    PUTU32(p+8,v);
 273                 v = (u32)(Z.lo);        PUTU32(p+12,v);
 274 #endif
 275         }
 276         else {
 277                 Xi[0] = Z.hi;
 278                 Xi[1] = Z.lo;
 279         }
 280 }
 281 #define GCM_MUL(ctx,Xi)   gcm_gmult_8bit(ctx->Xi.u,ctx->Htable)
 282
 283 #elif   TABLE_BITS==4
 284
 285 static void gcm_init_4bit(u128 Htable[16], u64 H[2])
 286 {
 287         int  i;
 288         u128 V;
 289
 290         Htable[0].hi = 0;
 291         Htable[0].lo = 0;
 292         V.hi = H[0];
 293         V.lo = H[1];
 294
 295         for (Htable[8]=V, i=4; i>0; i>>=1) {
 296                 if (sizeof(size_t)==8) {
 297                         u64 T = U64(0xe100000000000000) & (0-(V.lo&1));
 298                         V.lo  = (V.hi<<63)|(V.lo>>1);
 299                         V.hi  = (V.hi>>1 )^T;
 300                 }
 301                 else {
 302                         u32 T = 0xe1000000U & (0-(u32)(V.lo&1));
 303                         V.lo  = (V.hi<<63)|(V.lo>>1);
 304                         V.hi  = (V.hi>>1 )^((u64)T<<32);
 305                 }
 306                 Htable[i] = V;
 307         }
 308
 309 #if defined(OPENSSL_SMALL_FOOTPRINT)
 310         for (i=2; i<16; i<<=1) {
 311                 u128 *Hi = Htable+i;
 312                 int   j;
 313                 for (V=*Hi, j=1; j<i; ++j) {
 314                         Hi[j].hi = V.hi^Htable[j].hi;
 315                         Hi[j].lo = V.lo^Htable[j].lo;
 316                 }
 317         }
 318 #else
 319         Htable[3].hi  = V.hi^Htable[2].hi, Htable[3].lo  = V.lo^Htable[2].lo;
 320         V=Htable[4];
 321         Htable[5].hi  = V.hi^Htable[1].hi, Htable[5].lo  = V.lo^Htable[1].lo;
 322         Htable[6].hi  = V.hi^Htable[2].hi, Htable[6].lo  = V.lo^Htable[2].lo;
 323         Htable[7].hi  = V.hi^Htable[3].hi, Htable[7].lo  = V.lo^Htable[3].lo;
 324         V=Htable[8];
 325         Htable[9].hi  = V.hi^Htable[1].hi, Htable[9].lo  = V.lo^Htable[1].lo;
 326         Htable[10].hi = V.hi^Htable[2].hi, Htable[10].lo = V.lo^Htable[2].lo;
 327         Htable[11].hi = V.hi^Htable[3].hi, Htable[11].lo = V.lo^Htable[3].lo;
 328         Htable[12].hi = V.hi^Htable[4].hi, Htable[12].lo = V.lo^Htable[4].lo;
 329         Htable[13].hi = V.hi^Htable[5].hi, Htable[13].lo = V.lo^Htable[5].lo;
 330         Htable[14].hi = V.hi^Htable[6].hi, Htable[14].lo = V.lo^Htable[6].lo;
 331         Htable[15].hi = V.hi^Htable[7].hi, Htable[15].lo = V.lo^Htable[7].lo;
 332 #endif
 333 }
 334
 335 #ifndef GHASH_ASM
 336 static const size_t rem_4bit[16] = {
 337         PACK(0x0000), PACK(0x1C20), PACK(0x3840), PACK(0x2460),
 338         PACK(0x7080), PACK(0x6CA0), PACK(0x48C0), PACK(0x54E0),
 339         PACK(0xE100), PACK(0xFD20), PACK(0xD940), PACK(0xC560),
 340         PACK(0x9180), PACK(0x8DA0), PACK(0xA9C0), PACK(0xB5E0) };
 341
 342 static void gcm_gmult_4bit(u64 Xi[2], const u128 Htable[16])
 343 {
 344         u128 Z;
 345         int cnt = 15;
 346         size_t rem, nlo, nhi;
 347         const union { long one; char little; } is_endian = {1};
 348
 349         nlo  = ((const u8 *)Xi)[15];
 350         nhi  = nlo>>4;
 351         nlo &= 0xf;
 352
 353         Z.hi = Htable[nlo].hi;
 354         Z.lo = Htable[nlo].lo;
 355
 356         while (1) {
 357                 rem  = (size_t)Z.lo&0xf;
 358                 Z.lo = (Z.hi<<60)|(Z.lo>>4);
 359                 Z.hi = (Z.hi>>4);
 360                 if (sizeof(size_t)==8)
 361                         Z.hi ^= rem_4bit[rem];
 362                 else
 363                         Z.hi ^= (u64)rem_4bit[rem]<<32;
 364
 365                 Z.hi ^= Htable[nhi].hi;
 366                 Z.lo ^= Htable[nhi].lo;
 367
 368                 if (--cnt<0)            break;
 369
 370                 nlo  = ((const u8 *)Xi)[cnt];
 371                 nhi  = nlo>>4;
 372                 nlo &= 0xf;
 373
 374                 rem  = (size_t)Z.lo&0xf;
 375                 Z.lo = (Z.hi<<60)|(Z.lo>>4);
 376                 Z.hi = (Z.hi>>4);
 377                 if (sizeof(size_t)==8)
 378                         Z.hi ^= rem_4bit[rem];
 379                 else
 380                         Z.hi ^= (u64)rem_4bit[rem]<<32;
 381
 382                 Z.hi ^= Htable[nlo].hi;
 383                 Z.lo ^= Htable[nlo].lo;
 384         }
 385
 386         if (is_endian.little) {
 387 #ifdef BSWAP8
 388                 Xi[0] = BSWAP8(Z.hi);
 389                 Xi[1] = BSWAP8(Z.lo);
 390 #else
 391                 u8 *p = (u8 *)Xi;
 392                 u32 v;
 393                 v = (u32)(Z.hi>>32);    PUTU32(p,v);
 394                 v = (u32)(Z.hi);        PUTU32(p+4,v);
 395                 v = (u32)(Z.lo>>32);    PUTU32(p+8,v);
 396                 v = (u32)(Z.lo);        PUTU32(p+12,v);
 397 #endif
 398         }
 399         else {
 400                 Xi[0] = Z.hi;
 401                 Xi[1] = Z.lo;
 402         }
 403 }
 404
 405 #if !defined(OPENSSL_SMALL_FOOTPRINT)
 406 /*
 407  * Streamed gcm_mult_4bit, see CRYPTO_gcm128_[en|de]crypt for
 408  * details... Compiler-generated code doesn't seem to give any
 409  * performance improvement, at least not on x86[_64]. It's here
 410  * mostly as reference and a placeholder for possible future
 411  * non-trivial optimization[s]...
 412  */
 413 static void gcm_ghash_4bit(u64 Xi[2],const u128 Htable[16],
 414                                 const u8 *inp,size_t len)
 415 {
 416     u128 Z;
 417     int cnt;
 418     size_t rem, nlo, nhi;
 419     const union { long one; char little; } is_endian = {1};
 420
 421     do {
 422         cnt  = 15;
 423         nlo  = ((const u8 *)Xi)[15];
 424         nlo ^= inp[15];
 425         nhi  = nlo>>4;
 426         nlo &= 0xf;
 427
 428         Z.hi = Htable[nlo].hi;
 429         Z.lo = Htable[nlo].lo;
 430
 431         while (1) {
 432                 rem  = (size_t)Z.lo&0xf;
 433                 Z.lo = (Z.hi<<60)|(Z.lo>>4);
 434                 Z.hi = (Z.hi>>4);
 435                 if (sizeof(size_t)==8)
 436                         Z.hi ^= rem_4bit[rem];
 437                 else
 438                         Z.hi ^= (u64)rem_4bit[rem]<<32;
 439
 440                 Z.hi ^= Htable[nhi].hi;
 441                 Z.lo ^= Htable[nhi].lo;
 442
 443                 if (--cnt<0)            break;
 444
 445                 nlo  = ((const u8 *)Xi)[cnt];
 446                 nlo ^= inp[cnt];
 447                 nhi  = nlo>>4;
 448                 nlo &= 0xf;
 449
 450                 rem  = (size_t)Z.lo&0xf;
 451                 Z.lo = (Z.hi<<60)|(Z.lo>>4);
 452                 Z.hi = (Z.hi>>4);
 453                 if (sizeof(size_t)==8)
 454                         Z.hi ^= rem_4bit[rem];
 455                 else
 456                         Z.hi ^= (u64)rem_4bit[rem]<<32;
 457
 458                 Z.hi ^= Htable[nlo].hi;
 459                 Z.lo ^= Htable[nlo].lo;
 460         }
 461
 462         if (is_endian.little) {
 463 #ifdef BSWAP8
 464                 Xi[0] = BSWAP8(Z.hi);
 465                 Xi[1] = BSWAP8(Z.lo);
 466 #else
 467                 u8 *p = (u8 *)Xi;
 468                 u32 v;
 469                 v = (u32)(Z.hi>>32);    PUTU32(p,v);
 470                 v = (u32)(Z.hi);        PUTU32(p+4,v);
 471                 v = (u32)(Z.lo>>32);    PUTU32(p+8,v);
 472                 v = (u32)(Z.lo);        PUTU32(p+12,v);
 473 #endif
 474         }
 475         else {
 476                 Xi[0] = Z.hi;
 477                 Xi[1] = Z.lo;
 478         }
 479     } while (inp+=16, len-=16);
 480 }
 481 #endif
 482 #else
 483 void gcm_gmult_4bit(u64 Xi[2],const u128 Htable[16]);
 484 void gcm_ghash_4bit(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
 485 #endif
 486
 487 #define GCM_MUL(ctx,Xi)   gcm_gmult_4bit(ctx->Xi.u,ctx->Htable)
 488 #if defined(GHASH_ASM) || !defined(OPENSSL_SMALL_FOOTPRINT)
 489 #define GHASH(in,len,ctx) gcm_ghash_4bit((ctx)->Xi.u,(ctx)->Htable,in,len)
 490 /* GHASH_CHUNK is "stride parameter" missioned to mitigate cache
 491  * trashing effect. In other words idea is to hash data while it's
 492  * still in L1 cache after encryption pass... */
 493 #define GHASH_CHUNK       1024
 494 #endif
 495
 496 #else   /* TABLE_BITS */
 497
 498 static void gcm_gmult_1bit(u64 Xi[2],const u64 H[2])
 499 {
 500         u128 V,Z = { 0,0 };
 501         long X;
 502         int  i,j;
 503         const long *xi = (const long *)Xi;
 504         const union { long one; char little; } is_endian = {1};
 505
 506         V.hi = H[0];    /* H is in host byte order, no byte swapping */
 507         V.lo = H[1];
 508
 509         for (j=0; j<16/sizeof(long); ++j) {
 510                 if (is_endian.little) {
 511                         if (sizeof(long)==8) {
 512 #ifdef BSWAP8
 513                                 X = (long)(BSWAP8(xi[j]));
 514 #else
 515                                 const u8 *p = (const u8 *)(xi+j);
 516                                 X = (long)((u64)GETU32(p)<<32|GETU32(p+4));
 517 #endif
 518                         }
 519                         else {
 520                                 const u8 *p = (const u8 *)(xi+j);
 521                                 X = (long)GETU32(p);
 522                         }
 523                 }
 524                 else
 525                         X = xi[j];
 526
 527                 for (i=0; i<8*sizeof(long); ++i, X<<=1) {
 528                         u64 M = (u64)(X>>(8*sizeof(long)-1));
 529                         Z.hi ^= V.hi&M;
 530                         Z.lo ^= V.lo&M;
 531
 532                         if (sizeof(size_t)==8) {
 533                                 u64 T = U64(0xe100000000000000) & (0-(V.lo&1));
 534                                 V.lo  = (V.hi<<63)|(V.lo>>1);
 535                                 V.hi  = (V.hi>>1 )^T;
 536                         }
 537                         else {
 538                                 u32 T = 0xe1000000U & (0-(u32)(V.lo&1));
 539                                 V.lo  = (V.hi<<63)|(V.lo>>1);
 540                                 V.hi  = (V.hi>>1 )^((u64)T<<32);
 541                         }
 542
 543                 }
 544         }
 545
 546         if (is_endian.little) {
 547 #ifdef BSWAP8
 548                 Xi[0] = BSWAP8(Z.hi);
 549                 Xi[1] = BSWAP8(Z.lo);
 550 #else
 551                 u8 *p = (u8 *)Xi;
 552                 u32 v;
 553                 v = (u32)(Z.hi>>32);    PUTU32(p,v);
 554                 v = (u32)(Z.hi);        PUTU32(p+4,v);
 555                 v = (u32)(Z.lo>>32);    PUTU32(p+8,v);
 556                 v = (u32)(Z.lo);        PUTU32(p+12,v);
 557 #endif
 558         }
 559         else {
 560                 Xi[0] = Z.hi;
 561                 Xi[1] = Z.lo;
 562         }
 563 }
 564 #define GCM_MUL(ctx,Xi)   gcm_gmult_1bit(ctx->Xi.u,ctx->H.u)
 565
 566 #endif
 567
 568 typedef struct {
 569         /* Following 6 names follow names in GCM specification */
 570         union { u64 u[2]; u32 d[4]; u8 c[16]; } Yi,EKi,EK0,
 571                                                 Xi,H,
 572                                                 len;
 573         /* Pre-computed table used by gcm_gmult_* */
 574 #if TABLE_BITS==8
 575         u128 Htable[256];
 576 #else
 577         u128 Htable[16];
 578 #endif
 579         unsigned int res, ctr;
 580         block128_f block;
 581         void *key;
 582 } GCM128_CONTEXT;
 583
 584 void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx,void *key,block128_f block)
 585 {
 586         const union { long one; char little; } is_endian = {1};
 587
 588         memset(ctx,0,sizeof(*ctx));
 589         ctx->block = block;
 590         ctx->key   = key;
 591
 592         (*block)(ctx->H.c,ctx->H.c,key);
 593
 594         if (is_endian.little) {
 595                 /* H is stored in host byte order */
 596 #ifdef BSWAP8
 597                 ctx->H.u[0] = BSWAP8(ctx->H.u[0]);
 598                 ctx->H.u[1] = BSWAP8(ctx->H.u[1]);
 599 #else
 600                 u8 *p = ctx->H.c;
 601                 u64 hi,lo;
 602                 hi = (u64)GETU32(p)  <<32|GETU32(p+4);
 603                 lo = (u64)GETU32(p+8)<<32|GETU32(p+12);
 604                 ctx->H.u[0] = hi;
 605                 ctx->H.u[1] = lo;
 606 #endif
 607         }
 608
 609 #if     TABLE_BITS==8
 610         gcm_init_8bit(ctx->Htable,ctx->H.u);
 611 #elif   TABLE_BITS==4
 612         gcm_init_4bit(ctx->Htable,ctx->H.u);
 613 #endif
 614 }
 615
 616 void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx,const unsigned char *iv,size_t len)
 617 {
 618         const union { long one; char little; } is_endian = {1};
 619
 620         ctx->Yi.u[0]  = 0;
 621         ctx->Yi.u[1]  = 0;
 622         ctx->Xi.u[0]  = 0;
 623         ctx->Xi.u[1]  = 0;
 624         ctx->len.u[0] = 0;
 625         ctx->len.u[1] = 0;
 626         ctx->res = 0;
 627
 628         if (len==12) {
 629                 memcpy(ctx->Yi.c,iv,12);
 630                 ctx->Yi.c[15]=1;
 631                 ctx->ctr=1;
 632         }
 633         else {
 634                 size_t i;
 635                 u64 len0 = len;
 636
 637                 while (len>=16) {
 638                         for (i=0; i<16; ++i) ctx->Yi.c[i] ^= iv[i];
 639                         GCM_MUL(ctx,Yi);
 640                         iv += 16;
 641                         len -= 16;
 642                 }
 643                 if (len) {
 644                         for (i=0; i<len; ++i) ctx->Yi.c[i] ^= iv[i];
 645                         GCM_MUL(ctx,Yi);
 646                 }
 647                 len0 <<= 3;
 648                 if (is_endian.little) {
 649 #ifdef BSWAP8
 650                         ctx->Yi.u[1]  ^= BSWAP8(len0);
 651 #else
 652                         ctx->Yi.c[8]  ^= (u8)(len0>>56);
 653                         ctx->Yi.c[9]  ^= (u8)(len0>>48);
 654                         ctx->Yi.c[10] ^= (u8)(len0>>40);
 655                         ctx->Yi.c[11] ^= (u8)(len0>>32);
 656                         ctx->Yi.c[12] ^= (u8)(len0>>24);
 657                         ctx->Yi.c[13] ^= (u8)(len0>>16);
 658                         ctx->Yi.c[14] ^= (u8)(len0>>8);
 659                         ctx->Yi.c[15] ^= (u8)(len0);
 660 #endif
 661                 }
 662                 else
 663                         ctx->Yi.u[1]  ^= len0;
 664
 665                 GCM_MUL(ctx,Yi);
 666
 667                 if (is_endian.little)
 668                         ctx->ctr = GETU32(ctx->Yi.c+12);
 669                 else
 670                         ctx->ctr = ctx->Yi.d[3];
 671         }
 672
 673         (*ctx->block)(ctx->Yi.c,ctx->EK0.c,ctx->key);
 674         ++ctx->ctr;
 675         if (is_endian.little)
 676                 PUTU32(ctx->Yi.c+12,ctx->ctr);
 677         else
 678                 ctx->Yi.d[3] = ctx->ctr;
 679 }
 680
 681 void CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx,const unsigned char *aad,size_t len)
 682 {
 683         size_t i;
 684
 685         ctx->len.u[0] += len;
 686
 687 #ifdef GHASH
 688         if ((i = (len&(size_t)-16))) {
 689                 GHASH(aad,i,ctx);
 690                 aad += i;
 691                 len -= i;
 692         }
 693 #else
 694         while (len>=16) {
 695                 for (i=0; i<16; ++i) ctx->Xi.c[i] ^= aad[i];
 696                 GCM_MUL(ctx,Xi);
 697                 aad += 16;
 698                 len -= 16;
 699         }
 700 #endif
 701         if (len) {
 702                 for (i=0; i<len; ++i) ctx->Xi.c[i] ^= aad[i];
 703                 GCM_MUL(ctx,Xi);
 704         }
 705 }
 706
 707 void CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
 708                 const unsigned char *in, unsigned char *out,
 709                 size_t len)
 710 {
 711         const union { long one; char little; } is_endian = {1};
 712         unsigned int n, ctr;
 713         size_t i;
 714
 715         ctx->len.u[1] += len;
 716         n   = ctx->res;
 717         ctr = ctx->ctr;
 718
 719 #if !defined(OPENSSL_SMALL_FOOTPRINT)
 720         if (16%sizeof(size_t) == 0) do {        /* always true actually */
 721                 if (n) {
 722                         while (n && len) {
 723                                 ctx->Xi.c[n] ^= *(out++) = *(in++)^ctx->EKi.c[n];
 724                                 --len;
 725                                 n = (n+1)%16;
 726                         }
 727                         if (n==0) GCM_MUL(ctx,Xi);
 728                         else {
 729                                 ctx->res = n;
 730                                 return;
 731                         }
 732                 }
 733 #if defined(STRICT_ALIGNMENT)
 734                 if (((size_t)in|(size_t)out)%sizeof(size_t) != 0)
 735                         break;
 736 #endif
 737 #if defined(GHASH) && defined(GHASH_CHUNK)
 738                 while (len>=GHASH_CHUNK) {
 739                     size_t j=GHASH_CHUNK;
 740
 741                     while (j) {
 742                         (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
 743                         ++ctr;
 744                         if (is_endian.little)
 745                                 PUTU32(ctx->Yi.c+12,ctr);
 746                         else
 747                                 ctx->Yi.d[3] = ctr;
 748                         for (i=0; i<16; i+=sizeof(size_t))
 749                                 *(size_t *)(out+i) =
 750                                 *(size_t *)(in+i)^*(size_t *)(ctx->EKi.c+i);
 751                         out += 16;
 752                         in  += 16;
 753                         j   -= 16;
 754                     }
 755                     GHASH(out-GHASH_CHUNK,GHASH_CHUNK,ctx);
 756                     len -= GHASH_CHUNK;
 757                 }
 758                 if ((i = (len&(size_t)-16))) {
 759                     size_t j=i;
 760
 761                     while (len>=16) {
 762                         (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
 763                         ++ctr;
 764                         if (is_endian.little)
 765                                 PUTU32(ctx->Yi.c+12,ctr);
 766                         else
 767                                 ctx->Yi.d[3] = ctr;
 768                         for (i=0; i<16; i+=sizeof(size_t))
 769                                 *(size_t *)(out+i) =
 770                                 *(size_t *)(in+i)^*(size_t *)(ctx->EKi.c+i);
 771                         out += 16;
 772                         in  += 16;
 773                         len -= 16;
 774                     }
 775                     GHASH(out-j,j,ctx);
 776                 }
 777 #else
 778                 while (len>=16) {
 779                         (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
 780                         ++ctr;
 781                         if (is_endian.little)
 782                                 PUTU32(ctx->Yi.c+12,ctr);
 783                         else
 784                                 ctx->Yi.d[3] = ctr;
 785                         for (i=0; i<16; i+=sizeof(size_t))
 786                                 *(size_t *)(ctx->Xi.c+i) ^=
 787                                 *(size_t *)(out+i) =
 788                                 *(size_t *)(in+i)^*(size_t *)(ctx->EKi.c+i);
 789                         GCM_MUL(ctx,Xi);
 790                         out += 16;
 791                         in  += 16;
 792                         len -= 16;
 793                 }
 794 #endif
 795                 if (len) {
 796                         (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
 797                         ++ctr;
 798                         if (is_endian.little)
 799                                 PUTU32(ctx->Yi.c+12,ctr);
 800                         else
 801                                 ctx->Yi.d[3] = ctr;
 802                         while (len--) {
 803                                 ctx->Xi.c[n] ^= out[n] = in[n]^ctx->EKi.c[n];
 804                                 ++n;
 805                         }
 806                 }
 807
 808                 ctx->res = n;
 809                 ctx->ctr = ctr;
 810                 return;
 811         } while(0);
 812 #endif
 813         for (i=0;i<len;++i) {
 814                 if (n==0) {
 815                         (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
 816                         ++ctr;
 817                         if (is_endian.little)
 818                                 PUTU32(ctx->Yi.c+12,ctr);
 819                         else
 820                                 ctx->Yi.d[3] = ctr;
 821                 }
 822                 ctx->Xi.c[n] ^= out[i] = in[i]^ctx->EKi.c[n];
 823                 n = (n+1)%16;
 824                 if (n==0)
 825                         GCM_MUL(ctx,Xi);
 826         }
 827
 828         ctx->res = n;
 829         ctx->ctr = ctr;
 830 }
 831
 832 void CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
 833                 const unsigned char *in, unsigned char *out,
 834                 size_t len)
 835 {
 836         const union { long one; char little; } is_endian = {1};
 837         unsigned int n, ctr;
 838         size_t i;
 839
 840         ctx->len.u[1] += len;
 841         n   = ctx->res;
 842         ctr = ctx->ctr;
 843
 844 #if !defined(OPENSSL_SMALL_FOOTPRINT)
 845         if (16%sizeof(size_t) == 0) do {        /* always true actually */
 846                 if (n) {
 847                         while (n && len) {
 848                                 u8 c = *(in++);
 849                                 *(out++) = c^ctx->EKi.c[n];
 850                                 ctx->Xi.c[n] ^= c;
 851                                 --len;
 852                                 n = (n+1)%16;
 853                         }
 854                         if (n==0) GCM_MUL (ctx,Xi);
 855                         else {
 856                                 ctx->res = n;
 857                                 return;
 858                         }
 859                 }
 860 #if defined(STRICT_ALIGNMENT)
 861                 if (((size_t)in|(size_t)out)%sizeof(size_t) != 0)
 862                         break;
 863 #endif
 864 #if defined(GHASH) && defined(GHASH_CHUNK)
 865                 while (len>=GHASH_CHUNK) {
 866                     size_t j=GHASH_CHUNK;
 867
 868                     GHASH(in,GHASH_CHUNK,ctx);
 869                     while (j) {
 870                         (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
 871                         ++ctr;
 872                         if (is_endian.little)
 873                                 PUTU32(ctx->Yi.c+12,ctr);
 874                         else
 875                                 ctx->Yi.d[3] = ctr;
 876                         for (i=0; i<16; i+=sizeof(size_t))
 877                                 *(size_t *)(out+i) =
 878                                 *(size_t *)(in+i)^*(size_t *)(ctx->EKi.c+i);
 879                         out += 16;
 880                         in  += 16;
 881                         j   -= 16;
 882                     }
 883                     len -= GHASH_CHUNK;
 884                 }
 885                 if ((i = (len&(size_t)-16))) {
 886                     GHASH(in,i,ctx);
 887                     while (len>=16) {
 888                         (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
 889                         ++ctr;
 890                         if (is_endian.little)
 891                                 PUTU32(ctx->Yi.c+12,ctr);
 892                         else
 893                                 ctx->Yi.d[3] = ctr;
 894                         for (i=0; i<16; i+=sizeof(size_t))
 895                                 *(size_t *)(out+i) =
 896                                 *(size_t *)(in+i)^*(size_t *)(ctx->EKi.c+i);
 897                         out += 16;
 898                         in  += 16;
 899                         len -= 16;
 900                     }
 901                 }
 902 #else
 903                 while (len>=16) {
 904                         (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
 905                         ++ctr;
 906                         if (is_endian.little)
 907                                 PUTU32(ctx->Yi.c+12,ctr);
 908                         else
 909                                 ctx->Yi.d[3] = ctr;
 910                         for (i=0; i<16; i+=sizeof(size_t)) {
 911                                 size_t c = *(size_t *)(in+i);
 912                                 *(size_t *)(out+i) = c^*(size_t *)(ctx->EKi.c+i);
 913                                 *(size_t *)(ctx->Xi.c+i) ^= c;
 914                         }
 915                         GCM_MUL(ctx,Xi);
 916                         out += 16;
 917                         in  += 16;
 918                         len -= 16;
 919                 }
 920 #endif
 921                 if (len) {
 922                         (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
 923                         ++ctr;
 924                         if (is_endian.little)
 925                                 PUTU32(ctx->Yi.c+12,ctr);
 926                         else
 927                                 ctx->Yi.d[3] = ctr;
 928                         while (len--) {
 929                                 u8 c = in[n];
 930                                 ctx->Xi.c[n] ^= c;
 931                                 out[n] = c^ctx->EKi.c[n];
 932                                 ++n;
 933                         }
 934                 }
 935
 936                 ctx->res = n;
 937                 ctx->ctr = ctr;
 938                 return;
 939         } while(0);
 940 #endif
 941         for (i=0;i<len;++i) {
 942                 u8 c;
 943                 if (n==0) {
 944                         (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
 945                         ++ctr;
 946                         if (is_endian.little)
 947                                 PUTU32(ctx->Yi.c+12,ctr);
 948                         else
 949                                 ctx->Yi.d[3] = ctr;
 950                 }
 951                 c = in[i];
 952                 out[i] ^= ctx->EKi.c[n];
 953                 ctx->Xi.c[n] ^= c;
 954                 n = (n+1)%16;
 955                 if (n==0)
 956                         GCM_MUL(ctx,Xi);
 957         }
 958
 959         ctx->res = n;
 960         ctx->ctr = ctr;
 961 }
 962
 963 void CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx)
 964 {
 965         const union { long one; char little; } is_endian = {1};
 966         u64 alen = ctx->len.u[0]<<3;
 967         u64 clen = ctx->len.u[1]<<3;
 968
 969         if (ctx->res)
 970                 GCM_MUL(ctx,Xi);
 971
 972         if (is_endian.little) {
 973 #ifdef BSWAP8
 974                 alen = BSWAP8(alen);
 975                 clen = BSWAP8(clen);
 976 #else
 977                 u8 *p = ctx->len.c;
 978
 979                 ctx->len.u[0] = alen;
 980                 ctx->len.u[1] = clen;
 981
 982                 alen = (u64)GETU32(p)  <<32|GETU32(p+4);
 983                 clen = (u64)GETU32(p+8)<<32|GETU32(p+12);
 984 #endif
 985         }
 986
 987         ctx->Xi.u[0] ^= alen;
 988         ctx->Xi.u[1] ^= clen;
 989         GCM_MUL(ctx,Xi);
 990
 991         ctx->Xi.u[0] ^= ctx->EK0.u[0];
 992         ctx->Xi.u[1] ^= ctx->EK0.u[1];
 993 }
 994
 995 #if defined(SELFTEST)
 996 #include <stdio.h>
 997 #include <openssl/aes.h>
 998
 999 /* Test Case 1 */
1000 static const u8 K1[16],
1001                 *P1=NULL,
1002                 *A1=NULL,
1003                 IV1[12],
1004                 *C1=NULL,
1005                 T1[]=  {0x58,0xe2,0xfc,0xce,0xfa,0x7e,0x30,0x61,0x36,0x7f,0x1d,0x57,0xa4,0xe7,0x45,0x5a};
1006
1007 /* Test Case 2 */
1008 #define K2 K1
1009 #define A2 A1
1010 #define IV2 IV1
1011 static const u8 P2[16],
1012                 C2[]=  {0x03,0x88,0xda,0xce,0x60,0xb6,0xa3,0x92,0xf3,0x28,0xc2,0xb9,0x71,0xb2,0xfe,0x78},
1013                 T2[]=  {0xab,0x6e,0x47,0xd4,0x2c,0xec,0x13,0xbd,0xf5,0x3a,0x67,0xb2,0x12,0x57,0xbd,0xdf};
1014
1015 /* Test Case 3 */
1016 #define A3 A2
1017 static const u8 K3[]=  {0xfe,0xff,0xe9,0x92,0x86,0x65,0x73,0x1c,0x6d,0x6a,0x8f,0x94,0x67,0x30,0x83,0x08},
1018                 P3[]=  {0xd9,0x31,0x32,0x25,0xf8,0x84,0x06,0xe5,0xa5,0x59,0x09,0xc5,0xaf,0xf5,0x26,0x9a,
1019                         0x86,0xa7,0xa9,0x53,0x15,0x34,0xf7,0xda,0x2e,0x4c,0x30,0x3d,0x8a,0x31,0x8a,0x72,
1020                         0x1c,0x3c,0x0c,0x95,0x95,0x68,0x09,0x53,0x2f,0xcf,0x0e,0x24,0x49,0xa6,0xb5,0x25,
1021                         0xb1,0x6a,0xed,0xf5,0xaa,0x0d,0xe6,0x57,0xba,0x63,0x7b,0x39,0x1a,0xaf,0xd2,0x55},
1022                 IV3[]= {0xca,0xfe,0xba,0xbe,0xfa,0xce,0xdb,0xad,0xde,0xca,0xf8,0x88},
1023                 C3[]=  {0x42,0x83,0x1e,0xc2,0x21,0x77,0x74,0x24,0x4b,0x72,0x21,0xb7,0x84,0xd0,0xd4,0x9c,
1024                         0xe3,0xaa,0x21,0x2f,0x2c,0x02,0xa4,0xe0,0x35,0xc1,0x7e,0x23,0x29,0xac,0xa1,0x2e,
1025                         0x21,0xd5,0x14,0xb2,0x54,0x66,0x93,0x1c,0x7d,0x8f,0x6a,0x5a,0xac,0x84,0xaa,0x05,
1026                         0x1b,0xa3,0x0b,0x39,0x6a,0x0a,0xac,0x97,0x3d,0x58,0xe0,0x91,0x47,0x3f,0x59,0x85},
1027                 T3[]=  {0x4d,0x5c,0x2a,0xf3,0x27,0xcd,0x64,0xa6,0x2c,0xf3,0x5a,0xbd,0x2b,0xa6,0xfa,0xb4,};
1028
1029 /* Test Case 4 */
1030 #define K4 K3
1031 #define IV4 IV3
1032 static const u8 P4[]=  {0xd9,0x31,0x32,0x25,0xf8,0x84,0x06,0xe5,0xa5,0x59,0x09,0xc5,0xaf,0xf5,0x26,0x9a,
1033                         0x86,0xa7,0xa9,0x53,0x15,0x34,0xf7,0xda,0x2e,0x4c,0x30,0x3d,0x8a,0x31,0x8a,0x72,
1034                         0x1c,0x3c,0x0c,0x95,0x95,0x68,0x09,0x53,0x2f,0xcf,0x0e,0x24,0x49,0xa6,0xb5,0x25,
1035                         0xb1,0x6a,0xed,0xf5,0xaa,0x0d,0xe6,0x57,0xba,0x63,0x7b,0x39},
1036                 A4[]=  {0xfe,0xed,0xfa,0xce,0xde,0xad,0xbe,0xef,0xfe,0xed,0xfa,0xce,0xde,0xad,0xbe,0xef,
1037                         0xab,0xad,0xda,0xd2},
1038                 C4[]=  {0x42,0x83,0x1e,0xc2,0x21,0x77,0x74,0x24,0x4b,0x72,0x21,0xb7,0x84,0xd0,0xd4,0x9c,
1039                         0xe3,0xaa,0x21,0x2f,0x2c,0x02,0xa4,0xe0,0x35,0xc1,0x7e,0x23,0x29,0xac,0xa1,0x2e,
1040                         0x21,0xd5,0x14,0xb2,0x54,0x66,0x93,0x1c,0x7d,0x8f,0x6a,0x5a,0xac,0x84,0xaa,0x05,
1041                         0x1b,0xa3,0x0b,0x39,0x6a,0x0a,0xac,0x97,0x3d,0x58,0xe0,0x91},
1042                 T4[]=  {0x5b,0xc9,0x4f,0xbc,0x32,0x21,0xa5,0xdb,0x94,0xfa,0xe9,0x5a,0xe7,0x12,0x1a,0x47};
1043
1044 /* Test Case 5 */
1045 #define K5 K4
1046 #define P5 P4
1047 static const u8 A5[]=  {0xfe,0xed,0xfa,0xce,0xde,0xad,0xbe,0xef,0xfe,0xed,0xfa,0xce,0xde,0xad,0xbe,0xef,
1048                         0xab,0xad,0xda,0xd2},
1049                 IV5[]= {0xca,0xfe,0xba,0xbe,0xfa,0xce,0xdb,0xad},
1050                 C5[]=  {0x61,0x35,0x3b,0x4c,0x28,0x06,0x93,0x4a,0x77,0x7f,0xf5,0x1f,0xa2,0x2a,0x47,0x55,
1051                         0x69,0x9b,0x2a,0x71,0x4f,0xcd,0xc6,0xf8,0x37,0x66,0xe5,0xf9,0x7b,0x6c,0x74,0x23,
1052                         0x73,0x80,0x69,0x00,0xe4,0x9f,0x24,0xb2,0x2b,0x09,0x75,0x44,0xd4,0x89,0x6b,0x42,
1053                         0x49,0x89,0xb5,0xe1,0xeb,0xac,0x0f,0x07,0xc2,0x3f,0x45,0x98},
1054                 T5[]=  {0x36,0x12,0xd2,0xe7,0x9e,0x3b,0x07,0x85,0x56,0x1b,0xe1,0x4a,0xac,0xa2,0xfc,0xcb};
1055
1056 /* Test Case 6 */
1057 #define K6 K5
1058 #define P6 P5
1059 #define A6 A5
1060 static const u8 IV6[]= {0x93,0x13,0x22,0x5d,0xf8,0x84,0x06,0xe5,0x55,0x90,0x9c,0x5a,0xff,0x52,0x69,0xaa,
1061                         0x6a,0x7a,0x95,0x38,0x53,0x4f,0x7d,0xa1,0xe4,0xc3,0x03,0xd2,0xa3,0x18,0xa7,0x28,
1062                         0xc3,0xc0,0xc9,0x51,0x56,0x80,0x95,0x39,0xfc,0xf0,0xe2,0x42,0x9a,0x6b,0x52,0x54,
1063                         0x16,0xae,0xdb,0xf5,0xa0,0xde,0x6a,0x57,0xa6,0x37,0xb3,0x9b},
1064                 C6[]=  {0x8c,0xe2,0x49,0x98,0x62,0x56,0x15,0xb6,0x03,0xa0,0x33,0xac,0xa1,0x3f,0xb8,0x94,
1065                         0xbe,0x91,0x12,0xa5,0xc3,0xa2,0x11,0xa8,0xba,0x26,0x2a,0x3c,0xca,0x7e,0x2c,0xa7,
1066                         0x01,0xe4,0xa9,0xa4,0xfb,0xa4,0x3c,0x90,0xcc,0xdc,0xb2,0x81,0xd4,0x8c,0x7c,0x6f,
1067                         0xd6,0x28,0x75,0xd2,0xac,0xa4,0x17,0x03,0x4c,0x34,0xae,0xe5},
1068                 T6[]=  {0x61,0x9c,0xc5,0xae,0xff,0xfe,0x0b,0xfa,0x46,0x2a,0xf4,0x3c,0x16,0x99,0xd0,0x50};
1069
1070 /* Test Case 7 */
1071 static const u8 K7[24],
1072                 *P7=NULL,
1073                 *A7=NULL,
1074                 IV7[12],
1075                 *C7=NULL,
1076                 T7[]=  {0xcd,0x33,0xb2,0x8a,0xc7,0x73,0xf7,0x4b,0xa0,0x0e,0xd1,0xf3,0x12,0x57,0x24,0x35};
1077
1078 /* Test Case 8 */
1079 #define K8 K7
1080 #define IV8 IV7
1081 #define A8 A7
1082 static const u8 P8[16],
1083                 C8[]=  {0x98,0xe7,0x24,0x7c,0x07,0xf0,0xfe,0x41,0x1c,0x26,0x7e,0x43,0x84,0xb0,0xf6,0x00},
1084                 T8[]=  {0x2f,0xf5,0x8d,0x80,0x03,0x39,0x27,0xab,0x8e,0xf4,0xd4,0x58,0x75,0x14,0xf0,0xfb};
1085
1086 /* Test Case 9 */
1087 #define A9 A8
1088 static const u8 K9[]=  {0xfe,0xff,0xe9,0x92,0x86,0x65,0x73,0x1c,0x6d,0x6a,0x8f,0x94,0x67,0x30,0x83,0x08,
1089                         0xfe,0xff,0xe9,0x92,0x86,0x65,0x73,0x1c},
1090                 P9[]=  {0xd9,0x31,0x32,0x25,0xf8,0x84,0x06,0xe5,0xa5,0x59,0x09,0xc5,0xaf,0xf5,0x26,0x9a,
1091                         0x86,0xa7,0xa9,0x53,0x15,0x34,0xf7,0xda,0x2e,0x4c,0x30,0x3d,0x8a,0x31,0x8a,0x72,
1092                         0x1c,0x3c,0x0c,0x95,0x95,0x68,0x09,0x53,0x2f,0xcf,0x0e,0x24,0x49,0xa6,0xb5,0x25,
1093                         0xb1,0x6a,0xed,0xf5,0xaa,0x0d,0xe6,0x57,0xba,0x63,0x7b,0x39,0x1a,0xaf,0xd2,0x55},
1094                 IV9[]= {0xca,0xfe,0xba,0xbe,0xfa,0xce,0xdb,0xad,0xde,0xca,0xf8,0x88},
1095                 C9[]=  {0x39,0x80,0xca,0x0b,0x3c,0x00,0xe8,0x41,0xeb,0x06,0xfa,0xc4,0x87,0x2a,0x27,0x57,
1096                         0x85,0x9e,0x1c,0xea,0xa6,0xef,0xd9,0x84,0x62,0x85,0x93,0xb4,0x0c,0xa1,0xe1,0x9c,
1097                         0x7d,0x77,0x3d,0x00,0xc1,0x44,0xc5,0x25,0xac,0x61,0x9d,0x18,0xc8,0x4a,0x3f,0x47,
1098                         0x18,0xe2,0x44,0x8b,0x2f,0xe3,0x24,0xd9,0xcc,0xda,0x27,0x10,0xac,0xad,0xe2,0x56},
1099                 T9[]=  {0x99,0x24,0xa7,0xc8,0x58,0x73,0x36,0xbf,0xb1,0x18,0x02,0x4d,0xb8,0x67,0x4a,0x14};
1100
1101 /* Test Case 10 */
1102 #define K10 K9
1103 #define IV10 IV9
1104 static const u8 P10[]= {0xd9,0x31,0x32,0x25,0xf8,0x84,0x06,0xe5,0xa5,0x59,0x09,0xc5,0xaf,0xf5,0x26,0x9a,
1105                         0x86,0xa7,0xa9,0x53,0x15,0x34,0xf7,0xda,0x2e,0x4c,0x30,0x3d,0x8a,0x31,0x8a,0x72,
1106                         0x1c,0x3c,0x0c,0x95,0x95,0x68,0x09,0x53,0x2f,0xcf,0x0e,0x24,0x49,0xa6,0xb5,0x25,
1107                         0xb1,0x6a,0xed,0xf5,0xaa,0x0d,0xe6,0x57,0xba,0x63,0x7b,0x39},
1108                 A10[]= {0xfe,0xed,0xfa,0xce,0xde,0xad,0xbe,0xef,0xfe,0xed,0xfa,0xce,0xde,0xad,0xbe,0xef,
1109                         0xab,0xad,0xda,0xd2},
1110                 C10[]= {0x39,0x80,0xca,0x0b,0x3c,0x00,0xe8,0x41,0xeb,0x06,0xfa,0xc4,0x87,0x2a,0x27,0x57,
1111                         0x85,0x9e,0x1c,0xea,0xa6,0xef,0xd9,0x84,0x62,0x85,0x93,0xb4,0x0c,0xa1,0xe1,0x9c,
1112                         0x7d,0x77,0x3d,0x00,0xc1,0x44,0xc5,0x25,0xac,0x61,0x9d,0x18,0xc8,0x4a,0x3f,0x47,
1113                         0x18,0xe2,0x44,0x8b,0x2f,0xe3,0x24,0xd9,0xcc,0xda,0x27,0x10},
1114                 T10[]= {0x25,0x19,0x49,0x8e,0x80,0xf1,0x47,0x8f,0x37,0xba,0x55,0xbd,0x6d,0x27,0x61,0x8c};
1115
1116 /* Test Case 11 */
1117 #define K11 K10
1118 #define P11 P10
1119 #define A11 A10
1120 static const u8 IV11[]={0xca,0xfe,0xba,0xbe,0xfa,0xce,0xdb,0xad},
1121                 C11[]= {0x0f,0x10,0xf5,0x99,0xae,0x14,0xa1,0x54,0xed,0x24,0xb3,0x6e,0x25,0x32,0x4d,0xb8,
1122                         0xc5,0x66,0x63,0x2e,0xf2,0xbb,0xb3,0x4f,0x83,0x47,0x28,0x0f,0xc4,0x50,0x70,0x57,
1123                         0xfd,0xdc,0x29,0xdf,0x9a,0x47,0x1f,0x75,0xc6,0x65,0x41,0xd4,0xd4,0xda,0xd1,0xc9,
1124                         0xe9,0x3a,0x19,0xa5,0x8e,0x8b,0x47,0x3f,0xa0,0xf0,0x62,0xf7},
1125                 T11[]= {0x65,0xdc,0xc5,0x7f,0xcf,0x62,0x3a,0x24,0x09,0x4f,0xcc,0xa4,0x0d,0x35,0x33,0xf8};
1126
1127 /* Test Case 12 */
1128 #define K12 K11
1129 #define P12 P11
1130 #define A12 A11
1131 static const u8 IV12[]={0x93,0x13,0x22,0x5d,0xf8,0x84,0x06,0xe5,0x55,0x90,0x9c,0x5a,0xff,0x52,0x69,0xaa,
1132                         0x6a,0x7a,0x95,0x38,0x53,0x4f,0x7d,0xa1,0xe4,0xc3,0x03,0xd2,0xa3,0x18,0xa7,0x28,
1133                         0xc3,0xc0,0xc9,0x51,0x56,0x80,0x95,0x39,0xfc,0xf0,0xe2,0x42,0x9a,0x6b,0x52,0x54,
1134                         0x16,0xae,0xdb,0xf5,0xa0,0xde,0x6a,0x57,0xa6,0x37,0xb3,0x9b},
1135                 C12[]= {0xd2,0x7e,0x88,0x68,0x1c,0xe3,0x24,0x3c,0x48,0x30,0x16,0x5a,0x8f,0xdc,0xf9,0xff,
1136                         0x1d,0xe9,0xa1,0xd8,0xe6,0xb4,0x47,0xef,0x6e,0xf7,0xb7,0x98,0x28,0x66,0x6e,0x45,
1137                         0x81,0xe7,0x90,0x12,0xaf,0x34,0xdd,0xd9,0xe2,0xf0,0x37,0x58,0x9b,0x29,0x2d,0xb3,
1138                         0xe6,0x7c,0x03,0x67,0x45,0xfa,0x22,0xe7,0xe9,0xb7,0x37,0x3b},
1139                 T12[]= {0xdc,0xf5,0x66,0xff,0x29,0x1c,0x25,0xbb,0xb8,0x56,0x8f,0xc3,0xd3,0x76,0xa6,0xd9};
1140
1141 /* Test Case 13 */
1142 static const u8 K13[32],
1143                 *P13=NULL,
1144                 *A13=NULL,
1145                 IV13[12],
1146                 *C13=NULL,
1147                 T13[]={0x53,0x0f,0x8a,0xfb,0xc7,0x45,0x36,0xb9,0xa9,0x63,0xb4,0xf1,0xc4,0xcb,0x73,0x8b};
1148
1149 /* Test Case 14 */
1150 #define K14 K13
1151 #define A14 A13
1152 static const u8 P14[16],
1153                 IV14[12],
1154                 C14[]= {0xce,0xa7,0x40,0x3d,0x4d,0x60,0x6b,0x6e,0x07,0x4e,0xc5,0xd3,0xba,0xf3,0x9d,0x18},
1155                 T14[]= {0xd0,0xd1,0xc8,0xa7,0x99,0x99,0x6b,0xf0,0x26,0x5b,0x98,0xb5,0xd4,0x8a,0xb9,0x19};
1156
1157 /* Test Case 15 */
1158 #define A15 A14
1159 static const u8 K15[]= {0xfe,0xff,0xe9,0x92,0x86,0x65,0x73,0x1c,0x6d,0x6a,0x8f,0x94,0x67,0x30,0x83,0x08,
1160                         0xfe,0xff,0xe9,0x92,0x86,0x65,0x73,0x1c,0x6d,0x6a,0x8f,0x94,0x67,0x30,0x83,0x08},
1161                 P15[]= {0xd9,0x31,0x32,0x25,0xf8,0x84,0x06,0xe5,0xa5,0x59,0x09,0xc5,0xaf,0xf5,0x26,0x9a,
1162                         0x86,0xa7,0xa9,0x53,0x15,0x34,0xf7,0xda,0x2e,0x4c,0x30,0x3d,0x8a,0x31,0x8a,0x72,
1163                         0x1c,0x3c,0x0c,0x95,0x95,0x68,0x09,0x53,0x2f,0xcf,0x0e,0x24,0x49,0xa6,0xb5,0x25,
1164                         0xb1,0x6a,0xed,0xf5,0xaa,0x0d,0xe6,0x57,0xba,0x63,0x7b,0x39,0x1a,0xaf,0xd2,0x55},
1165                 IV15[]={0xca,0xfe,0xba,0xbe,0xfa,0xce,0xdb,0xad,0xde,0xca,0xf8,0x88},
1166                 C15[]= {0x52,0x2d,0xc1,0xf0,0x99,0x56,0x7d,0x07,0xf4,0x7f,0x37,0xa3,0x2a,0x84,0x42,0x7d,
1167                         0x64,0x3a,0x8c,0xdc,0xbf,0xe5,0xc0,0xc9,0x75,0x98,0xa2,0xbd,0x25,0x55,0xd1,0xaa,
1168                         0x8c,0xb0,0x8e,0x48,0x59,0x0d,0xbb,0x3d,0xa7,0xb0,0x8b,0x10,0x56,0x82,0x88,0x38,
1169                         0xc5,0xf6,0x1e,0x63,0x93,0xba,0x7a,0x0a,0xbc,0xc9,0xf6,0x62,0x89,0x80,0x15,0xad},
1170                 T15[]= {0xb0,0x94,0xda,0xc5,0xd9,0x34,0x71,0xbd,0xec,0x1a,0x50,0x22,0x70,0xe3,0xcc,0x6c};
1171
1172 /* Test Case 16 */
1173 #define K16 K15
1174 #define IV16 IV15
1175 static const u8 P16[]= {0xd9,0x31,0x32,0x25,0xf8,0x84,0x06,0xe5,0xa5,0x59,0x09,0xc5,0xaf,0xf5,0x26,0x9a,
1176                         0x86,0xa7,0xa9,0x53,0x15,0x34,0xf7,0xda,0x2e,0x4c,0x30,0x3d,0x8a,0x31,0x8a,0x72,
1177                         0x1c,0x3c,0x0c,0x95,0x95,0x68,0x09,0x53,0x2f,0xcf,0x0e,0x24,0x49,0xa6,0xb5,0x25,
1178                         0xb1,0x6a,0xed,0xf5,0xaa,0x0d,0xe6,0x57,0xba,0x63,0x7b,0x39},
1179                 A16[]= {0xfe,0xed,0xfa,0xce,0xde,0xad,0xbe,0xef,0xfe,0xed,0xfa,0xce,0xde,0xad,0xbe,0xef,
1180                         0xab,0xad,0xda,0xd2},
1181                 C16[]= {0x52,0x2d,0xc1,0xf0,0x99,0x56,0x7d,0x07,0xf4,0x7f,0x37,0xa3,0x2a,0x84,0x42,0x7d,
1182                         0x64,0x3a,0x8c,0xdc,0xbf,0xe5,0xc0,0xc9,0x75,0x98,0xa2,0xbd,0x25,0x55,0xd1,0xaa,
1183                         0x8c,0xb0,0x8e,0x48,0x59,0x0d,0xbb,0x3d,0xa7,0xb0,0x8b,0x10,0x56,0x82,0x88,0x38,
1184                         0xc5,0xf6,0x1e,0x63,0x93,0xba,0x7a,0x0a,0xbc,0xc9,0xf6,0x62},
1185                 T16[]= {0x76,0xfc,0x6e,0xce,0x0f,0x4e,0x17,0x68,0xcd,0xdf,0x88,0x53,0xbb,0x2d,0x55,0x1b};
1186
1187 /* Test Case 17 */
1188 #define K17 K16
1189 #define P17 P16
1190 #define A17 A16
1191 static const u8 IV17[]={0xca,0xfe,0xba,0xbe,0xfa,0xce,0xdb,0xad},
1192                 C17[]= {0xc3,0x76,0x2d,0xf1,0xca,0x78,0x7d,0x32,0xae,0x47,0xc1,0x3b,0xf1,0x98,0x44,0xcb,
1193                         0xaf,0x1a,0xe1,0x4d,0x0b,0x97,0x6a,0xfa,0xc5,0x2f,0xf7,0xd7,0x9b,0xba,0x9d,0xe0,
1194                         0xfe,0xb5,0x82,0xd3,0x39,0x34,0xa4,0xf0,0x95,0x4c,0xc2,0x36,0x3b,0xc7,0x3f,0x78,
1195                         0x62,0xac,0x43,0x0e,0x64,0xab,0xe4,0x99,0xf4,0x7c,0x9b,0x1f},
1196                 T17[]= {0x3a,0x33,0x7d,0xbf,0x46,0xa7,0x92,0xc4,0x5e,0x45,0x49,0x13,0xfe,0x2e,0xa8,0xf2};
1197
1198 /* Test Case 18 */
1199 #define K18 K17
1200 #define P18 P17
1201 #define A18 A17
1202 static const u8 IV18[]={0x93,0x13,0x22,0x5d,0xf8,0x84,0x06,0xe5,0x55,0x90,0x9c,0x5a,0xff,0x52,0x69,0xaa,
1203                         0x6a,0x7a,0x95,0x38,0x53,0x4f,0x7d,0xa1,0xe4,0xc3,0x03,0xd2,0xa3,0x18,0xa7,0x28,
1204                         0xc3,0xc0,0xc9,0x51,0x56,0x80,0x95,0x39,0xfc,0xf0,0xe2,0x42,0x9a,0x6b,0x52,0x54,
1205                         0x16,0xae,0xdb,0xf5,0xa0,0xde,0x6a,0x57,0xa6,0x37,0xb3,0x9b},
1206                 C18[]= {0x5a,0x8d,0xef,0x2f,0x0c,0x9e,0x53,0xf1,0xf7,0x5d,0x78,0x53,0x65,0x9e,0x2a,0x20,
1207                         0xee,0xb2,0xb2,0x2a,0xaf,0xde,0x64,0x19,0xa0,0x58,0xab,0x4f,0x6f,0x74,0x6b,0xf4,
1208                         0x0f,0xc0,0xc3,0xb7,0x80,0xf2,0x44,0x45,0x2d,0xa3,0xeb,0xf1,0xc5,0xd8,0x2c,0xde,
1209                         0xa2,0x41,0x89,0x97,0x20,0x0e,0xf8,0x2e,0x44,0xae,0x7e,0x3f},
1210                 T18[]= {0xa4,0x4a,0x82,0x66,0xee,0x1c,0x8e,0xb0,0xc8,0xb5,0xd4,0xcf,0x5a,0xe9,0xf1,0x9a};
1211
1212 #define TEST_CASE(n)    do {                                    \
1213         u8 out[sizeof(P##n)];                                   \
1214         AES_set_encrypt_key(K##n,sizeof(K##n)*8,&key);          \
1215         CRYPTO_gcm128_init(&ctx,&key,(block128_f)AES_encrypt);  \
1216         CRYPTO_gcm128_setiv(&ctx,IV##n,sizeof(IV##n));          \
1217         if (A##n) CRYPTO_gcm128_aad(&ctx,A##n,sizeof(A##n));    \
1218         if (P##n) CRYPTO_gcm128_encrypt(&ctx,P##n,out,sizeof(out));     \
1219         CRYPTO_gcm128_finish(&ctx);                             \
1220         if (memcmp(ctx.Xi.c,T##n,16) || (C##n && memcmp(out,C##n,sizeof(out)))) \
1221                 ret++, printf ("encrypt test#%d failed.\n",n);\
1222         CRYPTO_gcm128_setiv(&ctx,IV##n,sizeof(IV##n));          \
1223         if (A##n) CRYPTO_gcm128_aad(&ctx,A##n,sizeof(A##n));    \
1224         if (C##n) CRYPTO_gcm128_decrypt(&ctx,C##n,out,sizeof(out));     \
1225         CRYPTO_gcm128_finish(&ctx);                             \
1226         if (memcmp(ctx.Xi.c,T##n,16) || (P##n && memcmp(out,P##n,sizeof(out)))) \
1227                 ret++, printf ("decrypt test#%d failed.\n",n);\
1228         } while(0)
1229
1230 int main()
1231 {
1232         GCM128_CONTEXT ctx;
1233         AES_KEY key;
1234         int ret=0;
1235
1236         TEST_CASE(1);
1237         TEST_CASE(2);
1238         TEST_CASE(3);
1239         TEST_CASE(4);
1240         TEST_CASE(5);
1241         TEST_CASE(6);
1242         TEST_CASE(7);
1243         TEST_CASE(8);
1244         TEST_CASE(9);
1245         TEST_CASE(10);
1246         TEST_CASE(11);
1247         TEST_CASE(12);
1248         TEST_CASE(13);
1249         TEST_CASE(14);
1250         TEST_CASE(15);
1251         TEST_CASE(16);
1252         TEST_CASE(17);
1253         TEST_CASE(18);
1254
1255 #ifdef OPENSSL_CPUID_OBJ
1256         {
1257         size_t start,stop,gcm_t,ctr_t,OPENSSL_rdtsc();
1258         union { u64 u; u8 c[1024]; } buf;
1259
1260         AES_set_encrypt_key(K1,sizeof(K1)*8,&key);
1261         CRYPTO_gcm128_init(&ctx,&key,(block128_f)AES_encrypt);
1262         CRYPTO_gcm128_setiv(&ctx,IV1,sizeof(IV1));
1263
1264         CRYPTO_gcm128_encrypt(&ctx,buf.c,buf.c,sizeof(buf));
1265         start = OPENSSL_rdtsc();
1266         CRYPTO_gcm128_encrypt(&ctx,buf.c,buf.c,sizeof(buf));
1267         gcm_t = OPENSSL_rdtsc() - start;
1268
1269         CRYPTO_ctr128_encrypt(buf.c,buf.c,sizeof(buf),
1270                         &key,ctx.Yi.c,ctx.EKi.c,&ctx.res,
1271                         (block128_f)AES_encrypt);
1272         start = OPENSSL_rdtsc();
1273         CRYPTO_ctr128_encrypt(buf.c,buf.c,sizeof(buf),
1274                         &key,ctx.Yi.c,ctx.EKi.c,&ctx.res,
1275                         (block128_f)AES_encrypt);
1276         ctr_t = OPENSSL_rdtsc() - start;
1277
1278         printf("%.2f-%.2f=%.2f\n",
1279                         gcm_t/(double)sizeof(buf),
1280                         ctr_t/(double)sizeof(buf),
1281                         (gcm_t-ctr_t)/(double)sizeof(buf));
1282 #ifdef GHASH
1283         GHASH(buf.c,sizeof(buf),&ctx);
1284         start = OPENSSL_rdtsc();
1285         GHASH(buf.c,sizeof(buf),&ctx);
1286         gcm_t = OPENSSL_rdtsc() - start;
1287         printf("%.2f\n",gcm_t/(double)sizeof(buf));
1288 #endif
1289         }
1290 #endif
1291
1292         return ret;
1293 }
1294 #endif