ghash-x86.pl: engage original MMX version in no-sse2 builds.
[openssl.git] / crypto / modes / gcm128.c
index acae5e95acc360b3b6745e0933a396116654933e..f77aa972bab47601774600edfbb73f9e9d892479 100644 (file)
@@ -47,6 +47,8 @@
  * ====================================================================
  */
 
+#define OPENSSL_FIPSAPI
+
 #include <openssl/crypto.h>
 #include "modes_lcl.h"
 #include <string.h>
@@ -58,8 +60,6 @@
 #endif
 #include <assert.h>
 
-typedef struct { u64 hi,lo; } u128;
-
 #if defined(BSWAP4) && defined(STRICT_ALIGNMENT)
 /* redefine, because alignment is ensured */
 #undef GETU32
@@ -82,9 +82,6 @@ typedef struct { u64 hi,lo; } u128;
        } \
 } while(0)
 
-#ifdef TABLE_BITS
-#undef TABLE_BITS
-#endif
 /*
  * Even though permitted values for TABLE_BITS are 8, 4 and 1, it should
  * never be set to 8. 8 is effectively reserved for testing purposes.
@@ -116,9 +113,9 @@ typedef struct { u64 hi,lo; } u128;
  * - larger table has larger cache footprint, which can affect
  *   performance of other code paths (not necessarily even from same
  *   thread in Hyper-Threading world);
+ *
+ * Value of 1 is not appropriate for performance reasons.
  */
-#define        TABLE_BITS 4
-
 #if    TABLE_BITS==8
 
 static void gcm_init_8bit(u128 Htable[256], u64 H[2])
@@ -145,12 +142,13 @@ static void gcm_init_8bit(u128 Htable[256], u64 H[2])
        }
 }
 
-static void gcm_gmult_8bit(u64 Xi[2], u128 Htable[256])
+static void gcm_gmult_8bit(u64 Xi[2], const u128 Htable[256])
 {
        u128 Z = { 0, 0};
        const u8 *xi = (const u8 *)Xi+15;
        size_t rem, n = *xi;
        const union { long one; char little; } is_endian = {1};
+       __fips_constseg
        static const size_t rem_8bit[256] = {
                PACK(0x0000), PACK(0x01C2), PACK(0x0384), PACK(0x0246),
                PACK(0x0708), PACK(0x06CA), PACK(0x048C), PACK(0x054E),
@@ -329,6 +327,7 @@ static void gcm_init_4bit(u128 Htable[16], u64 H[2])
 }
 
 #ifndef GHASH_ASM
+__fips_constseg
 static const size_t rem_4bit[16] = {
        PACK(0x0000), PACK(0x1C20), PACK(0x3840), PACK(0x2460),
        PACK(0x7080), PACK(0x6CA0), PACK(0x48C0), PACK(0x54E0),
@@ -464,6 +463,7 @@ static void gcm_ghash_4bit(u64 Xi[2],const u128 Htable[16],
      */ 
     u128 Hshr4[16];    /* Htable shifted right by 4 bits */
     u8   Hshl4[16];    /* Htable shifted left  by 4 bits */
+    __fips_constseg
     static const unsigned short rem_8bit[256] = {
        0x0000, 0x01C2, 0x0384, 0x0246, 0x0708, 0x06CA, 0x048C, 0x054E,
        0x0E10, 0x0FD2, 0x0D94, 0x0C56, 0x0918, 0x08DA, 0x0A9C, 0x0B5E,
@@ -645,47 +645,45 @@ static void gcm_gmult_1bit(u64 Xi[2],const u64 H[2])
 
 #endif
 
-struct gcm128_context {
-       /* Following 6 names follow names in GCM specification */
-       union { u64 u[2]; u32 d[4]; u8 c[16]; } Yi,EKi,EK0,
-                                               Xi,H,len;
-       /* Pre-computed table used by gcm_gmult_* */
-#if TABLE_BITS==8
-       u128 Htable[256];
-#else
-       u128 Htable[16];
-       void (*gmult)(u64 Xi[2],const u128 Htable[16]);
-       void (*ghash)(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
-#endif
-       unsigned int res, pad;
-       block128_f block;
-       void *key;
-};
-
-#if    TABLE_BITS==4 && defined(GHASH_ASM) && !defined(I386_ONLY) && \
+#if    TABLE_BITS==4 && defined(GHASH_ASM)
+# if   !defined(I386_ONLY) && \
        (defined(__i386)        || defined(__i386__)    || \
         defined(__x86_64)      || defined(__x86_64__)  || \
         defined(_M_IX86)       || defined(_M_AMD64)    || defined(_M_X64))
-# define GHASH_ASM_IAX
+#  define GHASH_ASM_X86_OR_64
+#  define GCM_FUNCREF_4BIT
 extern unsigned int OPENSSL_ia32cap_P[2];
 
 void gcm_init_clmul(u128 Htable[16],const u64 Xi[2]);
 void gcm_gmult_clmul(u64 Xi[2],const u128 Htable[16]);
 void gcm_ghash_clmul(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
 
-# if   defined(__i386) || defined(__i386__) || defined(_M_IX86)
-#  define GHASH_ASM_X86
+#  if  defined(__i386) || defined(__i386__) || defined(_M_IX86)
+#   define GHASH_ASM_X86
 void gcm_gmult_4bit_mmx(u64 Xi[2],const u128 Htable[16]);
 void gcm_ghash_4bit_mmx(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
 
 void gcm_gmult_4bit_x86(u64 Xi[2],const u128 Htable[16]);
 void gcm_ghash_4bit_x86(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
+#  endif
+# elif defined(__arm__) || defined(__arm)
+#  include "arm_arch.h"
+#  if __ARM_ARCH__>=7
+#   define GHASH_ASM_ARM
+#   define GCM_FUNCREF_4BIT
+void gcm_gmult_neon(u64 Xi[2],const u128 Htable[16]);
+void gcm_ghash_neon(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
+#  endif
 # endif
+#endif
 
+#ifdef GCM_FUNCREF_4BIT
 # undef  GCM_MUL
-# define GCM_MUL(ctx,Xi)   (*((ctx)->gmult))(ctx->Xi.u,ctx->Htable)
-# undef  GHASH
-# define GHASH(ctx,in,len) (*((ctx)->ghash))((ctx)->Xi.u,(ctx)->Htable,in,len)
+# define GCM_MUL(ctx,Xi)       (*gcm_gmult_p)(ctx->Xi.u,ctx->Htable)
+# ifdef GHASH
+#  undef  GHASH
+#  define GHASH(ctx,in,len)    (*gcm_ghash_p)(ctx->Xi.u,ctx->Htable,in,len)
+# endif
 #endif
 
 void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx,void *key,block128_f block)
@@ -716,16 +714,23 @@ void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx,void *key,block128_f block)
 #if    TABLE_BITS==8
        gcm_init_8bit(ctx->Htable,ctx->H.u);
 #elif  TABLE_BITS==4
-# if   defined(GHASH_ASM_IAX)                  /* both x86 and x86_64 */
-       if (OPENSSL_ia32cap_P[1]&(1<<1)) {
+# if   defined(GHASH_ASM_X86_OR_64)
+#  if  !defined(GHASH_ASM_X86) || defined(OPENSSL_IA32_SSE2)
+       if (OPENSSL_ia32cap_P[0]&(1<<24) &&     /* check FXSR bit */
+           OPENSSL_ia32cap_P[1]&(1<<1) ) {     /* check PCLMULQDQ bit */
                gcm_init_clmul(ctx->Htable,ctx->H.u);
                ctx->gmult = gcm_gmult_clmul;
                ctx->ghash = gcm_ghash_clmul;
                return;
        }
+#  endif
        gcm_init_4bit(ctx->Htable,ctx->H.u);
 #  if  defined(GHASH_ASM_X86)                  /* x86 only */
-       if (OPENSSL_ia32cap_P[0]&(1<<23)) {
+#   if defined(OPENSSL_IA32_SSE2)
+       if (OPENSSL_ia32cap_P[0]&(1<<25)) {     /* check SSE bit */
+#   else
+       if (OPENSSL_ia32cap_P[0]&(1<<23)) {     /* check MMX bit */
+#   endif
                ctx->gmult = gcm_gmult_4bit_mmx;
                ctx->ghash = gcm_ghash_4bit_mmx;
        } else {
@@ -736,6 +741,15 @@ void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx,void *key,block128_f block)
        ctx->gmult = gcm_gmult_4bit;
        ctx->ghash = gcm_ghash_4bit;
 #  endif
+# elif defined(GHASH_ASM_ARM)
+       if (OPENSSL_armcap_P & ARMV7_NEON) {
+               ctx->gmult = gcm_gmult_neon;
+               ctx->ghash = gcm_ghash_neon;
+       } else {
+               gcm_init_4bit(ctx->Htable,ctx->H.u);
+               ctx->gmult = gcm_gmult_4bit;
+               ctx->ghash = gcm_ghash_4bit;
+       }
 # else
        gcm_init_4bit(ctx->Htable,ctx->H.u);
 # endif
@@ -746,14 +760,18 @@ void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx,const unsigned char *iv,size_t len)
 {
        const union { long one; char little; } is_endian = {1};
        unsigned int ctr;
+#ifdef GCM_FUNCREF_4BIT
+       void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16])    = ctx->gmult;
+#endif
 
        ctx->Yi.u[0]  = 0;
        ctx->Yi.u[1]  = 0;
        ctx->Xi.u[0]  = 0;
        ctx->Xi.u[1]  = 0;
-       ctx->len.u[0] = 0;
-       ctx->len.u[1] = 0;
-       ctx->res = 0;
+       ctx->len.u[0] = 0;      /* AAD length */
+       ctx->len.u[1] = 0;      /* message length */
+       ctx->ares = 0;
+       ctx->mres = 0;
 
        if (len==12) {
                memcpy(ctx->Yi.c,iv,12);
@@ -808,11 +826,39 @@ void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx,const unsigned char *iv,size_t len)
                ctx->Yi.d[3] = ctr;
 }
 
-void CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx,const unsigned char *aad,size_t len)
+int CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx,const unsigned char *aad,size_t len)
 {
        size_t i;
+       unsigned int n;
+       u64 alen = ctx->len.u[0];
+#ifdef GCM_FUNCREF_4BIT
+       void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16])    = ctx->gmult;
+# ifdef GHASH
+       void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],
+                               const u8 *inp,size_t len)       = ctx->ghash;
+# endif
+#endif
+
+       if (ctx->len.u[1]) return -2;
 
-       ctx->len.u[0] += len;
+       alen += len;
+       if (alen>(U64(1)<<61) || (sizeof(len)==8 && alen<len))
+               return -1;
+       ctx->len.u[0] = alen;
+
+       n = ctx->ares;
+       if (n) {
+               while (n && len) {
+                       ctx->Xi.c[n] ^= *(aad++);
+                       --len;
+                       n = (n+1)%16;
+               }
+               if (n==0) GCM_MUL(ctx,Xi);
+               else {
+                       ctx->ares = n;
+                       return 0;
+               }
+       }
 
 #ifdef GHASH
        if ((i = (len&(size_t)-16))) {
@@ -829,26 +875,52 @@ void CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx,const unsigned char *aad,size_t len)
        }
 #endif
        if (len) {
+               n = (unsigned int)len;
                for (i=0; i<len; ++i) ctx->Xi.c[i] ^= aad[i];
-               GCM_MUL(ctx,Xi);
        }
+
+       ctx->ares = n;
+       return 0;
 }
 
-void CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
+int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
                const unsigned char *in, unsigned char *out,
                size_t len)
 {
        const union { long one; char little; } is_endian = {1};
        unsigned int n, ctr;
        size_t i;
+       u64        mlen  = ctx->len.u[1];
+       block128_f block = ctx->block;
+       void      *key   = ctx->key;
+#ifdef GCM_FUNCREF_4BIT
+       void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16])    = ctx->gmult;
+# ifdef GHASH
+       void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],
+                               const u8 *inp,size_t len)       = ctx->ghash;
+# endif
+#endif
+
+#if 0
+       n = (unsigned int)mlen%16; /* alternative to ctx->mres */
+#endif
+       mlen += len;
+       if (mlen>((U64(1)<<36)-32) || (sizeof(len)==8 && mlen<len))
+               return -1;
+       ctx->len.u[1] = mlen;
+
+       if (ctx->ares) {
+               /* First call to encrypt finalizes GHASH(AAD) */
+               GCM_MUL(ctx,Xi);
+               ctx->ares = 0;
+       }
 
-       ctx->len.u[1] += len;
-       n   = ctx->res;
        if (is_endian.little)
                ctr = GETU32(ctx->Yi.c+12);
        else
                ctr = ctx->Yi.d[3];
 
+       n = ctx->mres;
 #if !defined(OPENSSL_SMALL_FOOTPRINT)
        if (16%sizeof(size_t) == 0) do {        /* always true actually */
                if (n) {
@@ -859,8 +931,8 @@ void CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
                        }
                        if (n==0) GCM_MUL(ctx,Xi);
                        else {
-                               ctx->res = n;
-                               return;
+                               ctx->mres = n;
+                               return 0;
                        }
                }
 #if defined(STRICT_ALIGNMENT)
@@ -872,7 +944,7 @@ void CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
                    size_t j=GHASH_CHUNK;
 
                    while (j) {
-                       (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+                       (*block)(ctx->Yi.c,ctx->EKi.c,key);
                        ++ctr;
                        if (is_endian.little)
                                PUTU32(ctx->Yi.c+12,ctr);
@@ -892,7 +964,7 @@ void CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
                    size_t j=i;
 
                    while (len>=16) {
-                       (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+                       (*block)(ctx->Yi.c,ctx->EKi.c,key);
                        ++ctr;
                        if (is_endian.little)
                                PUTU32(ctx->Yi.c+12,ctr);
@@ -909,7 +981,7 @@ void CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
                }
 #else
                while (len>=16) {
-                       (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+                       (*block)(ctx->Yi.c,ctx->EKi.c,key);
                        ++ctr;
                        if (is_endian.little)
                                PUTU32(ctx->Yi.c+12,ctr);
@@ -926,7 +998,7 @@ void CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
                }
 #endif
                if (len) {
-                       (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+                       (*block)(ctx->Yi.c,ctx->EKi.c,key);
                        ++ctr;
                        if (is_endian.little)
                                PUTU32(ctx->Yi.c+12,ctr);
@@ -938,13 +1010,13 @@ void CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
                        }
                }
 
-               ctx->res = n;
-               return;
+               ctx->mres = n;
+               return 0;
        } while(0);
 #endif
        for (i=0;i<len;++i) {
                if (n==0) {
-                       (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+                       (*block)(ctx->Yi.c,ctx->EKi.c,key);
                        ++ctr;
                        if (is_endian.little)
                                PUTU32(ctx->Yi.c+12,ctr);
@@ -957,24 +1029,45 @@ void CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
                        GCM_MUL(ctx,Xi);
        }
 
-       ctx->res = n;
+       ctx->mres = n;
+       return 0;
 }
 
-void CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
+int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
                const unsigned char *in, unsigned char *out,
                size_t len)
 {
        const union { long one; char little; } is_endian = {1};
        unsigned int n, ctr;
        size_t i;
+       u64        mlen  = ctx->len.u[1];
+       block128_f block = ctx->block;
+       void      *key   = ctx->key;
+#ifdef GCM_FUNCREF_4BIT
+       void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16])    = ctx->gmult;
+# ifdef GHASH
+       void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],
+                               const u8 *inp,size_t len)       = ctx->ghash;
+# endif
+#endif
+
+       mlen += len;
+       if (mlen>((U64(1)<<36)-32) || (sizeof(len)==8 && mlen<len))
+               return -1;
+       ctx->len.u[1] = mlen;
+
+       if (ctx->ares) {
+               /* First call to decrypt finalizes GHASH(AAD) */
+               GCM_MUL(ctx,Xi);
+               ctx->ares = 0;
+       }
 
-       ctx->len.u[1] += len;
-       n   = ctx->res;
        if (is_endian.little)
                ctr = GETU32(ctx->Yi.c+12);
        else
                ctr = ctx->Yi.d[3];
 
+       n = ctx->mres;
 #if !defined(OPENSSL_SMALL_FOOTPRINT)
        if (16%sizeof(size_t) == 0) do {        /* always true actually */
                if (n) {
@@ -987,8 +1080,8 @@ void CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
                        }
                        if (n==0) GCM_MUL (ctx,Xi);
                        else {
-                               ctx->res = n;
-                               return;
+                               ctx->mres = n;
+                               return 0;
                        }
                }
 #if defined(STRICT_ALIGNMENT)
@@ -1001,7 +1094,7 @@ void CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
 
                    GHASH(ctx,in,GHASH_CHUNK);
                    while (j) {
-                       (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+                       (*block)(ctx->Yi.c,ctx->EKi.c,key);
                        ++ctr;
                        if (is_endian.little)
                                PUTU32(ctx->Yi.c+12,ctr);
@@ -1019,7 +1112,7 @@ void CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
                if ((i = (len&(size_t)-16))) {
                    GHASH(ctx,in,i);
                    while (len>=16) {
-                       (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+                       (*block)(ctx->Yi.c,ctx->EKi.c,key);
                        ++ctr;
                        if (is_endian.little)
                                PUTU32(ctx->Yi.c+12,ctr);
@@ -1035,7 +1128,7 @@ void CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
                }
 #else
                while (len>=16) {
-                       (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+                       (*block)(ctx->Yi.c,ctx->EKi.c,key);
                        ++ctr;
                        if (is_endian.little)
                                PUTU32(ctx->Yi.c+12,ctr);
@@ -1053,7 +1146,7 @@ void CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
                }
 #endif
                if (len) {
-                       (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+                       (*block)(ctx->Yi.c,ctx->EKi.c,key);
                        ++ctr;
                        if (is_endian.little)
                                PUTU32(ctx->Yi.c+12,ctr);
@@ -1067,14 +1160,14 @@ void CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
                        }
                }
 
-               ctx->res = n;
-               return;
+               ctx->mres = n;
+               return 0;
        } while(0);
 #endif
        for (i=0;i<len;++i) {
                u8 c;
                if (n==0) {
-                       (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+                       (*block)(ctx->Yi.c,ctx->EKi.c,key);
                        ++ctr;
                        if (is_endian.little)
                                PUTU32(ctx->Yi.c+12,ctr);
@@ -1089,24 +1182,44 @@ void CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
                        GCM_MUL(ctx,Xi);
        }
 
-       ctx->res = n;
+       ctx->mres = n;
+       return 0;
 }
 
-void CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
+int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
                const unsigned char *in, unsigned char *out,
                size_t len, ctr128_f stream)
 {
        const union { long one; char little; } is_endian = {1};
        unsigned int n, ctr;
        size_t i;
+       u64   mlen = ctx->len.u[1];
+       void *key  = ctx->key;
+#ifdef GCM_FUNCREF_4BIT
+       void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16])    = ctx->gmult;
+# ifdef GHASH
+       void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],
+                               const u8 *inp,size_t len)       = ctx->ghash;
+# endif
+#endif
+
+       mlen += len;
+       if (mlen>((U64(1)<<36)-32) || (sizeof(len)==8 && mlen<len))
+               return -1;
+       ctx->len.u[1] = mlen;
+
+       if (ctx->ares) {
+               /* First call to encrypt finalizes GHASH(AAD) */
+               GCM_MUL(ctx,Xi);
+               ctx->ares = 0;
+       }
 
-       ctx->len.u[1] += len;
-       n   = ctx->res;
        if (is_endian.little)
                ctr = GETU32(ctx->Yi.c+12);
        else
                ctr = ctx->Yi.d[3];
 
+       n = ctx->mres;
        if (n) {
                while (n && len) {
                        ctx->Xi.c[n] ^= *(out++) = *(in++)^ctx->EKi.c[n];
@@ -1115,13 +1228,13 @@ void CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
                }
                if (n==0) GCM_MUL(ctx,Xi);
                else {
-                       ctx->res = n;
-                       return;
+                       ctx->mres = n;
+                       return 0;
                }
        }
 #if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT)
        while (len>=GHASH_CHUNK) {
-               (*stream)(in,out,GHASH_CHUNK/16,ctx->key,ctx->Yi.c);
+               (*stream)(in,out,GHASH_CHUNK/16,key,ctx->Yi.c);
                ctr += GHASH_CHUNK/16;
                if (is_endian.little)
                        PUTU32(ctx->Yi.c+12,ctr);
@@ -1136,7 +1249,7 @@ void CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
        if ((i = (len&(size_t)-16))) {
                size_t j=i/16;
 
-               (*stream)(in,out,j,ctx->key,ctx->Yi.c);
+               (*stream)(in,out,j,key,ctx->Yi.c);
                ctr += (unsigned int)j;
                if (is_endian.little)
                        PUTU32(ctx->Yi.c+12,ctr);
@@ -1156,7 +1269,7 @@ void CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
 #endif
        }
        if (len) {
-               (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+               (*ctx->block)(ctx->Yi.c,ctx->EKi.c,key);
                ++ctr;
                if (is_endian.little)
                        PUTU32(ctx->Yi.c+12,ctr);
@@ -1168,24 +1281,44 @@ void CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
                }
        }
 
-       ctx->res = n;
+       ctx->mres = n;
+       return 0;
 }
 
-void CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
+int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
                const unsigned char *in, unsigned char *out,
                size_t len,ctr128_f stream)
 {
        const union { long one; char little; } is_endian = {1};
        unsigned int n, ctr;
        size_t i;
+       u64   mlen = ctx->len.u[1];
+       void *key  = ctx->key;
+#ifdef GCM_FUNCREF_4BIT
+       void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16])    = ctx->gmult;
+# ifdef GHASH
+       void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],
+                               const u8 *inp,size_t len)       = ctx->ghash;
+# endif
+#endif
+
+       mlen += len;
+       if (mlen>((U64(1)<<36)-32) || (sizeof(len)==8 && mlen<len))
+               return -1;
+       ctx->len.u[1] = mlen;
+
+       if (ctx->ares) {
+               /* First call to decrypt finalizes GHASH(AAD) */
+               GCM_MUL(ctx,Xi);
+               ctx->ares = 0;
+       }
 
-       ctx->len.u[1] += len;
-       n   = ctx->res;
        if (is_endian.little)
                ctr = GETU32(ctx->Yi.c+12);
        else
                ctr = ctx->Yi.d[3];
 
+       n = ctx->mres;
        if (n) {
                while (n && len) {
                        u8 c = *(in++);
@@ -1196,14 +1329,14 @@ void CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
                }
                if (n==0) GCM_MUL (ctx,Xi);
                else {
-                       ctx->res = n;
-                       return;
+                       ctx->mres = n;
+                       return 0;
                }
        }
 #if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT)
        while (len>=GHASH_CHUNK) {
                GHASH(ctx,in,GHASH_CHUNK);
-               (*stream)(in,out,GHASH_CHUNK/16,ctx->key,ctx->Yi.c);
+               (*stream)(in,out,GHASH_CHUNK/16,key,ctx->Yi.c);
                ctr += GHASH_CHUNK/16;
                if (is_endian.little)
                        PUTU32(ctx->Yi.c+12,ctr);
@@ -1229,7 +1362,7 @@ void CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
                j   = i/16;
                in -= i;
 #endif
-               (*stream)(in,out,j,ctx->key,ctx->Yi.c);
+               (*stream)(in,out,j,key,ctx->Yi.c);
                ctr += (unsigned int)j;
                if (is_endian.little)
                        PUTU32(ctx->Yi.c+12,ctr);
@@ -1240,7 +1373,7 @@ void CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
                len -= i;
        }
        if (len) {
-               (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+               (*ctx->block)(ctx->Yi.c,ctx->EKi.c,key);
                ++ctr;
                if (is_endian.little)
                        PUTU32(ctx->Yi.c+12,ctr);
@@ -1254,7 +1387,8 @@ void CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
                }
        }
 
-       ctx->res = n;
+       ctx->mres = n;
+       return 0;
 }
 
 int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx,const unsigned char *tag,
@@ -1263,8 +1397,11 @@ int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx,const unsigned char *tag,
        const union { long one; char little; } is_endian = {1};
        u64 alen = ctx->len.u[0]<<3;
        u64 clen = ctx->len.u[1]<<3;
+#ifdef GCM_FUNCREF_4BIT
+       void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16])    = ctx->gmult;
+#endif
 
-       if (ctx->res)
+       if (ctx->mres)
                GCM_MUL(ctx,Xi);
 
        if (is_endian.little) {
@@ -1295,6 +1432,12 @@ int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx,const unsigned char *tag,
                return -1;
 }
 
+void CRYPTO_gcm128_tag(GCM128_CONTEXT *ctx, unsigned char *tag, size_t len)
+{
+       CRYPTO_gcm128_finish(ctx, NULL, 0);
+       memcpy(tag, ctx->Xi.c, len<=sizeof(ctx->Xi.c)?len:sizeof(ctx->Xi.c));
+}
+
 GCM128_CONTEXT *CRYPTO_gcm128_new(void *key, block128_f block)
 {
        GCM128_CONTEXT *ret;
@@ -1365,9 +1508,8 @@ static const u8   P4[]=  {0xd9,0x31,0x32,0x25,0xf8,0x84,0x06,0xe5,0xa5,0x59,0x09,0
 /* Test Case 5 */
 #define K5 K4
 #define P5 P4
-static const u8        A5[]=  {0xfe,0xed,0xfa,0xce,0xde,0xad,0xbe,0xef,0xfe,0xed,0xfa,0xce,0xde,0xad,0xbe,0xef,
-                       0xab,0xad,0xda,0xd2},
-               IV5[]= {0xca,0xfe,0xba,0xbe,0xfa,0xce,0xdb,0xad},
+#define A5 A4
+static const u8        IV5[]= {0xca,0xfe,0xba,0xbe,0xfa,0xce,0xdb,0xad},
                C5[]=  {0x61,0x35,0x3b,0x4c,0x28,0x06,0x93,0x4a,0x77,0x7f,0xf5,0x1f,0xa2,0x2a,0x47,0x55,
                        0x69,0x9b,0x2a,0x71,0x4f,0xcd,0xc6,0xf8,0x37,0x66,0xe5,0xf9,0x7b,0x6c,0x74,0x23,
                        0x73,0x80,0x69,0x00,0xe4,0x9f,0x24,0xb2,0x2b,0x09,0x75,0x44,0xd4,0x89,0x6b,0x42,
@@ -1591,11 +1733,11 @@ int main()
        gcm_t = OPENSSL_rdtsc() - start;
 
        CRYPTO_ctr128_encrypt(buf.c,buf.c,sizeof(buf),
-                       &key,ctx.Yi.c,ctx.EKi.c,&ctx.res,
+                       &key,ctx.Yi.c,ctx.EKi.c,&ctx.mres,
                        (block128_f)AES_encrypt);
        start = OPENSSL_rdtsc();
        CRYPTO_ctr128_encrypt(buf.c,buf.c,sizeof(buf),
-                       &key,ctx.Yi.c,ctx.EKi.c,&ctx.res,
+                       &key,ctx.Yi.c,ctx.EKi.c,&ctx.mres,
                        (block128_f)AES_encrypt);
        ctr_t = OPENSSL_rdtsc() - start;