sha1-x86_64.pl: update from master.

diff --git a/crypto/sha/asm/sha1-x86_64.pl b/crypto/sha/asm/sha1-x86_64.pl
index cfdc45cce6dad0ecf274e15de8b8cef1a4c9f7c5..88919003107d5551b6bc09bd1ebb9080bcb815bc 100755 (executable)
--- a/crypto/sha/asm/sha1-x86_64.pl
+++ b/crypto/sha/asm/sha1-x86_64.pl
@@ -1,7 +1,7 @@
 #!/usr/bin/env perl
 #
 # ====================================================================
 #!/usr/bin/env perl
 #
 # ====================================================================

-# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL

 # project. The module is, however, dual licensed under OpenSSL and
 # CRYPTOGAMS licenses depending on where you obtain it. For further
 # details see http://www.openssl.org/~appro/cryptogams/.
 # project. The module is, however, dual licensed under OpenSSL and
 # CRYPTOGAMS licenses depending on where you obtain it. For further
 # details see http://www.openssl.org/~appro/cryptogams/.


@@ -55,11 +55,14 @@
 #
 #              x86_64          SSSE3           AVX
 # P4           9.8             -
 #
 #              x86_64          SSSE3           AVX
 # P4           9.8             -

-# Opteron      6.6             -
-# Core2                6.7             6.1/+10%        -
-# Atom         11.0            9.7/+13%        -
-# Westmere     7.1             5.6/+27%        -
-# Sandy Bridge 7.9             6.3/+25%        5.2/+51%
+# Opteron      6.65            -
+# Core2                6.70            6.05/+11%       -
+# Westmere     7.08            5.49/+29%       -
+# Sandy Bridge 7.93            6.16/+28%       4.99/+59%
+# Ivy Bridge   6.30            4.63/+36%       4.60/+37%
+# Bulldozer    10.9            5.95/+82%
+# VIA Nano     10.2            7.46/+37%
+# Atom         11.0            9.61/+14%

 
 $flavour = shift;
 $output  = shift;
 
 $flavour = shift;
 $output  = shift;


@@ -295,9 +298,11 @@ ___
 my $Xi=4;
 my @X=map("%xmm$_",(4..7,0..3));
 my @Tx=map("%xmm$_",(8..10));
 my $Xi=4;
 my @X=map("%xmm$_",(4..7,0..3));
 my @Tx=map("%xmm$_",(8..10));

+my $Kx="%xmm11";

 my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp");   # size optimization
 my @T=("%esi","%edi");
 my $j=0;
 my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp");   # size optimization
 my @T=("%esi","%edi");
 my $j=0;

+my $rx=0;

 my $K_XX_XX="%r11";
 
 my $_rol=sub { &rol(@_) };
 my $K_XX_XX="%r11";
 
 my $_rol=sub { &rol(@_) };


@@ -311,7 +316,7 @@ _ssse3_shortcut:
        push    %rbx
        push    %rbp
        push    %r12
        push    %rbx
        push    %rbp
        push    %r12

-       lea     `-64-($win64?5*16:0)`(%rsp),%rsp
+       lea     `-64-($win64?6*16:0)`(%rsp),%rsp

 ___
 $code.=<<___ if ($win64);
        movaps  %xmm6,64+0(%rsp)
 ___
 $code.=<<___ if ($win64);
        movaps  %xmm6,64+0(%rsp)


@@ -319,6 +324,7 @@ $code.=<<___ if ($win64);
        movaps  %xmm8,64+32(%rsp)
        movaps  %xmm9,64+48(%rsp)
        movaps  %xmm10,64+64(%rsp)
        movaps  %xmm8,64+32(%rsp)
        movaps  %xmm9,64+48(%rsp)
        movaps  %xmm10,64+64(%rsp)

+       movaps  %xmm11,64+80(%rsp)

 .Lprologue_ssse3:
 ___
 $code.=<<___;
 .Lprologue_ssse3:
 ___
 $code.=<<___;


@@ -336,6 +342,9 @@ $code.=<<___;
        mov     12($ctx),$D
        mov     $B,@T[0]                # magic seed
        mov     16($ctx),$E
        mov     12($ctx),$D
        mov     $B,@T[0]                # magic seed
        mov     16($ctx),$E

+       mov     $C,@T[1]
+       xor     $D,@T[1]
+       and     @T[1],@T[0]

 
        movdqa  64($K_XX_XX),@X[2]      # pbswap mask
        movdqa  0($K_XX_XX),@Tx[1]      # K_00_19
 
        movdqa  64($K_XX_XX),@X[2]      # pbswap mask
        movdqa  0($K_XX_XX),@Tx[1]      # K_00_19


@@ -451,7 +460,7 @@ sub Xupdate_ssse3_16_31()           # recall that $Xi starts wtih 4
 sub Xupdate_ssse3_32_79()
 { use integer;
   my $body = shift;
 sub Xupdate_ssse3_32_79()
 { use integer;
   my $body = shift;

-  my @insns = (&$body,&$body,&$body,&$body);   # 32 to 48 instructions
+  my @insns = (&$body,&$body,&$body,&$body);   # 32 to 44 instructions

   my ($a,$b,$c,$d,$e);
 
        &movdqa (@Tx[0],@X[-1&7])       if ($Xi==8);
   my ($a,$b,$c,$d,$e);
 
        &movdqa (@Tx[0],@X[-1&7])       if ($Xi==8);


@@ -583,51 +592,66 @@ sub Xtail_ssse3()
        foreach (@insns) { eval; }
 }
 
        foreach (@insns) { eval; }
 }
 

-sub body_00_19 () {
+sub body_00_19 () {    # ((c^d)&b)^d
+       # on start @T[0]=(c^d)&b
+       return &body_20_39() if ($rx==19); $rx++;

        (
        '($a,$b,$c,$d,$e)=@V;'.
        (
        '($a,$b,$c,$d,$e)=@V;'.

-       '&add   ($e,eval(4*($j&15))."(%rsp)");',        # X[]+K xfer
-       '&xor   ($c,$d);',
-       '&mov   (@T[1],$a);',   # $b in next round
-       '&$_rol ($a,5);',
-       '&and   (@T[0],$c);',   # ($b&($c^$d))
-       '&xor   ($c,$d);',      # restore $c
-       '&xor   (@T[0],$d);',
-       '&add   ($e,$a);',
-       '&$_ror ($b,$j?7:2);',  # $b>>>2
-       '&add   ($e,@T[0]);'    .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
+       '&$_ror ($b,$j?7:2)',   # $b>>>2
+       '&xor   (@T[0],$d)',
+       '&mov   (@T[1],$a)',    # $b for next round
+
+       '&add   ($e,eval(4*($j&15))."(%rsp)")', # X[]+K xfer
+       '&xor   ($b,$c)',       # $c^$d for next round
+
+       '&$_rol ($a,5)',
+       '&add   ($e,@T[0])',
+       '&and   (@T[1],$b)',    # ($b&($c^$d)) for next round
+
+       '&xor   ($b,$c)',       # restore $b
+       '&add   ($e,$a);'       .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'

        );
 }
 
        );
 }
 

-sub body_20_39 () {
+sub body_20_39 () {    # b^d^c
+       # on entry @T[0]=b^d
+       return &body_40_59() if ($rx==39); $rx++;

        (
        '($a,$b,$c,$d,$e)=@V;'.
        (
        '($a,$b,$c,$d,$e)=@V;'.

-       '&add   ($e,eval(4*($j++&15))."(%rsp)");',      # X[]+K xfer
-       '&xor   (@T[0],$d);',   # ($b^$d)
-       '&mov   (@T[1],$a);',   # $b in next round
-       '&$_rol ($a,5);',
-       '&xor   (@T[0],$c);',   # ($b^$d^$c)
-       '&add   ($e,$a);',
-       '&$_ror ($b,7);',       # $b>>>2
-       '&add   ($e,@T[0]);'    .'unshift(@V,pop(@V)); unshift(@T,pop(@T));'
+       '&add   ($e,eval(4*($j&15))."(%rsp)")', # X[]+K xfer
+       '&xor   (@T[0],$d)      if($j==19);'.
+       '&xor   (@T[0],$c)      if($j> 19)',    # ($b^$d^$c)
+       '&mov   (@T[1],$a)',    # $b for next round
+
+       '&$_rol ($a,5)',
+       '&add   ($e,@T[0])',
+       '&xor   (@T[1],$c)      if ($j< 79)',   # $b^$d for next round
+
+       '&$_ror ($b,7)',        # $b>>>2
+       '&add   ($e,$a);'       .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'

        );
 }
 
        );
 }
 

-sub body_40_59 () {
+sub body_40_59 () {    # ((b^c)&(c^d))^c
+       # on entry @T[0]=(b^c), (c^=d)
+       $rx++;

        (
        '($a,$b,$c,$d,$e)=@V;'.
        (
        '($a,$b,$c,$d,$e)=@V;'.

-       '&mov   (@T[1],$c);',
-       '&xor   ($c,$d);',
-       '&add   ($e,eval(4*($j++&15))."(%rsp)");',      # X[]+K xfer
-       '&and   (@T[1],$d);',
-       '&and   (@T[0],$c);',   # ($b&($c^$d))
-       '&$_ror ($b,7);',       # $b>>>2
-       '&add   ($e,@T[1]);',
-       '&mov   (@T[1],$a);',   # $b in next round
-       '&$_rol ($a,5);',
-       '&add   ($e,@T[0]);',
-       '&xor   ($c,$d);',      # restore $c
-       '&add   ($e,$a);'       .'unshift(@V,pop(@V)); unshift(@T,pop(@T));'
+       '&add   ($e,eval(4*($j&15))."(%rsp)")', # X[]+K xfer
+       '&and   (@T[0],$c)      if ($j>=40)',   # (b^c)&(c^d)
+       '&xor   ($c,$d)         if ($j>=40)',   # restore $c
+
+       '&$_ror ($b,7)',        # $b>>>2
+       '&mov   (@T[1],$a)',    # $b for next round
+       '&xor   (@T[0],$c)',
+
+       '&$_rol ($a,5)',
+       '&add   ($e,@T[0])',
+       '&xor   (@T[1],$c)      if ($j==59);'.
+       '&xor   (@T[1],$b)      if ($j< 59)',   # b^c for next round
+
+       '&xor   ($b,$c)         if ($j< 59)',   # c^d for next round
+       '&add   ($e,$a);'       .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'

        );
 }
 $code.=<<___;
        );
 }
 $code.=<<___;


@@ -668,8 +692,11 @@ $code.=<<___;
        mov     @T[0],4($ctx)
        mov     @T[0],$B                        # magic seed
        mov     $C,8($ctx)
        mov     @T[0],4($ctx)
        mov     @T[0],$B                        # magic seed
        mov     $C,8($ctx)

+       mov     $C,@T[1]

        mov     $D,12($ctx)
        mov     $D,12($ctx)

+       xor     $D,@T[1]

        mov     $E,16($ctx)
        mov     $E,16($ctx)

+       and     @T[1],@T[0]

        jmp     .Loop_ssse3
 
 .align 16
        jmp     .Loop_ssse3
 
 .align 16


@@ -699,9 +726,10 @@ $code.=<<___ if ($win64);
        movaps  64+32(%rsp),%xmm8
        movaps  64+48(%rsp),%xmm9
        movaps  64+64(%rsp),%xmm10
        movaps  64+32(%rsp),%xmm8
        movaps  64+48(%rsp),%xmm9
        movaps  64+64(%rsp),%xmm10

+       movaps  64+80(%rsp),%xmm11

 ___
 $code.=<<___;
 ___
 $code.=<<___;

-       lea     `64+($win64?5*16:0)`(%rsp),%rsi
+       lea     `64+($win64?6*16:0)`(%rsp),%rsi

        mov     0(%rsi),%r12
        mov     8(%rsi),%rbp
        mov     16(%rsi),%rbx
        mov     0(%rsi),%r12
        mov     8(%rsi),%rbp
        mov     16(%rsi),%rbx


@@ -712,13 +740,13 @@ $code.=<<___;
 ___
 
 if ($avx) {
 ___
 
 if ($avx) {

-my $Xi=4;
-my @X=map("%xmm$_",(4..7,0..3));
-my @Tx=map("%xmm$_",(8..10));
-my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp");   # size optimization
-my @T=("%esi","%edi");
-my $j=0;
-my $K_XX_XX="%r11";
+$Xi=4;                         # reset variables
+@X=map("%xmm$_",(4..7,0..3));
+@Tx=map("%xmm$_",(8..10));
+$j=0;
+$rx=0;
+
+my $done_avx_label=".Ldone_avx";

 
 my $_rol=sub { &shld(@_[0],@_) };
 my $_ror=sub { &shrd(@_[0],@_) };
 
 my $_rol=sub { &shld(@_[0],@_) };
 my $_ror=sub { &shrd(@_[0],@_) };


@@ -731,7 +759,7 @@ _avx_shortcut:
        push    %rbx
        push    %rbp
        push    %r12
        push    %rbx
        push    %rbp
        push    %r12

-       lea     `-64-($win64?5*16:0)`(%rsp),%rsp
+       lea     `-64-($win64?6*16:0)`(%rsp),%rsp

 ___
 $code.=<<___ if ($win64);
        movaps  %xmm6,64+0(%rsp)
 ___
 $code.=<<___ if ($win64);
        movaps  %xmm6,64+0(%rsp)


@@ -739,6 +767,7 @@ $code.=<<___ if ($win64);
        movaps  %xmm8,64+32(%rsp)
        movaps  %xmm9,64+48(%rsp)
        movaps  %xmm10,64+64(%rsp)
        movaps  %xmm8,64+32(%rsp)
        movaps  %xmm9,64+48(%rsp)
        movaps  %xmm10,64+64(%rsp)

+       movaps  %xmm11,64+80(%rsp)

 .Lprologue_avx:
 ___
 $code.=<<___;
 .Lprologue_avx:
 ___
 $code.=<<___;


@@ -757,9 +786,12 @@ $code.=<<___;
        mov     12($ctx),$D
        mov     $B,@T[0]                # magic seed
        mov     16($ctx),$E
        mov     12($ctx),$D
        mov     $B,@T[0]                # magic seed
        mov     16($ctx),$E

+       mov     $C,@T[1]
+       xor     $D,@T[1]
+       and     @T[1],@T[0]

 
        vmovdqa 64($K_XX_XX),@X[2]      # pbswap mask
 
        vmovdqa 64($K_XX_XX),@X[2]      # pbswap mask

-       vmovdqa 0($K_XX_XX),@Tx[1]      # K_00_19
+       vmovdqa 0($K_XX_XX),$Kx         # K_00_19

        vmovdqu 0($inp),@X[-4&7]        # load input to %xmm[0-3]
        vmovdqu 16($inp),@X[-3&7]
        vmovdqu 32($inp),@X[-2&7]
        vmovdqu 0($inp),@X[-4&7]        # load input to %xmm[0-3]
        vmovdqu 16($inp),@X[-3&7]
        vmovdqu 32($inp),@X[-2&7]


@@ -769,9 +801,9 @@ $code.=<<___;
        vpshufb @X[2],@X[-3&7],@X[-3&7]
        vpshufb @X[2],@X[-2&7],@X[-2&7]
        vpshufb @X[2],@X[-1&7],@X[-1&7]
        vpshufb @X[2],@X[-3&7],@X[-3&7]
        vpshufb @X[2],@X[-2&7],@X[-2&7]
        vpshufb @X[2],@X[-1&7],@X[-1&7]

-       vpaddd  @Tx[1],@X[-4&7],@X[0]   # add K_00_19
-       vpaddd  @Tx[1],@X[-3&7],@X[1]
-       vpaddd  @Tx[1],@X[-2&7],@X[2]
+       vpaddd  $Kx,@X[-4&7],@X[0]      # add K_00_19
+       vpaddd  $Kx,@X[-3&7],@X[1]
+       vpaddd  $Kx,@X[-2&7],@X[2]

        vmovdqa @X[0],0(%rsp)           # X[]+K xfer to IALU
        vmovdqa @X[1],16(%rsp)
        vmovdqa @X[2],32(%rsp)
        vmovdqa @X[0],0(%rsp)           # X[]+K xfer to IALU
        vmovdqa @X[1],16(%rsp)
        vmovdqa @X[2],32(%rsp)


@@ -790,10 +822,10 @@ sub Xupdate_avx_16_31()           # recall that $Xi starts wtih 4
         eval(shift(@insns));
         eval(shift(@insns));
 
         eval(shift(@insns));
         eval(shift(@insns));
 

-         &vpaddd       (@Tx[1],@Tx[1],@X[-1&7]);
+         &vpaddd       (@Tx[1],$Kx,@X[-1&7]);

         eval(shift(@insns));
         eval(shift(@insns));
         eval(shift(@insns));
         eval(shift(@insns));

-       &vpsrldq(@Tx[0],@X[-1&7],4);    # "X[-3]", 3 dwords
+       &vpsrldq(@Tx[0],@X[-1&7],4);            # "X[-3]", 3 dwords

         eval(shift(@insns));
         eval(shift(@insns));
        &vpxor  (@X[0],@X[0],@X[-4&7]);         # "X[0]"^="X[-16]"
         eval(shift(@insns));
         eval(shift(@insns));
        &vpxor  (@X[0],@X[0],@X[-4&7]);         # "X[0]"^="X[-16]"


@@ -843,7 +875,7 @@ sub Xupdate_avx_16_31()             # recall that $Xi starts wtih 4
        &vpxor  (@X[0],@X[0],@Tx[2]);           # "X[0]"^=("X[0]">>96)<<<2
         eval(shift(@insns));
         eval(shift(@insns));
        &vpxor  (@X[0],@X[0],@Tx[2]);           # "X[0]"^=("X[0]">>96)<<<2
         eval(shift(@insns));
         eval(shift(@insns));

-         &vmovdqa      (@Tx[2],eval(16*(($Xi)/5))."($K_XX_XX)");       # K_XX_XX
+         &vmovdqa      ($Kx,eval(16*(($Xi)/5))."($K_XX_XX)")   if ($Xi%5==0);  # K_XX_XX

         eval(shift(@insns));
         eval(shift(@insns));
 
         eval(shift(@insns));
         eval(shift(@insns));
 


@@ -851,13 +883,12 @@ sub Xupdate_avx_16_31()           # recall that $Xi starts wtih 4
         foreach (@insns) { eval; }     # remaining instructions [if any]
 
   $Xi++;       push(@X,shift(@X));     # "rotate" X[]
         foreach (@insns) { eval; }     # remaining instructions [if any]
 
   $Xi++;       push(@X,shift(@X));     # "rotate" X[]

-               push(@Tx,shift(@Tx));

 }
 
 sub Xupdate_avx_32_79()
 { use integer;
   my $body = shift;
 }
 
 sub Xupdate_avx_32_79()
 { use integer;
   my $body = shift;

-  my @insns = (&$body,&$body,&$body,&$body);   # 32 to 48 instructions
+  my @insns = (&$body,&$body,&$body,&$body);   # 32 to 44 instructions

   my ($a,$b,$c,$d,$e);
 
        &vpalignr(@Tx[0],@X[-1&7],@X[-2&7],8);  # compose "X[-6]"
   my ($a,$b,$c,$d,$e);
 
        &vpalignr(@Tx[0],@X[-1&7],@X[-2&7],8);  # compose "X[-6]"


@@ -870,12 +901,8 @@ sub Xupdate_avx_32_79()
        &vpxor  (@X[0],@X[0],@X[-7&7]);         # "X[0]"^="X[-28]"
         eval(shift(@insns));
         eval(shift(@insns))    if (@insns[0] !~ /&ro[rl]/);
        &vpxor  (@X[0],@X[0],@X[-7&7]);         # "X[0]"^="X[-28]"
         eval(shift(@insns));
         eval(shift(@insns))    if (@insns[0] !~ /&ro[rl]/);

-       if ($Xi%5) {
-         &vmovdqa      (@Tx[2],@Tx[1]);# "perpetuate" K_XX_XX...
-       } else {                        # ... or load next one
-         &vmovdqa      (@Tx[2],eval(16*($Xi/5))."($K_XX_XX)");
-       }
-         &vpaddd       (@Tx[1],@Tx[1],@X[-1&7]);
+         &vpaddd       (@Tx[1],$Kx,@X[-1&7]);
+         &vmovdqa      ($Kx,eval(16*($Xi/5))."($K_XX_XX)")     if ($Xi%5==0);

         eval(shift(@insns));           # ror
         eval(shift(@insns));
 
         eval(shift(@insns));           # ror
         eval(shift(@insns));
 


@@ -905,7 +932,6 @@ sub Xupdate_avx_32_79()
        &vpor   (@X[0],@X[0],@Tx[0]);           # "X[0]"<<<=2
         eval(shift(@insns));           # body_20_39
         eval(shift(@insns));
        &vpor   (@X[0],@X[0],@Tx[0]);           # "X[0]"<<<=2
         eval(shift(@insns));           # body_20_39
         eval(shift(@insns));

-         &vmovdqa      (@Tx[1],@X[0])  if ($Xi<19);

         eval(shift(@insns));
         eval(shift(@insns));           # rol
         eval(shift(@insns));
         eval(shift(@insns));
         eval(shift(@insns));           # rol
         eval(shift(@insns));


@@ -916,7 +942,6 @@ sub Xupdate_avx_32_79()
         foreach (@insns) { eval; }     # remaining instructions
 
   $Xi++;       push(@X,shift(@X));     # "rotate" X[]
         foreach (@insns) { eval; }     # remaining instructions
 
   $Xi++;       push(@X,shift(@X));     # "rotate" X[]

-               push(@Tx,shift(@Tx));

 }
 
 sub Xuplast_avx_80()
 }
 
 sub Xuplast_avx_80()


@@ -926,23 +951,21 @@ sub Xuplast_avx_80()
   my ($a,$b,$c,$d,$e);
 
         eval(shift(@insns));
   my ($a,$b,$c,$d,$e);
 
         eval(shift(@insns));

-         &vpaddd       (@Tx[1],@Tx[1],@X[-1&7]);
+         &vpaddd       (@Tx[1],$Kx,@X[-1&7]);

         eval(shift(@insns));
         eval(shift(@insns));
         eval(shift(@insns));
         eval(shift(@insns));
 
         eval(shift(@insns));
         eval(shift(@insns));
         eval(shift(@insns));
         eval(shift(@insns));
 

-         &movdqa       (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer IALU
+         &vmovdqa      (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer IALU

 
         foreach (@insns) { eval; }             # remaining instructions
 
        &cmp    ($inp,$num);
 
         foreach (@insns) { eval; }             # remaining instructions
 
        &cmp    ($inp,$num);

-       &je     (".Ldone_avx");
-
-       unshift(@Tx,pop(@Tx));
+       &je     ($done_avx_label);

 
        &vmovdqa(@X[2],"64($K_XX_XX)");         # pbswap mask
 
        &vmovdqa(@X[2],"64($K_XX_XX)");         # pbswap mask

-       &vmovdqa(@Tx[1],"0($K_XX_XX)");         # K_00_19
+       &vmovdqa($Kx,"0($K_XX_XX)");            # K_00_19

        &vmovdqu(@X[-4&7],"0($inp)");           # load input
        &vmovdqu(@X[-3&7],"16($inp)");
        &vmovdqu(@X[-2&7],"32($inp)");
        &vmovdqu(@X[-4&7],"0($inp)");           # load input
        &vmovdqu(@X[-3&7],"16($inp)");
        &vmovdqu(@X[-2&7],"32($inp)");


@@ -964,7 +987,7 @@ sub Xloop_avx()
        &vpshufb(@X[($Xi-3)&7],@X[($Xi-3)&7],@X[2]);
         eval(shift(@insns));
         eval(shift(@insns));
        &vpshufb(@X[($Xi-3)&7],@X[($Xi-3)&7],@X[2]);
         eval(shift(@insns));
         eval(shift(@insns));

-       &vpaddd (@X[$Xi&7],@X[($Xi-4)&7],@Tx[1]);
+       &vpaddd (@X[$Xi&7],@X[($Xi-4)&7],$Kx);

         eval(shift(@insns));
         eval(shift(@insns));
         eval(shift(@insns));
         eval(shift(@insns));
         eval(shift(@insns));
         eval(shift(@insns));


@@ -1024,12 +1047,15 @@ $code.=<<___;
        mov     @T[0],4($ctx)
        mov     @T[0],$B                        # magic seed
        mov     $C,8($ctx)
        mov     @T[0],4($ctx)
        mov     @T[0],$B                        # magic seed
        mov     $C,8($ctx)

+       mov     $C,@T[1]

        mov     $D,12($ctx)
        mov     $D,12($ctx)

+       xor     $D,@T[1]

        mov     $E,16($ctx)
        mov     $E,16($ctx)

+       and     @T[1],@T[0]

        jmp     .Loop_avx
 
 .align 16
        jmp     .Loop_avx
 
 .align 16

-.Ldone_avx:
+$done_avx_label:

 ___
                                $j=$saved_j; @V=@saved_V;
 
 ___
                                $j=$saved_j; @V=@saved_V;
 


@@ -1057,9 +1083,10 @@ $code.=<<___ if ($win64);
        movaps  64+32(%rsp),%xmm8
        movaps  64+48(%rsp),%xmm9
        movaps  64+64(%rsp),%xmm10
        movaps  64+32(%rsp),%xmm8
        movaps  64+48(%rsp),%xmm9
        movaps  64+64(%rsp),%xmm10

+       movaps  64+80(%rsp),%xmm11

 ___
 $code.=<<___;
 ___
 $code.=<<___;

-       lea     `64+($win64?5*16:0)`(%rsp),%rsi
+       lea     `64+($win64?6*16:0)`(%rsp),%rsi

        mov     0(%rsi),%r12
        mov     8(%rsi),%rbp
        mov     16(%rsi),%rbx
        mov     0(%rsi),%r12
        mov     8(%rsi),%rbp
        mov     16(%rsi),%rbx


@@ -1170,9 +1197,9 @@ ssse3_handler:
 
        lea     64(%rax),%rsi
        lea     512($context),%rdi      # &context.Xmm6
 
        lea     64(%rax),%rsi
        lea     512($context),%rdi      # &context.Xmm6

-       mov     \$10,%ecx
+       mov     \$12,%ecx

        .long   0xa548f3fc              # cld; rep movsq
        .long   0xa548f3fc              # cld; rep movsq

-       lea     `24+64+5*16`(%rax),%rax # adjust stack pointer
+       lea     `24+64+6*16`(%rax),%rax # adjust stack pointer

 
        mov     -8(%rax),%rbx
        mov     -16(%rax),%rbp
 
        mov     -8(%rax),%rbx
        mov     -16(%rax),%rbp