crypto/sha/asm/sha256-mb-x86_64.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2013-2016 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the OpenSSL license (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9
  10 # ====================================================================
  11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  12 # project. The module is, however, dual licensed under OpenSSL and
  13 # CRYPTOGAMS licenses depending on where you obtain it. For further
  14 # details see http://www.openssl.org/~appro/cryptogams/.
  15 # ====================================================================
  16
  17 # Multi-buffer SHA256 procedure processes n buffers in parallel by
  18 # placing buffer data to designated lane of SIMD register. n is
  19 # naturally limited to 4 on pre-AVX2 processors and to 8 on
  20 # AVX2-capable processors such as Haswell.
  21 #
  22 #               this    +aesni(i)       sha256  aesni-sha256    gain(iv)
  23 # -------------------------------------------------------------------
  24 # Westmere(ii)  23.3/n  +1.28=7.11(n=4) 12.3    +3.75=16.1      +126%
  25 # Atom(ii)      38.7/n  +3.93=13.6(n=4) 20.8    +5.69=26.5      +95%
  26 # Sandy Bridge  (20.5   +5.15=25.7)/n   11.6    13.0            +103%
  27 # Ivy Bridge    (20.4   +5.14=25.5)/n   10.3    11.6            +82%
  28 # Haswell(iii)  (21.0   +5.00=26.0)/n   7.80    8.79            +170%
  29 # Skylake       (18.9   +5.00=23.9)/n   7.70    8.17            +170%
  30 # Bulldozer     (21.6   +5.76=27.4)/n   13.6    13.7            +100%
  31 #
  32 # (i)   multi-block CBC encrypt with 128-bit key;
  33 # (ii)  (HASH+AES)/n does not apply to Westmere for n>3 and Atom,
  34 #       because of lower AES-NI instruction throughput, nor is there
  35 #       AES-NI-SHA256 stitch for these processors;
  36 # (iii) "this" is for n=8, when we gather twice as much data, result
  37 #       for n=4 is 20.3+4.44=24.7;
  38 # (iv)  presented improvement coefficients are asymptotic limits and
  39 #       in real-life application are somewhat lower, e.g. for 2KB
  40 #       fragments they range from 75% to 130% (on Haswell);
  41
  42 $flavour = shift;
  43 $output  = shift;
  44 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
  45
  46 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  47
  48 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  49 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  50 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
  51 die "can't locate x86_64-xlate.pl";
  52
  53 $avx=0;
  54
  55 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  56                 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  57         $avx = ($1>=2.19) + ($1>=2.22);
  58 }
  59
  60 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
  61            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
  62         $avx = ($1>=2.09) + ($1>=2.10);
  63 }
  64
  65 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
  66            `ml64 2>&1` =~ /Version ([0-9]+)\./) {
  67         $avx = ($1>=10) + ($1>=11);
  68 }
  69
  70 if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) {
  71         $avx = ($2>=3.0) + ($2>3.0);
  72 }
  73
  74 open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
  75 *STDOUT=*OUT;
  76
  77 # void sha256_multi_block (
  78 #     struct {  unsigned int A[8];
  79 #               unsigned int B[8];
  80 #               unsigned int C[8];
  81 #               unsigned int D[8];
  82 #               unsigned int E[8];
  83 #               unsigned int F[8];
  84 #               unsigned int G[8];
  85 #               unsigned int H[8];      } *ctx,
  86 #     struct {  void *ptr; int blocks;  } inp[8],
  87 #     int num);         /* 1 or 2 */
  88 #
  89 $ctx="%rdi";    # 1st arg
  90 $inp="%rsi";    # 2nd arg
  91 $num="%edx";    # 3rd arg
  92 @ptr=map("%r$_",(8..11));
  93 $Tbl="%rbp";
  94
  95 @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("%xmm$_",(8..15));
  96 ($t1,$t2,$t3,$axb,$bxc,$Xi,$Xn,$sigma)=map("%xmm$_",(0..7));
  97
  98 $REG_SZ=16;
  99
 100 sub Xi_off {
 101 my $off = shift;
 102
 103     $off %= 16; $off *= $REG_SZ;
 104     $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)";
 105 }
 106
 107 sub ROUND_00_15 {
 108 my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
 109
 110 $code.=<<___ if ($i<15);
 111         movd            `4*$i`(@ptr[0]),$Xi
 112         movd            `4*$i`(@ptr[1]),$t1
 113         movd            `4*$i`(@ptr[2]),$t2
 114         movd            `4*$i`(@ptr[3]),$t3
 115         punpckldq       $t2,$Xi
 116         punpckldq       $t3,$t1
 117         punpckldq       $t1,$Xi
 118 ___
 119 $code.=<<___ if ($i==15);
 120         movd            `4*$i`(@ptr[0]),$Xi
 121          lea            `16*4`(@ptr[0]),@ptr[0]
 122         movd            `4*$i`(@ptr[1]),$t1
 123          lea            `16*4`(@ptr[1]),@ptr[1]
 124         movd            `4*$i`(@ptr[2]),$t2
 125          lea            `16*4`(@ptr[2]),@ptr[2]
 126         movd            `4*$i`(@ptr[3]),$t3
 127          lea            `16*4`(@ptr[3]),@ptr[3]
 128         punpckldq       $t2,$Xi
 129         punpckldq       $t3,$t1
 130         punpckldq       $t1,$Xi
 131 ___
 132 $code.=<<___;
 133         movdqa  $e,$sigma
 134         `"pshufb        $Xn,$Xi"                if ($i<=15 && ($i&1)==0)`
 135         movdqa  $e,$t3
 136         `"pshufb        $Xn,$Xi"                if ($i<=15 && ($i&1)==1)`
 137         psrld   \$6,$sigma
 138         movdqa  $e,$t2
 139         pslld   \$7,$t3
 140         movdqa  $Xi,`&Xi_off($i)`
 141          paddd  $h,$Xi                          # Xi+=h
 142
 143         psrld   \$11,$t2
 144         pxor    $t3,$sigma
 145         pslld   \$21-7,$t3
 146          paddd  `32*($i%8)-128`($Tbl),$Xi       # Xi+=K[round]
 147         pxor    $t2,$sigma
 148
 149         psrld   \$25-11,$t2
 150          movdqa $e,$t1
 151          `"prefetcht0   63(@ptr[0])"            if ($i==15)`
 152         pxor    $t3,$sigma
 153          movdqa $e,$axb                         # borrow $axb
 154         pslld   \$26-21,$t3
 155          pandn  $g,$t1
 156          pand   $f,$axb
 157         pxor    $t2,$sigma
 158
 159          `"prefetcht0   63(@ptr[1])"            if ($i==15)`
 160         movdqa  $a,$t2
 161         pxor    $t3,$sigma                      # Sigma1(e)
 162         movdqa  $a,$t3
 163         psrld   \$2,$t2
 164         paddd   $sigma,$Xi                      # Xi+=Sigma1(e)
 165          pxor   $axb,$t1                        # Ch(e,f,g)
 166          movdqa $b,$axb
 167         movdqa  $a,$sigma
 168         pslld   \$10,$t3
 169          pxor   $a,$axb                         # a^b, b^c in next round
 170
 171          `"prefetcht0   63(@ptr[2])"            if ($i==15)`
 172         psrld   \$13,$sigma
 173         pxor    $t3,$t2
 174          paddd  $t1,$Xi                         # Xi+=Ch(e,f,g)
 175         pslld   \$19-10,$t3
 176          pand   $axb,$bxc
 177         pxor    $sigma,$t2
 178
 179          `"prefetcht0   63(@ptr[3])"            if ($i==15)`
 180         psrld   \$22-13,$sigma
 181         pxor    $t3,$t2
 182          movdqa $b,$h
 183         pslld   \$30-19,$t3
 184         pxor    $t2,$sigma
 185          pxor   $bxc,$h                         # h=Maj(a,b,c)=Ch(a^b,c,b)
 186          paddd  $Xi,$d                          # d+=Xi
 187         pxor    $t3,$sigma                      # Sigma0(a)
 188
 189         paddd   $Xi,$h                          # h+=Xi
 190         paddd   $sigma,$h                       # h+=Sigma0(a)
 191 ___
 192 $code.=<<___ if (($i%8)==7);
 193         lea     `32*8`($Tbl),$Tbl
 194 ___
 195         ($axb,$bxc)=($bxc,$axb);
 196 }
 197
 198 sub ROUND_16_XX {
 199 my $i=shift;
 200
 201 $code.=<<___;
 202         movdqa  `&Xi_off($i+1)`,$Xn
 203         paddd   `&Xi_off($i+9)`,$Xi             # Xi+=X[i+9]
 204
 205         movdqa  $Xn,$sigma
 206         movdqa  $Xn,$t2
 207         psrld   \$3,$sigma
 208         movdqa  $Xn,$t3
 209
 210         psrld   \$7,$t2
 211         movdqa  `&Xi_off($i+14)`,$t1
 212         pslld   \$14,$t3
 213         pxor    $t2,$sigma
 214         psrld   \$18-7,$t2
 215         movdqa  $t1,$axb                        # borrow $axb
 216         pxor    $t3,$sigma
 217         pslld   \$25-14,$t3
 218         pxor    $t2,$sigma
 219         psrld   \$10,$t1
 220         movdqa  $axb,$t2
 221
 222         psrld   \$17,$axb
 223         pxor    $t3,$sigma                      # sigma0(X[i+1])
 224         pslld   \$13,$t2
 225          paddd  $sigma,$Xi                      # Xi+=sigma0(e)
 226         pxor    $axb,$t1
 227         psrld   \$19-17,$axb
 228         pxor    $t2,$t1
 229         pslld   \$15-13,$t2
 230         pxor    $axb,$t1
 231         pxor    $t2,$t1                         # sigma0(X[i+14])
 232         paddd   $t1,$Xi                         # Xi+=sigma1(X[i+14])
 233 ___
 234         &ROUND_00_15($i,@_);
 235         ($Xi,$Xn)=($Xn,$Xi);
 236 }
 237
 238 $code.=<<___;
 239 .text
 240
 241 .extern OPENSSL_ia32cap_P
 242
 243 .globl  sha256_multi_block
 244 .type   sha256_multi_block,\@function,3
 245 .align  32
 246 sha256_multi_block:
 247 .cfi_startproc
 248         mov     OPENSSL_ia32cap_P+4(%rip),%rcx
 249         bt      \$61,%rcx                       # check SHA bit
 250         jc      _shaext_shortcut
 251 ___
 252 $code.=<<___ if ($avx);
 253         test    \$`1<<28`,%ecx
 254         jnz     _avx_shortcut
 255 ___
 256 $code.=<<___;
 257         mov     %rsp,%rax
 258 .cfi_def_cfa_register   %rax
 259         push    %rbx
 260 .cfi_push       %rbx
 261         push    %rbp
 262 .cfi_push       %rbp
 263 ___
 264 $code.=<<___ if ($win64);
 265         lea     -0xa8(%rsp),%rsp
 266         movaps  %xmm6,(%rsp)
 267         movaps  %xmm7,0x10(%rsp)
 268         movaps  %xmm8,0x20(%rsp)
 269         movaps  %xmm9,0x30(%rsp)
 270         movaps  %xmm10,-0x78(%rax)
 271         movaps  %xmm11,-0x68(%rax)
 272         movaps  %xmm12,-0x58(%rax)
 273         movaps  %xmm13,-0x48(%rax)
 274         movaps  %xmm14,-0x38(%rax)
 275         movaps  %xmm15,-0x28(%rax)
 276 ___
 277 $code.=<<___;
 278         sub     \$`$REG_SZ*18`, %rsp
 279         and     \$-256,%rsp
 280         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
 281 .cfi_cfa_expression     %rsp+`$REG_SZ*17`,deref,+8
 282 .Lbody:
 283         lea     K256+128(%rip),$Tbl
 284         lea     `$REG_SZ*16`(%rsp),%rbx
 285         lea     0x80($ctx),$ctx                 # size optimization
 286
 287 .Loop_grande:
 288         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
 289         xor     $num,$num
 290 ___
 291 for($i=0;$i<4;$i++) {
 292     $code.=<<___;
 293         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
 294         mov     `16*$i+8`($inp),%ecx            # number of blocks
 295         cmp     $num,%ecx
 296         cmovg   %ecx,$num                       # find maximum
 297         test    %ecx,%ecx
 298         mov     %ecx,`4*$i`(%rbx)               # initialize counters
 299         cmovle  $Tbl,@ptr[$i]                   # cancel input
 300 ___
 301 }
 302 $code.=<<___;
 303         test    $num,$num
 304         jz      .Ldone
 305
 306         movdqu  0x00-0x80($ctx),$A              # load context
 307          lea    128(%rsp),%rax
 308         movdqu  0x20-0x80($ctx),$B
 309         movdqu  0x40-0x80($ctx),$C
 310         movdqu  0x60-0x80($ctx),$D
 311         movdqu  0x80-0x80($ctx),$E
 312         movdqu  0xa0-0x80($ctx),$F
 313         movdqu  0xc0-0x80($ctx),$G
 314         movdqu  0xe0-0x80($ctx),$H
 315         movdqu  .Lpbswap(%rip),$Xn
 316         jmp     .Loop
 317
 318 .align  32
 319 .Loop:
 320         movdqa  $C,$bxc
 321         pxor    $B,$bxc                         # magic seed
 322 ___
 323 for($i=0;$i<16;$i++)    { &ROUND_00_15($i,@V); unshift(@V,pop(@V)); }
 324 $code.=<<___;
 325         movdqu  `&Xi_off($i)`,$Xi
 326         mov     \$3,%ecx
 327         jmp     .Loop_16_xx
 328 .align  32
 329 .Loop_16_xx:
 330 ___
 331 for(;$i<32;$i++)        { &ROUND_16_XX($i,@V); unshift(@V,pop(@V)); }
 332 $code.=<<___;
 333         dec     %ecx
 334         jnz     .Loop_16_xx
 335
 336         mov     \$1,%ecx
 337         lea     K256+128(%rip),$Tbl
 338
 339         movdqa  (%rbx),$sigma                   # pull counters
 340         cmp     4*0(%rbx),%ecx                  # examine counters
 341         pxor    $t1,$t1
 342         cmovge  $Tbl,@ptr[0]                    # cancel input
 343         cmp     4*1(%rbx),%ecx
 344         movdqa  $sigma,$Xn
 345         cmovge  $Tbl,@ptr[1]
 346         cmp     4*2(%rbx),%ecx
 347         pcmpgtd $t1,$Xn                         # mask value
 348         cmovge  $Tbl,@ptr[2]
 349         cmp     4*3(%rbx),%ecx
 350         paddd   $Xn,$sigma                      # counters--
 351         cmovge  $Tbl,@ptr[3]
 352
 353         movdqu  0x00-0x80($ctx),$t1
 354         pand    $Xn,$A
 355         movdqu  0x20-0x80($ctx),$t2
 356         pand    $Xn,$B
 357         movdqu  0x40-0x80($ctx),$t3
 358         pand    $Xn,$C
 359         movdqu  0x60-0x80($ctx),$Xi
 360         pand    $Xn,$D
 361         paddd   $t1,$A
 362         movdqu  0x80-0x80($ctx),$t1
 363         pand    $Xn,$E
 364         paddd   $t2,$B
 365         movdqu  0xa0-0x80($ctx),$t2
 366         pand    $Xn,$F
 367         paddd   $t3,$C
 368         movdqu  0xc0-0x80($ctx),$t3
 369         pand    $Xn,$G
 370         paddd   $Xi,$D
 371         movdqu  0xe0-0x80($ctx),$Xi
 372         pand    $Xn,$H
 373         paddd   $t1,$E
 374         paddd   $t2,$F
 375         movdqu  $A,0x00-0x80($ctx)
 376         paddd   $t3,$G
 377         movdqu  $B,0x20-0x80($ctx)
 378         paddd   $Xi,$H
 379         movdqu  $C,0x40-0x80($ctx)
 380         movdqu  $D,0x60-0x80($ctx)
 381         movdqu  $E,0x80-0x80($ctx)
 382         movdqu  $F,0xa0-0x80($ctx)
 383         movdqu  $G,0xc0-0x80($ctx)
 384         movdqu  $H,0xe0-0x80($ctx)
 385
 386         movdqa  $sigma,(%rbx)                   # save counters
 387         movdqa  .Lpbswap(%rip),$Xn
 388         dec     $num
 389         jnz     .Loop
 390
 391         mov     `$REG_SZ*17+8`(%rsp),$num
 392         lea     $REG_SZ($ctx),$ctx
 393         lea     `16*$REG_SZ/4`($inp),$inp
 394         dec     $num
 395         jnz     .Loop_grande
 396
 397 .Ldone:
 398         mov     `$REG_SZ*17`(%rsp),%rax         # original %rsp
 399 .cfi_def_cfa    %rax,8
 400 ___
 401 $code.=<<___ if ($win64);
 402         movaps  -0xb8(%rax),%xmm6
 403         movaps  -0xa8(%rax),%xmm7
 404         movaps  -0x98(%rax),%xmm8
 405         movaps  -0x88(%rax),%xmm9
 406         movaps  -0x78(%rax),%xmm10
 407         movaps  -0x68(%rax),%xmm11
 408         movaps  -0x58(%rax),%xmm12
 409         movaps  -0x48(%rax),%xmm13
 410         movaps  -0x38(%rax),%xmm14
 411         movaps  -0x28(%rax),%xmm15
 412 ___
 413 $code.=<<___;
 414         mov     -16(%rax),%rbp
 415 .cfi_restore    %rbp
 416         mov     -8(%rax),%rbx
 417 .cfi_restore    %rbx
 418         lea     (%rax),%rsp
 419 .cfi_def_cfa_register   %rsp
 420 .Lepilogue:
 421         ret
 422 .cfi_endproc
 423 .size   sha256_multi_block,.-sha256_multi_block
 424 ___
 425                                                 {{{
 426 my ($Wi,$TMP0,$TMP1,$TMPx,$ABEF0,$CDGH0,$ABEF1,$CDGH1)=map("%xmm$_",(0..3,12..15));
 427 my @MSG0=map("%xmm$_",(4..7));
 428 my @MSG1=map("%xmm$_",(8..11));
 429
 430 $code.=<<___;
 431 .type   sha256_multi_block_shaext,\@function,3
 432 .align  32
 433 sha256_multi_block_shaext:
 434 .cfi_startproc
 435 _shaext_shortcut:
 436         mov     %rsp,%rax
 437 .cfi_def_cfa_register   %rax
 438         push    %rbx
 439 .cfi_push       %rbx
 440         push    %rbp
 441 .cfi_push       %rbp
 442 ___
 443 $code.=<<___ if ($win64);
 444         lea     -0xa8(%rsp),%rsp
 445         movaps  %xmm6,(%rsp)
 446         movaps  %xmm7,0x10(%rsp)
 447         movaps  %xmm8,0x20(%rsp)
 448         movaps  %xmm9,0x30(%rsp)
 449         movaps  %xmm10,-0x78(%rax)
 450         movaps  %xmm11,-0x68(%rax)
 451         movaps  %xmm12,-0x58(%rax)
 452         movaps  %xmm13,-0x48(%rax)
 453         movaps  %xmm14,-0x38(%rax)
 454         movaps  %xmm15,-0x28(%rax)
 455 ___
 456 $code.=<<___;
 457         sub     \$`$REG_SZ*18`,%rsp
 458         shl     \$1,$num                        # we process pair at a time
 459         and     \$-256,%rsp
 460         lea     0x80($ctx),$ctx                 # size optimization
 461         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
 462 .Lbody_shaext:
 463         lea     `$REG_SZ*16`(%rsp),%rbx
 464         lea     K256_shaext+0x80(%rip),$Tbl
 465
 466 .Loop_grande_shaext:
 467         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
 468         xor     $num,$num
 469 ___
 470 for($i=0;$i<2;$i++) {
 471     $code.=<<___;
 472         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
 473         mov     `16*$i+8`($inp),%ecx            # number of blocks
 474         cmp     $num,%ecx
 475         cmovg   %ecx,$num                       # find maximum
 476         test    %ecx,%ecx
 477         mov     %ecx,`4*$i`(%rbx)               # initialize counters
 478         cmovle  %rsp,@ptr[$i]                   # cancel input
 479 ___
 480 }
 481 $code.=<<___;
 482         test    $num,$num
 483         jz      .Ldone_shaext
 484
 485         movq            0x00-0x80($ctx),$ABEF0          # A1.A0
 486         movq            0x20-0x80($ctx),@MSG0[0]        # B1.B0
 487         movq            0x40-0x80($ctx),$CDGH0          # C1.C0
 488         movq            0x60-0x80($ctx),@MSG0[1]        # D1.D0
 489         movq            0x80-0x80($ctx),@MSG1[0]        # E1.E0
 490         movq            0xa0-0x80($ctx),@MSG1[1]        # F1.F0
 491         movq            0xc0-0x80($ctx),@MSG1[2]        # G1.G0
 492         movq            0xe0-0x80($ctx),@MSG1[3]        # H1.H0
 493
 494         punpckldq       @MSG0[0],$ABEF0                 # B1.A1.B0.A0
 495         punpckldq       @MSG0[1],$CDGH0                 # D1.C1.D0.C0
 496         punpckldq       @MSG1[1],@MSG1[0]               # F1.E1.F0.E0
 497         punpckldq       @MSG1[3],@MSG1[2]               # H1.G1.H0.G0
 498         movdqa          K256_shaext-0x10(%rip),$TMPx    # byte swap
 499
 500         movdqa          $ABEF0,$ABEF1
 501         movdqa          $CDGH0,$CDGH1
 502         punpcklqdq      @MSG1[0],$ABEF0                 # F0.E0.B0.A0
 503         punpcklqdq      @MSG1[2],$CDGH0                 # H0.G0.D0.C0
 504         punpckhqdq      @MSG1[0],$ABEF1                 # F1.E1.B1.A1
 505         punpckhqdq      @MSG1[2],$CDGH1                 # H1.G1.D1.C1
 506
 507         pshufd          \$0b00011011,$ABEF0,$ABEF0
 508         pshufd          \$0b00011011,$CDGH0,$CDGH0
 509         pshufd          \$0b00011011,$ABEF1,$ABEF1
 510         pshufd          \$0b00011011,$CDGH1,$CDGH1
 511         jmp             .Loop_shaext
 512
 513 .align  32
 514 .Loop_shaext:
 515         movdqu          0x00(@ptr[0]),@MSG0[0]
 516          movdqu         0x00(@ptr[1]),@MSG1[0]
 517         movdqu          0x10(@ptr[0]),@MSG0[1]
 518          movdqu         0x10(@ptr[1]),@MSG1[1]
 519         movdqu          0x20(@ptr[0]),@MSG0[2]
 520         pshufb          $TMPx,@MSG0[0]
 521          movdqu         0x20(@ptr[1]),@MSG1[2]
 522          pshufb         $TMPx,@MSG1[0]
 523         movdqu          0x30(@ptr[0]),@MSG0[3]
 524         lea             0x40(@ptr[0]),@ptr[0]
 525          movdqu         0x30(@ptr[1]),@MSG1[3]
 526          lea            0x40(@ptr[1]),@ptr[1]
 527
 528         movdqa          0*16-0x80($Tbl),$Wi
 529         pshufb          $TMPx,@MSG0[1]
 530         paddd           @MSG0[0],$Wi
 531         pxor            $ABEF0,@MSG0[0]         # black magic
 532         movdqa          $Wi,$TMP0
 533          movdqa         0*16-0x80($Tbl),$TMP1
 534          pshufb         $TMPx,@MSG1[1]
 535          paddd          @MSG1[0],$TMP1
 536         movdqa          $CDGH0,0x50(%rsp)       # offload
 537         sha256rnds2     $ABEF0,$CDGH0           # 0-3
 538          pxor           $ABEF1,@MSG1[0]         # black magic
 539          movdqa         $TMP1,$Wi
 540          movdqa         $CDGH1,0x70(%rsp)
 541          sha256rnds2    $ABEF1,$CDGH1           # 0-3
 542         pshufd          \$0x0e,$TMP0,$Wi
 543         pxor            $ABEF0,@MSG0[0]         # black magic
 544         movdqa          $ABEF0,0x40(%rsp)       # offload
 545         sha256rnds2     $CDGH0,$ABEF0
 546          pshufd         \$0x0e,$TMP1,$Wi
 547          pxor           $ABEF1,@MSG1[0]         # black magic
 548          movdqa         $ABEF1,0x60(%rsp)
 549         movdqa          1*16-0x80($Tbl),$TMP0
 550         paddd           @MSG0[1],$TMP0
 551         pshufb          $TMPx,@MSG0[2]
 552          sha256rnds2    $CDGH1,$ABEF1
 553
 554         movdqa          $TMP0,$Wi
 555          movdqa         1*16-0x80($Tbl),$TMP1
 556          paddd          @MSG1[1],$TMP1
 557         sha256rnds2     $ABEF0,$CDGH0           # 4-7
 558          movdqa         $TMP1,$Wi
 559         prefetcht0      127(@ptr[0])
 560         pshufb          $TMPx,@MSG0[3]
 561          pshufb         $TMPx,@MSG1[2]
 562          prefetcht0     127(@ptr[1])
 563          sha256rnds2    $ABEF1,$CDGH1           # 4-7
 564         pshufd          \$0x0e,$TMP0,$Wi
 565          pshufb         $TMPx,@MSG1[3]
 566         sha256msg1      @MSG0[1],@MSG0[0]
 567         sha256rnds2     $CDGH0,$ABEF0
 568          pshufd         \$0x0e,$TMP1,$Wi
 569         movdqa          2*16-0x80($Tbl),$TMP0
 570         paddd           @MSG0[2],$TMP0
 571          sha256rnds2    $CDGH1,$ABEF1
 572
 573         movdqa          $TMP0,$Wi
 574          movdqa         2*16-0x80($Tbl),$TMP1
 575          paddd          @MSG1[2],$TMP1
 576         sha256rnds2     $ABEF0,$CDGH0           # 8-11
 577          sha256msg1     @MSG1[1],@MSG1[0]
 578          movdqa         $TMP1,$Wi
 579         movdqa          @MSG0[3],$TMPx
 580          sha256rnds2    $ABEF1,$CDGH1           # 8-11
 581         pshufd          \$0x0e,$TMP0,$Wi
 582         palignr         \$4,@MSG0[2],$TMPx
 583         paddd           $TMPx,@MSG0[0]
 584          movdqa         @MSG1[3],$TMPx
 585          palignr        \$4,@MSG1[2],$TMPx
 586         sha256msg1      @MSG0[2],@MSG0[1]
 587         sha256rnds2     $CDGH0,$ABEF0
 588          pshufd         \$0x0e,$TMP1,$Wi
 589         movdqa          3*16-0x80($Tbl),$TMP0
 590         paddd           @MSG0[3],$TMP0
 591          sha256rnds2    $CDGH1,$ABEF1
 592          sha256msg1     @MSG1[2],@MSG1[1]
 593
 594         movdqa          $TMP0,$Wi
 595          movdqa         3*16-0x80($Tbl),$TMP1
 596          paddd          $TMPx,@MSG1[0]
 597          paddd          @MSG1[3],$TMP1
 598         sha256msg2      @MSG0[3],@MSG0[0]
 599         sha256rnds2     $ABEF0,$CDGH0           # 12-15
 600          movdqa         $TMP1,$Wi
 601         movdqa          @MSG0[0],$TMPx
 602         palignr         \$4,@MSG0[3],$TMPx
 603          sha256rnds2    $ABEF1,$CDGH1           # 12-15
 604          sha256msg2     @MSG1[3],@MSG1[0]
 605         pshufd          \$0x0e,$TMP0,$Wi
 606         paddd           $TMPx,@MSG0[1]
 607          movdqa         @MSG1[0],$TMPx
 608          palignr        \$4,@MSG1[3],$TMPx
 609         sha256msg1      @MSG0[3],@MSG0[2]
 610         sha256rnds2     $CDGH0,$ABEF0
 611          pshufd         \$0x0e,$TMP1,$Wi
 612         movdqa          4*16-0x80($Tbl),$TMP0
 613         paddd           @MSG0[0],$TMP0
 614          sha256rnds2    $CDGH1,$ABEF1
 615          sha256msg1     @MSG1[3],@MSG1[2]
 616 ___
 617 for($i=4;$i<16-3;$i++) {
 618 $code.=<<___;
 619         movdqa          $TMP0,$Wi
 620          movdqa         $i*16-0x80($Tbl),$TMP1
 621          paddd          $TMPx,@MSG1[1]
 622          paddd          @MSG1[0],$TMP1
 623         sha256msg2      @MSG0[0],@MSG0[1]
 624         sha256rnds2     $ABEF0,$CDGH0           # 16-19...
 625          movdqa         $TMP1,$Wi
 626         movdqa          @MSG0[1],$TMPx
 627         palignr         \$4,@MSG0[0],$TMPx
 628          sha256rnds2    $ABEF1,$CDGH1           # 16-19...
 629          sha256msg2     @MSG1[0],@MSG1[1]
 630         pshufd          \$0x0e,$TMP0,$Wi
 631         paddd           $TMPx,@MSG0[2]
 632          movdqa         @MSG1[1],$TMPx
 633          palignr        \$4,@MSG1[0],$TMPx
 634         sha256msg1      @MSG0[0],@MSG0[3]
 635         sha256rnds2     $CDGH0,$ABEF0
 636          pshufd         \$0x0e,$TMP1,$Wi
 637         movdqa          `($i+1)*16`-0x80($Tbl),$TMP0
 638         paddd           @MSG0[1],$TMP0
 639          sha256rnds2    $CDGH1,$ABEF1
 640          sha256msg1     @MSG1[0],@MSG1[3]
 641 ___
 642         push(@MSG0,shift(@MSG0));       push(@MSG1,shift(@MSG1));
 643 }
 644 $code.=<<___;
 645         movdqa          $TMP0,$Wi
 646          movdqa         13*16-0x80($Tbl),$TMP1
 647          paddd          $TMPx,@MSG1[1]
 648          paddd          @MSG1[0],$TMP1
 649         sha256msg2      @MSG0[0],@MSG0[1]
 650         sha256rnds2     $ABEF0,$CDGH0           # 52-55
 651          movdqa         $TMP1,$Wi
 652         movdqa          @MSG0[1],$TMPx
 653         palignr         \$4,@MSG0[0],$TMPx
 654          sha256rnds2    $ABEF1,$CDGH1           # 52-55
 655          sha256msg2     @MSG1[0],@MSG1[1]
 656         pshufd          \$0x0e,$TMP0,$Wi
 657         paddd           $TMPx,@MSG0[2]
 658          movdqa         @MSG1[1],$TMPx
 659          palignr        \$4,@MSG1[0],$TMPx
 660         nop
 661         sha256rnds2     $CDGH0,$ABEF0
 662          pshufd         \$0x0e,$TMP1,$Wi
 663         movdqa          14*16-0x80($Tbl),$TMP0
 664         paddd           @MSG0[1],$TMP0
 665          sha256rnds2    $CDGH1,$ABEF1
 666
 667         movdqa          $TMP0,$Wi
 668          movdqa         14*16-0x80($Tbl),$TMP1
 669          paddd          $TMPx,@MSG1[2]
 670          paddd          @MSG1[1],$TMP1
 671         sha256msg2      @MSG0[1],@MSG0[2]
 672         nop
 673         sha256rnds2     $ABEF0,$CDGH0           # 56-59
 674          movdqa         $TMP1,$Wi
 675           mov           \$1,%ecx
 676           pxor          @MSG0[1],@MSG0[1]       # zero
 677          sha256rnds2    $ABEF1,$CDGH1           # 56-59
 678          sha256msg2     @MSG1[1],@MSG1[2]
 679         pshufd          \$0x0e,$TMP0,$Wi
 680         movdqa          15*16-0x80($Tbl),$TMP0
 681         paddd           @MSG0[2],$TMP0
 682           movq          (%rbx),@MSG0[2]         # pull counters
 683           nop
 684         sha256rnds2     $CDGH0,$ABEF0
 685          pshufd         \$0x0e,$TMP1,$Wi
 686          movdqa         15*16-0x80($Tbl),$TMP1
 687          paddd          @MSG1[2],$TMP1
 688          sha256rnds2    $CDGH1,$ABEF1
 689
 690         movdqa          $TMP0,$Wi
 691           cmp           4*0(%rbx),%ecx          # examine counters
 692           cmovge        %rsp,@ptr[0]            # cancel input
 693           cmp           4*1(%rbx),%ecx
 694           cmovge        %rsp,@ptr[1]
 695           pshufd        \$0x00,@MSG0[2],@MSG1[0]
 696         sha256rnds2     $ABEF0,$CDGH0           # 60-63
 697          movdqa         $TMP1,$Wi
 698           pshufd        \$0x55,@MSG0[2],@MSG1[1]
 699           movdqa        @MSG0[2],@MSG1[2]
 700          sha256rnds2    $ABEF1,$CDGH1           # 60-63
 701         pshufd          \$0x0e,$TMP0,$Wi
 702           pcmpgtd       @MSG0[1],@MSG1[0]
 703           pcmpgtd       @MSG0[1],@MSG1[1]
 704         sha256rnds2     $CDGH0,$ABEF0
 705          pshufd         \$0x0e,$TMP1,$Wi
 706           pcmpgtd       @MSG0[1],@MSG1[2]       # counter mask
 707           movdqa        K256_shaext-0x10(%rip),$TMPx
 708          sha256rnds2    $CDGH1,$ABEF1
 709
 710         pand            @MSG1[0],$CDGH0
 711          pand           @MSG1[1],$CDGH1
 712         pand            @MSG1[0],$ABEF0
 713          pand           @MSG1[1],$ABEF1
 714         paddd           @MSG0[2],@MSG1[2]       # counters--
 715
 716         paddd           0x50(%rsp),$CDGH0
 717          paddd          0x70(%rsp),$CDGH1
 718         paddd           0x40(%rsp),$ABEF0
 719          paddd          0x60(%rsp),$ABEF1
 720
 721         movq            @MSG1[2],(%rbx)         # save counters
 722         dec             $num
 723         jnz             .Loop_shaext
 724
 725         mov             `$REG_SZ*17+8`(%rsp),$num
 726
 727         pshufd          \$0b00011011,$ABEF0,$ABEF0
 728         pshufd          \$0b00011011,$CDGH0,$CDGH0
 729         pshufd          \$0b00011011,$ABEF1,$ABEF1
 730         pshufd          \$0b00011011,$CDGH1,$CDGH1
 731
 732         movdqa          $ABEF0,@MSG0[0]
 733         movdqa          $CDGH0,@MSG0[1]
 734         punpckldq       $ABEF1,$ABEF0                   # B1.B0.A1.A0
 735         punpckhdq       $ABEF1,@MSG0[0]                 # F1.F0.E1.E0
 736         punpckldq       $CDGH1,$CDGH0                   # D1.D0.C1.C0
 737         punpckhdq       $CDGH1,@MSG0[1]                 # H1.H0.G1.G0
 738
 739         movq            $ABEF0,0x00-0x80($ctx)          # A1.A0
 740         psrldq          \$8,$ABEF0
 741         movq            @MSG0[0],0x80-0x80($ctx)        # E1.E0
 742         psrldq          \$8,@MSG0[0]
 743         movq            $ABEF0,0x20-0x80($ctx)          # B1.B0
 744         movq            @MSG0[0],0xa0-0x80($ctx)        # F1.F0
 745
 746         movq            $CDGH0,0x40-0x80($ctx)          # C1.C0
 747         psrldq          \$8,$CDGH0
 748         movq            @MSG0[1],0xc0-0x80($ctx)        # G1.G0
 749         psrldq          \$8,@MSG0[1]
 750         movq            $CDGH0,0x60-0x80($ctx)          # D1.D0
 751         movq            @MSG0[1],0xe0-0x80($ctx)        # H1.H0
 752
 753         lea     `$REG_SZ/2`($ctx),$ctx
 754         lea     `16*2`($inp),$inp
 755         dec     $num
 756         jnz     .Loop_grande_shaext
 757
 758 .Ldone_shaext:
 759         #mov    `$REG_SZ*17`(%rsp),%rax         # original %rsp
 760 ___
 761 $code.=<<___ if ($win64);
 762         movaps  -0xb8(%rax),%xmm6
 763         movaps  -0xa8(%rax),%xmm7
 764         movaps  -0x98(%rax),%xmm8
 765         movaps  -0x88(%rax),%xmm9
 766         movaps  -0x78(%rax),%xmm10
 767         movaps  -0x68(%rax),%xmm11
 768         movaps  -0x58(%rax),%xmm12
 769         movaps  -0x48(%rax),%xmm13
 770         movaps  -0x38(%rax),%xmm14
 771         movaps  -0x28(%rax),%xmm15
 772 ___
 773 $code.=<<___;
 774         mov     -16(%rax),%rbp
 775 .cfi_restore    %rbp
 776         mov     -8(%rax),%rbx
 777 .cfi_restore    %rbx
 778         lea     (%rax),%rsp
 779 .cfi_def_cfa_register   %rsp
 780 .Lepilogue_shaext:
 781         ret
 782 .cfi_endproc
 783 .size   sha256_multi_block_shaext,.-sha256_multi_block_shaext
 784 ___
 785                                                 }}}
 786                                                 if ($avx) {{{
 787 sub ROUND_00_15_avx {
 788 my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
 789
 790 $code.=<<___ if ($i<15 && $REG_SZ==16);
 791         vmovd           `4*$i`(@ptr[0]),$Xi
 792         vmovd           `4*$i`(@ptr[1]),$t1
 793         vpinsrd         \$1,`4*$i`(@ptr[2]),$Xi,$Xi
 794         vpinsrd         \$1,`4*$i`(@ptr[3]),$t1,$t1
 795         vpunpckldq      $t1,$Xi,$Xi
 796         vpshufb         $Xn,$Xi,$Xi
 797 ___
 798 $code.=<<___ if ($i==15 && $REG_SZ==16);
 799         vmovd           `4*$i`(@ptr[0]),$Xi
 800          lea            `16*4`(@ptr[0]),@ptr[0]
 801         vmovd           `4*$i`(@ptr[1]),$t1
 802          lea            `16*4`(@ptr[1]),@ptr[1]
 803         vpinsrd         \$1,`4*$i`(@ptr[2]),$Xi,$Xi
 804          lea            `16*4`(@ptr[2]),@ptr[2]
 805         vpinsrd         \$1,`4*$i`(@ptr[3]),$t1,$t1
 806          lea            `16*4`(@ptr[3]),@ptr[3]
 807         vpunpckldq      $t1,$Xi,$Xi
 808         vpshufb         $Xn,$Xi,$Xi
 809 ___
 810 $code.=<<___ if ($i<15 && $REG_SZ==32);
 811         vmovd           `4*$i`(@ptr[0]),$Xi
 812         vmovd           `4*$i`(@ptr[4]),$t1
 813         vmovd           `4*$i`(@ptr[1]),$t2
 814         vmovd           `4*$i`(@ptr[5]),$t3
 815         vpinsrd         \$1,`4*$i`(@ptr[2]),$Xi,$Xi
 816         vpinsrd         \$1,`4*$i`(@ptr[6]),$t1,$t1
 817         vpinsrd         \$1,`4*$i`(@ptr[3]),$t2,$t2
 818         vpunpckldq      $t2,$Xi,$Xi
 819         vpinsrd         \$1,`4*$i`(@ptr[7]),$t3,$t3
 820         vpunpckldq      $t3,$t1,$t1
 821         vinserti128     $t1,$Xi,$Xi
 822         vpshufb         $Xn,$Xi,$Xi
 823 ___
 824 $code.=<<___ if ($i==15 && $REG_SZ==32);
 825         vmovd           `4*$i`(@ptr[0]),$Xi
 826          lea            `16*4`(@ptr[0]),@ptr[0]
 827         vmovd           `4*$i`(@ptr[4]),$t1
 828          lea            `16*4`(@ptr[4]),@ptr[4]
 829         vmovd           `4*$i`(@ptr[1]),$t2
 830          lea            `16*4`(@ptr[1]),@ptr[1]
 831         vmovd           `4*$i`(@ptr[5]),$t3
 832          lea            `16*4`(@ptr[5]),@ptr[5]
 833         vpinsrd         \$1,`4*$i`(@ptr[2]),$Xi,$Xi
 834          lea            `16*4`(@ptr[2]),@ptr[2]
 835         vpinsrd         \$1,`4*$i`(@ptr[6]),$t1,$t1
 836          lea            `16*4`(@ptr[6]),@ptr[6]
 837         vpinsrd         \$1,`4*$i`(@ptr[3]),$t2,$t2
 838          lea            `16*4`(@ptr[3]),@ptr[3]
 839         vpunpckldq      $t2,$Xi,$Xi
 840         vpinsrd         \$1,`4*$i`(@ptr[7]),$t3,$t3
 841          lea            `16*4`(@ptr[7]),@ptr[7]
 842         vpunpckldq      $t3,$t1,$t1
 843         vinserti128     $t1,$Xi,$Xi
 844         vpshufb         $Xn,$Xi,$Xi
 845 ___
 846 $code.=<<___;
 847         vpsrld  \$6,$e,$sigma
 848         vpslld  \$26,$e,$t3
 849         vmovdqu $Xi,`&Xi_off($i)`
 850          vpaddd $h,$Xi,$Xi                      # Xi+=h
 851
 852         vpsrld  \$11,$e,$t2
 853         vpxor   $t3,$sigma,$sigma
 854         vpslld  \$21,$e,$t3
 855          vpaddd `32*($i%8)-128`($Tbl),$Xi,$Xi   # Xi+=K[round]
 856         vpxor   $t2,$sigma,$sigma
 857
 858         vpsrld  \$25,$e,$t2
 859         vpxor   $t3,$sigma,$sigma
 860          `"prefetcht0   63(@ptr[0])"            if ($i==15)`
 861         vpslld  \$7,$e,$t3
 862          vpandn $g,$e,$t1
 863          vpand  $f,$e,$axb                      # borrow $axb
 864          `"prefetcht0   63(@ptr[1])"            if ($i==15)`
 865         vpxor   $t2,$sigma,$sigma
 866
 867         vpsrld  \$2,$a,$h                       # borrow $h
 868         vpxor   $t3,$sigma,$sigma               # Sigma1(e)
 869          `"prefetcht0   63(@ptr[2])"            if ($i==15)`
 870         vpslld  \$30,$a,$t2
 871          vpxor  $axb,$t1,$t1                    # Ch(e,f,g)
 872          vpxor  $a,$b,$axb                      # a^b, b^c in next round
 873          `"prefetcht0   63(@ptr[3])"            if ($i==15)`
 874         vpxor   $t2,$h,$h
 875         vpaddd  $sigma,$Xi,$Xi                  # Xi+=Sigma1(e)
 876
 877         vpsrld  \$13,$a,$t2
 878          `"prefetcht0   63(@ptr[4])"            if ($i==15 && $REG_SZ==32)`
 879         vpslld  \$19,$a,$t3
 880          vpaddd $t1,$Xi,$Xi                     # Xi+=Ch(e,f,g)
 881          vpand  $axb,$bxc,$bxc
 882          `"prefetcht0   63(@ptr[5])"            if ($i==15 && $REG_SZ==32)`
 883         vpxor   $t2,$h,$sigma
 884
 885         vpsrld  \$22,$a,$t2
 886         vpxor   $t3,$sigma,$sigma
 887          `"prefetcht0   63(@ptr[6])"            if ($i==15 && $REG_SZ==32)`
 888         vpslld  \$10,$a,$t3
 889          vpxor  $bxc,$b,$h                      # h=Maj(a,b,c)=Ch(a^b,c,b)
 890          vpaddd $Xi,$d,$d                       # d+=Xi
 891          `"prefetcht0   63(@ptr[7])"            if ($i==15 && $REG_SZ==32)`
 892         vpxor   $t2,$sigma,$sigma
 893         vpxor   $t3,$sigma,$sigma               # Sigma0(a)
 894
 895         vpaddd  $Xi,$h,$h                       # h+=Xi
 896         vpaddd  $sigma,$h,$h                    # h+=Sigma0(a)
 897 ___
 898 $code.=<<___ if (($i%8)==7);
 899         add     \$`32*8`,$Tbl
 900 ___
 901         ($axb,$bxc)=($bxc,$axb);
 902 }
 903
 904 sub ROUND_16_XX_avx {
 905 my $i=shift;
 906
 907 $code.=<<___;
 908         vmovdqu `&Xi_off($i+1)`,$Xn
 909         vpaddd  `&Xi_off($i+9)`,$Xi,$Xi         # Xi+=X[i+9]
 910
 911         vpsrld  \$3,$Xn,$sigma
 912         vpsrld  \$7,$Xn,$t2
 913         vpslld  \$25,$Xn,$t3
 914         vpxor   $t2,$sigma,$sigma
 915         vpsrld  \$18,$Xn,$t2
 916         vpxor   $t3,$sigma,$sigma
 917         vpslld  \$14,$Xn,$t3
 918         vmovdqu `&Xi_off($i+14)`,$t1
 919         vpsrld  \$10,$t1,$axb                   # borrow $axb
 920
 921         vpxor   $t2,$sigma,$sigma
 922         vpsrld  \$17,$t1,$t2
 923         vpxor   $t3,$sigma,$sigma               # sigma0(X[i+1])
 924         vpslld  \$15,$t1,$t3
 925          vpaddd $sigma,$Xi,$Xi                  # Xi+=sigma0(e)
 926         vpxor   $t2,$axb,$sigma
 927         vpsrld  \$19,$t1,$t2
 928         vpxor   $t3,$sigma,$sigma
 929         vpslld  \$13,$t1,$t3
 930         vpxor   $t2,$sigma,$sigma
 931         vpxor   $t3,$sigma,$sigma               # sigma0(X[i+14])
 932         vpaddd  $sigma,$Xi,$Xi                  # Xi+=sigma1(X[i+14])
 933 ___
 934         &ROUND_00_15_avx($i,@_);
 935         ($Xi,$Xn)=($Xn,$Xi);
 936 }
 937
 938 $code.=<<___;
 939 .type   sha256_multi_block_avx,\@function,3
 940 .align  32
 941 sha256_multi_block_avx:
 942 .cfi_startproc
 943 _avx_shortcut:
 944 ___
 945 $code.=<<___ if ($avx>1);
 946         shr     \$32,%rcx
 947         cmp     \$2,$num
 948         jb      .Lavx
 949         test    \$`1<<5`,%ecx
 950         jnz     _avx2_shortcut
 951         jmp     .Lavx
 952 .align  32
 953 .Lavx:
 954 ___
 955 $code.=<<___;
 956         mov     %rsp,%rax
 957 .cfi_def_cfa_register   %rax
 958         push    %rbx
 959 .cfi_push       %rbx
 960         push    %rbp
 961 .cfi_push       %rbp
 962 ___
 963 $code.=<<___ if ($win64);
 964         lea     -0xa8(%rsp),%rsp
 965         movaps  %xmm6,(%rsp)
 966         movaps  %xmm7,0x10(%rsp)
 967         movaps  %xmm8,0x20(%rsp)
 968         movaps  %xmm9,0x30(%rsp)
 969         movaps  %xmm10,-0x78(%rax)
 970         movaps  %xmm11,-0x68(%rax)
 971         movaps  %xmm12,-0x58(%rax)
 972         movaps  %xmm13,-0x48(%rax)
 973         movaps  %xmm14,-0x38(%rax)
 974         movaps  %xmm15,-0x28(%rax)
 975 ___
 976 $code.=<<___;
 977         sub     \$`$REG_SZ*18`, %rsp
 978         and     \$-256,%rsp
 979         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
 980 .cfi_cfa_expression     %rsp+`$REG_SZ*17`,deref,+8
 981 .Lbody_avx:
 982         lea     K256+128(%rip),$Tbl
 983         lea     `$REG_SZ*16`(%rsp),%rbx
 984         lea     0x80($ctx),$ctx                 # size optimization
 985
 986 .Loop_grande_avx:
 987         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
 988         xor     $num,$num
 989 ___
 990 for($i=0;$i<4;$i++) {
 991     $code.=<<___;
 992         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
 993         mov     `16*$i+8`($inp),%ecx            # number of blocks
 994         cmp     $num,%ecx
 995         cmovg   %ecx,$num                       # find maximum
 996         test    %ecx,%ecx
 997         mov     %ecx,`4*$i`(%rbx)               # initialize counters
 998         cmovle  $Tbl,@ptr[$i]                   # cancel input
 999 ___
1000 }
1001 $code.=<<___;
1002         test    $num,$num
1003         jz      .Ldone_avx
1004
1005         vmovdqu 0x00-0x80($ctx),$A              # load context
1006          lea    128(%rsp),%rax
1007         vmovdqu 0x20-0x80($ctx),$B
1008         vmovdqu 0x40-0x80($ctx),$C
1009         vmovdqu 0x60-0x80($ctx),$D
1010         vmovdqu 0x80-0x80($ctx),$E
1011         vmovdqu 0xa0-0x80($ctx),$F
1012         vmovdqu 0xc0-0x80($ctx),$G
1013         vmovdqu 0xe0-0x80($ctx),$H
1014         vmovdqu .Lpbswap(%rip),$Xn
1015         jmp     .Loop_avx
1016
1017 .align  32
1018 .Loop_avx:
1019         vpxor   $B,$C,$bxc                      # magic seed
1020 ___
1021 for($i=0;$i<16;$i++)    { &ROUND_00_15_avx($i,@V); unshift(@V,pop(@V)); }
1022 $code.=<<___;
1023         vmovdqu `&Xi_off($i)`,$Xi
1024         mov     \$3,%ecx
1025         jmp     .Loop_16_xx_avx
1026 .align  32
1027 .Loop_16_xx_avx:
1028 ___
1029 for(;$i<32;$i++)        { &ROUND_16_XX_avx($i,@V); unshift(@V,pop(@V)); }
1030 $code.=<<___;
1031         dec     %ecx
1032         jnz     .Loop_16_xx_avx
1033
1034         mov     \$1,%ecx
1035         lea     K256+128(%rip),$Tbl
1036 ___
1037 for($i=0;$i<4;$i++) {
1038     $code.=<<___;
1039         cmp     `4*$i`(%rbx),%ecx               # examine counters
1040         cmovge  $Tbl,@ptr[$i]                   # cancel input
1041 ___
1042 }
1043 $code.=<<___;
1044         vmovdqa (%rbx),$sigma                   # pull counters
1045         vpxor   $t1,$t1,$t1
1046         vmovdqa $sigma,$Xn
1047         vpcmpgtd $t1,$Xn,$Xn                    # mask value
1048         vpaddd  $Xn,$sigma,$sigma               # counters--
1049
1050         vmovdqu 0x00-0x80($ctx),$t1
1051         vpand   $Xn,$A,$A
1052         vmovdqu 0x20-0x80($ctx),$t2
1053         vpand   $Xn,$B,$B
1054         vmovdqu 0x40-0x80($ctx),$t3
1055         vpand   $Xn,$C,$C
1056         vmovdqu 0x60-0x80($ctx),$Xi
1057         vpand   $Xn,$D,$D
1058         vpaddd  $t1,$A,$A
1059         vmovdqu 0x80-0x80($ctx),$t1
1060         vpand   $Xn,$E,$E
1061         vpaddd  $t2,$B,$B
1062         vmovdqu 0xa0-0x80($ctx),$t2
1063         vpand   $Xn,$F,$F
1064         vpaddd  $t3,$C,$C
1065         vmovdqu 0xc0-0x80($ctx),$t3
1066         vpand   $Xn,$G,$G
1067         vpaddd  $Xi,$D,$D
1068         vmovdqu 0xe0-0x80($ctx),$Xi
1069         vpand   $Xn,$H,$H
1070         vpaddd  $t1,$E,$E
1071         vpaddd  $t2,$F,$F
1072         vmovdqu $A,0x00-0x80($ctx)
1073         vpaddd  $t3,$G,$G
1074         vmovdqu $B,0x20-0x80($ctx)
1075         vpaddd  $Xi,$H,$H
1076         vmovdqu $C,0x40-0x80($ctx)
1077         vmovdqu $D,0x60-0x80($ctx)
1078         vmovdqu $E,0x80-0x80($ctx)
1079         vmovdqu $F,0xa0-0x80($ctx)
1080         vmovdqu $G,0xc0-0x80($ctx)
1081         vmovdqu $H,0xe0-0x80($ctx)
1082
1083         vmovdqu $sigma,(%rbx)                   # save counters
1084         vmovdqu .Lpbswap(%rip),$Xn
1085         dec     $num
1086         jnz     .Loop_avx
1087
1088         mov     `$REG_SZ*17+8`(%rsp),$num
1089         lea     $REG_SZ($ctx),$ctx
1090         lea     `16*$REG_SZ/4`($inp),$inp
1091         dec     $num
1092         jnz     .Loop_grande_avx
1093
1094 .Ldone_avx:
1095         mov     `$REG_SZ*17`(%rsp),%rax         # original %rsp
1096 .cfi_def_cfa    %rax,8
1097         vzeroupper
1098 ___
1099 $code.=<<___ if ($win64);
1100         movaps  -0xb8(%rax),%xmm6
1101         movaps  -0xa8(%rax),%xmm7
1102         movaps  -0x98(%rax),%xmm8
1103         movaps  -0x88(%rax),%xmm9
1104         movaps  -0x78(%rax),%xmm10
1105         movaps  -0x68(%rax),%xmm11
1106         movaps  -0x58(%rax),%xmm12
1107         movaps  -0x48(%rax),%xmm13
1108         movaps  -0x38(%rax),%xmm14
1109         movaps  -0x28(%rax),%xmm15
1110 ___
1111 $code.=<<___;
1112         mov     -16(%rax),%rbp
1113 .cfi_restore    %rbp
1114         mov     -8(%rax),%rbx
1115 .cfi_restore    %rbx
1116         lea     (%rax),%rsp
1117 .cfi_def_cfa_register   %rsp
1118 .Lepilogue_avx:
1119         ret
1120 .cfi_endproc
1121 .size   sha256_multi_block_avx,.-sha256_multi_block_avx
1122 ___
1123                                                 if ($avx>1) {
1124 $code =~ s/\`([^\`]*)\`/eval $1/gem;
1125
1126 $REG_SZ=32;
1127 @ptr=map("%r$_",(12..15,8..11));
1128
1129 @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("%ymm$_",(8..15));
1130 ($t1,$t2,$t3,$axb,$bxc,$Xi,$Xn,$sigma)=map("%ymm$_",(0..7));
1131
1132 $code.=<<___;
1133 .type   sha256_multi_block_avx2,\@function,3
1134 .align  32
1135 sha256_multi_block_avx2:
1136 .cfi_startproc
1137 _avx2_shortcut:
1138         mov     %rsp,%rax
1139 .cfi_def_cfa_register   %rax
1140         push    %rbx
1141 .cfi_push       %rbx
1142         push    %rbp
1143 .cfi_push       %rbp
1144         push    %r12
1145 .cfi_push       %r12
1146         push    %r13
1147 .cfi_push       %r13
1148         push    %r14
1149 .cfi_push       %r14
1150         push    %r15
1151 .cfi_push       %r15
1152 ___
1153 $code.=<<___ if ($win64);
1154         lea     -0xa8(%rsp),%rsp
1155         movaps  %xmm6,(%rsp)
1156         movaps  %xmm7,0x10(%rsp)
1157         movaps  %xmm8,0x20(%rsp)
1158         movaps  %xmm9,0x30(%rsp)
1159         movaps  %xmm10,0x40(%rsp)
1160         movaps  %xmm11,0x50(%rsp)
1161         movaps  %xmm12,-0x78(%rax)
1162         movaps  %xmm13,-0x68(%rax)
1163         movaps  %xmm14,-0x58(%rax)
1164         movaps  %xmm15,-0x48(%rax)
1165 ___
1166 $code.=<<___;
1167         sub     \$`$REG_SZ*18`, %rsp
1168         and     \$-256,%rsp
1169         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
1170 .cfi_cfa_expression     %rsp+`$REG_SZ*17`,deref,+8
1171 .Lbody_avx2:
1172         lea     K256+128(%rip),$Tbl
1173         lea     0x80($ctx),$ctx                 # size optimization
1174
1175 .Loop_grande_avx2:
1176         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
1177         xor     $num,$num
1178         lea     `$REG_SZ*16`(%rsp),%rbx
1179 ___
1180 for($i=0;$i<8;$i++) {
1181     $code.=<<___;
1182         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
1183         mov     `16*$i+8`($inp),%ecx            # number of blocks
1184         cmp     $num,%ecx
1185         cmovg   %ecx,$num                       # find maximum
1186         test    %ecx,%ecx
1187         mov     %ecx,`4*$i`(%rbx)               # initialize counters
1188         cmovle  $Tbl,@ptr[$i]                   # cancel input
1189 ___
1190 }
1191 $code.=<<___;
1192         vmovdqu 0x00-0x80($ctx),$A              # load context
1193          lea    128(%rsp),%rax
1194         vmovdqu 0x20-0x80($ctx),$B
1195          lea    256+128(%rsp),%rbx
1196         vmovdqu 0x40-0x80($ctx),$C
1197         vmovdqu 0x60-0x80($ctx),$D
1198         vmovdqu 0x80-0x80($ctx),$E
1199         vmovdqu 0xa0-0x80($ctx),$F
1200         vmovdqu 0xc0-0x80($ctx),$G
1201         vmovdqu 0xe0-0x80($ctx),$H
1202         vmovdqu .Lpbswap(%rip),$Xn
1203         jmp     .Loop_avx2
1204
1205 .align  32
1206 .Loop_avx2:
1207         vpxor   $B,$C,$bxc                      # magic seed
1208 ___
1209 for($i=0;$i<16;$i++)    { &ROUND_00_15_avx($i,@V); unshift(@V,pop(@V)); }
1210 $code.=<<___;
1211         vmovdqu `&Xi_off($i)`,$Xi
1212         mov     \$3,%ecx
1213         jmp     .Loop_16_xx_avx2
1214 .align  32
1215 .Loop_16_xx_avx2:
1216 ___
1217 for(;$i<32;$i++)        { &ROUND_16_XX_avx($i,@V); unshift(@V,pop(@V)); }
1218 $code.=<<___;
1219         dec     %ecx
1220         jnz     .Loop_16_xx_avx2
1221
1222         mov     \$1,%ecx
1223         lea     `$REG_SZ*16`(%rsp),%rbx
1224         lea     K256+128(%rip),$Tbl
1225 ___
1226 for($i=0;$i<8;$i++) {
1227     $code.=<<___;
1228         cmp     `4*$i`(%rbx),%ecx               # examine counters
1229         cmovge  $Tbl,@ptr[$i]                   # cancel input
1230 ___
1231 }
1232 $code.=<<___;
1233         vmovdqa (%rbx),$sigma                   # pull counters
1234         vpxor   $t1,$t1,$t1
1235         vmovdqa $sigma,$Xn
1236         vpcmpgtd $t1,$Xn,$Xn                    # mask value
1237         vpaddd  $Xn,$sigma,$sigma               # counters--
1238
1239         vmovdqu 0x00-0x80($ctx),$t1
1240         vpand   $Xn,$A,$A
1241         vmovdqu 0x20-0x80($ctx),$t2
1242         vpand   $Xn,$B,$B
1243         vmovdqu 0x40-0x80($ctx),$t3
1244         vpand   $Xn,$C,$C
1245         vmovdqu 0x60-0x80($ctx),$Xi
1246         vpand   $Xn,$D,$D
1247         vpaddd  $t1,$A,$A
1248         vmovdqu 0x80-0x80($ctx),$t1
1249         vpand   $Xn,$E,$E
1250         vpaddd  $t2,$B,$B
1251         vmovdqu 0xa0-0x80($ctx),$t2
1252         vpand   $Xn,$F,$F
1253         vpaddd  $t3,$C,$C
1254         vmovdqu 0xc0-0x80($ctx),$t3
1255         vpand   $Xn,$G,$G
1256         vpaddd  $Xi,$D,$D
1257         vmovdqu 0xe0-0x80($ctx),$Xi
1258         vpand   $Xn,$H,$H
1259         vpaddd  $t1,$E,$E
1260         vpaddd  $t2,$F,$F
1261         vmovdqu $A,0x00-0x80($ctx)
1262         vpaddd  $t3,$G,$G
1263         vmovdqu $B,0x20-0x80($ctx)
1264         vpaddd  $Xi,$H,$H
1265         vmovdqu $C,0x40-0x80($ctx)
1266         vmovdqu $D,0x60-0x80($ctx)
1267         vmovdqu $E,0x80-0x80($ctx)
1268         vmovdqu $F,0xa0-0x80($ctx)
1269         vmovdqu $G,0xc0-0x80($ctx)
1270         vmovdqu $H,0xe0-0x80($ctx)
1271
1272         vmovdqu $sigma,(%rbx)                   # save counters
1273         lea     256+128(%rsp),%rbx
1274         vmovdqu .Lpbswap(%rip),$Xn
1275         dec     $num
1276         jnz     .Loop_avx2
1277
1278         #mov    `$REG_SZ*17+8`(%rsp),$num
1279         #lea    $REG_SZ($ctx),$ctx
1280         #lea    `16*$REG_SZ/4`($inp),$inp
1281         #dec    $num
1282         #jnz    .Loop_grande_avx2
1283
1284 .Ldone_avx2:
1285         mov     `$REG_SZ*17`(%rsp),%rax         # original %rsp
1286 .cfi_def_cfa    %rax,8
1287         vzeroupper
1288 ___
1289 $code.=<<___ if ($win64);
1290         movaps  -0xd8(%rax),%xmm6
1291         movaps  -0xc8(%rax),%xmm7
1292         movaps  -0xb8(%rax),%xmm8
1293         movaps  -0xa8(%rax),%xmm9
1294         movaps  -0x98(%rax),%xmm10
1295         movaps  -0x88(%rax),%xmm11
1296         movaps  -0x78(%rax),%xmm12
1297         movaps  -0x68(%rax),%xmm13
1298         movaps  -0x58(%rax),%xmm14
1299         movaps  -0x48(%rax),%xmm15
1300 ___
1301 $code.=<<___;
1302         mov     -48(%rax),%r15
1303 .cfi_restore    %r15
1304         mov     -40(%rax),%r14
1305 .cfi_restore    %r14
1306         mov     -32(%rax),%r13
1307 .cfi_restore    %r13
1308         mov     -24(%rax),%r12
1309 .cfi_restore    %r12
1310         mov     -16(%rax),%rbp
1311 .cfi_restore    %rbp
1312         mov     -8(%rax),%rbx
1313 .cfi_restore    %rbx
1314         lea     (%rax),%rsp
1315 .cfi_def_cfa_register   %rsp
1316 .Lepilogue_avx2:
1317         ret
1318 .cfi_endproc
1319 .size   sha256_multi_block_avx2,.-sha256_multi_block_avx2
1320 ___
1321                                         }       }}}
1322 $code.=<<___;
1323 .align  256
1324 K256:
1325 ___
1326 sub TABLE {
1327     foreach (@_) {
1328         $code.=<<___;
1329         .long   $_,$_,$_,$_
1330         .long   $_,$_,$_,$_
1331 ___
1332     }
1333 }
1334 &TABLE( 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,
1335         0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5,
1336         0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,
1337         0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174,
1338         0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,
1339         0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da,
1340         0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,
1341         0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967,
1342         0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,
1343         0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85,
1344         0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,
1345         0xd192e819,0xd6990624,0xf40e3585,0x106aa070,
1346         0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,
1347         0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3,
1348         0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,
1349         0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 );
1350 $code.=<<___;
1351 .Lpbswap:
1352         .long   0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f     # pbswap
1353         .long   0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f     # pbswap
1354 K256_shaext:
1355         .long   0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
1356         .long   0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
1357         .long   0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
1358         .long   0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
1359         .long   0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
1360         .long   0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
1361         .long   0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
1362         .long   0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
1363         .long   0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
1364         .long   0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
1365         .long   0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
1366         .long   0xd192e819,0xd6990624,0xf40e3585,0x106aa070
1367         .long   0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
1368         .long   0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
1369         .long   0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
1370         .long   0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
1371         .asciz  "SHA256 multi-block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
1372 ___
1373
1374 if ($win64) {
1375 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1376 #               CONTEXT *context,DISPATCHER_CONTEXT *disp)
1377 $rec="%rcx";
1378 $frame="%rdx";
1379 $context="%r8";
1380 $disp="%r9";
1381
1382 $code.=<<___;
1383 .extern __imp_RtlVirtualUnwind
1384 .type   se_handler,\@abi-omnipotent
1385 .align  16
1386 se_handler:
1387         push    %rsi
1388         push    %rdi
1389         push    %rbx
1390         push    %rbp
1391         push    %r12
1392         push    %r13
1393         push    %r14
1394         push    %r15
1395         pushfq
1396         sub     \$64,%rsp
1397
1398         mov     120($context),%rax      # pull context->Rax
1399         mov     248($context),%rbx      # pull context->Rip
1400
1401         mov     8($disp),%rsi           # disp->ImageBase
1402         mov     56($disp),%r11          # disp->HandlerData
1403
1404         mov     0(%r11),%r10d           # HandlerData[0]
1405         lea     (%rsi,%r10),%r10        # end of prologue label
1406         cmp     %r10,%rbx               # context->Rip<.Lbody
1407         jb      .Lin_prologue
1408
1409         mov     152($context),%rax      # pull context->Rsp
1410
1411         mov     4(%r11),%r10d           # HandlerData[1]
1412         lea     (%rsi,%r10),%r10        # epilogue label
1413         cmp     %r10,%rbx               # context->Rip>=.Lepilogue
1414         jae     .Lin_prologue
1415
1416         mov     `16*17`(%rax),%rax      # pull saved stack pointer
1417
1418         mov     -8(%rax),%rbx
1419         mov     -16(%rax),%rbp
1420         mov     %rbx,144($context)      # restore context->Rbx
1421         mov     %rbp,160($context)      # restore context->Rbp
1422
1423         lea     -24-10*16(%rax),%rsi
1424         lea     512($context),%rdi      # &context.Xmm6
1425         mov     \$20,%ecx
1426         .long   0xa548f3fc              # cld; rep movsq
1427
1428 .Lin_prologue:
1429         mov     8(%rax),%rdi
1430         mov     16(%rax),%rsi
1431         mov     %rax,152($context)      # restore context->Rsp
1432         mov     %rsi,168($context)      # restore context->Rsi
1433         mov     %rdi,176($context)      # restore context->Rdi
1434
1435         mov     40($disp),%rdi          # disp->ContextRecord
1436         mov     $context,%rsi           # context
1437         mov     \$154,%ecx              # sizeof(CONTEXT)
1438         .long   0xa548f3fc              # cld; rep movsq
1439
1440         mov     $disp,%rsi
1441         xor     %rcx,%rcx               # arg1, UNW_FLAG_NHANDLER
1442         mov     8(%rsi),%rdx            # arg2, disp->ImageBase
1443         mov     0(%rsi),%r8             # arg3, disp->ControlPc
1444         mov     16(%rsi),%r9            # arg4, disp->FunctionEntry
1445         mov     40(%rsi),%r10           # disp->ContextRecord
1446         lea     56(%rsi),%r11           # &disp->HandlerData
1447         lea     24(%rsi),%r12           # &disp->EstablisherFrame
1448         mov     %r10,32(%rsp)           # arg5
1449         mov     %r11,40(%rsp)           # arg6
1450         mov     %r12,48(%rsp)           # arg7
1451         mov     %rcx,56(%rsp)           # arg8, (NULL)
1452         call    *__imp_RtlVirtualUnwind(%rip)
1453
1454         mov     \$1,%eax                # ExceptionContinueSearch
1455         add     \$64,%rsp
1456         popfq
1457         pop     %r15
1458         pop     %r14
1459         pop     %r13
1460         pop     %r12
1461         pop     %rbp
1462         pop     %rbx
1463         pop     %rdi
1464         pop     %rsi
1465         ret
1466 .size   se_handler,.-se_handler
1467 ___
1468 $code.=<<___ if ($avx>1);
1469 .type   avx2_handler,\@abi-omnipotent
1470 .align  16
1471 avx2_handler:
1472         push    %rsi
1473         push    %rdi
1474         push    %rbx
1475         push    %rbp
1476         push    %r12
1477         push    %r13
1478         push    %r14
1479         push    %r15
1480         pushfq
1481         sub     \$64,%rsp
1482
1483         mov     120($context),%rax      # pull context->Rax
1484         mov     248($context),%rbx      # pull context->Rip
1485
1486         mov     8($disp),%rsi           # disp->ImageBase
1487         mov     56($disp),%r11          # disp->HandlerData
1488
1489         mov     0(%r11),%r10d           # HandlerData[0]
1490         lea     (%rsi,%r10),%r10        # end of prologue label
1491         cmp     %r10,%rbx               # context->Rip<body label
1492         jb      .Lin_prologue
1493
1494         mov     152($context),%rax      # pull context->Rsp
1495
1496         mov     4(%r11),%r10d           # HandlerData[1]
1497         lea     (%rsi,%r10),%r10        # epilogue label
1498         cmp     %r10,%rbx               # context->Rip>=epilogue label
1499         jae     .Lin_prologue
1500
1501         mov     `32*17`($context),%rax  # pull saved stack pointer
1502
1503         mov     -8(%rax),%rbx
1504         mov     -16(%rax),%rbp
1505         mov     -24(%rax),%r12
1506         mov     -32(%rax),%r13
1507         mov     -40(%rax),%r14
1508         mov     -48(%rax),%r15
1509         mov     %rbx,144($context)      # restore context->Rbx
1510         mov     %rbp,160($context)      # restore context->Rbp
1511         mov     %r12,216($context)      # restore cotnext->R12
1512         mov     %r13,224($context)      # restore cotnext->R13
1513         mov     %r14,232($context)      # restore cotnext->R14
1514         mov     %r15,240($context)      # restore cotnext->R15
1515
1516         lea     -56-10*16(%rax),%rsi
1517         lea     512($context),%rdi      # &context.Xmm6
1518         mov     \$20,%ecx
1519         .long   0xa548f3fc              # cld; rep movsq
1520
1521         jmp     .Lin_prologue
1522 .size   avx2_handler,.-avx2_handler
1523 ___
1524 $code.=<<___;
1525 .section        .pdata
1526 .align  4
1527         .rva    .LSEH_begin_sha256_multi_block
1528         .rva    .LSEH_end_sha256_multi_block
1529         .rva    .LSEH_info_sha256_multi_block
1530         .rva    .LSEH_begin_sha256_multi_block_shaext
1531         .rva    .LSEH_end_sha256_multi_block_shaext
1532         .rva    .LSEH_info_sha256_multi_block_shaext
1533 ___
1534 $code.=<<___ if ($avx);
1535         .rva    .LSEH_begin_sha256_multi_block_avx
1536         .rva    .LSEH_end_sha256_multi_block_avx
1537         .rva    .LSEH_info_sha256_multi_block_avx
1538 ___
1539 $code.=<<___ if ($avx>1);
1540         .rva    .LSEH_begin_sha256_multi_block_avx2
1541         .rva    .LSEH_end_sha256_multi_block_avx2
1542         .rva    .LSEH_info_sha256_multi_block_avx2
1543 ___
1544 $code.=<<___;
1545 .section        .xdata
1546 .align  8
1547 .LSEH_info_sha256_multi_block:
1548         .byte   9,0,0,0
1549         .rva    se_handler
1550         .rva    .Lbody,.Lepilogue                       # HandlerData[]
1551 .LSEH_info_sha256_multi_block_shaext:
1552         .byte   9,0,0,0
1553         .rva    se_handler
1554         .rva    .Lbody_shaext,.Lepilogue_shaext         # HandlerData[]
1555 ___
1556 $code.=<<___ if ($avx);
1557 .LSEH_info_sha256_multi_block_avx:
1558         .byte   9,0,0,0
1559         .rva    se_handler
1560         .rva    .Lbody_avx,.Lepilogue_avx               # HandlerData[]
1561 ___
1562 $code.=<<___ if ($avx>1);
1563 .LSEH_info_sha256_multi_block_avx2:
1564         .byte   9,0,0,0
1565         .rva    avx2_handler
1566         .rva    .Lbody_avx2,.Lepilogue_avx2             # HandlerData[]
1567 ___
1568 }
1569 ####################################################################
1570
1571 sub rex {
1572   local *opcode=shift;
1573   my ($dst,$src)=@_;
1574   my $rex=0;
1575
1576     $rex|=0x04                  if ($dst>=8);
1577     $rex|=0x01                  if ($src>=8);
1578     unshift @opcode,$rex|0x40   if ($rex);
1579 }
1580
1581 sub sha256op38 {
1582     my $instr = shift;
1583     my %opcodelet = (
1584                 "sha256rnds2" => 0xcb,
1585                 "sha256msg1"  => 0xcc,
1586                 "sha256msg2"  => 0xcd   );
1587
1588     if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1589       my @opcode=(0x0f,0x38);
1590         rex(\@opcode,$2,$1);
1591         push @opcode,$opcodelet{$instr};
1592         push @opcode,0xc0|($1&7)|(($2&7)<<3);           # ModR/M
1593         return ".byte\t".join(',',@opcode);
1594     } else {
1595         return $instr."\t".@_[0];
1596     }
1597 }
1598
1599 foreach (split("\n",$code)) {
1600         s/\`([^\`]*)\`/eval($1)/ge;
1601
1602         s/\b(sha256[^\s]*)\s+(.*)/sha256op38($1,$2)/geo         or
1603
1604         s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go          or
1605         s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go         or
1606         s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go    or
1607         s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go        or
1608         s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go            or
1609         s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go;
1610
1611         print $_,"\n";
1612 }
1613
1614 close STDOUT;