crypto/sha/asm/sha256-mb-x86_64.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2013-2016 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the OpenSSL license (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9
  10 # ====================================================================
  11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  12 # project. The module is, however, dual licensed under OpenSSL and
  13 # CRYPTOGAMS licenses depending on where you obtain it. For further
  14 # details see http://www.openssl.org/~appro/cryptogams/.
  15 # ====================================================================
  16
  17 # Multi-buffer SHA256 procedure processes n buffers in parallel by
  18 # placing buffer data to designated lane of SIMD register. n is
  19 # naturally limited to 4 on pre-AVX2 processors and to 8 on
  20 # AVX2-capable processors such as Haswell.
  21 #
  22 #               this    +aesni(i)       sha256  aesni-sha256    gain(iv)
  23 # -------------------------------------------------------------------
  24 # Westmere(ii)  23.3/n  +1.28=7.11(n=4) 12.3    +3.75=16.1      +126%
  25 # Atom(ii)      38.7/n  +3.93=13.6(n=4) 20.8    +5.69=26.5      +95%
  26 # Sandy Bridge  (20.5   +5.15=25.7)/n   11.6    13.0            +103%
  27 # Ivy Bridge    (20.4   +5.14=25.5)/n   10.3    11.6            +82%
  28 # Haswell(iii)  (21.0   +5.00=26.0)/n   7.80    8.79            +170%
  29 # Skylake       (18.9   +5.00=23.9)/n   7.70    8.17            +170%
  30 # Bulldozer     (21.6   +5.76=27.4)/n   13.6    13.7            +100%
  31 #
  32 # (i)   multi-block CBC encrypt with 128-bit key;
  33 # (ii)  (HASH+AES)/n does not apply to Westmere for n>3 and Atom,
  34 #       because of lower AES-NI instruction throughput, nor is there
  35 #       AES-NI-SHA256 stitch for these processors;
  36 # (iii) "this" is for n=8, when we gather twice as much data, result
  37 #       for n=4 is 20.3+4.44=24.7;
  38 # (iv)  presented improvement coefficients are asymptotic limits and
  39 #       in real-life application are somewhat lower, e.g. for 2KB
  40 #       fragments they range from 75% to 130% (on Haswell);
  41
  42 $flavour = shift;
  43 $output  = shift;
  44 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
  45
  46 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  47
  48 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  49 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  50 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
  51 die "can't locate x86_64-xlate.pl";
  52
  53 $avx=0;
  54
  55 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  56                 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  57         $avx = ($1>=2.19) + ($1>=2.22);
  58 }
  59
  60 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
  61            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
  62         $avx = ($1>=2.09) + ($1>=2.10);
  63 }
  64
  65 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
  66            `ml64 2>&1` =~ /Version ([0-9]+)\./) {
  67         $avx = ($1>=10) + ($1>=11);
  68 }
  69
  70 if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) {
  71         $avx = ($2>=3.0) + ($2>3.0);
  72 }
  73
  74 open OUT,"| \"$^X\" $xlate $flavour $output";
  75 *STDOUT=*OUT;
  76
  77 # void sha256_multi_block (
  78 #     struct {  unsigned int A[8];
  79 #               unsigned int B[8];
  80 #               unsigned int C[8];
  81 #               unsigned int D[8];
  82 #               unsigned int E[8];
  83 #               unsigned int F[8];
  84 #               unsigned int G[8];
  85 #               unsigned int H[8];      } *ctx,
  86 #     struct {  void *ptr; int blocks;  } inp[8],
  87 #     int num);         /* 1 or 2 */
  88 #
  89 $ctx="%rdi";    # 1st arg
  90 $inp="%rsi";    # 2nd arg
  91 $num="%edx";    # 3rd arg
  92 @ptr=map("%r$_",(8..11));
  93 $Tbl="%rbp";
  94
  95 @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("%xmm$_",(8..15));
  96 ($t1,$t2,$t3,$axb,$bxc,$Xi,$Xn,$sigma)=map("%xmm$_",(0..7));
  97
  98 $REG_SZ=16;
  99
 100 sub Xi_off {
 101 my $off = shift;
 102
 103     $off %= 16; $off *= $REG_SZ;
 104     $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)";
 105 }
 106
 107 sub ROUND_00_15 {
 108 my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
 109
 110 $code.=<<___ if ($i<15);
 111         movd            `4*$i`(@ptr[0]),$Xi
 112         movd            `4*$i`(@ptr[1]),$t1
 113         movd            `4*$i`(@ptr[2]),$t2
 114         movd            `4*$i`(@ptr[3]),$t3
 115         punpckldq       $t2,$Xi
 116         punpckldq       $t3,$t1
 117         punpckldq       $t1,$Xi
 118 ___
 119 $code.=<<___ if ($i==15);
 120         movd            `4*$i`(@ptr[0]),$Xi
 121          lea            `16*4`(@ptr[0]),@ptr[0]
 122         movd            `4*$i`(@ptr[1]),$t1
 123          lea            `16*4`(@ptr[1]),@ptr[1]
 124         movd            `4*$i`(@ptr[2]),$t2
 125          lea            `16*4`(@ptr[2]),@ptr[2]
 126         movd            `4*$i`(@ptr[3]),$t3
 127          lea            `16*4`(@ptr[3]),@ptr[3]
 128         punpckldq       $t2,$Xi
 129         punpckldq       $t3,$t1
 130         punpckldq       $t1,$Xi
 131 ___
 132 $code.=<<___;
 133         movdqa  $e,$sigma
 134         `"pshufb        $Xn,$Xi"                if ($i<=15 && ($i&1)==0)`
 135         movdqa  $e,$t3
 136         `"pshufb        $Xn,$Xi"                if ($i<=15 && ($i&1)==1)`
 137         psrld   \$6,$sigma
 138         movdqa  $e,$t2
 139         pslld   \$7,$t3
 140         movdqa  $Xi,`&Xi_off($i)`
 141          paddd  $h,$Xi                          # Xi+=h
 142
 143         psrld   \$11,$t2
 144         pxor    $t3,$sigma
 145         pslld   \$21-7,$t3
 146          paddd  `32*($i%8)-128`($Tbl),$Xi       # Xi+=K[round]
 147         pxor    $t2,$sigma
 148
 149         psrld   \$25-11,$t2
 150          movdqa $e,$t1
 151          `"prefetcht0   63(@ptr[0])"            if ($i==15)`
 152         pxor    $t3,$sigma
 153          movdqa $e,$axb                         # borrow $axb
 154         pslld   \$26-21,$t3
 155          pandn  $g,$t1
 156          pand   $f,$axb
 157         pxor    $t2,$sigma
 158
 159          `"prefetcht0   63(@ptr[1])"            if ($i==15)`
 160         movdqa  $a,$t2
 161         pxor    $t3,$sigma                      # Sigma1(e)
 162         movdqa  $a,$t3
 163         psrld   \$2,$t2
 164         paddd   $sigma,$Xi                      # Xi+=Sigma1(e)
 165          pxor   $axb,$t1                        # Ch(e,f,g)
 166          movdqa $b,$axb
 167         movdqa  $a,$sigma
 168         pslld   \$10,$t3
 169          pxor   $a,$axb                         # a^b, b^c in next round
 170
 171          `"prefetcht0   63(@ptr[2])"            if ($i==15)`
 172         psrld   \$13,$sigma
 173         pxor    $t3,$t2
 174          paddd  $t1,$Xi                         # Xi+=Ch(e,f,g)
 175         pslld   \$19-10,$t3
 176          pand   $axb,$bxc
 177         pxor    $sigma,$t2
 178
 179          `"prefetcht0   63(@ptr[3])"            if ($i==15)`
 180         psrld   \$22-13,$sigma
 181         pxor    $t3,$t2
 182          movdqa $b,$h
 183         pslld   \$30-19,$t3
 184         pxor    $t2,$sigma
 185          pxor   $bxc,$h                         # h=Maj(a,b,c)=Ch(a^b,c,b)
 186          paddd  $Xi,$d                          # d+=Xi
 187         pxor    $t3,$sigma                      # Sigma0(a)
 188
 189         paddd   $Xi,$h                          # h+=Xi
 190         paddd   $sigma,$h                       # h+=Sigma0(a)
 191 ___
 192 $code.=<<___ if (($i%8)==7);
 193         lea     `32*8`($Tbl),$Tbl
 194 ___
 195         ($axb,$bxc)=($bxc,$axb);
 196 }
 197
 198 sub ROUND_16_XX {
 199 my $i=shift;
 200
 201 $code.=<<___;
 202         movdqa  `&Xi_off($i+1)`,$Xn
 203         paddd   `&Xi_off($i+9)`,$Xi             # Xi+=X[i+9]
 204
 205         movdqa  $Xn,$sigma
 206         movdqa  $Xn,$t2
 207         psrld   \$3,$sigma
 208         movdqa  $Xn,$t3
 209
 210         psrld   \$7,$t2
 211         movdqa  `&Xi_off($i+14)`,$t1
 212         pslld   \$14,$t3
 213         pxor    $t2,$sigma
 214         psrld   \$18-7,$t2
 215         movdqa  $t1,$axb                        # borrow $axb
 216         pxor    $t3,$sigma
 217         pslld   \$25-14,$t3
 218         pxor    $t2,$sigma
 219         psrld   \$10,$t1
 220         movdqa  $axb,$t2
 221
 222         psrld   \$17,$axb
 223         pxor    $t3,$sigma                      # sigma0(X[i+1])
 224         pslld   \$13,$t2
 225          paddd  $sigma,$Xi                      # Xi+=sigma0(e)
 226         pxor    $axb,$t1
 227         psrld   \$19-17,$axb
 228         pxor    $t2,$t1
 229         pslld   \$15-13,$t2
 230         pxor    $axb,$t1
 231         pxor    $t2,$t1                         # sigma0(X[i+14])
 232         paddd   $t1,$Xi                         # Xi+=sigma1(X[i+14])
 233 ___
 234         &ROUND_00_15($i,@_);
 235         ($Xi,$Xn)=($Xn,$Xi);
 236 }
 237
 238 $code.=<<___;
 239 .text
 240
 241 .extern OPENSSL_ia32cap_P
 242
 243 .globl  sha256_multi_block
 244 .type   sha256_multi_block,\@function,3
 245 .align  32
 246 sha256_multi_block:
 247         mov     OPENSSL_ia32cap_P+4(%rip),%rcx
 248         bt      \$61,%rcx                       # check SHA bit
 249         jc      _shaext_shortcut
 250 ___
 251 $code.=<<___ if ($avx);
 252         test    \$`1<<28`,%ecx
 253         jnz     _avx_shortcut
 254 ___
 255 $code.=<<___;
 256         mov     %rsp,%rax
 257         push    %rbx
 258         push    %rbp
 259 ___
 260 $code.=<<___ if ($win64);
 261         lea     -0xa8(%rsp),%rsp
 262         movaps  %xmm6,(%rsp)
 263         movaps  %xmm7,0x10(%rsp)
 264         movaps  %xmm8,0x20(%rsp)
 265         movaps  %xmm9,0x30(%rsp)
 266         movaps  %xmm10,-0x78(%rax)
 267         movaps  %xmm11,-0x68(%rax)
 268         movaps  %xmm12,-0x58(%rax)
 269         movaps  %xmm13,-0x48(%rax)
 270         movaps  %xmm14,-0x38(%rax)
 271         movaps  %xmm15,-0x28(%rax)
 272 ___
 273 $code.=<<___;
 274         sub     \$`$REG_SZ*18`, %rsp
 275         and     \$-256,%rsp
 276         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
 277 .Lbody:
 278         lea     K256+128(%rip),$Tbl
 279         lea     `$REG_SZ*16`(%rsp),%rbx
 280         lea     0x80($ctx),$ctx                 # size optimization
 281
 282 .Loop_grande:
 283         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
 284         xor     $num,$num
 285 ___
 286 for($i=0;$i<4;$i++) {
 287     $code.=<<___;
 288         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
 289         mov     `16*$i+8`($inp),%ecx            # number of blocks
 290         cmp     $num,%ecx
 291         cmovg   %ecx,$num                       # find maximum
 292         test    %ecx,%ecx
 293         mov     %ecx,`4*$i`(%rbx)               # initialize counters
 294         cmovle  $Tbl,@ptr[$i]                   # cancel input
 295 ___
 296 }
 297 $code.=<<___;
 298         test    $num,$num
 299         jz      .Ldone
 300
 301         movdqu  0x00-0x80($ctx),$A              # load context
 302          lea    128(%rsp),%rax
 303         movdqu  0x20-0x80($ctx),$B
 304         movdqu  0x40-0x80($ctx),$C
 305         movdqu  0x60-0x80($ctx),$D
 306         movdqu  0x80-0x80($ctx),$E
 307         movdqu  0xa0-0x80($ctx),$F
 308         movdqu  0xc0-0x80($ctx),$G
 309         movdqu  0xe0-0x80($ctx),$H
 310         movdqu  .Lpbswap(%rip),$Xn
 311         jmp     .Loop
 312
 313 .align  32
 314 .Loop:
 315         movdqa  $C,$bxc
 316         pxor    $B,$bxc                         # magic seed
 317 ___
 318 for($i=0;$i<16;$i++)    { &ROUND_00_15($i,@V); unshift(@V,pop(@V)); }
 319 $code.=<<___;
 320         movdqu  `&Xi_off($i)`,$Xi
 321         mov     \$3,%ecx
 322         jmp     .Loop_16_xx
 323 .align  32
 324 .Loop_16_xx:
 325 ___
 326 for(;$i<32;$i++)        { &ROUND_16_XX($i,@V); unshift(@V,pop(@V)); }
 327 $code.=<<___;
 328         dec     %ecx
 329         jnz     .Loop_16_xx
 330
 331         mov     \$1,%ecx
 332         lea     K256+128(%rip),$Tbl
 333
 334         movdqa  (%rbx),$sigma                   # pull counters
 335         cmp     4*0(%rbx),%ecx                  # examine counters
 336         pxor    $t1,$t1
 337         cmovge  $Tbl,@ptr[0]                    # cancel input
 338         cmp     4*1(%rbx),%ecx
 339         movdqa  $sigma,$Xn
 340         cmovge  $Tbl,@ptr[1]
 341         cmp     4*2(%rbx),%ecx
 342         pcmpgtd $t1,$Xn                         # mask value
 343         cmovge  $Tbl,@ptr[2]
 344         cmp     4*3(%rbx),%ecx
 345         paddd   $Xn,$sigma                      # counters--
 346         cmovge  $Tbl,@ptr[3]
 347
 348         movdqu  0x00-0x80($ctx),$t1
 349         pand    $Xn,$A
 350         movdqu  0x20-0x80($ctx),$t2
 351         pand    $Xn,$B
 352         movdqu  0x40-0x80($ctx),$t3
 353         pand    $Xn,$C
 354         movdqu  0x60-0x80($ctx),$Xi
 355         pand    $Xn,$D
 356         paddd   $t1,$A
 357         movdqu  0x80-0x80($ctx),$t1
 358         pand    $Xn,$E
 359         paddd   $t2,$B
 360         movdqu  0xa0-0x80($ctx),$t2
 361         pand    $Xn,$F
 362         paddd   $t3,$C
 363         movdqu  0xc0-0x80($ctx),$t3
 364         pand    $Xn,$G
 365         paddd   $Xi,$D
 366         movdqu  0xe0-0x80($ctx),$Xi
 367         pand    $Xn,$H
 368         paddd   $t1,$E
 369         paddd   $t2,$F
 370         movdqu  $A,0x00-0x80($ctx)
 371         paddd   $t3,$G
 372         movdqu  $B,0x20-0x80($ctx)
 373         paddd   $Xi,$H
 374         movdqu  $C,0x40-0x80($ctx)
 375         movdqu  $D,0x60-0x80($ctx)
 376         movdqu  $E,0x80-0x80($ctx)
 377         movdqu  $F,0xa0-0x80($ctx)
 378         movdqu  $G,0xc0-0x80($ctx)
 379         movdqu  $H,0xe0-0x80($ctx)
 380
 381         movdqa  $sigma,(%rbx)                   # save counters
 382         movdqa  .Lpbswap(%rip),$Xn
 383         dec     $num
 384         jnz     .Loop
 385
 386         mov     `$REG_SZ*17+8`(%rsp),$num
 387         lea     $REG_SZ($ctx),$ctx
 388         lea     `16*$REG_SZ/4`($inp),$inp
 389         dec     $num
 390         jnz     .Loop_grande
 391
 392 .Ldone:
 393         mov     `$REG_SZ*17`(%rsp),%rax         # original %rsp
 394 ___
 395 $code.=<<___ if ($win64);
 396         movaps  -0xb8(%rax),%xmm6
 397         movaps  -0xa8(%rax),%xmm7
 398         movaps  -0x98(%rax),%xmm8
 399         movaps  -0x88(%rax),%xmm9
 400         movaps  -0x78(%rax),%xmm10
 401         movaps  -0x68(%rax),%xmm11
 402         movaps  -0x58(%rax),%xmm12
 403         movaps  -0x48(%rax),%xmm13
 404         movaps  -0x38(%rax),%xmm14
 405         movaps  -0x28(%rax),%xmm15
 406 ___
 407 $code.=<<___;
 408         mov     -16(%rax),%rbp
 409         mov     -8(%rax),%rbx
 410         lea     (%rax),%rsp
 411 .Lepilogue:
 412         ret
 413 .size   sha256_multi_block,.-sha256_multi_block
 414 ___
 415                                                 {{{
 416 my ($Wi,$TMP0,$TMP1,$TMPx,$ABEF0,$CDGH0,$ABEF1,$CDGH1)=map("%xmm$_",(0..3,12..15));
 417 my @MSG0=map("%xmm$_",(4..7));
 418 my @MSG1=map("%xmm$_",(8..11));
 419
 420 $code.=<<___;
 421 .type   sha256_multi_block_shaext,\@function,3
 422 .align  32
 423 sha256_multi_block_shaext:
 424 _shaext_shortcut:
 425         mov     %rsp,%rax
 426         push    %rbx
 427         push    %rbp
 428 ___
 429 $code.=<<___ if ($win64);
 430         lea     -0xa8(%rsp),%rsp
 431         movaps  %xmm6,(%rsp)
 432         movaps  %xmm7,0x10(%rsp)
 433         movaps  %xmm8,0x20(%rsp)
 434         movaps  %xmm9,0x30(%rsp)
 435         movaps  %xmm10,-0x78(%rax)
 436         movaps  %xmm11,-0x68(%rax)
 437         movaps  %xmm12,-0x58(%rax)
 438         movaps  %xmm13,-0x48(%rax)
 439         movaps  %xmm14,-0x38(%rax)
 440         movaps  %xmm15,-0x28(%rax)
 441 ___
 442 $code.=<<___;
 443         sub     \$`$REG_SZ*18`,%rsp
 444         shl     \$1,$num                        # we process pair at a time
 445         and     \$-256,%rsp
 446         lea     0x80($ctx),$ctx                 # size optimization
 447         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
 448 .Lbody_shaext:
 449         lea     `$REG_SZ*16`(%rsp),%rbx
 450         lea     K256_shaext+0x80(%rip),$Tbl
 451
 452 .Loop_grande_shaext:
 453         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
 454         xor     $num,$num
 455 ___
 456 for($i=0;$i<2;$i++) {
 457     $code.=<<___;
 458         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
 459         mov     `16*$i+8`($inp),%ecx            # number of blocks
 460         cmp     $num,%ecx
 461         cmovg   %ecx,$num                       # find maximum
 462         test    %ecx,%ecx
 463         mov     %ecx,`4*$i`(%rbx)               # initialize counters
 464         cmovle  %rsp,@ptr[$i]                   # cancel input
 465 ___
 466 }
 467 $code.=<<___;
 468         test    $num,$num
 469         jz      .Ldone_shaext
 470
 471         movq            0x00-0x80($ctx),$ABEF0          # A1.A0
 472         movq            0x20-0x80($ctx),@MSG0[0]        # B1.B0
 473         movq            0x40-0x80($ctx),$CDGH0          # C1.C0
 474         movq            0x60-0x80($ctx),@MSG0[1]        # D1.D0
 475         movq            0x80-0x80($ctx),@MSG1[0]        # E1.E0
 476         movq            0xa0-0x80($ctx),@MSG1[1]        # F1.F0
 477         movq            0xc0-0x80($ctx),@MSG1[2]        # G1.G0
 478         movq            0xe0-0x80($ctx),@MSG1[3]        # H1.H0
 479
 480         punpckldq       @MSG0[0],$ABEF0                 # B1.A1.B0.A0
 481         punpckldq       @MSG0[1],$CDGH0                 # D1.C1.D0.C0
 482         punpckldq       @MSG1[1],@MSG1[0]               # F1.E1.F0.E0
 483         punpckldq       @MSG1[3],@MSG1[2]               # H1.G1.H0.G0
 484         movdqa          K256_shaext-0x10(%rip),$TMPx    # byte swap
 485
 486         movdqa          $ABEF0,$ABEF1
 487         movdqa          $CDGH0,$CDGH1
 488         punpcklqdq      @MSG1[0],$ABEF0                 # F0.E0.B0.A0
 489         punpcklqdq      @MSG1[2],$CDGH0                 # H0.G0.D0.C0
 490         punpckhqdq      @MSG1[0],$ABEF1                 # F1.E1.B1.A1
 491         punpckhqdq      @MSG1[2],$CDGH1                 # H1.G1.D1.C1
 492
 493         pshufd          \$0b00011011,$ABEF0,$ABEF0
 494         pshufd          \$0b00011011,$CDGH0,$CDGH0
 495         pshufd          \$0b00011011,$ABEF1,$ABEF1
 496         pshufd          \$0b00011011,$CDGH1,$CDGH1
 497         jmp             .Loop_shaext
 498
 499 .align  32
 500 .Loop_shaext:
 501         movdqu          0x00(@ptr[0]),@MSG0[0]
 502          movdqu         0x00(@ptr[1]),@MSG1[0]
 503         movdqu          0x10(@ptr[0]),@MSG0[1]
 504          movdqu         0x10(@ptr[1]),@MSG1[1]
 505         movdqu          0x20(@ptr[0]),@MSG0[2]
 506         pshufb          $TMPx,@MSG0[0]
 507          movdqu         0x20(@ptr[1]),@MSG1[2]
 508          pshufb         $TMPx,@MSG1[0]
 509         movdqu          0x30(@ptr[0]),@MSG0[3]
 510         lea             0x40(@ptr[0]),@ptr[0]
 511          movdqu         0x30(@ptr[1]),@MSG1[3]
 512          lea            0x40(@ptr[1]),@ptr[1]
 513
 514         movdqa          0*16-0x80($Tbl),$Wi
 515         pshufb          $TMPx,@MSG0[1]
 516         paddd           @MSG0[0],$Wi
 517         pxor            $ABEF0,@MSG0[0]         # black magic
 518         movdqa          $Wi,$TMP0
 519          movdqa         0*16-0x80($Tbl),$TMP1
 520          pshufb         $TMPx,@MSG1[1]
 521          paddd          @MSG1[0],$TMP1
 522         movdqa          $CDGH0,0x50(%rsp)       # offload
 523         sha256rnds2     $ABEF0,$CDGH0           # 0-3
 524          pxor           $ABEF1,@MSG1[0]         # black magic
 525          movdqa         $TMP1,$Wi
 526          movdqa         $CDGH1,0x70(%rsp)
 527          sha256rnds2    $ABEF1,$CDGH1           # 0-3
 528         pshufd          \$0x0e,$TMP0,$Wi
 529         pxor            $ABEF0,@MSG0[0]         # black magic
 530         movdqa          $ABEF0,0x40(%rsp)       # offload
 531         sha256rnds2     $CDGH0,$ABEF0
 532          pshufd         \$0x0e,$TMP1,$Wi
 533          pxor           $ABEF1,@MSG1[0]         # black magic
 534          movdqa         $ABEF1,0x60(%rsp)
 535         movdqa          1*16-0x80($Tbl),$TMP0
 536         paddd           @MSG0[1],$TMP0
 537         pshufb          $TMPx,@MSG0[2]
 538          sha256rnds2    $CDGH1,$ABEF1
 539
 540         movdqa          $TMP0,$Wi
 541          movdqa         1*16-0x80($Tbl),$TMP1
 542          paddd          @MSG1[1],$TMP1
 543         sha256rnds2     $ABEF0,$CDGH0           # 4-7
 544          movdqa         $TMP1,$Wi
 545         prefetcht0      127(@ptr[0])
 546         pshufb          $TMPx,@MSG0[3]
 547          pshufb         $TMPx,@MSG1[2]
 548          prefetcht0     127(@ptr[1])
 549          sha256rnds2    $ABEF1,$CDGH1           # 4-7
 550         pshufd          \$0x0e,$TMP0,$Wi
 551          pshufb         $TMPx,@MSG1[3]
 552         sha256msg1      @MSG0[1],@MSG0[0]
 553         sha256rnds2     $CDGH0,$ABEF0
 554          pshufd         \$0x0e,$TMP1,$Wi
 555         movdqa          2*16-0x80($Tbl),$TMP0
 556         paddd           @MSG0[2],$TMP0
 557          sha256rnds2    $CDGH1,$ABEF1
 558
 559         movdqa          $TMP0,$Wi
 560          movdqa         2*16-0x80($Tbl),$TMP1
 561          paddd          @MSG1[2],$TMP1
 562         sha256rnds2     $ABEF0,$CDGH0           # 8-11
 563          sha256msg1     @MSG1[1],@MSG1[0]
 564          movdqa         $TMP1,$Wi
 565         movdqa          @MSG0[3],$TMPx
 566          sha256rnds2    $ABEF1,$CDGH1           # 8-11
 567         pshufd          \$0x0e,$TMP0,$Wi
 568         palignr         \$4,@MSG0[2],$TMPx
 569         paddd           $TMPx,@MSG0[0]
 570          movdqa         @MSG1[3],$TMPx
 571          palignr        \$4,@MSG1[2],$TMPx
 572         sha256msg1      @MSG0[2],@MSG0[1]
 573         sha256rnds2     $CDGH0,$ABEF0
 574          pshufd         \$0x0e,$TMP1,$Wi
 575         movdqa          3*16-0x80($Tbl),$TMP0
 576         paddd           @MSG0[3],$TMP0
 577          sha256rnds2    $CDGH1,$ABEF1
 578          sha256msg1     @MSG1[2],@MSG1[1]
 579
 580         movdqa          $TMP0,$Wi
 581          movdqa         3*16-0x80($Tbl),$TMP1
 582          paddd          $TMPx,@MSG1[0]
 583          paddd          @MSG1[3],$TMP1
 584         sha256msg2      @MSG0[3],@MSG0[0]
 585         sha256rnds2     $ABEF0,$CDGH0           # 12-15
 586          movdqa         $TMP1,$Wi
 587         movdqa          @MSG0[0],$TMPx
 588         palignr         \$4,@MSG0[3],$TMPx
 589          sha256rnds2    $ABEF1,$CDGH1           # 12-15
 590          sha256msg2     @MSG1[3],@MSG1[0]
 591         pshufd          \$0x0e,$TMP0,$Wi
 592         paddd           $TMPx,@MSG0[1]
 593          movdqa         @MSG1[0],$TMPx
 594          palignr        \$4,@MSG1[3],$TMPx
 595         sha256msg1      @MSG0[3],@MSG0[2]
 596         sha256rnds2     $CDGH0,$ABEF0
 597          pshufd         \$0x0e,$TMP1,$Wi
 598         movdqa          4*16-0x80($Tbl),$TMP0
 599         paddd           @MSG0[0],$TMP0
 600          sha256rnds2    $CDGH1,$ABEF1
 601          sha256msg1     @MSG1[3],@MSG1[2]
 602 ___
 603 for($i=4;$i<16-3;$i++) {
 604 $code.=<<___;
 605         movdqa          $TMP0,$Wi
 606          movdqa         $i*16-0x80($Tbl),$TMP1
 607          paddd          $TMPx,@MSG1[1]
 608          paddd          @MSG1[0],$TMP1
 609         sha256msg2      @MSG0[0],@MSG0[1]
 610         sha256rnds2     $ABEF0,$CDGH0           # 16-19...
 611          movdqa         $TMP1,$Wi
 612         movdqa          @MSG0[1],$TMPx
 613         palignr         \$4,@MSG0[0],$TMPx
 614          sha256rnds2    $ABEF1,$CDGH1           # 16-19...
 615          sha256msg2     @MSG1[0],@MSG1[1]
 616         pshufd          \$0x0e,$TMP0,$Wi
 617         paddd           $TMPx,@MSG0[2]
 618          movdqa         @MSG1[1],$TMPx
 619          palignr        \$4,@MSG1[0],$TMPx
 620         sha256msg1      @MSG0[0],@MSG0[3]
 621         sha256rnds2     $CDGH0,$ABEF0
 622          pshufd         \$0x0e,$TMP1,$Wi
 623         movdqa          `($i+1)*16`-0x80($Tbl),$TMP0
 624         paddd           @MSG0[1],$TMP0
 625          sha256rnds2    $CDGH1,$ABEF1
 626          sha256msg1     @MSG1[0],@MSG1[3]
 627 ___
 628         push(@MSG0,shift(@MSG0));       push(@MSG1,shift(@MSG1));
 629 }
 630 $code.=<<___;
 631         movdqa          $TMP0,$Wi
 632          movdqa         13*16-0x80($Tbl),$TMP1
 633          paddd          $TMPx,@MSG1[1]
 634          paddd          @MSG1[0],$TMP1
 635         sha256msg2      @MSG0[0],@MSG0[1]
 636         sha256rnds2     $ABEF0,$CDGH0           # 52-55
 637          movdqa         $TMP1,$Wi
 638         movdqa          @MSG0[1],$TMPx
 639         palignr         \$4,@MSG0[0],$TMPx
 640          sha256rnds2    $ABEF1,$CDGH1           # 52-55
 641          sha256msg2     @MSG1[0],@MSG1[1]
 642         pshufd          \$0x0e,$TMP0,$Wi
 643         paddd           $TMPx,@MSG0[2]
 644          movdqa         @MSG1[1],$TMPx
 645          palignr        \$4,@MSG1[0],$TMPx
 646         nop
 647         sha256rnds2     $CDGH0,$ABEF0
 648          pshufd         \$0x0e,$TMP1,$Wi
 649         movdqa          14*16-0x80($Tbl),$TMP0
 650         paddd           @MSG0[1],$TMP0
 651          sha256rnds2    $CDGH1,$ABEF1
 652
 653         movdqa          $TMP0,$Wi
 654          movdqa         14*16-0x80($Tbl),$TMP1
 655          paddd          $TMPx,@MSG1[2]
 656          paddd          @MSG1[1],$TMP1
 657         sha256msg2      @MSG0[1],@MSG0[2]
 658         nop
 659         sha256rnds2     $ABEF0,$CDGH0           # 56-59
 660          movdqa         $TMP1,$Wi
 661           mov           \$1,%ecx
 662           pxor          @MSG0[1],@MSG0[1]       # zero
 663          sha256rnds2    $ABEF1,$CDGH1           # 56-59
 664          sha256msg2     @MSG1[1],@MSG1[2]
 665         pshufd          \$0x0e,$TMP0,$Wi
 666         movdqa          15*16-0x80($Tbl),$TMP0
 667         paddd           @MSG0[2],$TMP0
 668           movq          (%rbx),@MSG0[2]         # pull counters
 669           nop
 670         sha256rnds2     $CDGH0,$ABEF0
 671          pshufd         \$0x0e,$TMP1,$Wi
 672          movdqa         15*16-0x80($Tbl),$TMP1
 673          paddd          @MSG1[2],$TMP1
 674          sha256rnds2    $CDGH1,$ABEF1
 675
 676         movdqa          $TMP0,$Wi
 677           cmp           4*0(%rbx),%ecx          # examine counters
 678           cmovge        %rsp,@ptr[0]            # cancel input
 679           cmp           4*1(%rbx),%ecx
 680           cmovge        %rsp,@ptr[1]
 681           pshufd        \$0x00,@MSG0[2],@MSG1[0]
 682         sha256rnds2     $ABEF0,$CDGH0           # 60-63
 683          movdqa         $TMP1,$Wi
 684           pshufd        \$0x55,@MSG0[2],@MSG1[1]
 685           movdqa        @MSG0[2],@MSG1[2]
 686          sha256rnds2    $ABEF1,$CDGH1           # 60-63
 687         pshufd          \$0x0e,$TMP0,$Wi
 688           pcmpgtd       @MSG0[1],@MSG1[0]
 689           pcmpgtd       @MSG0[1],@MSG1[1]
 690         sha256rnds2     $CDGH0,$ABEF0
 691          pshufd         \$0x0e,$TMP1,$Wi
 692           pcmpgtd       @MSG0[1],@MSG1[2]       # counter mask
 693           movdqa        K256_shaext-0x10(%rip),$TMPx
 694          sha256rnds2    $CDGH1,$ABEF1
 695
 696         pand            @MSG1[0],$CDGH0
 697          pand           @MSG1[1],$CDGH1
 698         pand            @MSG1[0],$ABEF0
 699          pand           @MSG1[1],$ABEF1
 700         paddd           @MSG0[2],@MSG1[2]       # counters--
 701
 702         paddd           0x50(%rsp),$CDGH0
 703          paddd          0x70(%rsp),$CDGH1
 704         paddd           0x40(%rsp),$ABEF0
 705          paddd          0x60(%rsp),$ABEF1
 706
 707         movq            @MSG1[2],(%rbx)         # save counters
 708         dec             $num
 709         jnz             .Loop_shaext
 710
 711         mov             `$REG_SZ*17+8`(%rsp),$num
 712
 713         pshufd          \$0b00011011,$ABEF0,$ABEF0
 714         pshufd          \$0b00011011,$CDGH0,$CDGH0
 715         pshufd          \$0b00011011,$ABEF1,$ABEF1
 716         pshufd          \$0b00011011,$CDGH1,$CDGH1
 717
 718         movdqa          $ABEF0,@MSG0[0]
 719         movdqa          $CDGH0,@MSG0[1]
 720         punpckldq       $ABEF1,$ABEF0                   # B1.B0.A1.A0
 721         punpckhdq       $ABEF1,@MSG0[0]                 # F1.F0.E1.E0
 722         punpckldq       $CDGH1,$CDGH0                   # D1.D0.C1.C0
 723         punpckhdq       $CDGH1,@MSG0[1]                 # H1.H0.G1.G0
 724
 725         movq            $ABEF0,0x00-0x80($ctx)          # A1.A0
 726         psrldq          \$8,$ABEF0
 727         movq            @MSG0[0],0x80-0x80($ctx)        # E1.E0
 728         psrldq          \$8,@MSG0[0]
 729         movq            $ABEF0,0x20-0x80($ctx)          # B1.B0
 730         movq            @MSG0[0],0xa0-0x80($ctx)        # F1.F0
 731
 732         movq            $CDGH0,0x40-0x80($ctx)          # C1.C0
 733         psrldq          \$8,$CDGH0
 734         movq            @MSG0[1],0xc0-0x80($ctx)        # G1.G0
 735         psrldq          \$8,@MSG0[1]
 736         movq            $CDGH0,0x60-0x80($ctx)          # D1.D0
 737         movq            @MSG0[1],0xe0-0x80($ctx)        # H1.H0
 738
 739         lea     `$REG_SZ/2`($ctx),$ctx
 740         lea     `16*2`($inp),$inp
 741         dec     $num
 742         jnz     .Loop_grande_shaext
 743
 744 .Ldone_shaext:
 745         #mov    `$REG_SZ*17`(%rsp),%rax         # original %rsp
 746 ___
 747 $code.=<<___ if ($win64);
 748         movaps  -0xb8(%rax),%xmm6
 749         movaps  -0xa8(%rax),%xmm7
 750         movaps  -0x98(%rax),%xmm8
 751         movaps  -0x88(%rax),%xmm9
 752         movaps  -0x78(%rax),%xmm10
 753         movaps  -0x68(%rax),%xmm11
 754         movaps  -0x58(%rax),%xmm12
 755         movaps  -0x48(%rax),%xmm13
 756         movaps  -0x38(%rax),%xmm14
 757         movaps  -0x28(%rax),%xmm15
 758 ___
 759 $code.=<<___;
 760         mov     -16(%rax),%rbp
 761         mov     -8(%rax),%rbx
 762         lea     (%rax),%rsp
 763 .Lepilogue_shaext:
 764         ret
 765 .size   sha256_multi_block_shaext,.-sha256_multi_block_shaext
 766 ___
 767                                                 }}}
 768                                                 if ($avx) {{{
 769 sub ROUND_00_15_avx {
 770 my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
 771
 772 $code.=<<___ if ($i<15 && $REG_SZ==16);
 773         vmovd           `4*$i`(@ptr[0]),$Xi
 774         vmovd           `4*$i`(@ptr[1]),$t1
 775         vpinsrd         \$1,`4*$i`(@ptr[2]),$Xi,$Xi
 776         vpinsrd         \$1,`4*$i`(@ptr[3]),$t1,$t1
 777         vpunpckldq      $t1,$Xi,$Xi
 778         vpshufb         $Xn,$Xi,$Xi
 779 ___
 780 $code.=<<___ if ($i==15 && $REG_SZ==16);
 781         vmovd           `4*$i`(@ptr[0]),$Xi
 782          lea            `16*4`(@ptr[0]),@ptr[0]
 783         vmovd           `4*$i`(@ptr[1]),$t1
 784          lea            `16*4`(@ptr[1]),@ptr[1]
 785         vpinsrd         \$1,`4*$i`(@ptr[2]),$Xi,$Xi
 786          lea            `16*4`(@ptr[2]),@ptr[2]
 787         vpinsrd         \$1,`4*$i`(@ptr[3]),$t1,$t1
 788          lea            `16*4`(@ptr[3]),@ptr[3]
 789         vpunpckldq      $t1,$Xi,$Xi
 790         vpshufb         $Xn,$Xi,$Xi
 791 ___
 792 $code.=<<___ if ($i<15 && $REG_SZ==32);
 793         vmovd           `4*$i`(@ptr[0]),$Xi
 794         vmovd           `4*$i`(@ptr[4]),$t1
 795         vmovd           `4*$i`(@ptr[1]),$t2
 796         vmovd           `4*$i`(@ptr[5]),$t3
 797         vpinsrd         \$1,`4*$i`(@ptr[2]),$Xi,$Xi
 798         vpinsrd         \$1,`4*$i`(@ptr[6]),$t1,$t1
 799         vpinsrd         \$1,`4*$i`(@ptr[3]),$t2,$t2
 800         vpunpckldq      $t2,$Xi,$Xi
 801         vpinsrd         \$1,`4*$i`(@ptr[7]),$t3,$t3
 802         vpunpckldq      $t3,$t1,$t1
 803         vinserti128     $t1,$Xi,$Xi
 804         vpshufb         $Xn,$Xi,$Xi
 805 ___
 806 $code.=<<___ if ($i==15 && $REG_SZ==32);
 807         vmovd           `4*$i`(@ptr[0]),$Xi
 808          lea            `16*4`(@ptr[0]),@ptr[0]
 809         vmovd           `4*$i`(@ptr[4]),$t1
 810          lea            `16*4`(@ptr[4]),@ptr[4]
 811         vmovd           `4*$i`(@ptr[1]),$t2
 812          lea            `16*4`(@ptr[1]),@ptr[1]
 813         vmovd           `4*$i`(@ptr[5]),$t3
 814          lea            `16*4`(@ptr[5]),@ptr[5]
 815         vpinsrd         \$1,`4*$i`(@ptr[2]),$Xi,$Xi
 816          lea            `16*4`(@ptr[2]),@ptr[2]
 817         vpinsrd         \$1,`4*$i`(@ptr[6]),$t1,$t1
 818          lea            `16*4`(@ptr[6]),@ptr[6]
 819         vpinsrd         \$1,`4*$i`(@ptr[3]),$t2,$t2
 820          lea            `16*4`(@ptr[3]),@ptr[3]
 821         vpunpckldq      $t2,$Xi,$Xi
 822         vpinsrd         \$1,`4*$i`(@ptr[7]),$t3,$t3
 823          lea            `16*4`(@ptr[7]),@ptr[7]
 824         vpunpckldq      $t3,$t1,$t1
 825         vinserti128     $t1,$Xi,$Xi
 826         vpshufb         $Xn,$Xi,$Xi
 827 ___
 828 $code.=<<___;
 829         vpsrld  \$6,$e,$sigma
 830         vpslld  \$26,$e,$t3
 831         vmovdqu $Xi,`&Xi_off($i)`
 832          vpaddd $h,$Xi,$Xi                      # Xi+=h
 833
 834         vpsrld  \$11,$e,$t2
 835         vpxor   $t3,$sigma,$sigma
 836         vpslld  \$21,$e,$t3
 837          vpaddd `32*($i%8)-128`($Tbl),$Xi,$Xi   # Xi+=K[round]
 838         vpxor   $t2,$sigma,$sigma
 839
 840         vpsrld  \$25,$e,$t2
 841         vpxor   $t3,$sigma,$sigma
 842          `"prefetcht0   63(@ptr[0])"            if ($i==15)`
 843         vpslld  \$7,$e,$t3
 844          vpandn $g,$e,$t1
 845          vpand  $f,$e,$axb                      # borrow $axb
 846          `"prefetcht0   63(@ptr[1])"            if ($i==15)`
 847         vpxor   $t2,$sigma,$sigma
 848
 849         vpsrld  \$2,$a,$h                       # borrow $h
 850         vpxor   $t3,$sigma,$sigma               # Sigma1(e)
 851          `"prefetcht0   63(@ptr[2])"            if ($i==15)`
 852         vpslld  \$30,$a,$t2
 853          vpxor  $axb,$t1,$t1                    # Ch(e,f,g)
 854          vpxor  $a,$b,$axb                      # a^b, b^c in next round
 855          `"prefetcht0   63(@ptr[3])"            if ($i==15)`
 856         vpxor   $t2,$h,$h
 857         vpaddd  $sigma,$Xi,$Xi                  # Xi+=Sigma1(e)
 858
 859         vpsrld  \$13,$a,$t2
 860          `"prefetcht0   63(@ptr[4])"            if ($i==15 && $REG_SZ==32)`
 861         vpslld  \$19,$a,$t3
 862          vpaddd $t1,$Xi,$Xi                     # Xi+=Ch(e,f,g)
 863          vpand  $axb,$bxc,$bxc
 864          `"prefetcht0   63(@ptr[5])"            if ($i==15 && $REG_SZ==32)`
 865         vpxor   $t2,$h,$sigma
 866
 867         vpsrld  \$22,$a,$t2
 868         vpxor   $t3,$sigma,$sigma
 869          `"prefetcht0   63(@ptr[6])"            if ($i==15 && $REG_SZ==32)`
 870         vpslld  \$10,$a,$t3
 871          vpxor  $bxc,$b,$h                      # h=Maj(a,b,c)=Ch(a^b,c,b)
 872          vpaddd $Xi,$d,$d                       # d+=Xi
 873          `"prefetcht0   63(@ptr[7])"            if ($i==15 && $REG_SZ==32)`
 874         vpxor   $t2,$sigma,$sigma
 875         vpxor   $t3,$sigma,$sigma               # Sigma0(a)
 876
 877         vpaddd  $Xi,$h,$h                       # h+=Xi
 878         vpaddd  $sigma,$h,$h                    # h+=Sigma0(a)
 879 ___
 880 $code.=<<___ if (($i%8)==7);
 881         add     \$`32*8`,$Tbl
 882 ___
 883         ($axb,$bxc)=($bxc,$axb);
 884 }
 885
 886 sub ROUND_16_XX_avx {
 887 my $i=shift;
 888
 889 $code.=<<___;
 890         vmovdqu `&Xi_off($i+1)`,$Xn
 891         vpaddd  `&Xi_off($i+9)`,$Xi,$Xi         # Xi+=X[i+9]
 892
 893         vpsrld  \$3,$Xn,$sigma
 894         vpsrld  \$7,$Xn,$t2
 895         vpslld  \$25,$Xn,$t3
 896         vpxor   $t2,$sigma,$sigma
 897         vpsrld  \$18,$Xn,$t2
 898         vpxor   $t3,$sigma,$sigma
 899         vpslld  \$14,$Xn,$t3
 900         vmovdqu `&Xi_off($i+14)`,$t1
 901         vpsrld  \$10,$t1,$axb                   # borrow $axb
 902
 903         vpxor   $t2,$sigma,$sigma
 904         vpsrld  \$17,$t1,$t2
 905         vpxor   $t3,$sigma,$sigma               # sigma0(X[i+1])
 906         vpslld  \$15,$t1,$t3
 907          vpaddd $sigma,$Xi,$Xi                  # Xi+=sigma0(e)
 908         vpxor   $t2,$axb,$sigma
 909         vpsrld  \$19,$t1,$t2
 910         vpxor   $t3,$sigma,$sigma
 911         vpslld  \$13,$t1,$t3
 912         vpxor   $t2,$sigma,$sigma
 913         vpxor   $t3,$sigma,$sigma               # sigma0(X[i+14])
 914         vpaddd  $sigma,$Xi,$Xi                  # Xi+=sigma1(X[i+14])
 915 ___
 916         &ROUND_00_15_avx($i,@_);
 917         ($Xi,$Xn)=($Xn,$Xi);
 918 }
 919
 920 $code.=<<___;
 921 .type   sha256_multi_block_avx,\@function,3
 922 .align  32
 923 sha256_multi_block_avx:
 924 _avx_shortcut:
 925 ___
 926 $code.=<<___ if ($avx>1);
 927         shr     \$32,%rcx
 928         cmp     \$2,$num
 929         jb      .Lavx
 930         test    \$`1<<5`,%ecx
 931         jnz     _avx2_shortcut
 932         jmp     .Lavx
 933 .align  32
 934 .Lavx:
 935 ___
 936 $code.=<<___;
 937         mov     %rsp,%rax
 938         push    %rbx
 939         push    %rbp
 940 ___
 941 $code.=<<___ if ($win64);
 942         lea     -0xa8(%rsp),%rsp
 943         movaps  %xmm6,(%rsp)
 944         movaps  %xmm7,0x10(%rsp)
 945         movaps  %xmm8,0x20(%rsp)
 946         movaps  %xmm9,0x30(%rsp)
 947         movaps  %xmm10,-0x78(%rax)
 948         movaps  %xmm11,-0x68(%rax)
 949         movaps  %xmm12,-0x58(%rax)
 950         movaps  %xmm13,-0x48(%rax)
 951         movaps  %xmm14,-0x38(%rax)
 952         movaps  %xmm15,-0x28(%rax)
 953 ___
 954 $code.=<<___;
 955         sub     \$`$REG_SZ*18`, %rsp
 956         and     \$-256,%rsp
 957         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
 958 .Lbody_avx:
 959         lea     K256+128(%rip),$Tbl
 960         lea     `$REG_SZ*16`(%rsp),%rbx
 961         lea     0x80($ctx),$ctx                 # size optimization
 962
 963 .Loop_grande_avx:
 964         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
 965         xor     $num,$num
 966 ___
 967 for($i=0;$i<4;$i++) {
 968     $code.=<<___;
 969         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
 970         mov     `16*$i+8`($inp),%ecx            # number of blocks
 971         cmp     $num,%ecx
 972         cmovg   %ecx,$num                       # find maximum
 973         test    %ecx,%ecx
 974         mov     %ecx,`4*$i`(%rbx)               # initialize counters
 975         cmovle  $Tbl,@ptr[$i]                   # cancel input
 976 ___
 977 }
 978 $code.=<<___;
 979         test    $num,$num
 980         jz      .Ldone_avx
 981
 982         vmovdqu 0x00-0x80($ctx),$A              # load context
 983          lea    128(%rsp),%rax
 984         vmovdqu 0x20-0x80($ctx),$B
 985         vmovdqu 0x40-0x80($ctx),$C
 986         vmovdqu 0x60-0x80($ctx),$D
 987         vmovdqu 0x80-0x80($ctx),$E
 988         vmovdqu 0xa0-0x80($ctx),$F
 989         vmovdqu 0xc0-0x80($ctx),$G
 990         vmovdqu 0xe0-0x80($ctx),$H
 991         vmovdqu .Lpbswap(%rip),$Xn
 992         jmp     .Loop_avx
 993
 994 .align  32
 995 .Loop_avx:
 996         vpxor   $B,$C,$bxc                      # magic seed
 997 ___
 998 for($i=0;$i<16;$i++)    { &ROUND_00_15_avx($i,@V); unshift(@V,pop(@V)); }
 999 $code.=<<___;
1000         vmovdqu `&Xi_off($i)`,$Xi
1001         mov     \$3,%ecx
1002         jmp     .Loop_16_xx_avx
1003 .align  32
1004 .Loop_16_xx_avx:
1005 ___
1006 for(;$i<32;$i++)        { &ROUND_16_XX_avx($i,@V); unshift(@V,pop(@V)); }
1007 $code.=<<___;
1008         dec     %ecx
1009         jnz     .Loop_16_xx_avx
1010
1011         mov     \$1,%ecx
1012         lea     K256+128(%rip),$Tbl
1013 ___
1014 for($i=0;$i<4;$i++) {
1015     $code.=<<___;
1016         cmp     `4*$i`(%rbx),%ecx               # examine counters
1017         cmovge  $Tbl,@ptr[$i]                   # cancel input
1018 ___
1019 }
1020 $code.=<<___;
1021         vmovdqa (%rbx),$sigma                   # pull counters
1022         vpxor   $t1,$t1,$t1
1023         vmovdqa $sigma,$Xn
1024         vpcmpgtd $t1,$Xn,$Xn                    # mask value
1025         vpaddd  $Xn,$sigma,$sigma               # counters--
1026
1027         vmovdqu 0x00-0x80($ctx),$t1
1028         vpand   $Xn,$A,$A
1029         vmovdqu 0x20-0x80($ctx),$t2
1030         vpand   $Xn,$B,$B
1031         vmovdqu 0x40-0x80($ctx),$t3
1032         vpand   $Xn,$C,$C
1033         vmovdqu 0x60-0x80($ctx),$Xi
1034         vpand   $Xn,$D,$D
1035         vpaddd  $t1,$A,$A
1036         vmovdqu 0x80-0x80($ctx),$t1
1037         vpand   $Xn,$E,$E
1038         vpaddd  $t2,$B,$B
1039         vmovdqu 0xa0-0x80($ctx),$t2
1040         vpand   $Xn,$F,$F
1041         vpaddd  $t3,$C,$C
1042         vmovdqu 0xc0-0x80($ctx),$t3
1043         vpand   $Xn,$G,$G
1044         vpaddd  $Xi,$D,$D
1045         vmovdqu 0xe0-0x80($ctx),$Xi
1046         vpand   $Xn,$H,$H
1047         vpaddd  $t1,$E,$E
1048         vpaddd  $t2,$F,$F
1049         vmovdqu $A,0x00-0x80($ctx)
1050         vpaddd  $t3,$G,$G
1051         vmovdqu $B,0x20-0x80($ctx)
1052         vpaddd  $Xi,$H,$H
1053         vmovdqu $C,0x40-0x80($ctx)
1054         vmovdqu $D,0x60-0x80($ctx)
1055         vmovdqu $E,0x80-0x80($ctx)
1056         vmovdqu $F,0xa0-0x80($ctx)
1057         vmovdqu $G,0xc0-0x80($ctx)
1058         vmovdqu $H,0xe0-0x80($ctx)
1059
1060         vmovdqu $sigma,(%rbx)                   # save counters
1061         vmovdqu .Lpbswap(%rip),$Xn
1062         dec     $num
1063         jnz     .Loop_avx
1064
1065         mov     `$REG_SZ*17+8`(%rsp),$num
1066         lea     $REG_SZ($ctx),$ctx
1067         lea     `16*$REG_SZ/4`($inp),$inp
1068         dec     $num
1069         jnz     .Loop_grande_avx
1070
1071 .Ldone_avx:
1072         mov     `$REG_SZ*17`(%rsp),%rax         # original %rsp
1073         vzeroupper
1074 ___
1075 $code.=<<___ if ($win64);
1076         movaps  -0xb8(%rax),%xmm6
1077         movaps  -0xa8(%rax),%xmm7
1078         movaps  -0x98(%rax),%xmm8
1079         movaps  -0x88(%rax),%xmm9
1080         movaps  -0x78(%rax),%xmm10
1081         movaps  -0x68(%rax),%xmm11
1082         movaps  -0x58(%rax),%xmm12
1083         movaps  -0x48(%rax),%xmm13
1084         movaps  -0x38(%rax),%xmm14
1085         movaps  -0x28(%rax),%xmm15
1086 ___
1087 $code.=<<___;
1088         mov     -16(%rax),%rbp
1089         mov     -8(%rax),%rbx
1090         lea     (%rax),%rsp
1091 .Lepilogue_avx:
1092         ret
1093 .size   sha256_multi_block_avx,.-sha256_multi_block_avx
1094 ___
1095                                                 if ($avx>1) {
1096 $code =~ s/\`([^\`]*)\`/eval $1/gem;
1097
1098 $REG_SZ=32;
1099 @ptr=map("%r$_",(12..15,8..11));
1100
1101 @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("%ymm$_",(8..15));
1102 ($t1,$t2,$t3,$axb,$bxc,$Xi,$Xn,$sigma)=map("%ymm$_",(0..7));
1103
1104 $code.=<<___;
1105 .type   sha256_multi_block_avx2,\@function,3
1106 .align  32
1107 sha256_multi_block_avx2:
1108 _avx2_shortcut:
1109         mov     %rsp,%rax
1110         push    %rbx
1111         push    %rbp
1112         push    %r12
1113         push    %r13
1114         push    %r14
1115         push    %r15
1116 ___
1117 $code.=<<___ if ($win64);
1118         lea     -0xa8(%rsp),%rsp
1119         movaps  %xmm6,(%rsp)
1120         movaps  %xmm7,0x10(%rsp)
1121         movaps  %xmm8,0x20(%rsp)
1122         movaps  %xmm9,0x30(%rsp)
1123         movaps  %xmm10,0x40(%rsp)
1124         movaps  %xmm11,0x50(%rsp)
1125         movaps  %xmm12,-0x78(%rax)
1126         movaps  %xmm13,-0x68(%rax)
1127         movaps  %xmm14,-0x58(%rax)
1128         movaps  %xmm15,-0x48(%rax)
1129 ___
1130 $code.=<<___;
1131         sub     \$`$REG_SZ*18`, %rsp
1132         and     \$-256,%rsp
1133         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
1134 .Lbody_avx2:
1135         lea     K256+128(%rip),$Tbl
1136         lea     0x80($ctx),$ctx                 # size optimization
1137
1138 .Loop_grande_avx2:
1139         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
1140         xor     $num,$num
1141         lea     `$REG_SZ*16`(%rsp),%rbx
1142 ___
1143 for($i=0;$i<8;$i++) {
1144     $code.=<<___;
1145         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
1146         mov     `16*$i+8`($inp),%ecx            # number of blocks
1147         cmp     $num,%ecx
1148         cmovg   %ecx,$num                       # find maximum
1149         test    %ecx,%ecx
1150         mov     %ecx,`4*$i`(%rbx)               # initialize counters
1151         cmovle  $Tbl,@ptr[$i]                   # cancel input
1152 ___
1153 }
1154 $code.=<<___;
1155         vmovdqu 0x00-0x80($ctx),$A              # load context
1156          lea    128(%rsp),%rax
1157         vmovdqu 0x20-0x80($ctx),$B
1158          lea    256+128(%rsp),%rbx
1159         vmovdqu 0x40-0x80($ctx),$C
1160         vmovdqu 0x60-0x80($ctx),$D
1161         vmovdqu 0x80-0x80($ctx),$E
1162         vmovdqu 0xa0-0x80($ctx),$F
1163         vmovdqu 0xc0-0x80($ctx),$G
1164         vmovdqu 0xe0-0x80($ctx),$H
1165         vmovdqu .Lpbswap(%rip),$Xn
1166         jmp     .Loop_avx2
1167
1168 .align  32
1169 .Loop_avx2:
1170         vpxor   $B,$C,$bxc                      # magic seed
1171 ___
1172 for($i=0;$i<16;$i++)    { &ROUND_00_15_avx($i,@V); unshift(@V,pop(@V)); }
1173 $code.=<<___;
1174         vmovdqu `&Xi_off($i)`,$Xi
1175         mov     \$3,%ecx
1176         jmp     .Loop_16_xx_avx2
1177 .align  32
1178 .Loop_16_xx_avx2:
1179 ___
1180 for(;$i<32;$i++)        { &ROUND_16_XX_avx($i,@V); unshift(@V,pop(@V)); }
1181 $code.=<<___;
1182         dec     %ecx
1183         jnz     .Loop_16_xx_avx2
1184
1185         mov     \$1,%ecx
1186         lea     `$REG_SZ*16`(%rsp),%rbx
1187         lea     K256+128(%rip),$Tbl
1188 ___
1189 for($i=0;$i<8;$i++) {
1190     $code.=<<___;
1191         cmp     `4*$i`(%rbx),%ecx               # examine counters
1192         cmovge  $Tbl,@ptr[$i]                   # cancel input
1193 ___
1194 }
1195 $code.=<<___;
1196         vmovdqa (%rbx),$sigma                   # pull counters
1197         vpxor   $t1,$t1,$t1
1198         vmovdqa $sigma,$Xn
1199         vpcmpgtd $t1,$Xn,$Xn                    # mask value
1200         vpaddd  $Xn,$sigma,$sigma               # counters--
1201
1202         vmovdqu 0x00-0x80($ctx),$t1
1203         vpand   $Xn,$A,$A
1204         vmovdqu 0x20-0x80($ctx),$t2
1205         vpand   $Xn,$B,$B
1206         vmovdqu 0x40-0x80($ctx),$t3
1207         vpand   $Xn,$C,$C
1208         vmovdqu 0x60-0x80($ctx),$Xi
1209         vpand   $Xn,$D,$D
1210         vpaddd  $t1,$A,$A
1211         vmovdqu 0x80-0x80($ctx),$t1
1212         vpand   $Xn,$E,$E
1213         vpaddd  $t2,$B,$B
1214         vmovdqu 0xa0-0x80($ctx),$t2
1215         vpand   $Xn,$F,$F
1216         vpaddd  $t3,$C,$C
1217         vmovdqu 0xc0-0x80($ctx),$t3
1218         vpand   $Xn,$G,$G
1219         vpaddd  $Xi,$D,$D
1220         vmovdqu 0xe0-0x80($ctx),$Xi
1221         vpand   $Xn,$H,$H
1222         vpaddd  $t1,$E,$E
1223         vpaddd  $t2,$F,$F
1224         vmovdqu $A,0x00-0x80($ctx)
1225         vpaddd  $t3,$G,$G
1226         vmovdqu $B,0x20-0x80($ctx)
1227         vpaddd  $Xi,$H,$H
1228         vmovdqu $C,0x40-0x80($ctx)
1229         vmovdqu $D,0x60-0x80($ctx)
1230         vmovdqu $E,0x80-0x80($ctx)
1231         vmovdqu $F,0xa0-0x80($ctx)
1232         vmovdqu $G,0xc0-0x80($ctx)
1233         vmovdqu $H,0xe0-0x80($ctx)
1234
1235         vmovdqu $sigma,(%rbx)                   # save counters
1236         lea     256+128(%rsp),%rbx
1237         vmovdqu .Lpbswap(%rip),$Xn
1238         dec     $num
1239         jnz     .Loop_avx2
1240
1241         #mov    `$REG_SZ*17+8`(%rsp),$num
1242         #lea    $REG_SZ($ctx),$ctx
1243         #lea    `16*$REG_SZ/4`($inp),$inp
1244         #dec    $num
1245         #jnz    .Loop_grande_avx2
1246
1247 .Ldone_avx2:
1248         mov     `$REG_SZ*17`(%rsp),%rax         # original %rsp
1249         vzeroupper
1250 ___
1251 $code.=<<___ if ($win64);
1252         movaps  -0xd8(%rax),%xmm6
1253         movaps  -0xc8(%rax),%xmm7
1254         movaps  -0xb8(%rax),%xmm8
1255         movaps  -0xa8(%rax),%xmm9
1256         movaps  -0x98(%rax),%xmm10
1257         movaps  -0x88(%rax),%xmm11
1258         movaps  -0x78(%rax),%xmm12
1259         movaps  -0x68(%rax),%xmm13
1260         movaps  -0x58(%rax),%xmm14
1261         movaps  -0x48(%rax),%xmm15
1262 ___
1263 $code.=<<___;
1264         mov     -48(%rax),%r15
1265         mov     -40(%rax),%r14
1266         mov     -32(%rax),%r13
1267         mov     -24(%rax),%r12
1268         mov     -16(%rax),%rbp
1269         mov     -8(%rax),%rbx
1270         lea     (%rax),%rsp
1271 .Lepilogue_avx2:
1272         ret
1273 .size   sha256_multi_block_avx2,.-sha256_multi_block_avx2
1274 ___
1275                                         }       }}}
1276 $code.=<<___;
1277 .align  256
1278 K256:
1279 ___
1280 sub TABLE {
1281     foreach (@_) {
1282         $code.=<<___;
1283         .long   $_,$_,$_,$_
1284         .long   $_,$_,$_,$_
1285 ___
1286     }
1287 }
1288 &TABLE( 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,
1289         0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5,
1290         0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,
1291         0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174,
1292         0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,
1293         0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da,
1294         0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,
1295         0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967,
1296         0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,
1297         0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85,
1298         0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,
1299         0xd192e819,0xd6990624,0xf40e3585,0x106aa070,
1300         0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,
1301         0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3,
1302         0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,
1303         0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 );
1304 $code.=<<___;
1305 .Lpbswap:
1306         .long   0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f     # pbswap
1307         .long   0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f     # pbswap
1308 K256_shaext:
1309         .long   0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
1310         .long   0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
1311         .long   0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
1312         .long   0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
1313         .long   0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
1314         .long   0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
1315         .long   0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
1316         .long   0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
1317         .long   0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
1318         .long   0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
1319         .long   0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
1320         .long   0xd192e819,0xd6990624,0xf40e3585,0x106aa070
1321         .long   0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
1322         .long   0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
1323         .long   0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
1324         .long   0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
1325         .asciz  "SHA256 multi-block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
1326 ___
1327
1328 if ($win64) {
1329 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1330 #               CONTEXT *context,DISPATCHER_CONTEXT *disp)
1331 $rec="%rcx";
1332 $frame="%rdx";
1333 $context="%r8";
1334 $disp="%r9";
1335
1336 $code.=<<___;
1337 .extern __imp_RtlVirtualUnwind
1338 .type   se_handler,\@abi-omnipotent
1339 .align  16
1340 se_handler:
1341         push    %rsi
1342         push    %rdi
1343         push    %rbx
1344         push    %rbp
1345         push    %r12
1346         push    %r13
1347         push    %r14
1348         push    %r15
1349         pushfq
1350         sub     \$64,%rsp
1351
1352         mov     120($context),%rax      # pull context->Rax
1353         mov     248($context),%rbx      # pull context->Rip
1354
1355         mov     8($disp),%rsi           # disp->ImageBase
1356         mov     56($disp),%r11          # disp->HandlerData
1357
1358         mov     0(%r11),%r10d           # HandlerData[0]
1359         lea     (%rsi,%r10),%r10        # end of prologue label
1360         cmp     %r10,%rbx               # context->Rip<.Lbody
1361         jb      .Lin_prologue
1362
1363         mov     152($context),%rax      # pull context->Rsp
1364
1365         mov     4(%r11),%r10d           # HandlerData[1]
1366         lea     (%rsi,%r10),%r10        # epilogue label
1367         cmp     %r10,%rbx               # context->Rip>=.Lepilogue
1368         jae     .Lin_prologue
1369
1370         mov     `16*17`(%rax),%rax      # pull saved stack pointer
1371
1372         mov     -8(%rax),%rbx
1373         mov     -16(%rax),%rbp
1374         mov     %rbx,144($context)      # restore context->Rbx
1375         mov     %rbp,160($context)      # restore context->Rbp
1376
1377         lea     -24-10*16(%rax),%rsi
1378         lea     512($context),%rdi      # &context.Xmm6
1379         mov     \$20,%ecx
1380         .long   0xa548f3fc              # cld; rep movsq
1381
1382 .Lin_prologue:
1383         mov     8(%rax),%rdi
1384         mov     16(%rax),%rsi
1385         mov     %rax,152($context)      # restore context->Rsp
1386         mov     %rsi,168($context)      # restore context->Rsi
1387         mov     %rdi,176($context)      # restore context->Rdi
1388
1389         mov     40($disp),%rdi          # disp->ContextRecord
1390         mov     $context,%rsi           # context
1391         mov     \$154,%ecx              # sizeof(CONTEXT)
1392         .long   0xa548f3fc              # cld; rep movsq
1393
1394         mov     $disp,%rsi
1395         xor     %rcx,%rcx               # arg1, UNW_FLAG_NHANDLER
1396         mov     8(%rsi),%rdx            # arg2, disp->ImageBase
1397         mov     0(%rsi),%r8             # arg3, disp->ControlPc
1398         mov     16(%rsi),%r9            # arg4, disp->FunctionEntry
1399         mov     40(%rsi),%r10           # disp->ContextRecord
1400         lea     56(%rsi),%r11           # &disp->HandlerData
1401         lea     24(%rsi),%r12           # &disp->EstablisherFrame
1402         mov     %r10,32(%rsp)           # arg5
1403         mov     %r11,40(%rsp)           # arg6
1404         mov     %r12,48(%rsp)           # arg7
1405         mov     %rcx,56(%rsp)           # arg8, (NULL)
1406         call    *__imp_RtlVirtualUnwind(%rip)
1407
1408         mov     \$1,%eax                # ExceptionContinueSearch
1409         add     \$64,%rsp
1410         popfq
1411         pop     %r15
1412         pop     %r14
1413         pop     %r13
1414         pop     %r12
1415         pop     %rbp
1416         pop     %rbx
1417         pop     %rdi
1418         pop     %rsi
1419         ret
1420 .size   se_handler,.-se_handler
1421 ___
1422 $code.=<<___ if ($avx>1);
1423 .type   avx2_handler,\@abi-omnipotent
1424 .align  16
1425 avx2_handler:
1426         push    %rsi
1427         push    %rdi
1428         push    %rbx
1429         push    %rbp
1430         push    %r12
1431         push    %r13
1432         push    %r14
1433         push    %r15
1434         pushfq
1435         sub     \$64,%rsp
1436
1437         mov     120($context),%rax      # pull context->Rax
1438         mov     248($context),%rbx      # pull context->Rip
1439
1440         mov     8($disp),%rsi           # disp->ImageBase
1441         mov     56($disp),%r11          # disp->HandlerData
1442
1443         mov     0(%r11),%r10d           # HandlerData[0]
1444         lea     (%rsi,%r10),%r10        # end of prologue label
1445         cmp     %r10,%rbx               # context->Rip<body label
1446         jb      .Lin_prologue
1447
1448         mov     152($context),%rax      # pull context->Rsp
1449
1450         mov     4(%r11),%r10d           # HandlerData[1]
1451         lea     (%rsi,%r10),%r10        # epilogue label
1452         cmp     %r10,%rbx               # context->Rip>=epilogue label
1453         jae     .Lin_prologue
1454
1455         mov     `32*17`($context),%rax  # pull saved stack pointer
1456
1457         mov     -8(%rax),%rbx
1458         mov     -16(%rax),%rbp
1459         mov     -24(%rax),%r12
1460         mov     -32(%rax),%r13
1461         mov     -40(%rax),%r14
1462         mov     -48(%rax),%r15
1463         mov     %rbx,144($context)      # restore context->Rbx
1464         mov     %rbp,160($context)      # restore context->Rbp
1465         mov     %r12,216($context)      # restore cotnext->R12
1466         mov     %r13,224($context)      # restore cotnext->R13
1467         mov     %r14,232($context)      # restore cotnext->R14
1468         mov     %r15,240($context)      # restore cotnext->R15
1469
1470         lea     -56-10*16(%rax),%rsi
1471         lea     512($context),%rdi      # &context.Xmm6
1472         mov     \$20,%ecx
1473         .long   0xa548f3fc              # cld; rep movsq
1474
1475         jmp     .Lin_prologue
1476 .size   avx2_handler,.-avx2_handler
1477 ___
1478 $code.=<<___;
1479 .section        .pdata
1480 .align  4
1481         .rva    .LSEH_begin_sha256_multi_block
1482         .rva    .LSEH_end_sha256_multi_block
1483         .rva    .LSEH_info_sha256_multi_block
1484         .rva    .LSEH_begin_sha256_multi_block_shaext
1485         .rva    .LSEH_end_sha256_multi_block_shaext
1486         .rva    .LSEH_info_sha256_multi_block_shaext
1487 ___
1488 $code.=<<___ if ($avx);
1489         .rva    .LSEH_begin_sha256_multi_block_avx
1490         .rva    .LSEH_end_sha256_multi_block_avx
1491         .rva    .LSEH_info_sha256_multi_block_avx
1492 ___
1493 $code.=<<___ if ($avx>1);
1494         .rva    .LSEH_begin_sha256_multi_block_avx2
1495         .rva    .LSEH_end_sha256_multi_block_avx2
1496         .rva    .LSEH_info_sha256_multi_block_avx2
1497 ___
1498 $code.=<<___;
1499 .section        .xdata
1500 .align  8
1501 .LSEH_info_sha256_multi_block:
1502         .byte   9,0,0,0
1503         .rva    se_handler
1504         .rva    .Lbody,.Lepilogue                       # HandlerData[]
1505 .LSEH_info_sha256_multi_block_shaext:
1506         .byte   9,0,0,0
1507         .rva    se_handler
1508         .rva    .Lbody_shaext,.Lepilogue_shaext         # HandlerData[]
1509 ___
1510 $code.=<<___ if ($avx);
1511 .LSEH_info_sha256_multi_block_avx:
1512         .byte   9,0,0,0
1513         .rva    se_handler
1514         .rva    .Lbody_avx,.Lepilogue_avx               # HandlerData[]
1515 ___
1516 $code.=<<___ if ($avx>1);
1517 .LSEH_info_sha256_multi_block_avx2:
1518         .byte   9,0,0,0
1519         .rva    avx2_handler
1520         .rva    .Lbody_avx2,.Lepilogue_avx2             # HandlerData[]
1521 ___
1522 }
1523 ####################################################################
1524
1525 sub rex {
1526   local *opcode=shift;
1527   my ($dst,$src)=@_;
1528   my $rex=0;
1529
1530     $rex|=0x04                  if ($dst>=8);
1531     $rex|=0x01                  if ($src>=8);
1532     unshift @opcode,$rex|0x40   if ($rex);
1533 }
1534
1535 sub sha256op38 {
1536     my $instr = shift;
1537     my %opcodelet = (
1538                 "sha256rnds2" => 0xcb,
1539                 "sha256msg1"  => 0xcc,
1540                 "sha256msg2"  => 0xcd   );
1541
1542     if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1543       my @opcode=(0x0f,0x38);
1544         rex(\@opcode,$2,$1);
1545         push @opcode,$opcodelet{$instr};
1546         push @opcode,0xc0|($1&7)|(($2&7)<<3);           # ModR/M
1547         return ".byte\t".join(',',@opcode);
1548     } else {
1549         return $instr."\t".@_[0];
1550     }
1551 }
1552
1553 foreach (split("\n",$code)) {
1554         s/\`([^\`]*)\`/eval($1)/ge;
1555
1556         s/\b(sha256[^\s]*)\s+(.*)/sha256op38($1,$2)/geo         or
1557
1558         s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go          or
1559         s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go         or
1560         s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go    or
1561         s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go        or
1562         s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go            or
1563         s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go;
1564
1565         print $_,"\n";
1566 }
1567
1568 close STDOUT;