3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
10 # Multi-buffer SHA256 procedure processes n buffers in parallel by
11 # placing buffer data to designated lane of SIMD register. n is
12 # naturally limited to 4 on pre-AVX2 processors and to 8 on
13 # AVX2-capable processors such as Haswell.
15 # this +aesni(*) sha256 aesni-sha256 gain
16 # -------------------------------------------------------------------
17 # Westmere(**) 23.3/n +1.28=7.11(n=4) 12.3 +3.75=16.1 +126%
18 # Atom(**) 39.1/n +3.93=13.7(n=4) 20.8 +5.69=26.5 +93%
19 # Sandy Bridge (20.5 +5.15=25.7)/n 11.6 13.0 +103%
20 # Ivy Bridge (20.4 +5.14=25.5)/n 10.3 11.6 +82%
21 # Haswell(***) (21.0 +5.00=26.0)/n 7.80 8.79 +170%
22 # Bulldozer (21.6 +5.76=27.4)/n 13.6 13.7 +100%
24 # (*) multi-block CBC encrypt with 128-bit key;
25 # (**) (HASH+AES)/n does not apply to Westmere for n>3 and Atom,
26 # because of lower AES-NI instruction throughput, nor is there
27 # AES-NI-SHA256 stitch for these processors;
28 # (***) "this" is for n=8, when we gather twice as much data, result
29 # for n=4 is 20.3+4.44=24.7;
33 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
35 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
37 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
38 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
39 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
40 die "can't locate x86_64-xlate.pl";
44 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
45 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
46 $avx = ($1>=2.19) + ($1>=2.22);
49 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
50 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
51 $avx = ($1>=2.09) + ($1>=2.10);
54 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
55 `ml64 2>&1` =~ /Version ([0-9]+)\./) {
56 $avx = ($1>=10) + ($1>=11);
59 open OUT,"| \"$^X\" $xlate $flavour $output";
62 # void sha256_multi_block (
63 # struct { unsigned int A[8];
70 # unsigned int H[8]; } *ctx,
71 # struct { void *ptr; int blocks; } inp[8],
72 # int num); /* 1 or 2 */
74 $ctx="%rdi"; # 1st arg
75 $inp="%rsi"; # 2nd arg
76 $num="%edx"; # 3rd arg
77 @ptr=map("%r$_",(8..11));
80 @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("%xmm$_",(8..15));
81 ($t1,$t2,$t3,$axb,$bxc,$Xi,$Xn,$sigma)=map("%xmm$_",(0..7));
88 $off %= 16; $off *= $REG_SZ;
89 $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)";
93 my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
95 $code.=<<___ if ($i<15);
96 movd `4*$i`(@ptr[0]),$Xi
97 movd `4*$i`(@ptr[1]),$t1
98 movd `4*$i`(@ptr[2]),$t2
99 movd `4*$i`(@ptr[3]),$t3
105 $code.=<<___ if ($i==15);
106 movd `4*$i`(@ptr[0]),$Xi
107 lea `16*4`(@ptr[0]),@ptr[0]
108 movd `4*$i`(@ptr[1]),$t1
109 lea `16*4`(@ptr[1]),@ptr[1]
110 movd `4*$i`(@ptr[2]),$t2
111 lea `16*4`(@ptr[2]),@ptr[2]
112 movd `4*$i`(@ptr[3]),$t3
113 lea `16*4`(@ptr[3]),@ptr[3]
125 movdqa $Xi,`&Xi_off($i)`
131 paddd `32*($i%8)-128`($Tbl),$Xi # Xi+=K[round]
137 movdqa $e,$axb # borrow $axb
144 pxor $t3,$sigma # Sigma1(e)
147 paddd $sigma,$Xi # Xi+=Sigma1(e)
148 pxor $axb,$t1 # Ch(e,f,g)
152 pxor $a,$axb # a^b, b^c in next round
156 paddd $t1,$Xi # Xi+=Ch(e,f,g)
166 pxor $bxc,$h # h=Maj(a,b,c)=Ch(a^b,c,b)
168 pxor $t3,$sigma # Sigma0(a)
171 paddd $sigma,$h # h+=Sigma0(a)
173 $code.=<<___ if (($i%8)==7);
174 lea `32*8`($Tbl),$Tbl
176 ($axb,$bxc)=($bxc,$axb);
183 movdqa `&Xi_off($i+1)`,$Xn
184 paddd `&Xi_off($i+9)`,$Xi # Xi+=X[i+9]
192 movdqa `&Xi_off($i+14)`,$t1
196 movdqa $t1,$axb # borrow $axb
204 pxor $t3,$sigma # sigma0(X[i+1])
206 paddd $sigma,$Xi # Xi+=sigma0(e)
212 pxor $t2,$t1 # sigma0(X[i+14])
213 paddd $t1,$Xi # Xi+=sigma1(X[i+14])
222 .extern OPENSSL_ia32cap_P
224 .globl sha256_multi_block
225 .type sha256_multi_block,\@function,3
229 $code.=<<___ if ($avx);
230 mov OPENSSL_ia32cap_P+4(%rip),%rcx
239 $code.=<<___ if ($win64);
242 movaps %xmm7,0x10(%rsp)
243 movaps %xmm8,0x20(%rsp)
244 movaps %xmm9,0x30(%rsp)
245 movaps %xmm10,-0x78(%rax)
246 movaps %xmm11,-0x68(%rax)
247 movaps %xmm12,-0x58(%rax)
248 movaps %xmm13,-0x48(%rax)
249 movaps %xmm14,-0x38(%rax)
250 movaps %xmm15,-0x28(%rax)
253 sub \$`$REG_SZ*18`, %rsp
255 mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
256 lea K256+128(%rip),$Tbl
257 lea `$REG_SZ*16`(%rsp),%rbx
258 lea 0x80($ctx),$ctx # size optimization
261 mov $num,`$REG_SZ*17+8`(%rsp) # original $num
264 for($i=0;$i<4;$i++) {
266 mov `16*$i+0`($inp),@ptr[$i] # input pointer
267 mov `16*$i+8`($inp),%ecx # number of blocks
269 cmovg %ecx,$num # find maximum
271 mov %ecx,`4*$i`(%rbx) # initialize counters
272 cmovle $Tbl,@ptr[$i] # cancel input
279 movdqu 0x00-0x80($ctx),$A # load context
281 movdqu 0x20-0x80($ctx),$B
282 movdqu 0x40-0x80($ctx),$C
283 movdqu 0x60-0x80($ctx),$D
284 movdqu 0x80-0x80($ctx),$E
285 movdqu 0xa0-0x80($ctx),$F
286 movdqu 0xc0-0x80($ctx),$G
287 movdqu 0xe0-0x80($ctx),$H
288 movdqu .Lpbswap(%rip),$Xn
294 pxor $B,$bxc # magic seed
296 for($i=0;$i<16;$i++) { &ROUND_00_15($i,@V); unshift(@V,pop(@V)); }
298 movdqu `&Xi_off($i)`,$Xi
304 for(;$i<32;$i++) { &ROUND_16_XX($i,@V); unshift(@V,pop(@V)); }
310 lea K256+128(%rip),$Tbl
312 movdqa (%rbx),$sigma # pull counters
313 cmp 4*0(%rbx),%ecx # examine counters
315 cmovge $Tbl,@ptr[0] # cancel input
320 pcmpgtd $t1,$Xn # mask value
323 paddd $Xn,$sigma # counters--
326 movdqu 0x00-0x80($ctx),$t1
328 movdqu 0x20-0x80($ctx),$t2
330 movdqu 0x40-0x80($ctx),$t3
332 movdqu 0x60-0x80($ctx),$Xi
335 movdqu 0x80-0x80($ctx),$t1
338 movdqu 0xa0-0x80($ctx),$t2
341 movdqu 0xc0-0x80($ctx),$t3
344 movdqu 0xe0-0x80($ctx),$Xi
348 movdqu $A,0x00-0x80($ctx)
350 movdqu $B,0x20-0x80($ctx)
352 movdqu $C,0x40-0x80($ctx)
353 movdqu $D,0x60-0x80($ctx)
354 movdqu $E,0x80-0x80($ctx)
355 movdqu $F,0xa0-0x80($ctx)
356 movdqu $G,0xc0-0x80($ctx)
357 movdqu $H,0xe0-0x80($ctx)
359 movdqa $sigma,(%rbx) # save counters
360 movdqa .Lpbswap(%rip),$Xn
364 mov `$REG_SZ*17+8`(%rsp),$num
365 lea $REG_SZ($ctx),$ctx
366 lea `16*$REG_SZ/4`($inp),$inp
371 mov `$REG_SZ*17`(%rsp),%rax # orignal %rsp
373 $code.=<<___ if ($win64);
374 movaps -0xb8(%rax),%xmm6
375 movaps -0xa8(%rax),%xmm7
376 movaps -0x98(%rax),%xmm8
377 movaps -0x88(%rax),%xmm9
378 movaps -0x78(%rax),%xmm10
379 movaps -0x68(%rax),%xmm11
380 movaps -0x58(%rax),%xmm12
381 movaps -0x48(%rax),%xmm13
382 movaps -0x38(%rax),%xmm14
383 movaps -0x28(%rax),%xmm15
390 .size sha256_multi_block,.-sha256_multi_block
393 sub ROUND_00_15_avx {
394 my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
396 $code.=<<___ if ($i<15 && $REG_SZ==16);
397 vmovd `4*$i`(@ptr[0]),$Xi
398 vmovd `4*$i`(@ptr[1]),$t1
399 vpinsrd \$1,`4*$i`(@ptr[2]),$Xi,$Xi
400 vpinsrd \$1,`4*$i`(@ptr[3]),$t1,$t1
401 vpunpckldq $t1,$Xi,$Xi
404 $code.=<<___ if ($i==15 && $REG_SZ==16);
405 vmovd `4*$i`(@ptr[0]),$Xi
406 lea `16*4`(@ptr[0]),@ptr[0]
407 vmovd `4*$i`(@ptr[1]),$t1
408 lea `16*4`(@ptr[1]),@ptr[1]
409 vpinsrd \$1,`4*$i`(@ptr[2]),$Xi,$Xi
410 lea `16*4`(@ptr[2]),@ptr[2]
411 vpinsrd \$1,`4*$i`(@ptr[3]),$t1,$t1
412 lea `16*4`(@ptr[3]),@ptr[3]
413 vpunpckldq $t1,$Xi,$Xi
416 $code.=<<___ if ($i<15 && $REG_SZ==32);
417 vmovd `4*$i`(@ptr[0]),$Xi
418 vmovd `4*$i`(@ptr[4]),$t1
419 vmovd `4*$i`(@ptr[1]),$t2
420 vmovd `4*$i`(@ptr[5]),$t3
421 vpinsrd \$1,`4*$i`(@ptr[2]),$Xi,$Xi
422 vpinsrd \$1,`4*$i`(@ptr[6]),$t1,$t1
423 vpinsrd \$1,`4*$i`(@ptr[3]),$t2,$t2
424 vpunpckldq $t2,$Xi,$Xi
425 vpinsrd \$1,`4*$i`(@ptr[7]),$t3,$t3
426 vpunpckldq $t3,$t1,$t1
427 vinserti128 $t1,$Xi,$Xi
430 $code.=<<___ if ($i==15 && $REG_SZ==32);
431 vmovd `4*$i`(@ptr[0]),$Xi
432 lea `16*4`(@ptr[0]),@ptr[0]
433 vmovd `4*$i`(@ptr[4]),$t1
434 lea `16*4`(@ptr[4]),@ptr[4]
435 vmovd `4*$i`(@ptr[1]),$t2
436 lea `16*4`(@ptr[1]),@ptr[1]
437 vmovd `4*$i`(@ptr[5]),$t3
438 lea `16*4`(@ptr[5]),@ptr[5]
439 vpinsrd \$1,`4*$i`(@ptr[2]),$Xi,$Xi
440 lea `16*4`(@ptr[2]),@ptr[2]
441 vpinsrd \$1,`4*$i`(@ptr[6]),$t1,$t1
442 lea `16*4`(@ptr[6]),@ptr[6]
443 vpinsrd \$1,`4*$i`(@ptr[3]),$t2,$t2
444 lea `16*4`(@ptr[3]),@ptr[3]
445 vpunpckldq $t2,$Xi,$Xi
446 vpinsrd \$1,`4*$i`(@ptr[7]),$t3,$t3
447 lea `16*4`(@ptr[7]),@ptr[7]
448 vpunpckldq $t3,$t1,$t1
449 vinserti128 $t1,$Xi,$Xi
455 vmovdqu $Xi,`&Xi_off($i)`
456 vpaddd $h,$Xi,$Xi # Xi+=h
459 vpxor $t3,$sigma,$sigma
461 vpaddd `32*($i%8)-128`($Tbl),$Xi,$Xi # Xi+=K[round]
462 vpxor $t2,$sigma,$sigma
465 vpxor $t3,$sigma,$sigma
468 vpand $f,$e,$axb # borrow $axb
469 vpxor $t2,$sigma,$sigma
471 vpsrld \$2,$a,$h # borrow $h
472 vpxor $t3,$sigma,$sigma # Sigma1(e)
474 vpxor $axb,$t1,$t1 # Ch(e,f,g)
475 vpxor $a,$b,$axb # a^b, b^c in next round
477 vpaddd $sigma,$Xi,$Xi # Xi+=Sigma1(e)
481 vpaddd $t1,$Xi,$Xi # Xi+=Ch(e,f,g)
486 vpxor $t3,$sigma,$sigma
488 vpxor $bxc,$b,$h # h=Maj(a,b,c)=Ch(a^b,c,b)
489 vpaddd $Xi,$d,$d # d+=Xi
490 vpxor $t2,$sigma,$sigma
491 vpxor $t3,$sigma,$sigma # Sigma0(a)
493 vpaddd $Xi,$h,$h # h+=Xi
494 vpaddd $sigma,$h,$h # h+=Sigma0(a)
496 $code.=<<___ if (($i%8)==7);
499 ($axb,$bxc)=($bxc,$axb);
502 sub ROUND_16_XX_avx {
506 vmovdqu `&Xi_off($i+1)`,$Xn
507 vpaddd `&Xi_off($i+9)`,$Xi,$Xi # Xi+=X[i+9]
509 vpsrld \$3,$Xn,$sigma
512 vpxor $t2,$sigma,$sigma
514 vpxor $t3,$sigma,$sigma
516 vmovdqu `&Xi_off($i+14)`,$t1
517 vpsrld \$10,$t1,$axb # borrow $axb
519 vpxor $t2,$sigma,$sigma
521 vpxor $t3,$sigma,$sigma # sigma0(X[i+1])
523 vpaddd $sigma,$Xi,$Xi # Xi+=sigma0(e)
524 vpxor $t2,$axb,$sigma
526 vpxor $t3,$sigma,$sigma
528 vpxor $t2,$sigma,$sigma
529 vpxor $t3,$sigma,$sigma # sigma0(X[i+14])
530 vpaddd $sigma,$Xi,$Xi # Xi+=sigma1(X[i+14])
532 &ROUND_00_15_avx($i,@_);
537 .type sha256_multi_block_avx,\@function,3
539 sha256_multi_block_avx:
542 $code.=<<___ if ($avx>1);
557 $code.=<<___ if ($win64);
560 movaps %xmm7,0x10(%rsp)
561 movaps %xmm8,0x20(%rsp)
562 movaps %xmm9,0x30(%rsp)
563 movaps %xmm10,-0x78(%rax)
564 movaps %xmm11,-0x68(%rax)
565 movaps %xmm12,-0x58(%rax)
566 movaps %xmm13,-0x48(%rax)
567 movaps %xmm14,-0x38(%rax)
568 movaps %xmm15,-0x28(%rax)
571 sub \$`$REG_SZ*18`, %rsp
573 mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
574 lea K256+128(%rip),$Tbl
575 lea `$REG_SZ*16`(%rsp),%rbx
576 lea 0x80($ctx),$ctx # size optimization
579 mov $num,`$REG_SZ*17+8`(%rsp) # original $num
582 for($i=0;$i<4;$i++) {
584 mov `16*$i+0`($inp),@ptr[$i] # input pointer
585 mov `16*$i+8`($inp),%ecx # number of blocks
587 cmovg %ecx,$num # find maximum
589 mov %ecx,`4*$i`(%rbx) # initialize counters
590 cmovle $Tbl,@ptr[$i] # cancel input
597 vmovdqu 0x00-0x80($ctx),$A # load context
599 vmovdqu 0x20-0x80($ctx),$B
600 vmovdqu 0x40-0x80($ctx),$C
601 vmovdqu 0x60-0x80($ctx),$D
602 vmovdqu 0x80-0x80($ctx),$E
603 vmovdqu 0xa0-0x80($ctx),$F
604 vmovdqu 0xc0-0x80($ctx),$G
605 vmovdqu 0xe0-0x80($ctx),$H
606 vmovdqu .Lpbswap(%rip),$Xn
611 vpxor $B,$C,$bxc # magic seed
613 for($i=0;$i<16;$i++) { &ROUND_00_15_avx($i,@V); unshift(@V,pop(@V)); }
615 vmovdqu `&Xi_off($i)`,$Xi
621 for(;$i<32;$i++) { &ROUND_16_XX_avx($i,@V); unshift(@V,pop(@V)); }
627 lea K256+128(%rip),$Tbl
629 for($i=0;$i<4;$i++) {
631 cmp `4*$i`(%rbx),%ecx # examine counters
632 cmovge $Tbl,@ptr[$i] # cancel input
636 vmovdqa (%rbx),$sigma # pull counters
639 vpcmpgtd $t1,$Xn,$Xn # mask value
640 vpaddd $Xn,$sigma,$sigma # counters--
642 vmovdqu 0x00-0x80($ctx),$t1
644 vmovdqu 0x20-0x80($ctx),$t2
646 vmovdqu 0x40-0x80($ctx),$t3
648 vmovdqu 0x60-0x80($ctx),$Xi
651 vmovdqu 0x80-0x80($ctx),$t1
654 vmovdqu 0xa0-0x80($ctx),$t2
657 vmovdqu 0xc0-0x80($ctx),$t3
660 vmovdqu 0xe0-0x80($ctx),$Xi
664 vmovdqu $A,0x00-0x80($ctx)
666 vmovdqu $B,0x20-0x80($ctx)
668 vmovdqu $C,0x40-0x80($ctx)
669 vmovdqu $D,0x60-0x80($ctx)
670 vmovdqu $E,0x80-0x80($ctx)
671 vmovdqu $F,0xa0-0x80($ctx)
672 vmovdqu $G,0xc0-0x80($ctx)
673 vmovdqu $H,0xe0-0x80($ctx)
675 vmovdqu $sigma,(%rbx) # save counters
676 vmovdqu .Lpbswap(%rip),$Xn
680 mov `$REG_SZ*17+8`(%rsp),$num
681 lea $REG_SZ($ctx),$ctx
682 lea `16*$REG_SZ/4`($inp),$inp
687 mov `$REG_SZ*17`(%rsp),%rax # orignal %rsp
690 $code.=<<___ if ($win64);
691 movaps -0xb8(%rax),%xmm6
692 movaps -0xa8(%rax),%xmm7
693 movaps -0x98(%rax),%xmm8
694 movaps -0x88(%rax),%xmm9
695 movaps -0x78(%rax),%xmm10
696 movaps -0x68(%rax),%xmm11
697 movaps -0x58(%rax),%xmm12
698 movaps -0x48(%rax),%xmm13
699 movaps -0x38(%rax),%xmm14
700 movaps -0x28(%rax),%xmm15
707 .size sha256_multi_block_avx,.-sha256_multi_block_avx
710 $code =~ s/\`([^\`]*)\`/eval $1/gem;
713 @ptr=map("%r$_",(12..15,8..11));
715 @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("%ymm$_",(8..15));
716 ($t1,$t2,$t3,$axb,$bxc,$Xi,$Xn,$sigma)=map("%ymm$_",(0..7));
719 .type sha256_multi_block_avx2,\@function,3
721 sha256_multi_block_avx2:
731 $code.=<<___ if ($win64);
734 movaps %xmm7,0x10(%rsp)
735 movaps %xmm8,0x20(%rsp)
736 movaps %xmm9,0x30(%rsp)
737 movaps %xmm10,0x40(%rsp)
738 movaps %xmm11,0x50(%rsp)
739 movaps %xmm12,-0x78(%rax)
740 movaps %xmm13,-0x68(%rax)
741 movaps %xmm14,-0x58(%rax)
742 movaps %xmm15,-0x48(%rax)
745 sub \$`$REG_SZ*18`, %rsp
747 mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
748 lea K256+128(%rip),$Tbl
749 lea 0x80($ctx),$ctx # size optimization
752 mov $num,`$REG_SZ*17+8`(%rsp) # original $num
754 lea `$REG_SZ*16`(%rsp),%rbx
756 for($i=0;$i<8;$i++) {
758 mov `16*$i+0`($inp),@ptr[$i] # input pointer
759 mov `16*$i+8`($inp),%ecx # number of blocks
761 cmovg %ecx,$num # find maximum
763 mov %ecx,`4*$i`(%rbx) # initialize counters
764 cmovle $Tbl,@ptr[$i] # cancel input
768 vmovdqu 0x00-0x80($ctx),$A # load context
770 vmovdqu 0x20-0x80($ctx),$B
771 lea 256+128(%rsp),%rbx
772 vmovdqu 0x40-0x80($ctx),$C
773 vmovdqu 0x60-0x80($ctx),$D
774 vmovdqu 0x80-0x80($ctx),$E
775 vmovdqu 0xa0-0x80($ctx),$F
776 vmovdqu 0xc0-0x80($ctx),$G
777 vmovdqu 0xe0-0x80($ctx),$H
778 vmovdqu .Lpbswap(%rip),$Xn
783 vpxor $B,$C,$bxc # magic seed
785 for($i=0;$i<16;$i++) { &ROUND_00_15_avx($i,@V); unshift(@V,pop(@V)); }
787 vmovdqu `&Xi_off($i)`,$Xi
793 for(;$i<32;$i++) { &ROUND_16_XX_avx($i,@V); unshift(@V,pop(@V)); }
799 lea `$REG_SZ*16`(%rsp),%rbx
800 lea K256+128(%rip),$Tbl
802 for($i=0;$i<8;$i++) {
804 cmp `4*$i`(%rbx),%ecx # examine counters
805 cmovge $Tbl,@ptr[$i] # cancel input
809 vmovdqa (%rbx),$sigma # pull counters
812 vpcmpgtd $t1,$Xn,$Xn # mask value
813 vpaddd $Xn,$sigma,$sigma # counters--
815 vmovdqu 0x00-0x80($ctx),$t1
817 vmovdqu 0x20-0x80($ctx),$t2
819 vmovdqu 0x40-0x80($ctx),$t3
821 vmovdqu 0x60-0x80($ctx),$Xi
824 vmovdqu 0x80-0x80($ctx),$t1
827 vmovdqu 0xa0-0x80($ctx),$t2
830 vmovdqu 0xc0-0x80($ctx),$t3
833 vmovdqu 0xe0-0x80($ctx),$Xi
837 vmovdqu $A,0x00-0x80($ctx)
839 vmovdqu $B,0x20-0x80($ctx)
841 vmovdqu $C,0x40-0x80($ctx)
842 vmovdqu $D,0x60-0x80($ctx)
843 vmovdqu $E,0x80-0x80($ctx)
844 vmovdqu $F,0xa0-0x80($ctx)
845 vmovdqu $G,0xc0-0x80($ctx)
846 vmovdqu $H,0xe0-0x80($ctx)
848 vmovdqu $sigma,(%rbx) # save counters
849 lea 256+128(%rsp),%rbx
850 vmovdqu .Lpbswap(%rip),$Xn
854 #mov `$REG_SZ*17+8`(%rsp),$num
855 #lea $REG_SZ($ctx),$ctx
856 #lea `16*$REG_SZ/4`($inp),$inp
858 #jnz .Loop_grande_avx2
861 mov `$REG_SZ*17`(%rsp),%rax # orignal %rsp
864 $code.=<<___ if ($win64);
865 movaps -0xd8(%rax),%xmm6
866 movaps -0xc8(%rax),%xmm7
867 movaps -0xb8(%rax),%xmm8
868 movaps -0xa8(%rax),%xmm9
869 movaps -0x98(%rax),%xmm10
870 movaps -0x88(%rax),%xmm11
871 movaps -0x78(%rax),%xmm12
872 movaps -0x68(%rax),%xmm13
873 movaps -0x58(%rax),%xmm14
874 movaps -0x48(%rax),%xmm15
885 .size sha256_multi_block_avx2,.-sha256_multi_block_avx2
900 &TABLE( 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,
901 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5,
902 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,
903 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174,
904 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,
905 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da,
906 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,
907 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967,
908 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,
909 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85,
910 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,
911 0xd192e819,0xd6990624,0xf40e3585,0x106aa070,
912 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,
913 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3,
914 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,
915 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 );
918 .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap
919 .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap
922 foreach (split("\n",$code)) {
923 s/\`([^\`]*)\`/eval($1)/ge;
925 s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or
926 s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go or
927 s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go or
928 s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or
929 s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go or
930 s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go;