3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
10 # Multi-buffer SHA1 procedure processes n buffers in parallel by
11 # placing buffer data to designated lane of SIMD register. n is
12 # naturally limited to 4 on pre-AVX2 processors and to 8 on
13 # AVX2-capable processors such as Haswell.
15 # this +aesni(i) sha1 aesni-sha1 gain(iv)
16 # -------------------------------------------------------------------
17 # Westmere(ii) 10.7/n +1.28=3.96(n=4) 5.30 6.66 +68%
18 # Atom(ii) 18.1/n +3.93=8.46(n=4) 9.37 12.8 +51%
19 # Sandy Bridge (8.16 +5.15=13.3)/n 4.99 5.98 +80%
20 # Ivy Bridge (8.08 +5.14=13.2)/n 4.60 5.54 +68%
21 # Haswell(iii) (8.96 +5.00=14.0)/n 3.57 4.55 +160%
22 # Bulldozer (9.76 +5.76=15.5)/n 5.95 6.37 +64%
24 # (i) multi-block CBC encrypt with 128-bit key;
25 # (ii) (HASH+AES)/n does not apply to Westmere for n>3 and Atom,
26 # because of lower AES-NI instruction throughput;
27 # (iii) "this" is for n=8, when we gather twice as much data, result
28 # for n=4 is 8.00+4.44=12.4;
29 # (iv) presented improvement coefficients are asymptotic limits and
30 # in real-life application are somewhat lower, e.g. for 2KB
31 # fragments they range from 30% to 100% (on Haswell);
35 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
37 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
39 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
40 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
41 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
42 die "can't locate x86_64-xlate.pl";
46 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
47 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
48 $avx = ($1>=2.19) + ($1>=2.22);
51 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
52 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
53 $avx = ($1>=2.09) + ($1>=2.10);
56 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
57 `ml64 2>&1` =~ /Version ([0-9]+)\./) {
58 $avx = ($1>=10) + ($1>=11);
61 if (!$avx && `$ENV{CC} -v` =~ /LLVM ([3-9]\.[0-9]+)/) {
62 $avx = ($1>=3.0) + ($1>=3.1);
65 open OUT,"| \"$^X\" $xlate $flavour $output";
68 # void sha1_multi_block (
69 # struct { unsigned int A[8];
73 # unsigned int E[8]; } *ctx,
74 # struct { void *ptr; int blocks; } inp[8],
75 # int num); /* 1 or 2 */
77 $ctx="%rdi"; # 1st arg
78 $inp="%rsi"; # 2nd arg
80 @ptr=map("%r$_",(8..11));
83 @V=($A,$B,$C,$D,$E)=map("%xmm$_",(0..4));
84 ($t0,$t1,$t2,$t3,$tx)=map("%xmm$_",(5..9));
85 @Xi=map("%xmm$_",(10..14));
89 # Atom-specific optimization aiming to eliminate pshufb with high
90 # registers [and thus get rid of 48 cycles accumulated penalty]
91 @Xi=map("%xmm$_",(0..4));
92 ($tx,$t0,$t1,$t2,$t3)=map("%xmm$_",(5..9));
93 @V=($A,$B,$C,$D,$E)=map("%xmm$_",(10..14));
101 $off %= 16; $off *= $REG_SZ;
102 $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)";
106 my ($i,$a,$b,$c,$d,$e)=@_;
110 $code.=<<___ if ($i==0);
111 movd (@ptr[0]),@Xi[0]
112 lea `16*4`(@ptr[0]),@ptr[0]
113 movd (@ptr[1]),@Xi[2] # borrow @Xi[2]
114 lea `16*4`(@ptr[1]),@ptr[1]
115 movd (@ptr[2]),@Xi[3] # borrow @Xi[3]
116 lea `16*4`(@ptr[2]),@ptr[2]
117 movd (@ptr[3]),@Xi[4] # borrow @Xi[4]
118 lea `16*4`(@ptr[3]),@ptr[3]
119 punpckldq @Xi[3],@Xi[0]
120 movd `4*$j-16*4`(@ptr[0]),@Xi[1]
121 punpckldq @Xi[4],@Xi[2]
122 movd `4*$j-16*4`(@ptr[1]),$t3
123 punpckldq @Xi[2],@Xi[0]
124 movd `4*$j-16*4`(@ptr[2]),$t2
127 $code.=<<___ if ($i<14); # just load input
128 movd `4*$j-16*4`(@ptr[3]),$t1
131 paddd $K,$e # e+=K_00_19
141 movdqa @Xi[0],`&Xi_off($i)`
142 paddd @Xi[0],$e # e+=X[i]
143 movd `4*$k-16*4`(@ptr[0]),@Xi[2]
145 pxor $t1,$t0 # Ch(b,c,d)
148 por $t3,$t2 # rol(a,5)
149 movd `4*$k-16*4`(@ptr[1]),$t3
151 paddd $t0,$e # e+=Ch(b,c,d)
154 paddd $t2,$e # e+=rol(a,5)
156 movd `4*$j-16*4`(@ptr[2]),$t2
157 por $t1,$b # b=rol(b,30)
159 $code.=<<___ if ($i==14); # just load input
160 movd `4*$j-16*4`(@ptr[3]),$t1
163 paddd $K,$e # e+=K_00_19
168 prefetcht0 63(@ptr[0])
174 movdqa @Xi[0],`&Xi_off($i)`
175 paddd @Xi[0],$e # e+=X[i]
177 pxor $t1,$t0 # Ch(b,c,d)
179 prefetcht0 63(@ptr[1])
181 por $t3,$t2 # rol(a,5)
183 paddd $t0,$e # e+=Ch(b,c,d)
184 prefetcht0 63(@ptr[2])
187 paddd $t2,$e # e+=rol(a,5)
189 prefetcht0 63(@ptr[3])
190 por $t1,$b # b=rol(b,30)
192 $code.=<<___ if ($i>=13 && $i<15);
193 movdqa `&Xi_off($j+2)`,@Xi[3] # preload "X[2]"
195 $code.=<<___ if ($i>=15); # apply Xupdate
196 pxor @Xi[-2],@Xi[1] # "X[13]"
197 movdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]"
200 pxor `&Xi_off($j+8)`,@Xi[1]
201 paddd $K,$e # e+=K_00_19
213 movdqa @Xi[0],`&Xi_off($i)`
214 paddd @Xi[0],$e # e+=X[i]
216 pxor $t1,$t0 # Ch(b,c,d)
219 por $t3,$t2 # rol(a,5)
221 paddd $t0,$e # e+=Ch(b,c,d)
224 paddd $t2,$e # e+=rol(a,5)
225 por $tx,@Xi[1] # rol \$1,@Xi[1]
226 por $t1,$b # b=rol(b,30)
228 push(@Xi,shift(@Xi));
232 my ($i,$a,$b,$c,$d,$e)=@_;
235 $code.=<<___ if ($i<79);
236 pxor @Xi[-2],@Xi[1] # "X[13]"
237 movdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]"
241 pxor `&Xi_off($j+8)`,@Xi[1]
242 paddd $K,$e # e+=K_20_39
248 $code.=<<___ if ($i<72);
249 movdqa @Xi[0],`&Xi_off($i)`
251 $code.=<<___ if ($i<79);
252 paddd @Xi[0],$e # e+=X[i]
255 pxor $c,$t0 # Parity(b,c,d)
260 por $t3,$t2 # rol(a,5)
262 paddd $t0,$e # e+=Parity(b,c,d)
266 paddd $t2,$e # e+=rol(a,5)
267 por $tx,@Xi[1] # rol(@Xi[1],1)
268 por $t1,$b # b=rol(b,30)
270 $code.=<<___ if ($i==79);
272 paddd $K,$e # e+=K_20_39
278 paddd @Xi[0],$e # e+=X[i]
281 pxor $c,$t0 # Parity(b,c,d)
284 por $t3,$t2 # rol(a,5)
285 paddd $t0,$e # e+=Parity(b,c,d)
288 paddd $t2,$e # e+=rol(a,5)
289 por $t1,$b # b=rol(b,30)
291 push(@Xi,shift(@Xi));
295 my ($i,$a,$b,$c,$d,$e)=@_;
299 pxor @Xi[-2],@Xi[1] # "X[13]"
300 movdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]"
304 pxor `&Xi_off($j+8)`,@Xi[1]
306 paddd $K,$e # e+=K_40_59
317 movdqa @Xi[0],`&Xi_off($i)`
318 paddd @Xi[0],$e # e+=X[i]
319 por $t3,$t2 # rol(a,5)
326 paddd $t0,$e # e+=Maj(b,d,c)
329 paddd $t2,$e # e+=rol(a,5)
330 por $tx,@Xi[1] # rol(@X[1],1)
331 por $t1,$b # b=rol(b,30)
333 push(@Xi,shift(@Xi));
339 .extern OPENSSL_ia32cap_P
341 .globl sha1_multi_block
342 .type sha1_multi_block,\@function,3
345 mov OPENSSL_ia32cap_P+4(%rip),%rcx
346 bt \$61,%rcx # check SHA bit
349 $code.=<<___ if ($avx);
358 $code.=<<___ if ($win64);
361 movaps %xmm7,0x10(%rsp)
362 movaps %xmm8,0x20(%rsp)
363 movaps %xmm9,0x30(%rsp)
364 movaps %xmm10,-0x78(%rax)
365 movaps %xmm11,-0x68(%rax)
366 movaps %xmm12,-0x58(%rax)
367 movaps %xmm13,-0x48(%rax)
368 movaps %xmm14,-0x38(%rax)
369 movaps %xmm15,-0x28(%rax)
372 sub \$`$REG_SZ*18`,%rsp
374 mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
376 lea K_XX_XX(%rip),$Tbl
377 lea `$REG_SZ*16`(%rsp),%rbx
380 mov $num,`$REG_SZ*17+8`(%rsp) # original $num
383 for($i=0;$i<4;$i++) {
385 mov `16*$i+0`($inp),@ptr[$i] # input pointer
386 mov `16*$i+8`($inp),%ecx # number of blocks
388 cmovg %ecx,$num # find maximum
390 mov %ecx,`4*$i`(%rbx) # initialize counters
391 cmovle $Tbl,@ptr[$i] # cancel input
398 movdqu 0x00($ctx),$A # load context
404 movdqa 0x60($Tbl),$tx # pbswap_mask
405 movdqa -0x20($Tbl),$K # K_00_19
411 for($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); }
412 $code.=" movdqa 0x00($Tbl),$K\n"; # K_20_39
413 for(;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
414 $code.=" movdqa 0x20($Tbl),$K\n"; # K_40_59
415 for(;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); }
416 $code.=" movdqa 0x40($Tbl),$K\n"; # K_60_79
417 for(;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
419 movdqa (%rbx),@Xi[0] # pull counters
421 cmp 4*0(%rbx),%ecx # examinte counters
423 cmovge $Tbl,@ptr[0] # cancel input
428 pcmpgtd $t2,@Xi[1] # mask value
431 paddd @Xi[1],@Xi[0] # counters--
434 movdqu 0x00($ctx),$t0
436 movdqu 0x20($ctx),$t1
439 movdqu 0x40($ctx),$t2
442 movdqu 0x60($ctx),$t3
445 movdqu 0x80($ctx),$tx
455 movdqa @Xi[0],(%rbx) # save counters
456 movdqa 0x60($Tbl),$tx # pbswap_mask
457 movdqa -0x20($Tbl),$K # K_00_19
461 mov `$REG_SZ*17+8`(%rsp),$num
462 lea $REG_SZ($ctx),$ctx
463 lea `16*$REG_SZ/4`($inp),$inp
468 mov `$REG_SZ*17`(%rsp),%rax # orignal %rsp
470 $code.=<<___ if ($win64);
471 movaps -0xb8(%rax),%xmm6
472 movaps -0xa8(%rax),%xmm7
473 movaps -0x98(%rax),%xmm8
474 movaps -0x88(%rax),%xmm9
475 movaps -0x78(%rax),%xmm10
476 movaps -0x68(%rax),%xmm11
477 movaps -0x58(%rax),%xmm12
478 movaps -0x48(%rax),%xmm13
479 movaps -0x38(%rax),%xmm14
480 movaps -0x28(%rax),%xmm15
488 .size sha1_multi_block,.-sha1_multi_block
491 my ($ABCD0,$E0,$E0_,$BSWAP,$ABCD1,$E1,$E1_)=map("%xmm$_",(0..3,8..10));
492 my @MSG0=map("%xmm$_",(4..7));
493 my @MSG1=map("%xmm$_",(11..14));
496 .type sha1_multi_block_shaext,\@function,3
498 sha1_multi_block_shaext:
504 $code.=<<___ if ($win64);
507 movaps %xmm7,0x10(%rsp)
508 movaps %xmm8,0x20(%rsp)
509 movaps %xmm9,0x30(%rsp)
510 movaps %xmm10,-0x78(%rax)
511 movaps %xmm11,-0x68(%rax)
512 movaps %xmm12,-0x58(%rax)
513 movaps %xmm13,-0x48(%rax)
514 movaps %xmm14,-0x38(%rax)
515 movaps %xmm15,-0x28(%rax)
518 sub \$`$REG_SZ*18`,%rsp
519 shl \$1,$num # we process pair at a time
521 lea 0x40($ctx),$ctx # size optimization
522 mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
524 lea `$REG_SZ*16`(%rsp),%rbx
525 movdqa K_XX_XX+0x80(%rip),$BSWAP # byte-n-word swap
528 mov $num,`$REG_SZ*17+8`(%rsp) # orignal $num
531 for($i=0;$i<2;$i++) {
533 mov `16*$i+0`($inp),@ptr[$i] # input pointer
534 mov `16*$i+8`($inp),%ecx # number of blocks
536 cmovg %ecx,$num # find maximum
538 mov %ecx,`4*$i`(%rbx) # initialize counters
539 cmovle %rsp,@ptr[$i] # cancel input
546 movq 0x00-0x40($ctx),$ABCD0 # a1.a0
547 movq 0x20-0x40($ctx),@MSG0[0]# b1.b0
548 movq 0x40-0x40($ctx),@MSG0[1]# c1.c0
549 movq 0x60-0x40($ctx),@MSG0[2]# d1.d0
550 movq 0x80-0x40($ctx),@MSG0[3]# e1.e0
552 punpckldq @MSG0[0],$ABCD0 # b1.a1.b0.a0
553 punpckldq @MSG0[2],@MSG0[1] # d1.c1.d0.c0
556 punpcklqdq @MSG0[1],$ABCD0 # d0.c0.b0.a0
557 punpckhqdq @MSG0[1],$ABCD1 # d1.c1.b1.a1
559 pshufd \$0b00111111,@MSG0[3],$E0
560 pshufd \$0b01111111,@MSG0[3],$E1
561 pshufd \$0b00011011,$ABCD0,$ABCD0
562 pshufd \$0b00011011,$ABCD1,$ABCD1
567 movdqu 0x00(@ptr[0]),@MSG0[0]
568 movdqu 0x00(@ptr[1]),@MSG1[0]
569 movdqu 0x10(@ptr[0]),@MSG0[1]
570 movdqu 0x10(@ptr[1]),@MSG1[1]
571 movdqu 0x20(@ptr[0]),@MSG0[2]
572 pshufb $BSWAP,@MSG0[0]
573 movdqu 0x20(@ptr[1]),@MSG1[2]
574 pshufb $BSWAP,@MSG1[0]
575 movdqu 0x30(@ptr[0]),@MSG0[3]
576 lea 0x40(@ptr[0]),@ptr[0]
577 pshufb $BSWAP,@MSG0[1]
578 movdqu 0x30(@ptr[1]),@MSG1[3]
579 lea 0x40(@ptr[1]),@ptr[1]
580 pshufb $BSWAP,@MSG1[1]
582 movdqa $E0,0x50(%rsp) # offload
584 movdqa $E1,0x70(%rsp)
586 movdqa $ABCD0,0x40(%rsp) # offload
588 movdqa $ABCD1,0x60(%rsp)
590 sha1rnds4 \$0,$E0,$ABCD0 # 0-3
591 sha1nexte @MSG0[1],$E0_
592 sha1rnds4 \$0,$E1,$ABCD1 # 0-3
593 sha1nexte @MSG1[1],$E1_
594 pshufb $BSWAP,@MSG0[2]
595 prefetcht0 127(@ptr[0])
596 sha1msg1 @MSG0[1],@MSG0[0]
597 pshufb $BSWAP,@MSG1[2]
598 prefetcht0 127(@ptr[1])
599 sha1msg1 @MSG1[1],@MSG1[0]
601 pshufb $BSWAP,@MSG0[3]
603 pshufb $BSWAP,@MSG1[3]
605 sha1rnds4 \$0,$E0_,$ABCD0 # 4-7
606 sha1nexte @MSG0[2],$E0
607 sha1rnds4 \$0,$E1_,$ABCD1 # 4-7
608 sha1nexte @MSG1[2],$E1
609 pxor @MSG0[2],@MSG0[0]
610 sha1msg1 @MSG0[2],@MSG0[1]
611 pxor @MSG1[2],@MSG1[0]
612 sha1msg1 @MSG1[2],@MSG1[1]
614 for($i=2;$i<20-4;$i++) {
618 sha1rnds4 \$`int($i/5)`,$E0,$ABCD0 # 8-11
619 sha1nexte @MSG0[3],$E0_
620 sha1rnds4 \$`int($i/5)`,$E1,$ABCD1 # 8-11
621 sha1nexte @MSG1[3],$E1_
622 sha1msg2 @MSG0[3],@MSG0[0]
623 sha1msg2 @MSG1[3],@MSG1[0]
624 pxor @MSG0[3],@MSG0[1]
625 sha1msg1 @MSG0[3],@MSG0[2]
626 pxor @MSG1[3],@MSG1[1]
627 sha1msg1 @MSG1[3],@MSG1[2]
629 ($E0,$E0_)=($E0_,$E0); ($E1,$E1_)=($E1_,$E1);
630 push(@MSG0,shift(@MSG0)); push(@MSG1,shift(@MSG1));
635 sha1rnds4 \$3,$E0,$ABCD0 # 64-67
636 sha1nexte @MSG0[3],$E0_
637 sha1rnds4 \$3,$E1,$ABCD1 # 64-67
638 sha1nexte @MSG1[3],$E1_
639 sha1msg2 @MSG0[3],@MSG0[0]
640 sha1msg2 @MSG1[3],@MSG1[0]
641 pxor @MSG0[3],@MSG0[1]
642 pxor @MSG1[3],@MSG1[1]
645 pxor @MSG0[2],@MSG0[2] # zero
646 cmp 4*0(%rbx),%ecx # examine counters
647 cmovge %rsp,@ptr[0] # cancel input
651 sha1rnds4 \$3,$E0_,$ABCD0 # 68-71
652 sha1nexte @MSG0[0],$E0
653 sha1rnds4 \$3,$E1_,$ABCD1 # 68-71
654 sha1nexte @MSG1[0],$E1
655 sha1msg2 @MSG0[0],@MSG0[1]
656 sha1msg2 @MSG1[0],@MSG1[1]
660 movq (%rbx),@MSG0[0] # pull counters
664 sha1rnds4 \$3,$E0,$ABCD0 # 72-75
665 sha1nexte @MSG0[1],$E0_
666 sha1rnds4 \$3,$E1,$ABCD1 # 72-75
667 sha1nexte @MSG1[1],$E1_
669 pshufd \$0x00,@MSG0[0],@MSG1[2]
670 pshufd \$0x55,@MSG0[0],@MSG1[3]
671 movdqa @MSG0[0],@MSG0[1]
672 pcmpgtd @MSG0[2],@MSG1[2]
673 pcmpgtd @MSG0[2],@MSG1[3]
677 sha1rnds4 \$3,$E0_,$ABCD0 # 76-79
678 sha1nexte $MSG0[2],$E0
679 sha1rnds4 \$3,$E1_,$ABCD1 # 76-79
680 sha1nexte $MSG0[2],$E1
682 pcmpgtd @MSG0[2],@MSG0[1] # counter mask
687 paddd @MSG0[1],@MSG0[0] # counters--
689 paddd 0x40(%rsp),$ABCD0
691 paddd 0x60(%rsp),$ABCD1
694 movq @MSG0[0],(%rbx) # save counters
698 mov `$REG_SZ*17+8`(%rsp),$num
700 pshufd \$0b00011011,$ABCD0,$ABCD0
701 pshufd \$0b00011011,$ABCD1,$ABCD1
703 movdqa $ABCD0,@MSG0[0]
704 punpckldq $ABCD1,$ABCD0 # b1.b0.a1.a0
705 punpckhdq $ABCD1,@MSG0[0] # d1.d0.c1.c0
706 punpckhdq $E1,$E0 # e1.e0.xx.xx
707 movq $ABCD0,0x00-0x40($ctx) # a1.a0
709 movq @MSG0[0],0x40-0x40($ctx)# c1.c0
711 movq $ABCD0,0x20-0x40($ctx) # b1.b0
713 movq @MSG0[0],0x60-0x40($ctx)# d1.d0
714 movq $E0,0x80-0x40($ctx) # e1.e0
716 lea `$REG_SZ/2`($ctx),$ctx
717 lea `16*2`($inp),$inp
719 jnz .Loop_grande_shaext
722 #mov `$REG_SZ*17`(%rsp),%rax # original %rsp
724 $code.=<<___ if ($win64);
725 movaps -0xb8(%rax),%xmm6
726 movaps -0xa8(%rax),%xmm7
727 movaps -0x98(%rax),%xmm8
728 movaps -0x88(%rax),%xmm9
729 movaps -0x78(%rax),%xmm10
730 movaps -0x68(%rax),%xmm11
731 movaps -0x58(%rax),%xmm12
732 movaps -0x48(%rax),%xmm13
733 movaps -0x38(%rax),%xmm14
734 movaps -0x28(%rax),%xmm15
742 .size sha1_multi_block_shaext,.-sha1_multi_block_shaext
748 my ($i,$a,$b,$c,$d,$e)=@_;
751 my $vpack = $REG_SZ==16 ? "vpunpckldq" : "vinserti128";
752 my $ptr_n = $REG_SZ==16 ? @ptr[1] : @ptr[4];
754 $code.=<<___ if ($i==0 && $REG_SZ==16);
755 vmovd (@ptr[0]),@Xi[0]
756 lea `16*4`(@ptr[0]),@ptr[0]
757 vmovd (@ptr[1]),@Xi[2] # borrow Xi[2]
758 lea `16*4`(@ptr[1]),@ptr[1]
759 vpinsrd \$1,(@ptr[2]),@Xi[0],@Xi[0]
760 lea `16*4`(@ptr[2]),@ptr[2]
761 vpinsrd \$1,(@ptr[3]),@Xi[2],@Xi[2]
762 lea `16*4`(@ptr[3]),@ptr[3]
763 vmovd `4*$j-16*4`(@ptr[0]),@Xi[1]
764 vpunpckldq @Xi[2],@Xi[0],@Xi[0]
765 vmovd `4*$j-16*4`($ptr_n),$t3
766 vpshufb $tx,@Xi[0],@Xi[0]
768 $code.=<<___ if ($i<15 && $REG_SZ==16); # just load input
769 vpinsrd \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1]
770 vpinsrd \$1,`4*$j-16*4`(@ptr[3]),$t3,$t3
772 $code.=<<___ if ($i==0 && $REG_SZ==32);
773 vmovd (@ptr[0]),@Xi[0]
774 lea `16*4`(@ptr[0]),@ptr[0]
775 vmovd (@ptr[4]),@Xi[2] # borrow Xi[2]
776 lea `16*4`(@ptr[4]),@ptr[4]
778 lea `16*4`(@ptr[1]),@ptr[1]
780 lea `16*4`(@ptr[5]),@ptr[5]
781 vpinsrd \$1,(@ptr[2]),@Xi[0],@Xi[0]
782 lea `16*4`(@ptr[2]),@ptr[2]
783 vpinsrd \$1,(@ptr[6]),@Xi[2],@Xi[2]
784 lea `16*4`(@ptr[6]),@ptr[6]
785 vpinsrd \$1,(@ptr[3]),$t2,$t2
786 lea `16*4`(@ptr[3]),@ptr[3]
787 vpunpckldq $t2,@Xi[0],@Xi[0]
788 vpinsrd \$1,(@ptr[7]),$t1,$t1
789 lea `16*4`(@ptr[7]),@ptr[7]
790 vpunpckldq $t1,@Xi[2],@Xi[2]
791 vmovd `4*$j-16*4`(@ptr[0]),@Xi[1]
792 vinserti128 @Xi[2],@Xi[0],@Xi[0]
793 vmovd `4*$j-16*4`($ptr_n),$t3
794 vpshufb $tx,@Xi[0],@Xi[0]
796 $code.=<<___ if ($i<15 && $REG_SZ==32); # just load input
797 vmovd `4*$j-16*4`(@ptr[1]),$t2
798 vmovd `4*$j-16*4`(@ptr[5]),$t1
799 vpinsrd \$1,`4*$j-16*4`(@ptr[2]),@Xi[1],@Xi[1]
800 vpinsrd \$1,`4*$j-16*4`(@ptr[6]),$t3,$t3
801 vpinsrd \$1,`4*$j-16*4`(@ptr[3]),$t2,$t2
802 vpunpckldq $t2,@Xi[1],@Xi[1]
803 vpinsrd \$1,`4*$j-16*4`(@ptr[7]),$t1,$t1
804 vpunpckldq $t1,$t3,$t3
806 $code.=<<___ if ($i<14);
807 vpaddd $K,$e,$e # e+=K_00_19
812 vmovdqa @Xi[0],`&Xi_off($i)`
813 vpaddd @Xi[0],$e,$e # e+=X[i]
814 $vpack $t3,@Xi[1],@Xi[1]
816 vpxor $t1,$t0,$t0 # Ch(b,c,d)
817 vmovd `4*$k-16*4`(@ptr[0]),@Xi[2]
820 vpor $t3,$t2,$t2 # rol(a,5)
821 vmovd `4*$k-16*4`($ptr_n),$t3
822 vpaddd $t0,$e,$e # e+=Ch(b,c,d)
825 vpaddd $t2,$e,$e # e+=rol(a,5)
826 vpshufb $tx,@Xi[1],@Xi[1]
827 vpor $t1,$b,$b # b=rol(b,30)
829 $code.=<<___ if ($i==14);
830 vpaddd $K,$e,$e # e+=K_00_19
831 prefetcht0 63(@ptr[0])
836 vmovdqa @Xi[0],`&Xi_off($i)`
837 vpaddd @Xi[0],$e,$e # e+=X[i]
838 $vpack $t3,@Xi[1],@Xi[1]
840 prefetcht0 63(@ptr[1])
841 vpxor $t1,$t0,$t0 # Ch(b,c,d)
844 vpor $t3,$t2,$t2 # rol(a,5)
845 prefetcht0 63(@ptr[2])
846 vpaddd $t0,$e,$e # e+=Ch(b,c,d)
849 vpaddd $t2,$e,$e # e+=rol(a,5)
850 prefetcht0 63(@ptr[3])
851 vpshufb $tx,@Xi[1],@Xi[1]
852 vpor $t1,$b,$b # b=rol(b,30)
854 $code.=<<___ if ($i>=13 && $i<15);
855 vmovdqa `&Xi_off($j+2)`,@Xi[3] # preload "X[2]"
857 $code.=<<___ if ($i>=15); # apply Xupdate
858 vpxor @Xi[-2],@Xi[1],@Xi[1] # "X[13]"
859 vmovdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]"
861 vpaddd $K,$e,$e # e+=K_00_19
864 `"prefetcht0 63(@ptr[4])" if ($i==15 && $REG_SZ==32)`
867 vmovdqa @Xi[0],`&Xi_off($i)`
868 vpaddd @Xi[0],$e,$e # e+=X[i]
869 vpxor `&Xi_off($j+8)`,@Xi[1],@Xi[1]
871 vpxor $t1,$t0,$t0 # Ch(b,c,d)
872 vpxor @Xi[3],@Xi[1],@Xi[1]
873 `"prefetcht0 63(@ptr[5])" if ($i==15 && $REG_SZ==32)`
876 vpor $t3,$t2,$t2 # rol(a,5)
877 vpaddd $t0,$e,$e # e+=Ch(b,c,d)
878 `"prefetcht0 63(@ptr[6])" if ($i==15 && $REG_SZ==32)`
879 vpsrld \$31,@Xi[1],$tx
880 vpaddd @Xi[1],@Xi[1],@Xi[1]
883 `"prefetcht0 63(@ptr[7])" if ($i==15 && $REG_SZ==32)`
884 vpaddd $t2,$e,$e # e+=rol(a,5)
885 vpor $tx,@Xi[1],@Xi[1] # rol \$1,@Xi[1]
886 vpor $t1,$b,$b # b=rol(b,30)
888 push(@Xi,shift(@Xi));
892 my ($i,$a,$b,$c,$d,$e)=@_;
895 $code.=<<___ if ($i<79);
896 vpxor @Xi[-2],@Xi[1],@Xi[1] # "X[13]"
897 vmovdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]"
900 vpaddd $K,$e,$e # e+=K_20_39
903 $code.=<<___ if ($i<72);
904 vmovdqa @Xi[0],`&Xi_off($i)`
906 $code.=<<___ if ($i<79);
907 vpaddd @Xi[0],$e,$e # e+=X[i]
908 vpxor `&Xi_off($j+8)`,@Xi[1],@Xi[1]
910 vpxor $c,$t0,$t0 # Parity(b,c,d)
911 vpxor @Xi[3],@Xi[1],@Xi[1]
914 vpor $t3,$t2,$t2 # rol(a,5)
915 vpaddd $t0,$e,$e # e+=Parity(b,c,d)
916 vpsrld \$31,@Xi[1],$tx
917 vpaddd @Xi[1],@Xi[1],@Xi[1]
920 vpaddd $t2,$e,$e # e+=rol(a,5)
921 vpor $tx,@Xi[1],@Xi[1] # rol(@Xi[1],1)
922 vpor $t1,$b,$b # b=rol(b,30)
924 $code.=<<___ if ($i==79);
926 vpaddd $K,$e,$e # e+=K_20_39
930 vpaddd @Xi[0],$e,$e # e+=X[i]
931 vpxor $c,$t0,$t0 # Parity(b,c,d)
934 vpor $t3,$t2,$t2 # rol(a,5)
935 vpaddd $t0,$e,$e # e+=Parity(b,c,d)
938 vpaddd $t2,$e,$e # e+=rol(a,5)
939 vpor $t1,$b,$b # b=rol(b,30)
941 push(@Xi,shift(@Xi));
945 my ($i,$a,$b,$c,$d,$e)=@_;
949 vpxor @Xi[-2],@Xi[1],@Xi[1] # "X[13]"
950 vmovdqa `&Xi_off($j+2)`,@Xi[3] # "X[2]"
952 vpaddd $K,$e,$e # e+=K_40_59
955 vpxor `&Xi_off($j+8)`,@Xi[1],@Xi[1]
960 vpxor @Xi[3],@Xi[1],@Xi[1]
962 vmovdqu @Xi[0],`&Xi_off($i)`
963 vpaddd @Xi[0],$e,$e # e+=X[i]
964 vpor $t3,$t2,$t2 # rol(a,5)
965 vpsrld \$31,@Xi[1],$tx
967 vpaddd @Xi[1],@Xi[1],@Xi[1]
970 vpaddd $t0,$e,$e # e+=Maj(b,d,c)
973 vpaddd $t2,$e,$e # e+=rol(a,5)
974 vpor $tx,@Xi[1],@Xi[1] # rol(@X[1],1)
975 vpor $t1,$b,$b # b=rol(b,30)
977 push(@Xi,shift(@Xi));
981 .type sha1_multi_block_avx,\@function,3
983 sha1_multi_block_avx:
986 $code.=<<___ if ($avx>1);
1001 $code.=<<___ if ($win64);
1002 lea -0xa8(%rsp),%rsp
1004 movaps %xmm7,0x10(%rsp)
1005 movaps %xmm8,0x20(%rsp)
1006 movaps %xmm9,0x30(%rsp)
1007 movaps %xmm10,-0x78(%rax)
1008 movaps %xmm11,-0x68(%rax)
1009 movaps %xmm12,-0x58(%rax)
1010 movaps %xmm13,-0x48(%rax)
1011 movaps %xmm14,-0x38(%rax)
1012 movaps %xmm15,-0x28(%rax)
1015 sub \$`$REG_SZ*18`, %rsp
1017 mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
1019 lea K_XX_XX(%rip),$Tbl
1020 lea `$REG_SZ*16`(%rsp),%rbx
1024 mov $num,`$REG_SZ*17+8`(%rsp) # original $num
1027 for($i=0;$i<4;$i++) {
1029 mov `16*$i+0`($inp),@ptr[$i] # input pointer
1030 mov `16*$i+8`($inp),%ecx # number of blocks
1032 cmovg %ecx,$num # find maximum
1034 mov %ecx,`4*$i`(%rbx) # initialize counters
1035 cmovle $Tbl,@ptr[$i] # cancel input
1042 vmovdqu 0x00($ctx),$A # load context
1044 vmovdqu 0x20($ctx),$B
1045 vmovdqu 0x40($ctx),$C
1046 vmovdqu 0x60($ctx),$D
1047 vmovdqu 0x80($ctx),$E
1048 vmovdqu 0x60($Tbl),$tx # pbswap_mask
1054 $code.=" vmovdqa -0x20($Tbl),$K\n"; # K_00_19
1055 for($i=0;$i<20;$i++) { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); }
1056 $code.=" vmovdqa 0x00($Tbl),$K\n"; # K_20_39
1057 for(;$i<40;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1058 $code.=" vmovdqa 0x20($Tbl),$K\n"; # K_40_59
1059 for(;$i<60;$i++) { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); }
1060 $code.=" vmovdqa 0x40($Tbl),$K\n"; # K_60_79
1061 for(;$i<80;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1065 for($i=0;$i<4;$i++) {
1067 cmp `4*$i`(%rbx),%ecx # examine counters
1068 cmovge $Tbl,@ptr[$i] # cancel input
1072 vmovdqu (%rbx),$t0 # pull counters
1075 vpcmpgtd $t2,$t1,$t1 # mask value
1076 vpaddd $t1,$t0,$t0 # counters--
1080 vpaddd 0x00($ctx),$A,$A
1082 vpaddd 0x20($ctx),$B,$B
1084 vpaddd 0x40($ctx),$C,$C
1086 vpaddd 0x60($ctx),$D,$D
1087 vpaddd 0x80($ctx),$E,$E
1088 vmovdqu $A,0x00($ctx)
1089 vmovdqu $B,0x20($ctx)
1090 vmovdqu $C,0x40($ctx)
1091 vmovdqu $D,0x60($ctx)
1092 vmovdqu $E,0x80($ctx)
1094 vmovdqu $t0,(%rbx) # save counters
1095 vmovdqu 0x60($Tbl),$tx # pbswap_mask
1099 mov `$REG_SZ*17+8`(%rsp),$num
1100 lea $REG_SZ($ctx),$ctx
1101 lea `16*$REG_SZ/4`($inp),$inp
1103 jnz .Loop_grande_avx
1106 mov `$REG_SZ*17`(%rsp),%rax # orignal %rsp
1109 $code.=<<___ if ($win64);
1110 movaps -0xb8(%rax),%xmm6
1111 movaps -0xa8(%rax),%xmm7
1112 movaps -0x98(%rax),%xmm8
1113 movaps -0x88(%rax),%xmm9
1114 movaps -0x78(%rax),%xmm10
1115 movaps -0x68(%rax),%xmm11
1116 movaps -0x58(%rax),%xmm12
1117 movaps -0x48(%rax),%xmm13
1118 movaps -0x38(%rax),%xmm14
1119 movaps -0x28(%rax),%xmm15
1127 .size sha1_multi_block_avx,.-sha1_multi_block_avx
1131 $code =~ s/\`([^\`]*)\`/eval $1/gem;
1135 @ptr=map("%r$_",(12..15,8..11));
1137 @V=($A,$B,$C,$D,$E)=map("%ymm$_",(0..4));
1138 ($t0,$t1,$t2,$t3,$tx)=map("%ymm$_",(5..9));
1139 @Xi=map("%ymm$_",(10..14));
1143 .type sha1_multi_block_avx2,\@function,3
1145 sha1_multi_block_avx2:
1155 $code.=<<___ if ($win64);
1156 lea -0xa8(%rsp),%rsp
1158 movaps %xmm7,0x10(%rsp)
1159 movaps %xmm8,0x20(%rsp)
1160 movaps %xmm9,0x30(%rsp)
1161 movaps %xmm10,0x40(%rsp)
1162 movaps %xmm11,0x50(%rsp)
1163 movaps %xmm12,-0x78(%rax)
1164 movaps %xmm13,-0x68(%rax)
1165 movaps %xmm14,-0x58(%rax)
1166 movaps %xmm15,-0x48(%rax)
1169 sub \$`$REG_SZ*18`, %rsp
1171 mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
1173 lea K_XX_XX(%rip),$Tbl
1178 mov $num,`$REG_SZ*17+8`(%rsp) # original $num
1180 lea `$REG_SZ*16`(%rsp),%rbx
1182 for($i=0;$i<8;$i++) {
1184 mov `16*$i+0`($inp),@ptr[$i] # input pointer
1185 mov `16*$i+8`($inp),%ecx # number of blocks
1187 cmovg %ecx,$num # find maximum
1189 mov %ecx,`4*$i`(%rbx) # initialize counters
1190 cmovle $Tbl,@ptr[$i] # cancel input
1194 vmovdqu 0x00($ctx),$A # load context
1196 vmovdqu 0x20($ctx),$B
1197 lea 256+128(%rsp),%rbx
1198 vmovdqu 0x40($ctx),$C
1199 vmovdqu 0x60($ctx),$D
1200 vmovdqu 0x80($ctx),$E
1201 vmovdqu 0x60($Tbl),$tx # pbswap_mask
1207 $code.=" vmovdqa -0x20($Tbl),$K\n"; # K_00_19
1208 for($i=0;$i<20;$i++) { &BODY_00_19_avx($i,@V); unshift(@V,pop(@V)); }
1209 $code.=" vmovdqa 0x00($Tbl),$K\n"; # K_20_39
1210 for(;$i<40;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1211 $code.=" vmovdqa 0x20($Tbl),$K\n"; # K_40_59
1212 for(;$i<60;$i++) { &BODY_40_59_avx($i,@V); unshift(@V,pop(@V)); }
1213 $code.=" vmovdqa 0x40($Tbl),$K\n"; # K_60_79
1214 for(;$i<80;$i++) { &BODY_20_39_avx($i,@V); unshift(@V,pop(@V)); }
1217 lea `$REG_SZ*16`(%rsp),%rbx
1219 for($i=0;$i<8;$i++) {
1221 cmp `4*$i`(%rbx),%ecx # examine counters
1222 cmovge $Tbl,@ptr[$i] # cancel input
1226 vmovdqu (%rbx),$t0 # pull counters
1229 vpcmpgtd $t2,$t1,$t1 # mask value
1230 vpaddd $t1,$t0,$t0 # counters--
1234 vpaddd 0x00($ctx),$A,$A
1236 vpaddd 0x20($ctx),$B,$B
1238 vpaddd 0x40($ctx),$C,$C
1240 vpaddd 0x60($ctx),$D,$D
1241 vpaddd 0x80($ctx),$E,$E
1242 vmovdqu $A,0x00($ctx)
1243 vmovdqu $B,0x20($ctx)
1244 vmovdqu $C,0x40($ctx)
1245 vmovdqu $D,0x60($ctx)
1246 vmovdqu $E,0x80($ctx)
1248 vmovdqu $t0,(%rbx) # save counters
1249 lea 256+128(%rsp),%rbx
1250 vmovdqu 0x60($Tbl),$tx # pbswap_mask
1254 #mov `$REG_SZ*17+8`(%rsp),$num
1255 #lea $REG_SZ($ctx),$ctx
1256 #lea `16*$REG_SZ/4`($inp),$inp
1258 #jnz .Loop_grande_avx2
1261 mov `$REG_SZ*17`(%rsp),%rax # orignal %rsp
1264 $code.=<<___ if ($win64);
1265 movaps -0xd8(%rax),%xmm6
1266 movaps -0xc8(%rax),%xmm7
1267 movaps -0xb8(%rax),%xmm8
1268 movaps -0xa8(%rax),%xmm9
1269 movaps -0x98(%rax),%xmm10
1270 movaps -0x88(%rax),%xmm11
1271 movaps -0x78(%rax),%xmm12
1272 movaps -0x68(%rax),%xmm13
1273 movaps -0x58(%rax),%xmm14
1274 movaps -0x48(%rax),%xmm15
1286 .size sha1_multi_block_avx2,.-sha1_multi_block_avx2
1292 .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19
1293 .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19
1295 .long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 # K_20_39
1296 .long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 # K_20_39
1297 .long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59
1298 .long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59
1299 .long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79
1300 .long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79
1301 .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap
1302 .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap
1303 .byte 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0
1304 .asciz "SHA1 multi-block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
1308 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1309 # CONTEXT *context,DISPATCHER_CONTEXT *disp)
1316 .extern __imp_RtlVirtualUnwind
1317 .type se_handler,\@abi-omnipotent
1331 mov 120($context),%rax # pull context->Rax
1332 mov 248($context),%rbx # pull context->Rip
1334 mov 8($disp),%rsi # disp->ImageBase
1335 mov 56($disp),%r11 # disp->HandlerData
1337 mov 0(%r11),%r10d # HandlerData[0]
1338 lea (%rsi,%r10),%r10 # end of prologue label
1339 cmp %r10,%rbx # context->Rip<.Lbody
1342 mov 152($context),%rax # pull context->Rsp
1344 mov 4(%r11),%r10d # HandlerData[1]
1345 lea (%rsi,%r10),%r10 # epilogue label
1346 cmp %r10,%rbx # context->Rip>=.Lepilogue
1349 mov `16*17`(%rax),%rax # pull saved stack pointer
1353 mov %rbx,144($context) # restore context->Rbx
1354 mov %rbp,160($context) # restore context->Rbp
1356 lea -24-10*16(%rax),%rsi
1357 lea 512($context),%rdi # &context.Xmm6
1359 .long 0xa548f3fc # cld; rep movsq
1364 mov %rax,152($context) # restore context->Rsp
1365 mov %rsi,168($context) # restore context->Rsi
1366 mov %rdi,176($context) # restore context->Rdi
1368 mov 40($disp),%rdi # disp->ContextRecord
1369 mov $context,%rsi # context
1370 mov \$154,%ecx # sizeof(CONTEXT)
1371 .long 0xa548f3fc # cld; rep movsq
1374 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
1375 mov 8(%rsi),%rdx # arg2, disp->ImageBase
1376 mov 0(%rsi),%r8 # arg3, disp->ControlPc
1377 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
1378 mov 40(%rsi),%r10 # disp->ContextRecord
1379 lea 56(%rsi),%r11 # &disp->HandlerData
1380 lea 24(%rsi),%r12 # &disp->EstablisherFrame
1381 mov %r10,32(%rsp) # arg5
1382 mov %r11,40(%rsp) # arg6
1383 mov %r12,48(%rsp) # arg7
1384 mov %rcx,56(%rsp) # arg8, (NULL)
1385 call *__imp_RtlVirtualUnwind(%rip)
1387 mov \$1,%eax # ExceptionContinueSearch
1399 .size se_handler,.-se_handler
1401 $code.=<<___ if ($avx>1);
1402 .type avx2_handler,\@abi-omnipotent
1416 mov 120($context),%rax # pull context->Rax
1417 mov 248($context),%rbx # pull context->Rip
1419 mov 8($disp),%rsi # disp->ImageBase
1420 mov 56($disp),%r11 # disp->HandlerData
1422 mov 0(%r11),%r10d # HandlerData[0]
1423 lea (%rsi,%r10),%r10 # end of prologue label
1424 cmp %r10,%rbx # context->Rip<body label
1427 mov 152($context),%rax # pull context->Rsp
1429 mov 4(%r11),%r10d # HandlerData[1]
1430 lea (%rsi,%r10),%r10 # epilogue label
1431 cmp %r10,%rbx # context->Rip>=epilogue label
1434 mov `32*17`($context),%rax # pull saved stack pointer
1442 mov %rbx,144($context) # restore context->Rbx
1443 mov %rbp,160($context) # restore context->Rbp
1444 mov %r12,216($context) # restore cotnext->R12
1445 mov %r13,224($context) # restore cotnext->R13
1446 mov %r14,232($context) # restore cotnext->R14
1447 mov %r15,240($context) # restore cotnext->R15
1449 lea -56-10*16(%rax),%rsi
1450 lea 512($context),%rdi # &context.Xmm6
1452 .long 0xa548f3fc # cld; rep movsq
1455 .size avx2_handler,.-avx2_handler
1460 .rva .LSEH_begin_sha1_multi_block
1461 .rva .LSEH_end_sha1_multi_block
1462 .rva .LSEH_info_sha1_multi_block
1463 .rva .LSEH_begin_sha1_multi_block_shaext
1464 .rva .LSEH_end_sha1_multi_block_shaext
1465 .rva .LSEH_info_sha1_multi_block_shaext
1467 $code.=<<___ if ($avx);
1468 .rva .LSEH_begin_sha1_multi_block_avx
1469 .rva .LSEH_end_sha1_multi_block_avx
1470 .rva .LSEH_info_sha1_multi_block_avx
1472 $code.=<<___ if ($avx>1);
1473 .rva .LSEH_begin_sha1_multi_block_avx2
1474 .rva .LSEH_end_sha1_multi_block_avx2
1475 .rva .LSEH_info_sha1_multi_block_avx2
1480 .LSEH_info_sha1_multi_block:
1483 .rva .Lbody,.Lepilogue # HandlerData[]
1484 .LSEH_info_sha1_multi_block_shaext:
1487 .rva .Lbody_shaext,.Lepilogue_shaext # HandlerData[]
1489 $code.=<<___ if ($avx);
1490 .LSEH_info_sha1_multi_block_avx:
1493 .rva .Lbody_avx,.Lepilogue_avx # HandlerData[]
1495 $code.=<<___ if ($avx>1);
1496 .LSEH_info_sha1_multi_block_avx2:
1499 .rva .Lbody_avx2,.Lepilogue_avx2 # HandlerData[]
1502 ####################################################################
1505 local *opcode=shift;
1509 $rex|=0x04 if ($dst>=8);
1510 $rex|=0x01 if ($src>=8);
1511 unshift @opcode,$rex|0x40 if ($rex);
1515 if (@_[0] =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1516 my @opcode=(0x0f,0x3a,0xcc);
1517 rex(\@opcode,$3,$2);
1518 push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M
1520 push @opcode,$c=~/^0/?oct($c):$c;
1521 return ".byte\t".join(',',@opcode);
1523 return "sha1rnds4\t".@_[0];
1530 "sha1nexte" => 0xc8,
1532 "sha1msg2" => 0xca );
1534 if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1535 my @opcode=(0x0f,0x38);
1536 rex(\@opcode,$2,$1);
1537 push @opcode,$opcodelet{$instr};
1538 push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M
1539 return ".byte\t".join(',',@opcode);
1541 return $instr."\t".@_[0];
1545 foreach (split("\n",$code)) {
1546 s/\`([^\`]*)\`/eval($1)/ge;
1548 s/\b(sha1rnds4)\s+(.*)/sha1rnds4($2)/geo or
1549 s/\b(sha1[^\s]*)\s+(.*)/sha1op38($1,$2)/geo or
1551 s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or
1552 s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go or
1553 s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go or
1554 s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go or
1555 s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go or
1556 s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go;