Add support for Intel SHA extension.
[openssl.git] / crypto / sha / asm / sha256-mb-x86_64.pl
1 #!/usr/bin/env perl
2
3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9
10 # Multi-buffer SHA256 procedure processes n buffers in parallel by
11 # placing buffer data to designated lane of SIMD register. n is
12 # naturally limited to 4 on pre-AVX2 processors and to 8 on
13 # AVX2-capable processors such as Haswell.
14 #
15 #               this    +aesni(i)       sha256  aesni-sha256    gain(iv)
16 # -------------------------------------------------------------------
17 # Westmere(ii)  23.3/n  +1.28=7.11(n=4) 12.3    +3.75=16.1      +126%
18 # Atom(ii)      38.7/n  +3.93=13.6(n=4) 20.8    +5.69=26.5      +95%
19 # Sandy Bridge  (20.5   +5.15=25.7)/n   11.6    13.0            +103%
20 # Ivy Bridge    (20.4   +5.14=25.5)/n   10.3    11.6            +82%
21 # Haswell(iii)  (21.0   +5.00=26.0)/n   7.80    8.79            +170%
22 # Bulldozer     (21.6   +5.76=27.4)/n   13.6    13.7            +100%
23 #
24 # (i)   multi-block CBC encrypt with 128-bit key;
25 # (ii)  (HASH+AES)/n does not apply to Westmere for n>3 and Atom,
26 #       because of lower AES-NI instruction throughput, nor is there
27 #       AES-NI-SHA256 stitch for these processors;
28 # (iii) "this" is for n=8, when we gather twice as much data, result
29 #       for n=4 is 20.3+4.44=24.7;
30 # (iv)  presented improvement coefficients are asymptotic limits and
31 #       in real-life application are somewhat lower, e.g. for 2KB 
32 #       fragments they range from 75% to 130% (on Haswell);
33
34 $flavour = shift;
35 $output  = shift;
36 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
37
38 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
39
40 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
41 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
42 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
43 die "can't locate x86_64-xlate.pl";
44
45 $avx=0;
46
47 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
48                 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
49         $avx = ($1>=2.19) + ($1>=2.22);
50 }
51
52 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
53            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
54         $avx = ($1>=2.09) + ($1>=2.10);
55 }
56
57 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
58            `ml64 2>&1` =~ /Version ([0-9]+)\./) {
59         $avx = ($1>=10) + ($1>=11);
60 }
61
62 open OUT,"| \"$^X\" $xlate $flavour $output";
63 *STDOUT=*OUT;
64
65 # void sha256_multi_block (
66 #     struct {  unsigned int A[8];
67 #               unsigned int B[8];
68 #               unsigned int C[8];
69 #               unsigned int D[8];
70 #               unsigned int E[8];
71 #               unsigned int F[8];
72 #               unsigned int G[8];
73 #               unsigned int H[8];      } *ctx,
74 #     struct {  void *ptr; int blocks;  } inp[8],
75 #     int num);         /* 1 or 2 */
76 #
77 $ctx="%rdi";    # 1st arg
78 $inp="%rsi";    # 2nd arg
79 $num="%edx";    # 3rd arg
80 @ptr=map("%r$_",(8..11));
81 $Tbl="%rbp";
82
83 @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("%xmm$_",(8..15));
84 ($t1,$t2,$t3,$axb,$bxc,$Xi,$Xn,$sigma)=map("%xmm$_",(0..7));
85
86 $REG_SZ=16;
87
88 sub Xi_off {
89 my $off = shift;
90
91     $off %= 16; $off *= $REG_SZ;
92     $off<256 ? "$off-128(%rax)" : "$off-256-128(%rbx)";
93 }
94
95 sub ROUND_00_15 {
96 my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
97
98 $code.=<<___ if ($i<15);
99         movd            `4*$i`(@ptr[0]),$Xi
100         movd            `4*$i`(@ptr[1]),$t1
101         movd            `4*$i`(@ptr[2]),$t2
102         movd            `4*$i`(@ptr[3]),$t3
103         punpckldq       $t2,$Xi
104         punpckldq       $t3,$t1
105         punpckldq       $t1,$Xi
106 ___
107 $code.=<<___ if ($i==15);
108         movd            `4*$i`(@ptr[0]),$Xi
109          lea            `16*4`(@ptr[0]),@ptr[0]
110         movd            `4*$i`(@ptr[1]),$t1
111          lea            `16*4`(@ptr[1]),@ptr[1]
112         movd            `4*$i`(@ptr[2]),$t2
113          lea            `16*4`(@ptr[2]),@ptr[2]
114         movd            `4*$i`(@ptr[3]),$t3
115          lea            `16*4`(@ptr[3]),@ptr[3]
116         punpckldq       $t2,$Xi
117         punpckldq       $t3,$t1
118         punpckldq       $t1,$Xi
119 ___
120 $code.=<<___;
121         movdqa  $e,$sigma
122         `"pshufb        $Xn,$Xi"                if ($i<=15 && ($i&1)==0)`
123         movdqa  $e,$t3
124         `"pshufb        $Xn,$Xi"                if ($i<=15 && ($i&1)==1)`
125         psrld   \$6,$sigma
126         movdqa  $e,$t2
127         pslld   \$7,$t3
128         movdqa  $Xi,`&Xi_off($i)`
129          paddd  $h,$Xi                          # Xi+=h
130
131         psrld   \$11,$t2
132         pxor    $t3,$sigma
133         pslld   \$21-7,$t3
134          paddd  `32*($i%8)-128`($Tbl),$Xi       # Xi+=K[round]
135         pxor    $t2,$sigma
136
137         psrld   \$25-11,$t2
138          movdqa $e,$t1
139          `"prefetcht0   63(@ptr[0])"            if ($i==15)`
140         pxor    $t3,$sigma
141          movdqa $e,$axb                         # borrow $axb
142         pslld   \$26-21,$t3
143          pandn  $g,$t1
144          pand   $f,$axb
145         pxor    $t2,$sigma
146
147          `"prefetcht0   63(@ptr[1])"            if ($i==15)`
148         movdqa  $a,$t2
149         pxor    $t3,$sigma                      # Sigma1(e)
150         movdqa  $a,$t3
151         psrld   \$2,$t2
152         paddd   $sigma,$Xi                      # Xi+=Sigma1(e)
153          pxor   $axb,$t1                        # Ch(e,f,g)
154          movdqa $b,$axb
155         movdqa  $a,$sigma
156         pslld   \$10,$t3
157          pxor   $a,$axb                         # a^b, b^c in next round
158
159          `"prefetcht0   63(@ptr[2])"            if ($i==15)`
160         psrld   \$13,$sigma
161         pxor    $t3,$t2
162          paddd  $t1,$Xi                         # Xi+=Ch(e,f,g)
163         pslld   \$19-10,$t3
164          pand   $axb,$bxc
165         pxor    $sigma,$t2
166
167          `"prefetcht0   63(@ptr[3])"            if ($i==15)`
168         psrld   \$22-13,$sigma
169         pxor    $t3,$t2
170          movdqa $b,$h
171         pslld   \$30-19,$t3
172         pxor    $t2,$sigma
173          pxor   $bxc,$h                         # h=Maj(a,b,c)=Ch(a^b,c,b)
174          paddd  $Xi,$d                          # d+=Xi
175         pxor    $t3,$sigma                      # Sigma0(a)
176
177         paddd   $Xi,$h                          # h+=Xi
178         paddd   $sigma,$h                       # h+=Sigma0(a)
179 ___
180 $code.=<<___ if (($i%8)==7);
181         lea     `32*8`($Tbl),$Tbl
182 ___
183         ($axb,$bxc)=($bxc,$axb);
184 }
185
186 sub ROUND_16_XX {
187 my $i=shift;
188
189 $code.=<<___;
190         movdqa  `&Xi_off($i+1)`,$Xn
191         paddd   `&Xi_off($i+9)`,$Xi             # Xi+=X[i+9]
192
193         movdqa  $Xn,$sigma
194         movdqa  $Xn,$t2
195         psrld   \$3,$sigma
196         movdqa  $Xn,$t3
197
198         psrld   \$7,$t2
199         movdqa  `&Xi_off($i+14)`,$t1
200         pslld   \$14,$t3
201         pxor    $t2,$sigma
202         psrld   \$18-7,$t2
203         movdqa  $t1,$axb                        # borrow $axb
204         pxor    $t3,$sigma
205         pslld   \$25-14,$t3
206         pxor    $t2,$sigma
207         psrld   \$10,$t1
208         movdqa  $axb,$t2
209
210         psrld   \$17,$axb
211         pxor    $t3,$sigma                      # sigma0(X[i+1])
212         pslld   \$13,$t2
213          paddd  $sigma,$Xi                      # Xi+=sigma0(e)
214         pxor    $axb,$t1
215         psrld   \$19-17,$axb
216         pxor    $t2,$t1
217         pslld   \$15-13,$t2
218         pxor    $axb,$t1
219         pxor    $t2,$t1                         # sigma0(X[i+14])
220         paddd   $t1,$Xi                         # Xi+=sigma1(X[i+14])
221 ___
222         &ROUND_00_15($i,@_);
223         ($Xi,$Xn)=($Xn,$Xi);
224 }
225
226 $code.=<<___;
227 .text
228
229 .extern OPENSSL_ia32cap_P
230
231 .globl  sha256_multi_block
232 .type   sha256_multi_block,\@function,3
233 .align  32
234 sha256_multi_block:
235         mov     OPENSSL_ia32cap_P+4(%rip),%rcx
236         bt      \$61,%rcx                       # check SHA bit
237         jc      _shaext_shortcut
238 ___
239 $code.=<<___ if ($avx);
240         test    \$`1<<28`,%ecx
241         jnz     _avx_shortcut
242 ___
243 $code.=<<___;
244         mov     %rsp,%rax
245         push    %rbx
246         push    %rbp
247 ___
248 $code.=<<___ if ($win64);
249         lea     -0xa8(%rsp),%rsp
250         movaps  %xmm6,(%rsp)
251         movaps  %xmm7,0x10(%rsp)
252         movaps  %xmm8,0x20(%rsp)
253         movaps  %xmm9,0x30(%rsp)
254         movaps  %xmm10,-0x78(%rax)
255         movaps  %xmm11,-0x68(%rax)
256         movaps  %xmm12,-0x58(%rax)
257         movaps  %xmm13,-0x48(%rax)
258         movaps  %xmm14,-0x38(%rax)
259         movaps  %xmm15,-0x28(%rax)
260 ___
261 $code.=<<___;
262         sub     \$`$REG_SZ*18`, %rsp
263         and     \$-256,%rsp
264         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
265 .Lbody:
266         lea     K256+128(%rip),$Tbl
267         lea     `$REG_SZ*16`(%rsp),%rbx
268         lea     0x80($ctx),$ctx                 # size optimization
269
270 .Loop_grande:
271         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
272         xor     $num,$num
273 ___
274 for($i=0;$i<4;$i++) {
275     $code.=<<___;
276         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
277         mov     `16*$i+8`($inp),%ecx            # number of blocks
278         cmp     $num,%ecx
279         cmovg   %ecx,$num                       # find maximum
280         test    %ecx,%ecx
281         mov     %ecx,`4*$i`(%rbx)               # initialize counters
282         cmovle  $Tbl,@ptr[$i]                   # cancel input
283 ___
284 }
285 $code.=<<___;
286         test    $num,$num
287         jz      .Ldone
288
289         movdqu  0x00-0x80($ctx),$A              # load context
290          lea    128(%rsp),%rax
291         movdqu  0x20-0x80($ctx),$B
292         movdqu  0x40-0x80($ctx),$C
293         movdqu  0x60-0x80($ctx),$D
294         movdqu  0x80-0x80($ctx),$E
295         movdqu  0xa0-0x80($ctx),$F
296         movdqu  0xc0-0x80($ctx),$G
297         movdqu  0xe0-0x80($ctx),$H
298         movdqu  .Lpbswap(%rip),$Xn
299         jmp     .Loop
300
301 .align  32
302 .Loop:
303         movdqa  $C,$bxc
304         pxor    $B,$bxc                         # magic seed
305 ___
306 for($i=0;$i<16;$i++)    { &ROUND_00_15($i,@V); unshift(@V,pop(@V)); }
307 $code.=<<___;
308         movdqu  `&Xi_off($i)`,$Xi
309         mov     \$3,%ecx
310         jmp     .Loop_16_xx
311 .align  32
312 .Loop_16_xx:
313 ___
314 for(;$i<32;$i++)        { &ROUND_16_XX($i,@V); unshift(@V,pop(@V)); }
315 $code.=<<___;
316         dec     %ecx
317         jnz     .Loop_16_xx
318
319         mov     \$1,%ecx
320         lea     K256+128(%rip),$Tbl
321
322         movdqa  (%rbx),$sigma                   # pull counters
323         cmp     4*0(%rbx),%ecx                  # examine counters
324         pxor    $t1,$t1
325         cmovge  $Tbl,@ptr[0]                    # cancel input
326         cmp     4*1(%rbx),%ecx
327         movdqa  $sigma,$Xn
328         cmovge  $Tbl,@ptr[1]
329         cmp     4*2(%rbx),%ecx
330         pcmpgtd $t1,$Xn                         # mask value
331         cmovge  $Tbl,@ptr[2]
332         cmp     4*3(%rbx),%ecx
333         paddd   $Xn,$sigma                      # counters--
334         cmovge  $Tbl,@ptr[3]
335
336         movdqu  0x00-0x80($ctx),$t1
337         pand    $Xn,$A
338         movdqu  0x20-0x80($ctx),$t2
339         pand    $Xn,$B
340         movdqu  0x40-0x80($ctx),$t3
341         pand    $Xn,$C
342         movdqu  0x60-0x80($ctx),$Xi
343         pand    $Xn,$D
344         paddd   $t1,$A
345         movdqu  0x80-0x80($ctx),$t1
346         pand    $Xn,$E
347         paddd   $t2,$B
348         movdqu  0xa0-0x80($ctx),$t2
349         pand    $Xn,$F
350         paddd   $t3,$C
351         movdqu  0xc0-0x80($ctx),$t3
352         pand    $Xn,$G
353         paddd   $Xi,$D
354         movdqu  0xe0-0x80($ctx),$Xi
355         pand    $Xn,$H
356         paddd   $t1,$E
357         paddd   $t2,$F
358         movdqu  $A,0x00-0x80($ctx)
359         paddd   $t3,$G
360         movdqu  $B,0x20-0x80($ctx)
361         paddd   $Xi,$H
362         movdqu  $C,0x40-0x80($ctx)
363         movdqu  $D,0x60-0x80($ctx)
364         movdqu  $E,0x80-0x80($ctx)
365         movdqu  $F,0xa0-0x80($ctx)
366         movdqu  $G,0xc0-0x80($ctx)
367         movdqu  $H,0xe0-0x80($ctx)
368
369         movdqa  $sigma,(%rbx)                   # save counters
370         movdqa  .Lpbswap(%rip),$Xn
371         dec     $num
372         jnz     .Loop
373
374         mov     `$REG_SZ*17+8`(%rsp),$num
375         lea     $REG_SZ($ctx),$ctx
376         lea     `16*$REG_SZ/4`($inp),$inp
377         dec     $num
378         jnz     .Loop_grande
379
380 .Ldone:
381         mov     `$REG_SZ*17`(%rsp),%rax         # orignal %rsp
382 ___
383 $code.=<<___ if ($win64);
384         movaps  -0xb8(%rax),%xmm6
385         movaps  -0xa8(%rax),%xmm7
386         movaps  -0x98(%rax),%xmm8
387         movaps  -0x88(%rax),%xmm9
388         movaps  -0x78(%rax),%xmm10
389         movaps  -0x68(%rax),%xmm11
390         movaps  -0x58(%rax),%xmm12
391         movaps  -0x48(%rax),%xmm13
392         movaps  -0x38(%rax),%xmm14
393         movaps  -0x28(%rax),%xmm15
394 ___
395 $code.=<<___;
396         mov     -16(%rax),%rbp
397         mov     -8(%rax),%rbx
398         lea     (%rax),%rsp
399 .Lepilogue:
400         ret
401 .size   sha256_multi_block,.-sha256_multi_block
402 ___
403                                                 {{{
404 my ($Wi,$TMP0,$TMP1,$TMPx,$ABEF0,$CDGH0,$ABEF1,$CDGH1)=map("%xmm$_",(0..3,12..15));
405 my @MSG0=map("%xmm$_",(4..7));
406 my @MSG1=map("%xmm$_",(8..11));
407
408 $code.=<<___;
409 .type   sha256_multi_block_shaext,\@function,3
410 .align  32
411 sha256_multi_block_shaext:
412 _shaext_shortcut:
413         mov     %rsp,%rax
414         push    %rbx
415         push    %rbp
416 ___
417 $code.=<<___ if ($win64);
418         lea     -0xa8(%rsp),%rsp
419         movaps  %xmm6,(%rsp)
420         movaps  %xmm7,0x10(%rsp)
421         movaps  %xmm8,0x20(%rsp)
422         movaps  %xmm9,0x30(%rsp)
423         movaps  %xmm10,-0x78(%rax)
424         movaps  %xmm11,-0x68(%rax)
425         movaps  %xmm12,-0x58(%rax)
426         movaps  %xmm13,-0x48(%rax)
427         movaps  %xmm14,-0x38(%rax)
428         movaps  %xmm15,-0x28(%rax)
429 ___
430 $code.=<<___;
431         sub     \$`$REG_SZ*18`,%rsp
432         shl     \$1,$num                        # we process pair at a time
433         and     \$-256,%rsp
434         lea     0x80($ctx),$ctx                 # size optimization
435         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
436 .Lbody_shaext:
437         lea     `$REG_SZ*16`(%rsp),%rbx
438         lea     K256_shaext+0x80(%rip),$Tbl
439
440 .Loop_grande_shaext:
441         mov     $num,`$REG_SZ*17+8`(%rsp)       # orignal $num
442         xor     $num,$num
443 ___
444 for($i=0;$i<2;$i++) {
445     $code.=<<___;
446         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
447         mov     `16*$i+8`($inp),%ecx            # number of blocks
448         cmp     $num,%ecx
449         cmovg   %ecx,$num                       # find maximum
450         test    %ecx,%ecx
451         mov     %ecx,`4*$i`(%rbx)               # initialize counters
452         cmovle  %rsp,@ptr[$i]                   # cancel input
453 ___
454 }
455 $code.=<<___;
456         test    $num,$num
457         jz      .Ldone_shaext
458
459         movq            0x00-0x80($ctx),$ABEF0          # A1.A0
460         movq            0x20-0x80($ctx),@MSG0[0]        # B1.B0
461         movq            0x40-0x80($ctx),$CDGH0          # C1.C0
462         movq            0x60-0x80($ctx),@MSG0[1]        # D1.D0
463         movq            0x80-0x80($ctx),@MSG1[0]        # E1.E0
464         movq            0xa0-0x80($ctx),@MSG1[1]        # F1.F0
465         movq            0xc0-0x80($ctx),@MSG1[2]        # G1.G0
466         movq            0xe0-0x80($ctx),@MSG1[3]        # H1.H0
467
468         punpckldq       @MSG0[0],$ABEF0                 # B1.A1.B0.A0
469         punpckldq       @MSG0[1],$CDGH0                 # D1.C1.D0.C0
470         punpckldq       @MSG1[1],@MSG1[0]               # F1.E1.F0.E0
471         punpckldq       @MSG1[3],@MSG1[2]               # H1.G1.H0.G0
472         movdqa          K256_shaext-0x10(%rip),$TMPx    # byte swap
473
474         movdqa          $ABEF0,$ABEF1
475         movdqa          $CDGH0,$CDGH1
476         punpcklqdq      @MSG1[0],$ABEF0                 # F0.E0.B0.A0
477         punpcklqdq      @MSG1[2],$CDGH0                 # H0.G0.D0.C0
478         punpckhqdq      @MSG1[0],$ABEF1                 # F1.E1.B1.A1
479         punpckhqdq      @MSG1[2],$CDGH1                 # H1.G1.D1.C1
480
481         pshufd          \$0b00011011,$ABEF0,$ABEF0
482         pshufd          \$0b00011011,$CDGH0,$CDGH0
483         pshufd          \$0b00011011,$ABEF1,$ABEF1
484         pshufd          \$0b00011011,$CDGH1,$CDGH1
485         jmp             .Loop_shaext
486
487 .align  32
488 .Loop_shaext:
489         movdqu          0x00(@ptr[0]),@MSG0[0]
490          movdqu         0x00(@ptr[1]),@MSG1[0]
491         movdqu          0x10(@ptr[0]),@MSG0[1]
492          movdqu         0x10(@ptr[1]),@MSG1[1]
493         movdqu          0x20(@ptr[0]),@MSG0[2]
494         pshufb          $TMPx,@MSG0[0]
495          movdqu         0x20(@ptr[1]),@MSG1[2]
496          pshufb         $TMPx,@MSG1[0]
497         movdqu          0x30(@ptr[0]),@MSG0[3]
498         lea             0x40(@ptr[0]),@ptr[0]
499          movdqu         0x30(@ptr[1]),@MSG1[3]
500          lea            0x40(@ptr[1]),@ptr[1]
501
502         movdqa          0*16-0x80($Tbl),$Wi
503         pshufb          $TMPx,@MSG0[1]
504         paddd           @MSG0[0],$Wi
505         pxor            $ABEF0,@MSG0[0]         # black magic
506         movdqa          $Wi,$TMP0
507          movdqa         0*16-0x80($Tbl),$TMP1
508          pshufb         $TMPx,@MSG1[1]
509          paddd          @MSG1[0],$TMP1
510         movdqa          $CDGH0,0x50(%rsp)       # offload
511         sha256rnds2     $ABEF0,$CDGH0           # 0-3
512          pxor           $ABEF1,@MSG1[0]         # black magic
513          movdqa         $TMP1,$Wi
514          movdqa         $CDGH1,0x70(%rsp)
515          sha256rnds2    $ABEF1,$CDGH1           # 0-3
516         pshufd          \$0x0e,$TMP0,$Wi
517         pxor            $ABEF0,@MSG0[0]         # black magic
518         movdqa          $ABEF0,0x40(%rsp)       # offload
519         sha256rnds2     $CDGH0,$ABEF0
520          pshufd         \$0x0e,$TMP1,$Wi
521          pxor           $ABEF1,@MSG1[0]         # black magic
522          movdqa         $ABEF1,0x60(%rsp)
523         movdqa          1*16-0x80($Tbl),$TMP0
524         paddd           @MSG0[1],$TMP0
525         pshufb          $TMPx,@MSG0[2]
526          sha256rnds2    $CDGH1,$ABEF1
527
528         movdqa          $TMP0,$Wi
529          movdqa         1*16-0x80($Tbl),$TMP1
530          paddd          @MSG1[1],$TMP1
531         sha256rnds2     $ABEF0,$CDGH0           # 4-7
532          movdqa         $TMP1,$Wi
533         prefetcht0      127(@ptr[0])
534         pshufb          $TMPx,@MSG0[3]
535          pshufb         $TMPx,@MSG1[2]
536          prefetcht0     127(@ptr[1])
537          sha256rnds2    $ABEF1,$CDGH1           # 4-7
538         pshufd          \$0x0e,$TMP0,$Wi
539          pshufb         $TMPx,@MSG1[3]
540         sha256msg1      @MSG0[1],@MSG0[0]
541         sha256rnds2     $CDGH0,$ABEF0
542          pshufd         \$0x0e,$TMP1,$Wi
543         movdqa          2*16-0x80($Tbl),$TMP0
544         paddd           @MSG0[2],$TMP0
545          sha256rnds2    $CDGH1,$ABEF1
546
547         movdqa          $TMP0,$Wi
548          movdqa         2*16-0x80($Tbl),$TMP1
549          paddd          @MSG1[2],$TMP1
550         sha256rnds2     $ABEF0,$CDGH0           # 8-11
551          sha256msg1     @MSG1[1],@MSG1[0]
552          movdqa         $TMP1,$Wi
553         movdqa          @MSG0[3],$TMPx
554          sha256rnds2    $ABEF1,$CDGH1           # 8-11
555         pshufd          \$0x0e,$TMP0,$Wi
556         palignr         \$4,@MSG0[2],$TMPx
557         paddd           $TMPx,@MSG0[0]
558          movdqa         @MSG1[3],$TMPx
559          palignr        \$4,@MSG1[2],$TMPx
560         sha256msg1      @MSG0[2],@MSG0[1]
561         sha256rnds2     $CDGH0,$ABEF0
562          pshufd         \$0x0e,$TMP1,$Wi
563         movdqa          3*16-0x80($Tbl),$TMP0
564         paddd           @MSG0[3],$TMP0
565          sha256rnds2    $CDGH1,$ABEF1
566          sha256msg1     @MSG1[2],@MSG1[1]
567
568         movdqa          $TMP0,$Wi
569          movdqa         3*16-0x80($Tbl),$TMP1
570          paddd          $TMPx,@MSG1[0]
571          paddd          @MSG1[3],$TMP1
572         sha256msg2      @MSG0[3],@MSG0[0]
573         sha256rnds2     $ABEF0,$CDGH0           # 12-15
574          movdqa         $TMP1,$Wi
575         movdqa          @MSG0[0],$TMPx
576         palignr         \$4,@MSG0[3],$TMPx
577          sha256rnds2    $ABEF1,$CDGH1           # 12-15
578          sha256msg2     @MSG1[3],@MSG1[0]
579         pshufd          \$0x0e,$TMP0,$Wi
580         paddd           $TMPx,@MSG0[1]
581          movdqa         @MSG1[0],$TMPx
582          palignr        \$4,@MSG1[3],$TMPx
583         sha256msg1      @MSG0[3],@MSG0[2]
584         sha256rnds2     $CDGH0,$ABEF0
585          pshufd         \$0x0e,$TMP1,$Wi
586         movdqa          4*16-0x80($Tbl),$TMP0
587         paddd           @MSG0[0],$TMP0
588          sha256rnds2    $CDGH1,$ABEF1
589          sha256msg1     @MSG1[3],@MSG1[2]
590 ___
591 for($i=4;$i<16-3;$i++) {
592 $code.=<<___;
593         movdqa          $TMP0,$Wi
594          movdqa         $i*16-0x80($Tbl),$TMP1
595          paddd          $TMPx,@MSG1[1]
596          paddd          @MSG1[0],$TMP1
597         sha256msg2      @MSG0[0],@MSG0[1]
598         sha256rnds2     $ABEF0,$CDGH0           # 16-19...
599          movdqa         $TMP1,$Wi
600         movdqa          @MSG0[1],$TMPx
601         palignr         \$4,@MSG0[0],$TMPx
602          sha256rnds2    $ABEF1,$CDGH1           # 16-19...
603          sha256msg2     @MSG1[0],@MSG1[1]
604         pshufd          \$0x0e,$TMP0,$Wi
605         paddd           $TMPx,@MSG0[2]
606          movdqa         @MSG1[1],$TMPx
607          palignr        \$4,@MSG1[0],$TMPx
608         sha256msg1      @MSG0[0],@MSG0[3]
609         sha256rnds2     $CDGH0,$ABEF0
610          pshufd         \$0x0e,$TMP1,$Wi
611         movdqa          `($i+1)*16`-0x80($Tbl),$TMP0
612         paddd           @MSG0[1],$TMP0
613          sha256rnds2    $CDGH1,$ABEF1
614          sha256msg1     @MSG1[0],@MSG1[3]
615 ___
616         push(@MSG0,shift(@MSG0));       push(@MSG1,shift(@MSG1));
617 }
618 $code.=<<___;
619         movdqa          $TMP0,$Wi
620          movdqa         13*16-0x80($Tbl),$TMP1
621          paddd          $TMPx,@MSG1[1]
622          paddd          @MSG1[0],$TMP1
623         sha256msg2      @MSG0[0],@MSG0[1]
624         sha256rnds2     $ABEF0,$CDGH0           # 52-55
625          movdqa         $TMP1,$Wi
626         movdqa          @MSG0[1],$TMPx
627         palignr         \$4,@MSG0[0],$TMPx
628          sha256rnds2    $ABEF1,$CDGH1           # 52-55
629          sha256msg2     @MSG1[0],@MSG1[1]
630         pshufd          \$0x0e,$TMP0,$Wi
631         paddd           $TMPx,@MSG0[2]
632          movdqa         @MSG1[1],$TMPx
633          palignr        \$4,@MSG1[0],$TMPx
634         nop
635         sha256rnds2     $CDGH0,$ABEF0
636          pshufd         \$0x0e,$TMP1,$Wi
637         movdqa          14*16-0x80($Tbl),$TMP0
638         paddd           @MSG0[1],$TMP0
639          sha256rnds2    $CDGH1,$ABEF1
640
641         movdqa          $TMP0,$Wi
642          movdqa         14*16-0x80($Tbl),$TMP1
643          paddd          $TMPx,@MSG1[2]
644          paddd          @MSG1[1],$TMP1
645         sha256msg2      @MSG0[1],@MSG0[2]
646         nop
647         sha256rnds2     $ABEF0,$CDGH0           # 56-59
648          movdqa         $TMP1,$Wi
649           mov           \$1,%ecx
650           pxor          @MSG0[1],@MSG0[1]       # zero
651          sha256rnds2    $ABEF1,$CDGH1           # 56-59
652          sha256msg2     @MSG1[1],@MSG1[2]
653         pshufd          \$0x0e,$TMP0,$Wi
654         movdqa          15*16-0x80($Tbl),$TMP0
655         paddd           @MSG0[2],$TMP0
656           movq          (%rbx),@MSG0[2]         # pull counters
657           nop
658         sha256rnds2     $CDGH0,$ABEF0
659          pshufd         \$0x0e,$TMP1,$Wi
660          movdqa         15*16-0x80($Tbl),$TMP1
661          paddd          @MSG1[2],$TMP1
662          sha256rnds2    $CDGH1,$ABEF1
663
664         movdqa          $TMP0,$Wi
665           cmp           4*0(%rbx),%ecx          # examine counters
666           cmovge        %rsp,@ptr[0]            # cancel input
667           cmp           4*1(%rbx),%ecx
668           cmovge        %rsp,@ptr[1]
669           pshufd        \$0x00,@MSG0[2],@MSG1[0]
670         sha256rnds2     $ABEF0,$CDGH0           # 60-63
671          movdqa         $TMP1,$Wi
672           pshufd        \$0x55,@MSG0[2],@MSG1[1]
673           movdqa        @MSG0[2],@MSG1[2]
674          sha256rnds2    $ABEF1,$CDGH1           # 60-63
675         pshufd          \$0x0e,$TMP0,$Wi
676           pcmpgtd       @MSG0[1],@MSG1[0]
677           pcmpgtd       @MSG0[1],@MSG1[1]
678         sha256rnds2     $CDGH0,$ABEF0
679          pshufd         \$0x0e,$TMP1,$Wi
680           pcmpgtd       @MSG0[1],@MSG1[2]       # counter mask
681           movdqa        K256_shaext-0x10(%rip),$TMPx
682          sha256rnds2    $CDGH1,$ABEF1
683
684         pand            @MSG1[0],$CDGH0
685          pand           @MSG1[1],$CDGH1
686         pand            @MSG1[0],$ABEF0
687          pand           @MSG1[1],$ABEF1
688         paddd           @MSG0[2],@MSG1[2]       # counters--
689
690         paddd           0x50(%rsp),$CDGH0
691          paddd          0x70(%rsp),$CDGH1
692         paddd           0x40(%rsp),$ABEF0
693          paddd          0x60(%rsp),$ABEF1
694
695         movq            @MSG1[2],(%rbx)         # save counters
696         dec             $num
697         jnz             .Loop_shaext
698
699         mov             `$REG_SZ*17+8`(%rsp),$num
700
701         pshufd          \$0b00011011,$ABEF0,$ABEF0
702         pshufd          \$0b00011011,$CDGH0,$CDGH0
703         pshufd          \$0b00011011,$ABEF1,$ABEF1
704         pshufd          \$0b00011011,$CDGH1,$CDGH1
705
706         movdqa          $ABEF0,@MSG0[0]
707         movdqa          $CDGH0,@MSG0[1]
708         punpckldq       $ABEF1,$ABEF0                   # B1.B0.A1.A0
709         punpckhdq       $ABEF1,@MSG0[0]                 # F1.F0.E1.E0
710         punpckldq       $CDGH1,$CDGH0                   # D1.D0.C1.C0
711         punpckhdq       $CDGH1,@MSG0[1]                 # H1.H0.G1.G0
712
713         movq            $ABEF0,0x00-0x80($ctx)          # A1.A0
714         psrldq          \$8,$ABEF0
715         movq            @MSG0[0],0x80-0x80($ctx)        # E1.E0
716         psrldq          \$8,@MSG0[0]
717         movq            $ABEF0,0x20-0x80($ctx)          # B1.B0
718         movq            @MSG0[0],0xa0-0x80($ctx)        # F1.F0
719
720         movq            $CDGH0,0x40-0x80($ctx)          # C1.C0
721         psrldq          \$8,$CDGH0
722         movq            @MSG0[1],0xc0-0x80($ctx)        # G1.G0
723         psrldq          \$8,@MSG0[1]
724         movq            $CDGH0,0x60-0x80($ctx)          # D1.D0
725         movq            @MSG0[1],0xe0-0x80($ctx)        # H1.H0
726
727         lea     `$REG_SZ/2`($ctx),$ctx
728         lea     `16*2`($inp),$inp
729         dec     $num
730         jnz     .Loop_grande_shaext
731
732 .Ldone_shaext:
733         #mov    `$REG_SZ*17`(%rsp),%rax         # original %rsp
734 ___
735 $code.=<<___ if ($win64);
736         movaps  -0xb8(%rax),%xmm6
737         movaps  -0xa8(%rax),%xmm7
738         movaps  -0x98(%rax),%xmm8
739         movaps  -0x88(%rax),%xmm9
740         movaps  -0x78(%rax),%xmm10
741         movaps  -0x68(%rax),%xmm11
742         movaps  -0x58(%rax),%xmm12
743         movaps  -0x48(%rax),%xmm13
744         movaps  -0x38(%rax),%xmm14
745         movaps  -0x28(%rax),%xmm15
746 ___
747 $code.=<<___;
748         mov     -16(%rax),%rbp
749         mov     -8(%rax),%rbx
750         lea     (%rax),%rsp
751 .Lepilogue_shaext:
752         ret
753 .size   sha256_multi_block_shaext,.-sha256_multi_block_shaext
754 ___
755                                                 }}}
756                                                 if ($avx) {{{
757 sub ROUND_00_15_avx {
758 my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
759
760 $code.=<<___ if ($i<15 && $REG_SZ==16);
761         vmovd           `4*$i`(@ptr[0]),$Xi
762         vmovd           `4*$i`(@ptr[1]),$t1
763         vpinsrd         \$1,`4*$i`(@ptr[2]),$Xi,$Xi
764         vpinsrd         \$1,`4*$i`(@ptr[3]),$t1,$t1
765         vpunpckldq      $t1,$Xi,$Xi
766         vpshufb         $Xn,$Xi,$Xi
767 ___
768 $code.=<<___ if ($i==15 && $REG_SZ==16);
769         vmovd           `4*$i`(@ptr[0]),$Xi
770          lea            `16*4`(@ptr[0]),@ptr[0]
771         vmovd           `4*$i`(@ptr[1]),$t1
772          lea            `16*4`(@ptr[1]),@ptr[1]
773         vpinsrd         \$1,`4*$i`(@ptr[2]),$Xi,$Xi
774          lea            `16*4`(@ptr[2]),@ptr[2]
775         vpinsrd         \$1,`4*$i`(@ptr[3]),$t1,$t1
776          lea            `16*4`(@ptr[3]),@ptr[3]
777         vpunpckldq      $t1,$Xi,$Xi
778         vpshufb         $Xn,$Xi,$Xi
779 ___
780 $code.=<<___ if ($i<15 && $REG_SZ==32);
781         vmovd           `4*$i`(@ptr[0]),$Xi
782         vmovd           `4*$i`(@ptr[4]),$t1
783         vmovd           `4*$i`(@ptr[1]),$t2
784         vmovd           `4*$i`(@ptr[5]),$t3
785         vpinsrd         \$1,`4*$i`(@ptr[2]),$Xi,$Xi
786         vpinsrd         \$1,`4*$i`(@ptr[6]),$t1,$t1
787         vpinsrd         \$1,`4*$i`(@ptr[3]),$t2,$t2
788         vpunpckldq      $t2,$Xi,$Xi
789         vpinsrd         \$1,`4*$i`(@ptr[7]),$t3,$t3
790         vpunpckldq      $t3,$t1,$t1
791         vinserti128     $t1,$Xi,$Xi
792         vpshufb         $Xn,$Xi,$Xi
793 ___
794 $code.=<<___ if ($i==15 && $REG_SZ==32);
795         vmovd           `4*$i`(@ptr[0]),$Xi
796          lea            `16*4`(@ptr[0]),@ptr[0]
797         vmovd           `4*$i`(@ptr[4]),$t1
798          lea            `16*4`(@ptr[4]),@ptr[4]
799         vmovd           `4*$i`(@ptr[1]),$t2
800          lea            `16*4`(@ptr[1]),@ptr[1]
801         vmovd           `4*$i`(@ptr[5]),$t3
802          lea            `16*4`(@ptr[5]),@ptr[5]
803         vpinsrd         \$1,`4*$i`(@ptr[2]),$Xi,$Xi
804          lea            `16*4`(@ptr[2]),@ptr[2]
805         vpinsrd         \$1,`4*$i`(@ptr[6]),$t1,$t1
806          lea            `16*4`(@ptr[6]),@ptr[6]
807         vpinsrd         \$1,`4*$i`(@ptr[3]),$t2,$t2
808          lea            `16*4`(@ptr[3]),@ptr[3]
809         vpunpckldq      $t2,$Xi,$Xi
810         vpinsrd         \$1,`4*$i`(@ptr[7]),$t3,$t3
811          lea            `16*4`(@ptr[7]),@ptr[7]
812         vpunpckldq      $t3,$t1,$t1
813         vinserti128     $t1,$Xi,$Xi
814         vpshufb         $Xn,$Xi,$Xi
815 ___
816 $code.=<<___;
817         vpsrld  \$6,$e,$sigma
818         vpslld  \$26,$e,$t3
819         vmovdqu $Xi,`&Xi_off($i)`
820          vpaddd $h,$Xi,$Xi                      # Xi+=h
821
822         vpsrld  \$11,$e,$t2
823         vpxor   $t3,$sigma,$sigma
824         vpslld  \$21,$e,$t3
825          vpaddd `32*($i%8)-128`($Tbl),$Xi,$Xi   # Xi+=K[round]
826         vpxor   $t2,$sigma,$sigma
827
828         vpsrld  \$25,$e,$t2
829         vpxor   $t3,$sigma,$sigma
830          `"prefetcht0   63(@ptr[0])"            if ($i==15)`
831         vpslld  \$7,$e,$t3
832          vpandn $g,$e,$t1
833          vpand  $f,$e,$axb                      # borrow $axb
834          `"prefetcht0   63(@ptr[1])"            if ($i==15)`
835         vpxor   $t2,$sigma,$sigma
836
837         vpsrld  \$2,$a,$h                       # borrow $h
838         vpxor   $t3,$sigma,$sigma               # Sigma1(e)
839          `"prefetcht0   63(@ptr[2])"            if ($i==15)`
840         vpslld  \$30,$a,$t2
841          vpxor  $axb,$t1,$t1                    # Ch(e,f,g)
842          vpxor  $a,$b,$axb                      # a^b, b^c in next round
843          `"prefetcht0   63(@ptr[3])"            if ($i==15)`
844         vpxor   $t2,$h,$h
845         vpaddd  $sigma,$Xi,$Xi                  # Xi+=Sigma1(e)
846
847         vpsrld  \$13,$a,$t2
848          `"prefetcht0   63(@ptr[4])"            if ($i==15 && $REG_SZ==32)`
849         vpslld  \$19,$a,$t3
850          vpaddd $t1,$Xi,$Xi                     # Xi+=Ch(e,f,g)
851          vpand  $axb,$bxc,$bxc
852          `"prefetcht0   63(@ptr[5])"            if ($i==15 && $REG_SZ==32)`
853         vpxor   $t2,$h,$sigma
854
855         vpsrld  \$22,$a,$t2
856         vpxor   $t3,$sigma,$sigma
857          `"prefetcht0   63(@ptr[6])"            if ($i==15 && $REG_SZ==32)`
858         vpslld  \$10,$a,$t3
859          vpxor  $bxc,$b,$h                      # h=Maj(a,b,c)=Ch(a^b,c,b)
860          vpaddd $Xi,$d,$d                       # d+=Xi
861          `"prefetcht0   63(@ptr[7])"            if ($i==15 && $REG_SZ==32)`
862         vpxor   $t2,$sigma,$sigma
863         vpxor   $t3,$sigma,$sigma               # Sigma0(a)
864
865         vpaddd  $Xi,$h,$h                       # h+=Xi
866         vpaddd  $sigma,$h,$h                    # h+=Sigma0(a)
867 ___
868 $code.=<<___ if (($i%8)==7);
869         add     \$`32*8`,$Tbl
870 ___
871         ($axb,$bxc)=($bxc,$axb);
872 }
873
874 sub ROUND_16_XX_avx {
875 my $i=shift;
876
877 $code.=<<___;
878         vmovdqu `&Xi_off($i+1)`,$Xn
879         vpaddd  `&Xi_off($i+9)`,$Xi,$Xi         # Xi+=X[i+9]
880
881         vpsrld  \$3,$Xn,$sigma
882         vpsrld  \$7,$Xn,$t2
883         vpslld  \$25,$Xn,$t3
884         vpxor   $t2,$sigma,$sigma
885         vpsrld  \$18,$Xn,$t2
886         vpxor   $t3,$sigma,$sigma
887         vpslld  \$14,$Xn,$t3
888         vmovdqu `&Xi_off($i+14)`,$t1
889         vpsrld  \$10,$t1,$axb                   # borrow $axb
890
891         vpxor   $t2,$sigma,$sigma
892         vpsrld  \$17,$t1,$t2
893         vpxor   $t3,$sigma,$sigma               # sigma0(X[i+1])
894         vpslld  \$15,$t1,$t3
895          vpaddd $sigma,$Xi,$Xi                  # Xi+=sigma0(e)
896         vpxor   $t2,$axb,$sigma
897         vpsrld  \$19,$t1,$t2
898         vpxor   $t3,$sigma,$sigma
899         vpslld  \$13,$t1,$t3
900         vpxor   $t2,$sigma,$sigma
901         vpxor   $t3,$sigma,$sigma               # sigma0(X[i+14])
902         vpaddd  $sigma,$Xi,$Xi                  # Xi+=sigma1(X[i+14])
903 ___
904         &ROUND_00_15_avx($i,@_);
905         ($Xi,$Xn)=($Xn,$Xi);
906 }
907
908 $code.=<<___;
909 .type   sha256_multi_block_avx,\@function,3
910 .align  32
911 sha256_multi_block_avx:
912 _avx_shortcut:
913 ___
914 $code.=<<___ if ($avx>1);
915         shr     \$32,%rcx
916         cmp     \$2,$num
917         jb      .Lavx
918         test    \$`1<<5`,%ecx
919         jnz     _avx2_shortcut
920         jmp     .Lavx
921 .align  32
922 .Lavx:
923 ___
924 $code.=<<___;
925         mov     %rsp,%rax
926         push    %rbx
927         push    %rbp
928 ___
929 $code.=<<___ if ($win64);
930         lea     -0xa8(%rsp),%rsp
931         movaps  %xmm6,(%rsp)
932         movaps  %xmm7,0x10(%rsp)
933         movaps  %xmm8,0x20(%rsp)
934         movaps  %xmm9,0x30(%rsp)
935         movaps  %xmm10,-0x78(%rax)
936         movaps  %xmm11,-0x68(%rax)
937         movaps  %xmm12,-0x58(%rax)
938         movaps  %xmm13,-0x48(%rax)
939         movaps  %xmm14,-0x38(%rax)
940         movaps  %xmm15,-0x28(%rax)
941 ___
942 $code.=<<___;
943         sub     \$`$REG_SZ*18`, %rsp
944         and     \$-256,%rsp
945         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
946 .Lbody_avx:
947         lea     K256+128(%rip),$Tbl
948         lea     `$REG_SZ*16`(%rsp),%rbx
949         lea     0x80($ctx),$ctx                 # size optimization
950
951 .Loop_grande_avx:
952         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
953         xor     $num,$num
954 ___
955 for($i=0;$i<4;$i++) {
956     $code.=<<___;
957         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
958         mov     `16*$i+8`($inp),%ecx            # number of blocks
959         cmp     $num,%ecx
960         cmovg   %ecx,$num                       # find maximum
961         test    %ecx,%ecx
962         mov     %ecx,`4*$i`(%rbx)               # initialize counters
963         cmovle  $Tbl,@ptr[$i]                   # cancel input
964 ___
965 }
966 $code.=<<___;
967         test    $num,$num
968         jz      .Ldone_avx
969
970         vmovdqu 0x00-0x80($ctx),$A              # load context
971          lea    128(%rsp),%rax
972         vmovdqu 0x20-0x80($ctx),$B
973         vmovdqu 0x40-0x80($ctx),$C
974         vmovdqu 0x60-0x80($ctx),$D
975         vmovdqu 0x80-0x80($ctx),$E
976         vmovdqu 0xa0-0x80($ctx),$F
977         vmovdqu 0xc0-0x80($ctx),$G
978         vmovdqu 0xe0-0x80($ctx),$H
979         vmovdqu .Lpbswap(%rip),$Xn
980         jmp     .Loop_avx
981
982 .align  32
983 .Loop_avx:
984         vpxor   $B,$C,$bxc                      # magic seed
985 ___
986 for($i=0;$i<16;$i++)    { &ROUND_00_15_avx($i,@V); unshift(@V,pop(@V)); }
987 $code.=<<___;
988         vmovdqu `&Xi_off($i)`,$Xi
989         mov     \$3,%ecx
990         jmp     .Loop_16_xx_avx
991 .align  32
992 .Loop_16_xx_avx:
993 ___
994 for(;$i<32;$i++)        { &ROUND_16_XX_avx($i,@V); unshift(@V,pop(@V)); }
995 $code.=<<___;
996         dec     %ecx
997         jnz     .Loop_16_xx_avx
998
999         mov     \$1,%ecx
1000         lea     K256+128(%rip),$Tbl
1001 ___
1002 for($i=0;$i<4;$i++) {
1003     $code.=<<___;
1004         cmp     `4*$i`(%rbx),%ecx               # examine counters
1005         cmovge  $Tbl,@ptr[$i]                   # cancel input
1006 ___
1007 }
1008 $code.=<<___;
1009         vmovdqa (%rbx),$sigma                   # pull counters
1010         vpxor   $t1,$t1,$t1
1011         vmovdqa $sigma,$Xn
1012         vpcmpgtd $t1,$Xn,$Xn                    # mask value
1013         vpaddd  $Xn,$sigma,$sigma               # counters--
1014
1015         vmovdqu 0x00-0x80($ctx),$t1
1016         vpand   $Xn,$A,$A
1017         vmovdqu 0x20-0x80($ctx),$t2
1018         vpand   $Xn,$B,$B
1019         vmovdqu 0x40-0x80($ctx),$t3
1020         vpand   $Xn,$C,$C
1021         vmovdqu 0x60-0x80($ctx),$Xi
1022         vpand   $Xn,$D,$D
1023         vpaddd  $t1,$A,$A
1024         vmovdqu 0x80-0x80($ctx),$t1
1025         vpand   $Xn,$E,$E
1026         vpaddd  $t2,$B,$B
1027         vmovdqu 0xa0-0x80($ctx),$t2
1028         vpand   $Xn,$F,$F
1029         vpaddd  $t3,$C,$C
1030         vmovdqu 0xc0-0x80($ctx),$t3
1031         vpand   $Xn,$G,$G
1032         vpaddd  $Xi,$D,$D
1033         vmovdqu 0xe0-0x80($ctx),$Xi
1034         vpand   $Xn,$H,$H
1035         vpaddd  $t1,$E,$E
1036         vpaddd  $t2,$F,$F
1037         vmovdqu $A,0x00-0x80($ctx)
1038         vpaddd  $t3,$G,$G
1039         vmovdqu $B,0x20-0x80($ctx)
1040         vpaddd  $Xi,$H,$H
1041         vmovdqu $C,0x40-0x80($ctx)
1042         vmovdqu $D,0x60-0x80($ctx)
1043         vmovdqu $E,0x80-0x80($ctx)
1044         vmovdqu $F,0xa0-0x80($ctx)
1045         vmovdqu $G,0xc0-0x80($ctx)
1046         vmovdqu $H,0xe0-0x80($ctx)
1047
1048         vmovdqu $sigma,(%rbx)                   # save counters
1049         vmovdqu .Lpbswap(%rip),$Xn
1050         dec     $num
1051         jnz     .Loop_avx
1052
1053         mov     `$REG_SZ*17+8`(%rsp),$num
1054         lea     $REG_SZ($ctx),$ctx
1055         lea     `16*$REG_SZ/4`($inp),$inp
1056         dec     $num
1057         jnz     .Loop_grande_avx
1058
1059 .Ldone_avx:
1060         mov     `$REG_SZ*17`(%rsp),%rax         # orignal %rsp
1061         vzeroupper
1062 ___
1063 $code.=<<___ if ($win64);
1064         movaps  -0xb8(%rax),%xmm6
1065         movaps  -0xa8(%rax),%xmm7
1066         movaps  -0x98(%rax),%xmm8
1067         movaps  -0x88(%rax),%xmm9
1068         movaps  -0x78(%rax),%xmm10
1069         movaps  -0x68(%rax),%xmm11
1070         movaps  -0x58(%rax),%xmm12
1071         movaps  -0x48(%rax),%xmm13
1072         movaps  -0x38(%rax),%xmm14
1073         movaps  -0x28(%rax),%xmm15
1074 ___
1075 $code.=<<___;
1076         mov     -16(%rax),%rbp
1077         mov     -8(%rax),%rbx
1078         lea     (%rax),%rsp
1079 .Lepilogue_avx:
1080         ret
1081 .size   sha256_multi_block_avx,.-sha256_multi_block_avx
1082 ___
1083                                                 if ($avx>1) {
1084 $code =~ s/\`([^\`]*)\`/eval $1/gem;
1085
1086 $REG_SZ=32;
1087 @ptr=map("%r$_",(12..15,8..11));
1088
1089 @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("%ymm$_",(8..15));
1090 ($t1,$t2,$t3,$axb,$bxc,$Xi,$Xn,$sigma)=map("%ymm$_",(0..7));
1091
1092 $code.=<<___;
1093 .type   sha256_multi_block_avx2,\@function,3
1094 .align  32
1095 sha256_multi_block_avx2:
1096 _avx2_shortcut:
1097         mov     %rsp,%rax
1098         push    %rbx
1099         push    %rbp
1100         push    %r12
1101         push    %r13
1102         push    %r14
1103         push    %r15
1104 ___
1105 $code.=<<___ if ($win64);
1106         lea     -0xa8(%rsp),%rsp
1107         movaps  %xmm6,(%rsp)
1108         movaps  %xmm7,0x10(%rsp)
1109         movaps  %xmm8,0x20(%rsp)
1110         movaps  %xmm9,0x30(%rsp)
1111         movaps  %xmm10,0x40(%rsp)
1112         movaps  %xmm11,0x50(%rsp)
1113         movaps  %xmm12,-0x78(%rax)
1114         movaps  %xmm13,-0x68(%rax)
1115         movaps  %xmm14,-0x58(%rax)
1116         movaps  %xmm15,-0x48(%rax)
1117 ___
1118 $code.=<<___;
1119         sub     \$`$REG_SZ*18`, %rsp
1120         and     \$-256,%rsp
1121         mov     %rax,`$REG_SZ*17`(%rsp)         # original %rsp
1122 .Lbody_avx2:
1123         lea     K256+128(%rip),$Tbl
1124         lea     0x80($ctx),$ctx                 # size optimization
1125
1126 .Loop_grande_avx2:
1127         mov     $num,`$REG_SZ*17+8`(%rsp)       # original $num
1128         xor     $num,$num
1129         lea     `$REG_SZ*16`(%rsp),%rbx
1130 ___
1131 for($i=0;$i<8;$i++) {
1132     $code.=<<___;
1133         mov     `16*$i+0`($inp),@ptr[$i]        # input pointer
1134         mov     `16*$i+8`($inp),%ecx            # number of blocks
1135         cmp     $num,%ecx
1136         cmovg   %ecx,$num                       # find maximum
1137         test    %ecx,%ecx
1138         mov     %ecx,`4*$i`(%rbx)               # initialize counters
1139         cmovle  $Tbl,@ptr[$i]                   # cancel input
1140 ___
1141 }
1142 $code.=<<___;
1143         vmovdqu 0x00-0x80($ctx),$A              # load context
1144          lea    128(%rsp),%rax
1145         vmovdqu 0x20-0x80($ctx),$B
1146          lea    256+128(%rsp),%rbx
1147         vmovdqu 0x40-0x80($ctx),$C
1148         vmovdqu 0x60-0x80($ctx),$D
1149         vmovdqu 0x80-0x80($ctx),$E
1150         vmovdqu 0xa0-0x80($ctx),$F
1151         vmovdqu 0xc0-0x80($ctx),$G
1152         vmovdqu 0xe0-0x80($ctx),$H
1153         vmovdqu .Lpbswap(%rip),$Xn
1154         jmp     .Loop_avx2
1155
1156 .align  32
1157 .Loop_avx2:
1158         vpxor   $B,$C,$bxc                      # magic seed
1159 ___
1160 for($i=0;$i<16;$i++)    { &ROUND_00_15_avx($i,@V); unshift(@V,pop(@V)); }
1161 $code.=<<___;
1162         vmovdqu `&Xi_off($i)`,$Xi
1163         mov     \$3,%ecx
1164         jmp     .Loop_16_xx_avx2
1165 .align  32
1166 .Loop_16_xx_avx2:
1167 ___
1168 for(;$i<32;$i++)        { &ROUND_16_XX_avx($i,@V); unshift(@V,pop(@V)); }
1169 $code.=<<___;
1170         dec     %ecx
1171         jnz     .Loop_16_xx_avx2
1172
1173         mov     \$1,%ecx
1174         lea     `$REG_SZ*16`(%rsp),%rbx
1175         lea     K256+128(%rip),$Tbl
1176 ___
1177 for($i=0;$i<8;$i++) {
1178     $code.=<<___;
1179         cmp     `4*$i`(%rbx),%ecx               # examine counters
1180         cmovge  $Tbl,@ptr[$i]                   # cancel input
1181 ___
1182 }
1183 $code.=<<___;
1184         vmovdqa (%rbx),$sigma                   # pull counters
1185         vpxor   $t1,$t1,$t1
1186         vmovdqa $sigma,$Xn
1187         vpcmpgtd $t1,$Xn,$Xn                    # mask value
1188         vpaddd  $Xn,$sigma,$sigma               # counters--
1189
1190         vmovdqu 0x00-0x80($ctx),$t1
1191         vpand   $Xn,$A,$A
1192         vmovdqu 0x20-0x80($ctx),$t2
1193         vpand   $Xn,$B,$B
1194         vmovdqu 0x40-0x80($ctx),$t3
1195         vpand   $Xn,$C,$C
1196         vmovdqu 0x60-0x80($ctx),$Xi
1197         vpand   $Xn,$D,$D
1198         vpaddd  $t1,$A,$A
1199         vmovdqu 0x80-0x80($ctx),$t1
1200         vpand   $Xn,$E,$E
1201         vpaddd  $t2,$B,$B
1202         vmovdqu 0xa0-0x80($ctx),$t2
1203         vpand   $Xn,$F,$F
1204         vpaddd  $t3,$C,$C
1205         vmovdqu 0xc0-0x80($ctx),$t3
1206         vpand   $Xn,$G,$G
1207         vpaddd  $Xi,$D,$D
1208         vmovdqu 0xe0-0x80($ctx),$Xi
1209         vpand   $Xn,$H,$H
1210         vpaddd  $t1,$E,$E
1211         vpaddd  $t2,$F,$F
1212         vmovdqu $A,0x00-0x80($ctx)
1213         vpaddd  $t3,$G,$G
1214         vmovdqu $B,0x20-0x80($ctx)
1215         vpaddd  $Xi,$H,$H
1216         vmovdqu $C,0x40-0x80($ctx)
1217         vmovdqu $D,0x60-0x80($ctx)
1218         vmovdqu $E,0x80-0x80($ctx)
1219         vmovdqu $F,0xa0-0x80($ctx)
1220         vmovdqu $G,0xc0-0x80($ctx)
1221         vmovdqu $H,0xe0-0x80($ctx)
1222
1223         vmovdqu $sigma,(%rbx)                   # save counters
1224         lea     256+128(%rsp),%rbx
1225         vmovdqu .Lpbswap(%rip),$Xn
1226         dec     $num
1227         jnz     .Loop_avx2
1228
1229         #mov    `$REG_SZ*17+8`(%rsp),$num
1230         #lea    $REG_SZ($ctx),$ctx
1231         #lea    `16*$REG_SZ/4`($inp),$inp
1232         #dec    $num
1233         #jnz    .Loop_grande_avx2
1234
1235 .Ldone_avx2:
1236         mov     `$REG_SZ*17`(%rsp),%rax         # orignal %rsp
1237         vzeroupper
1238 ___
1239 $code.=<<___ if ($win64);
1240         movaps  -0xd8(%rax),%xmm6
1241         movaps  -0xc8(%rax),%xmm7
1242         movaps  -0xb8(%rax),%xmm8
1243         movaps  -0xa8(%rax),%xmm9
1244         movaps  -0x98(%rax),%xmm10
1245         movaps  -0x88(%rax),%xmm11
1246         movaps  -0x78(%rax),%xmm12
1247         movaps  -0x68(%rax),%xmm13
1248         movaps  -0x58(%rax),%xmm14
1249         movaps  -0x48(%rax),%xmm15
1250 ___
1251 $code.=<<___;
1252         mov     -48(%rax),%r15
1253         mov     -40(%rax),%r14
1254         mov     -32(%rax),%r13
1255         mov     -24(%rax),%r12
1256         mov     -16(%rax),%rbp
1257         mov     -8(%rax),%rbx
1258         lea     (%rax),%rsp
1259 .Lepilogue_avx2:
1260         ret
1261 .size   sha256_multi_block_avx2,.-sha256_multi_block_avx2
1262 ___
1263                                         }       }}}
1264 $code.=<<___;
1265 .align  256
1266 K256:
1267 ___
1268 sub TABLE {
1269     foreach (@_) {
1270         $code.=<<___;
1271         .long   $_,$_,$_,$_
1272         .long   $_,$_,$_,$_
1273 ___
1274     }
1275 }
1276 &TABLE( 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,
1277         0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5,
1278         0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,
1279         0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174,
1280         0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,
1281         0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da,
1282         0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,
1283         0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967,
1284         0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,
1285         0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85,
1286         0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,
1287         0xd192e819,0xd6990624,0xf40e3585,0x106aa070,
1288         0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,
1289         0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3,
1290         0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,
1291         0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 );
1292 $code.=<<___;
1293 .Lpbswap:
1294         .long   0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f     # pbswap
1295         .long   0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f     # pbswap
1296 K256_shaext:
1297         .long   0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
1298         .long   0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
1299         .long   0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
1300         .long   0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
1301         .long   0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
1302         .long   0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
1303         .long   0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
1304         .long   0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
1305         .long   0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
1306         .long   0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
1307         .long   0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
1308         .long   0xd192e819,0xd6990624,0xf40e3585,0x106aa070
1309         .long   0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
1310         .long   0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
1311         .long   0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
1312         .long   0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
1313         .asciz  "SHA256 multi-block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
1314 ___
1315
1316 if ($win64) {
1317 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1318 #               CONTEXT *context,DISPATCHER_CONTEXT *disp)
1319 $rec="%rcx";
1320 $frame="%rdx";
1321 $context="%r8";
1322 $disp="%r9";
1323
1324 $code.=<<___;
1325 .extern __imp_RtlVirtualUnwind
1326 .type   se_handler,\@abi-omnipotent
1327 .align  16
1328 se_handler:
1329         push    %rsi
1330         push    %rdi
1331         push    %rbx
1332         push    %rbp
1333         push    %r12
1334         push    %r13
1335         push    %r14
1336         push    %r15
1337         pushfq
1338         sub     \$64,%rsp
1339
1340         mov     120($context),%rax      # pull context->Rax
1341         mov     248($context),%rbx      # pull context->Rip
1342
1343         mov     8($disp),%rsi           # disp->ImageBase
1344         mov     56($disp),%r11          # disp->HandlerData
1345
1346         mov     0(%r11),%r10d           # HandlerData[0]
1347         lea     (%rsi,%r10),%r10        # end of prologue label
1348         cmp     %r10,%rbx               # context->Rip<.Lbody
1349         jb      .Lin_prologue
1350
1351         mov     152($context),%rax      # pull context->Rsp
1352
1353         mov     4(%r11),%r10d           # HandlerData[1]
1354         lea     (%rsi,%r10),%r10        # epilogue label
1355         cmp     %r10,%rbx               # context->Rip>=.Lepilogue
1356         jae     .Lin_prologue
1357
1358         mov     `16*17`(%rax),%rax      # pull saved stack pointer
1359
1360         mov     -8(%rax),%rbx
1361         mov     -16(%rax),%rbp
1362         mov     %rbx,144($context)      # restore context->Rbx
1363         mov     %rbp,160($context)      # restore context->Rbp
1364
1365         lea     -24-10*16(%rax),%rsi
1366         lea     512($context),%rdi      # &context.Xmm6
1367         mov     \$20,%ecx
1368         .long   0xa548f3fc              # cld; rep movsq
1369
1370 .Lin_prologue:
1371         mov     8(%rax),%rdi
1372         mov     16(%rax),%rsi
1373         mov     %rax,152($context)      # restore context->Rsp
1374         mov     %rsi,168($context)      # restore context->Rsi
1375         mov     %rdi,176($context)      # restore context->Rdi
1376
1377         mov     40($disp),%rdi          # disp->ContextRecord
1378         mov     $context,%rsi           # context
1379         mov     \$154,%ecx              # sizeof(CONTEXT)
1380         .long   0xa548f3fc              # cld; rep movsq
1381
1382         mov     $disp,%rsi
1383         xor     %rcx,%rcx               # arg1, UNW_FLAG_NHANDLER
1384         mov     8(%rsi),%rdx            # arg2, disp->ImageBase
1385         mov     0(%rsi),%r8             # arg3, disp->ControlPc
1386         mov     16(%rsi),%r9            # arg4, disp->FunctionEntry
1387         mov     40(%rsi),%r10           # disp->ContextRecord
1388         lea     56(%rsi),%r11           # &disp->HandlerData
1389         lea     24(%rsi),%r12           # &disp->EstablisherFrame
1390         mov     %r10,32(%rsp)           # arg5
1391         mov     %r11,40(%rsp)           # arg6
1392         mov     %r12,48(%rsp)           # arg7
1393         mov     %rcx,56(%rsp)           # arg8, (NULL)
1394         call    *__imp_RtlVirtualUnwind(%rip)
1395
1396         mov     \$1,%eax                # ExceptionContinueSearch
1397         add     \$64,%rsp
1398         popfq
1399         pop     %r15
1400         pop     %r14
1401         pop     %r13
1402         pop     %r12
1403         pop     %rbp
1404         pop     %rbx
1405         pop     %rdi
1406         pop     %rsi
1407         ret
1408 .size   se_handler,.-se_handler
1409 ___
1410 $code.=<<___ if ($avx>1);
1411 .type   avx2_handler,\@abi-omnipotent
1412 .align  16
1413 avx2_handler:
1414         push    %rsi
1415         push    %rdi
1416         push    %rbx
1417         push    %rbp
1418         push    %r12
1419         push    %r13
1420         push    %r14
1421         push    %r15
1422         pushfq
1423         sub     \$64,%rsp
1424
1425         mov     120($context),%rax      # pull context->Rax
1426         mov     248($context),%rbx      # pull context->Rip
1427
1428         mov     8($disp),%rsi           # disp->ImageBase
1429         mov     56($disp),%r11          # disp->HandlerData
1430
1431         mov     0(%r11),%r10d           # HandlerData[0]
1432         lea     (%rsi,%r10),%r10        # end of prologue label
1433         cmp     %r10,%rbx               # context->Rip<body label
1434         jb      .Lin_prologue
1435
1436         mov     152($context),%rax      # pull context->Rsp
1437
1438         mov     4(%r11),%r10d           # HandlerData[1]
1439         lea     (%rsi,%r10),%r10        # epilogue label
1440         cmp     %r10,%rbx               # context->Rip>=epilogue label
1441         jae     .Lin_prologue
1442
1443         mov     `32*17`($context),%rax  # pull saved stack pointer
1444
1445         mov     -8(%rax),%rbx
1446         mov     -16(%rax),%rbp
1447         mov     -24(%rax),%r12
1448         mov     -32(%rax),%r13
1449         mov     -40(%rax),%r14
1450         mov     -48(%rax),%r15
1451         mov     %rbx,144($context)      # restore context->Rbx
1452         mov     %rbp,160($context)      # restore context->Rbp
1453         mov     %r12,216($context)      # restore cotnext->R12
1454         mov     %r13,224($context)      # restore cotnext->R13
1455         mov     %r14,232($context)      # restore cotnext->R14
1456         mov     %r15,240($context)      # restore cotnext->R15
1457
1458         lea     -56-10*16(%rax),%rsi
1459         lea     512($context),%rdi      # &context.Xmm6
1460         mov     \$20,%ecx
1461         .long   0xa548f3fc              # cld; rep movsq
1462
1463         jmp     .Lin_prologue
1464 .size   avx2_handler,.-avx2_handler
1465 ___
1466 $code.=<<___;
1467 .section        .pdata
1468 .align  4
1469         .rva    .LSEH_begin_sha256_multi_block
1470         .rva    .LSEH_end_sha256_multi_block
1471         .rva    .LSEH_info_sha256_multi_block
1472         .rva    .LSEH_begin_sha256_multi_block_shaext
1473         .rva    .LSEH_end_sha256_multi_block_shaext
1474         .rva    .LSEH_info_sha256_multi_block_shaext
1475 ___
1476 $code.=<<___ if ($avx);
1477         .rva    .LSEH_begin_sha256_multi_block_avx
1478         .rva    .LSEH_end_sha256_multi_block_avx
1479         .rva    .LSEH_info_sha256_multi_block_avx
1480 ___
1481 $code.=<<___ if ($avx>1);
1482         .rva    .LSEH_begin_sha256_multi_block_avx2
1483         .rva    .LSEH_end_sha256_multi_block_avx2
1484         .rva    .LSEH_info_sha256_multi_block_avx2
1485 ___
1486 $code.=<<___;
1487 .section        .xdata
1488 .align  8
1489 .LSEH_info_sha256_multi_block:
1490         .byte   9,0,0,0
1491         .rva    se_handler
1492         .rva    .Lbody,.Lepilogue                       # HandlerData[]
1493 .LSEH_info_sha256_multi_block_shaext:
1494         .byte   9,0,0,0
1495         .rva    se_handler
1496         .rva    .Lbody_shaext,.Lepilogue_shaext         # HandlerData[]
1497 ___
1498 $code.=<<___ if ($avx);
1499 .LSEH_info_sha256_multi_block_avx:
1500         .byte   9,0,0,0
1501         .rva    se_handler
1502         .rva    .Lbody_avx,.Lepilogue_avx               # HandlerData[]
1503 ___
1504 $code.=<<___ if ($avx>1);
1505 .LSEH_info_sha256_multi_block_avx2:
1506         .byte   9,0,0,0
1507         .rva    avx2_handler
1508         .rva    .Lbody_avx2,.Lepilogue_avx2             # HandlerData[]
1509 ___
1510 }
1511 ####################################################################
1512
1513 sub rex {
1514   local *opcode=shift;
1515   my ($dst,$src)=@_;
1516   my $rex=0;
1517
1518     $rex|=0x04                  if ($dst>=8);
1519     $rex|=0x01                  if ($src>=8);
1520     unshift @opcode,$rex|0x40   if ($rex);
1521 }
1522
1523 sub sha256op38 {
1524     my $instr = shift;
1525     my %opcodelet = (
1526                 "sha256rnds2" => 0xcb,
1527                 "sha256msg1"  => 0xcc,
1528                 "sha256msg2"  => 0xcd   );
1529
1530     if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1531       my @opcode=(0x0f,0x38);
1532         rex(\@opcode,$2,$1);
1533         push @opcode,$opcodelet{$instr};
1534         push @opcode,0xc0|($1&7)|(($2&7)<<3);           # ModR/M
1535         return ".byte\t".join(',',@opcode);
1536     } else {
1537         return $instr."\t".@_[0];
1538     }
1539 }
1540
1541 foreach (split("\n",$code)) {
1542         s/\`([^\`]*)\`/eval($1)/ge;
1543
1544         s/\b(sha256[^\s]*)\s+(.*)/sha256op38($1,$2)/geo         or
1545
1546         s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go          or
1547         s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go         or
1548         s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+),%ymm([0-9]+)/$1$2%xmm$3,%xmm$4/go    or
1549         s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go        or
1550         s/\b(vinserti128)\b(\s+)%ymm/$1$2\$1,%xmm/go            or
1551         s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go;
1552
1553         print $_,"\n";
1554 }
1555
1556 close STDOUT;