x86_64 assembly pack: tolerate spaces in source directory name.
[openssl.git] / crypto / chacha / asm / chacha-x86_64.pl
1 #! /usr/bin/env perl
2 # Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
3 #
4 # Licensed under the OpenSSL license (the "License").  You may not use
5 # this file except in compliance with the License.  You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
8
9 #
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
16 #
17 # November 2014
18 #
19 # ChaCha20 for x86_64.
20 #
21 # Performance in cycles per byte out of large buffer.
22 #
23 #               IALU/gcc 4.8(i) 1xSSSE3/SSE2    4xSSSE3     8xAVX2
24 #
25 # P4            9.48/+99%       -/22.7(ii)      -
26 # Core2         7.83/+55%       7.90/8.08       4.35
27 # Westmere      7.19/+50%       5.60/6.70       3.00
28 # Sandy Bridge  8.31/+42%       5.45/6.76       2.72
29 # Ivy Bridge    6.71/+46%       5.40/6.49       2.41
30 # Haswell       5.92/+43%       5.20/6.45       2.42        1.23
31 # Silvermont    12.0/+33%       7.75/7.40       7.03(iii)
32 # Sledgehammer  7.28/+52%       -/14.2(ii)      -
33 # Bulldozer     9.66/+28%       9.85/11.1       3.06(iv)
34 # VIA Nano      10.5/+46%       6.72/8.60       6.05
35 #
36 # (i)   compared to older gcc 3.x one can observe >2x improvement on
37 #       most platforms;
38 # (ii)  as it can be seen, SSE2 performance is too low on legacy
39 #       processors; NxSSE2 results are naturally better, but not
40 #       impressively better than IALU ones, which is why you won't
41 #       find SSE2 code below;
42 # (iii) this is not optimal result for Atom because of MSROM
43 #       limitations, SSE2 can do better, but gain is considered too
44 #       low to justify the [maintenance] effort;
45 # (iv)  Bulldozer actually executes 4xXOP code path that delivers 2.20;
46
47 $flavour = shift;
48 $output  = shift;
49 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
50
51 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
52
53 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
54 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
55 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
56 die "can't locate x86_64-xlate.pl";
57
58 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
59                 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
60         $avx = ($1>=2.19) + ($1>=2.22);
61 }
62
63 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
64            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
65         $avx = ($1>=2.09) + ($1>=2.10);
66 }
67
68 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
69            `ml64 2>&1` =~ /Version ([0-9]+)\./) {
70         $avx = ($1>=10) + ($1>=11);
71 }
72
73 if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) {
74         $avx = ($2>=3.0) + ($2>3.0);
75 }
76
77 open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
78 *STDOUT=*OUT;
79
80 # input parameter block
81 ($out,$inp,$len,$key,$counter)=("%rdi","%rsi","%rdx","%rcx","%r8");
82
83 $code.=<<___;
84 .text
85
86 .extern OPENSSL_ia32cap_P
87
88 .align  64
89 .Lzero:
90 .long   0,0,0,0
91 .Lone:
92 .long   1,0,0,0
93 .Linc:
94 .long   0,1,2,3
95 .Lfour:
96 .long   4,4,4,4
97 .Lincy:
98 .long   0,2,4,6,1,3,5,7
99 .Leight:
100 .long   8,8,8,8,8,8,8,8
101 .Lrot16:
102 .byte   0x2,0x3,0x0,0x1, 0x6,0x7,0x4,0x5, 0xa,0xb,0x8,0x9, 0xe,0xf,0xc,0xd
103 .Lrot24:
104 .byte   0x3,0x0,0x1,0x2, 0x7,0x4,0x5,0x6, 0xb,0x8,0x9,0xa, 0xf,0xc,0xd,0xe
105 .Lsigma:
106 .asciz  "expand 32-byte k"
107 .asciz  "ChaCha20 for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
108 ___
109
110 sub AUTOLOAD()          # thunk [simplified] 32-bit style perlasm
111 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://;
112   my $arg = pop;
113     $arg = "\$$arg" if ($arg*1 eq $arg);
114     $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n";
115 }
116
117 @x=("%eax","%ebx","%ecx","%edx",map("%r${_}d",(8..11)),
118     "%nox","%nox","%nox","%nox",map("%r${_}d",(12..15)));
119 @t=("%esi","%edi");
120
121 sub ROUND {                     # critical path is 24 cycles per round
122 my ($a0,$b0,$c0,$d0)=@_;
123 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
124 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
125 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
126 my ($xc,$xc_)=map("\"$_\"",@t);
127 my @x=map("\"$_\"",@x);
128
129         # Consider order in which variables are addressed by their
130         # index:
131         #
132         #       a   b   c   d
133         #
134         #       0   4   8  12 < even round
135         #       1   5   9  13
136         #       2   6  10  14
137         #       3   7  11  15
138         #       0   5  10  15 < odd round
139         #       1   6  11  12
140         #       2   7   8  13
141         #       3   4   9  14
142         #
143         # 'a', 'b' and 'd's are permanently allocated in registers,
144         # @x[0..7,12..15], while 'c's are maintained in memory. If
145         # you observe 'c' column, you'll notice that pair of 'c's is
146         # invariant between rounds. This means that we have to reload
147         # them once per round, in the middle. This is why you'll see
148         # bunch of 'c' stores and loads in the middle, but none in
149         # the beginning or end.
150
151         # Normally instructions would be interleaved to favour in-order
152         # execution. Generally out-of-order cores manage it gracefully,
153         # but not this time for some reason. As in-order execution
154         # cores are dying breed, old Atom is the only one around,
155         # instructions are left uninterleaved. Besides, Atom is better
156         # off executing 1xSSSE3 code anyway...
157
158         (
159         "&add   (@x[$a0],@x[$b0])",     # Q1
160         "&xor   (@x[$d0],@x[$a0])",
161         "&rol   (@x[$d0],16)",
162          "&add  (@x[$a1],@x[$b1])",     # Q2
163          "&xor  (@x[$d1],@x[$a1])",
164          "&rol  (@x[$d1],16)",
165
166         "&add   ($xc,@x[$d0])",
167         "&xor   (@x[$b0],$xc)",
168         "&rol   (@x[$b0],12)",
169          "&add  ($xc_,@x[$d1])",
170          "&xor  (@x[$b1],$xc_)",
171          "&rol  (@x[$b1],12)",
172
173         "&add   (@x[$a0],@x[$b0])",
174         "&xor   (@x[$d0],@x[$a0])",
175         "&rol   (@x[$d0],8)",
176          "&add  (@x[$a1],@x[$b1])",
177          "&xor  (@x[$d1],@x[$a1])",
178          "&rol  (@x[$d1],8)",
179
180         "&add   ($xc,@x[$d0])",
181         "&xor   (@x[$b0],$xc)",
182         "&rol   (@x[$b0],7)",
183          "&add  ($xc_,@x[$d1])",
184          "&xor  (@x[$b1],$xc_)",
185          "&rol  (@x[$b1],7)",
186
187         "&mov   (\"4*$c0(%rsp)\",$xc)", # reload pair of 'c's
188          "&mov  (\"4*$c1(%rsp)\",$xc_)",
189         "&mov   ($xc,\"4*$c2(%rsp)\")",
190          "&mov  ($xc_,\"4*$c3(%rsp)\")",
191
192         "&add   (@x[$a2],@x[$b2])",     # Q3
193         "&xor   (@x[$d2],@x[$a2])",
194         "&rol   (@x[$d2],16)",
195          "&add  (@x[$a3],@x[$b3])",     # Q4
196          "&xor  (@x[$d3],@x[$a3])",
197          "&rol  (@x[$d3],16)",
198
199         "&add   ($xc,@x[$d2])",
200         "&xor   (@x[$b2],$xc)",
201         "&rol   (@x[$b2],12)",
202          "&add  ($xc_,@x[$d3])",
203          "&xor  (@x[$b3],$xc_)",
204          "&rol  (@x[$b3],12)",
205
206         "&add   (@x[$a2],@x[$b2])",
207         "&xor   (@x[$d2],@x[$a2])",
208         "&rol   (@x[$d2],8)",
209          "&add  (@x[$a3],@x[$b3])",
210          "&xor  (@x[$d3],@x[$a3])",
211          "&rol  (@x[$d3],8)",
212
213         "&add   ($xc,@x[$d2])",
214         "&xor   (@x[$b2],$xc)",
215         "&rol   (@x[$b2],7)",
216          "&add  ($xc_,@x[$d3])",
217          "&xor  (@x[$b3],$xc_)",
218          "&rol  (@x[$b3],7)"
219         );
220 }
221
222 ########################################################################
223 # Generic code path that handles all lengths on pre-SSSE3 processors.
224 $code.=<<___;
225 .globl  ChaCha20_ctr32
226 .type   ChaCha20_ctr32,\@function,5
227 .align  64
228 ChaCha20_ctr32:
229         cmp     \$0,$len
230         je      .Lno_data
231         mov     OPENSSL_ia32cap_P+4(%rip),%r10
232         test    \$`1<<(41-32)`,%r10d
233         jnz     .LChaCha20_ssse3
234
235         push    %rbx
236         push    %rbp
237         push    %r12
238         push    %r13
239         push    %r14
240         push    %r15
241         sub     \$64+24,%rsp
242
243         #movdqa .Lsigma(%rip),%xmm0
244         movdqu  ($key),%xmm1
245         movdqu  16($key),%xmm2
246         movdqu  ($counter),%xmm3
247         movdqa  .Lone(%rip),%xmm4
248
249         #movdqa %xmm0,4*0(%rsp)         # key[0]
250         movdqa  %xmm1,4*4(%rsp)         # key[1]
251         movdqa  %xmm2,4*8(%rsp)         # key[2]
252         movdqa  %xmm3,4*12(%rsp)        # key[3]
253         mov     $len,%rbp               # reassign $len
254         jmp     .Loop_outer
255
256 .align  32
257 .Loop_outer:
258         mov     \$0x61707865,@x[0]      # 'expa'
259         mov     \$0x3320646e,@x[1]      # 'nd 3'
260         mov     \$0x79622d32,@x[2]      # '2-by'
261         mov     \$0x6b206574,@x[3]      # 'te k'
262         mov     4*4(%rsp),@x[4]
263         mov     4*5(%rsp),@x[5]
264         mov     4*6(%rsp),@x[6]
265         mov     4*7(%rsp),@x[7]
266         movd    %xmm3,@x[12]
267         mov     4*13(%rsp),@x[13]
268         mov     4*14(%rsp),@x[14]
269         mov     4*15(%rsp),@x[15]
270
271         mov     %rbp,64+0(%rsp)         # save len
272         mov     \$10,%ebp
273         mov     $inp,64+8(%rsp)         # save inp
274         movq    %xmm2,%rsi              # "@x[8]"
275         mov     $out,64+16(%rsp)        # save out
276         mov     %rsi,%rdi
277         shr     \$32,%rdi               # "@x[9]"
278         jmp     .Loop
279
280 .align  32
281 .Loop:
282 ___
283         foreach (&ROUND (0, 4, 8,12)) { eval; }
284         foreach (&ROUND (0, 5,10,15)) { eval; }
285         &dec    ("%ebp");
286         &jnz    (".Loop");
287
288 $code.=<<___;
289         mov     @t[1],4*9(%rsp)         # modulo-scheduled
290         mov     @t[0],4*8(%rsp)
291         mov     64(%rsp),%rbp           # load len
292         movdqa  %xmm2,%xmm1
293         mov     64+8(%rsp),$inp         # load inp
294         paddd   %xmm4,%xmm3             # increment counter
295         mov     64+16(%rsp),$out        # load out
296
297         add     \$0x61707865,@x[0]      # 'expa'
298         add     \$0x3320646e,@x[1]      # 'nd 3'
299         add     \$0x79622d32,@x[2]      # '2-by'
300         add     \$0x6b206574,@x[3]      # 'te k'
301         add     4*4(%rsp),@x[4]
302         add     4*5(%rsp),@x[5]
303         add     4*6(%rsp),@x[6]
304         add     4*7(%rsp),@x[7]
305         add     4*12(%rsp),@x[12]
306         add     4*13(%rsp),@x[13]
307         add     4*14(%rsp),@x[14]
308         add     4*15(%rsp),@x[15]
309         paddd   4*8(%rsp),%xmm1
310
311         cmp     \$64,%rbp
312         jb      .Ltail
313
314         xor     4*0($inp),@x[0]         # xor with input
315         xor     4*1($inp),@x[1]
316         xor     4*2($inp),@x[2]
317         xor     4*3($inp),@x[3]
318         xor     4*4($inp),@x[4]
319         xor     4*5($inp),@x[5]
320         xor     4*6($inp),@x[6]
321         xor     4*7($inp),@x[7]
322         movdqu  4*8($inp),%xmm0
323         xor     4*12($inp),@x[12]
324         xor     4*13($inp),@x[13]
325         xor     4*14($inp),@x[14]
326         xor     4*15($inp),@x[15]
327         lea     4*16($inp),$inp         # inp+=64
328         pxor    %xmm1,%xmm0
329
330         movdqa  %xmm2,4*8(%rsp)
331         movd    %xmm3,4*12(%rsp)
332
333         mov     @x[0],4*0($out)         # write output
334         mov     @x[1],4*1($out)
335         mov     @x[2],4*2($out)
336         mov     @x[3],4*3($out)
337         mov     @x[4],4*4($out)
338         mov     @x[5],4*5($out)
339         mov     @x[6],4*6($out)
340         mov     @x[7],4*7($out)
341         movdqu  %xmm0,4*8($out)
342         mov     @x[12],4*12($out)
343         mov     @x[13],4*13($out)
344         mov     @x[14],4*14($out)
345         mov     @x[15],4*15($out)
346         lea     4*16($out),$out         # out+=64
347
348         sub     \$64,%rbp
349         jnz     .Loop_outer
350
351         jmp     .Ldone
352
353 .align  16
354 .Ltail:
355         mov     @x[0],4*0(%rsp)
356         mov     @x[1],4*1(%rsp)
357         xor     %rbx,%rbx
358         mov     @x[2],4*2(%rsp)
359         mov     @x[3],4*3(%rsp)
360         mov     @x[4],4*4(%rsp)
361         mov     @x[5],4*5(%rsp)
362         mov     @x[6],4*6(%rsp)
363         mov     @x[7],4*7(%rsp)
364         movdqa  %xmm1,4*8(%rsp)
365         mov     @x[12],4*12(%rsp)
366         mov     @x[13],4*13(%rsp)
367         mov     @x[14],4*14(%rsp)
368         mov     @x[15],4*15(%rsp)
369
370 .Loop_tail:
371         movzb   ($inp,%rbx),%eax
372         movzb   (%rsp,%rbx),%edx
373         lea     1(%rbx),%rbx
374         xor     %edx,%eax
375         mov     %al,-1($out,%rbx)
376         dec     %rbp
377         jnz     .Loop_tail
378
379 .Ldone:
380         add     \$64+24,%rsp
381         pop     %r15
382         pop     %r14
383         pop     %r13
384         pop     %r12
385         pop     %rbp
386         pop     %rbx
387 .Lno_data:
388         ret
389 .size   ChaCha20_ctr32,.-ChaCha20_ctr32
390 ___
391
392 ########################################################################
393 # SSSE3 code path that handles shorter lengths
394 {
395 my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("%xmm$_",(0..7));
396
397 sub SSSE3ROUND {        # critical path is 20 "SIMD ticks" per round
398         &paddd  ($a,$b);
399         &pxor   ($d,$a);
400         &pshufb ($d,$rot16);
401
402         &paddd  ($c,$d);
403         &pxor   ($b,$c);
404         &movdqa ($t,$b);
405         &psrld  ($b,20);
406         &pslld  ($t,12);
407         &por    ($b,$t);
408
409         &paddd  ($a,$b);
410         &pxor   ($d,$a);
411         &pshufb ($d,$rot24);
412
413         &paddd  ($c,$d);
414         &pxor   ($b,$c);
415         &movdqa ($t,$b);
416         &psrld  ($b,25);
417         &pslld  ($t,7);
418         &por    ($b,$t);
419 }
420
421 my $xframe = $win64 ? 32+32+8 : 24;
422
423 $code.=<<___;
424 .type   ChaCha20_ssse3,\@function,5
425 .align  32
426 ChaCha20_ssse3:
427 .LChaCha20_ssse3:
428 ___
429 $code.=<<___    if ($avx);
430         test    \$`1<<(43-32)`,%r10d
431         jnz     .LChaCha20_4xop         # XOP is fastest even if we use 1/4
432 ___
433 $code.=<<___;
434         cmp     \$128,$len              # we might throw away some data,
435         ja      .LChaCha20_4x           # but overall it won't be slower
436
437 .Ldo_sse3_after_all:
438         push    %rbx
439         push    %rbp
440         push    %r12
441         push    %r13
442         push    %r14
443         push    %r15
444
445         sub     \$64+$xframe,%rsp
446 ___
447 $code.=<<___    if ($win64);
448         movaps  %xmm6,64+32(%rsp)
449         movaps  %xmm7,64+48(%rsp)
450 ___
451 $code.=<<___;
452         movdqa  .Lsigma(%rip),$a
453         movdqu  ($key),$b
454         movdqu  16($key),$c
455         movdqu  ($counter),$d
456         movdqa  .Lrot16(%rip),$rot16
457         movdqa  .Lrot24(%rip),$rot24
458
459         movdqa  $a,0x00(%rsp)
460         movdqa  $b,0x10(%rsp)
461         movdqa  $c,0x20(%rsp)
462         movdqa  $d,0x30(%rsp)
463         mov     \$10,%ebp
464         jmp     .Loop_ssse3
465
466 .align  32
467 .Loop_outer_ssse3:
468         movdqa  .Lone(%rip),$d
469         movdqa  0x00(%rsp),$a
470         movdqa  0x10(%rsp),$b
471         movdqa  0x20(%rsp),$c
472         paddd   0x30(%rsp),$d
473         mov     \$10,%ebp
474         movdqa  $d,0x30(%rsp)
475         jmp     .Loop_ssse3
476
477 .align  32
478 .Loop_ssse3:
479 ___
480         &SSSE3ROUND();
481         &pshufd ($c,$c,0b01001110);
482         &pshufd ($b,$b,0b00111001);
483         &pshufd ($d,$d,0b10010011);
484         &nop    ();
485
486         &SSSE3ROUND();
487         &pshufd ($c,$c,0b01001110);
488         &pshufd ($b,$b,0b10010011);
489         &pshufd ($d,$d,0b00111001);
490
491         &dec    ("%ebp");
492         &jnz    (".Loop_ssse3");
493
494 $code.=<<___;
495         paddd   0x00(%rsp),$a
496         paddd   0x10(%rsp),$b
497         paddd   0x20(%rsp),$c
498         paddd   0x30(%rsp),$d
499
500         cmp     \$64,$len
501         jb      .Ltail_ssse3
502
503         movdqu  0x00($inp),$t
504         movdqu  0x10($inp),$t1
505         pxor    $t,$a                   # xor with input
506         movdqu  0x20($inp),$t
507         pxor    $t1,$b
508         movdqu  0x30($inp),$t1
509         lea     0x40($inp),$inp         # inp+=64
510         pxor    $t,$c
511         pxor    $t1,$d
512
513         movdqu  $a,0x00($out)           # write output
514         movdqu  $b,0x10($out)
515         movdqu  $c,0x20($out)
516         movdqu  $d,0x30($out)
517         lea     0x40($out),$out         # out+=64
518
519         sub     \$64,$len
520         jnz     .Loop_outer_ssse3
521
522         jmp     .Ldone_ssse3
523
524 .align  16
525 .Ltail_ssse3:
526         movdqa  $a,0x00(%rsp)
527         movdqa  $b,0x10(%rsp)
528         movdqa  $c,0x20(%rsp)
529         movdqa  $d,0x30(%rsp)
530         xor     %rbx,%rbx
531
532 .Loop_tail_ssse3:
533         movzb   ($inp,%rbx),%eax
534         movzb   (%rsp,%rbx),%ecx
535         lea     1(%rbx),%rbx
536         xor     %ecx,%eax
537         mov     %al,-1($out,%rbx)
538         dec     $len
539         jnz     .Loop_tail_ssse3
540
541 .Ldone_ssse3:
542 ___
543 $code.=<<___    if ($win64);
544         movaps  64+32(%rsp),%xmm6
545         movaps  64+48(%rsp),%xmm7
546 ___
547 $code.=<<___;
548         add     \$64+$xframe,%rsp
549         pop     %r15
550         pop     %r14
551         pop     %r13
552         pop     %r12
553         pop     %rbp
554         pop     %rbx
555         ret
556 .size   ChaCha20_ssse3,.-ChaCha20_ssse3
557 ___
558 }
559
560 ########################################################################
561 # SSSE3 code path that handles longer messages.
562 {
563 # assign variables to favor Atom front-end
564 my ($xd0,$xd1,$xd2,$xd3, $xt0,$xt1,$xt2,$xt3,
565     $xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3)=map("%xmm$_",(0..15));
566 my  @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
567         "%nox","%nox","%nox","%nox", $xd0,$xd1,$xd2,$xd3);
568
569 sub SSSE3_lane_ROUND {
570 my ($a0,$b0,$c0,$d0)=@_;
571 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
572 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
573 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
574 my ($xc,$xc_,$t0,$t1)=map("\"$_\"",$xt0,$xt1,$xt2,$xt3);
575 my @x=map("\"$_\"",@xx);
576
577         # Consider order in which variables are addressed by their
578         # index:
579         #
580         #       a   b   c   d
581         #
582         #       0   4   8  12 < even round
583         #       1   5   9  13
584         #       2   6  10  14
585         #       3   7  11  15
586         #       0   5  10  15 < odd round
587         #       1   6  11  12
588         #       2   7   8  13
589         #       3   4   9  14
590         #
591         # 'a', 'b' and 'd's are permanently allocated in registers,
592         # @x[0..7,12..15], while 'c's are maintained in memory. If
593         # you observe 'c' column, you'll notice that pair of 'c's is
594         # invariant between rounds. This means that we have to reload
595         # them once per round, in the middle. This is why you'll see
596         # bunch of 'c' stores and loads in the middle, but none in
597         # the beginning or end.
598
599         (
600         "&paddd         (@x[$a0],@x[$b0])",     # Q1
601          "&paddd        (@x[$a1],@x[$b1])",     # Q2
602         "&pxor          (@x[$d0],@x[$a0])",
603          "&pxor         (@x[$d1],@x[$a1])",
604         "&pshufb        (@x[$d0],$t1)",
605          "&pshufb       (@x[$d1],$t1)",
606
607         "&paddd         ($xc,@x[$d0])",
608          "&paddd        ($xc_,@x[$d1])",
609         "&pxor          (@x[$b0],$xc)",
610          "&pxor         (@x[$b1],$xc_)",
611         "&movdqa        ($t0,@x[$b0])",
612         "&pslld         (@x[$b0],12)",
613         "&psrld         ($t0,20)",
614          "&movdqa       ($t1,@x[$b1])",
615          "&pslld        (@x[$b1],12)",
616         "&por           (@x[$b0],$t0)",
617          "&psrld        ($t1,20)",
618         "&movdqa        ($t0,'(%r11)')",        # .Lrot24(%rip)
619          "&por          (@x[$b1],$t1)",
620
621         "&paddd         (@x[$a0],@x[$b0])",
622          "&paddd        (@x[$a1],@x[$b1])",
623         "&pxor          (@x[$d0],@x[$a0])",
624          "&pxor         (@x[$d1],@x[$a1])",
625         "&pshufb        (@x[$d0],$t0)",
626          "&pshufb       (@x[$d1],$t0)",
627
628         "&paddd         ($xc,@x[$d0])",
629          "&paddd        ($xc_,@x[$d1])",
630         "&pxor          (@x[$b0],$xc)",
631          "&pxor         (@x[$b1],$xc_)",
632         "&movdqa        ($t1,@x[$b0])",
633         "&pslld         (@x[$b0],7)",
634         "&psrld         ($t1,25)",
635          "&movdqa       ($t0,@x[$b1])",
636          "&pslld        (@x[$b1],7)",
637         "&por           (@x[$b0],$t1)",
638          "&psrld        ($t0,25)",
639         "&movdqa        ($t1,'(%r10)')",        # .Lrot16(%rip)
640          "&por          (@x[$b1],$t0)",
641
642         "&movdqa        (\"`16*($c0-8)`(%rsp)\",$xc)",  # reload pair of 'c's
643          "&movdqa       (\"`16*($c1-8)`(%rsp)\",$xc_)",
644         "&movdqa        ($xc,\"`16*($c2-8)`(%rsp)\")",
645          "&movdqa       ($xc_,\"`16*($c3-8)`(%rsp)\")",
646
647         "&paddd         (@x[$a2],@x[$b2])",     # Q3
648          "&paddd        (@x[$a3],@x[$b3])",     # Q4
649         "&pxor          (@x[$d2],@x[$a2])",
650          "&pxor         (@x[$d3],@x[$a3])",
651         "&pshufb        (@x[$d2],$t1)",
652          "&pshufb       (@x[$d3],$t1)",
653
654         "&paddd         ($xc,@x[$d2])",
655          "&paddd        ($xc_,@x[$d3])",
656         "&pxor          (@x[$b2],$xc)",
657          "&pxor         (@x[$b3],$xc_)",
658         "&movdqa        ($t0,@x[$b2])",
659         "&pslld         (@x[$b2],12)",
660         "&psrld         ($t0,20)",
661          "&movdqa       ($t1,@x[$b3])",
662          "&pslld        (@x[$b3],12)",
663         "&por           (@x[$b2],$t0)",
664          "&psrld        ($t1,20)",
665         "&movdqa        ($t0,'(%r11)')",        # .Lrot24(%rip)
666          "&por          (@x[$b3],$t1)",
667
668         "&paddd         (@x[$a2],@x[$b2])",
669          "&paddd        (@x[$a3],@x[$b3])",
670         "&pxor          (@x[$d2],@x[$a2])",
671          "&pxor         (@x[$d3],@x[$a3])",
672         "&pshufb        (@x[$d2],$t0)",
673          "&pshufb       (@x[$d3],$t0)",
674
675         "&paddd         ($xc,@x[$d2])",
676          "&paddd        ($xc_,@x[$d3])",
677         "&pxor          (@x[$b2],$xc)",
678          "&pxor         (@x[$b3],$xc_)",
679         "&movdqa        ($t1,@x[$b2])",
680         "&pslld         (@x[$b2],7)",
681         "&psrld         ($t1,25)",
682          "&movdqa       ($t0,@x[$b3])",
683          "&pslld        (@x[$b3],7)",
684         "&por           (@x[$b2],$t1)",
685          "&psrld        ($t0,25)",
686         "&movdqa        ($t1,'(%r10)')",        # .Lrot16(%rip)
687          "&por          (@x[$b3],$t0)"
688         );
689 }
690
691 my $xframe = $win64 ? 0xa0 : 0;
692
693 $code.=<<___;
694 .type   ChaCha20_4x,\@function,5
695 .align  32
696 ChaCha20_4x:
697 .LChaCha20_4x:
698         mov             %r10,%r11
699 ___
700 $code.=<<___    if ($avx>1);
701         shr             \$32,%r10               # OPENSSL_ia32cap_P+8
702         test            \$`1<<5`,%r10           # test AVX2
703         jnz             .LChaCha20_8x
704 ___
705 $code.=<<___;
706         cmp             \$192,$len
707         ja              .Lproceed4x
708
709         and             \$`1<<26|1<<22`,%r11    # isolate XSAVE+MOVBE
710         cmp             \$`1<<22`,%r11          # check for MOVBE without XSAVE
711         je              .Ldo_sse3_after_all     # to detect Atom
712
713 .Lproceed4x:
714         lea             -0x78(%rsp),%r11
715         sub             \$0x148+$xframe,%rsp
716 ___
717         ################ stack layout
718         # +0x00         SIMD equivalent of @x[8-12]
719         # ...
720         # +0x40         constant copy of key[0-2] smashed by lanes
721         # ...
722         # +0x100        SIMD counters (with nonce smashed by lanes)
723         # ...
724         # +0x140
725 $code.=<<___    if ($win64);
726         movaps          %xmm6,-0x30(%r11)
727         movaps          %xmm7,-0x20(%r11)
728         movaps          %xmm8,-0x10(%r11)
729         movaps          %xmm9,0x00(%r11)
730         movaps          %xmm10,0x10(%r11)
731         movaps          %xmm11,0x20(%r11)
732         movaps          %xmm12,0x30(%r11)
733         movaps          %xmm13,0x40(%r11)
734         movaps          %xmm14,0x50(%r11)
735         movaps          %xmm15,0x60(%r11)
736 ___
737 $code.=<<___;
738         movdqa          .Lsigma(%rip),$xa3      # key[0]
739         movdqu          ($key),$xb3             # key[1]
740         movdqu          16($key),$xt3           # key[2]
741         movdqu          ($counter),$xd3         # key[3]
742         lea             0x100(%rsp),%rcx        # size optimization
743         lea             .Lrot16(%rip),%r10
744         lea             .Lrot24(%rip),%r11
745
746         pshufd          \$0x00,$xa3,$xa0        # smash key by lanes...
747         pshufd          \$0x55,$xa3,$xa1
748         movdqa          $xa0,0x40(%rsp)         # ... and offload
749         pshufd          \$0xaa,$xa3,$xa2
750         movdqa          $xa1,0x50(%rsp)
751         pshufd          \$0xff,$xa3,$xa3
752         movdqa          $xa2,0x60(%rsp)
753         movdqa          $xa3,0x70(%rsp)
754
755         pshufd          \$0x00,$xb3,$xb0
756         pshufd          \$0x55,$xb3,$xb1
757         movdqa          $xb0,0x80-0x100(%rcx)
758         pshufd          \$0xaa,$xb3,$xb2
759         movdqa          $xb1,0x90-0x100(%rcx)
760         pshufd          \$0xff,$xb3,$xb3
761         movdqa          $xb2,0xa0-0x100(%rcx)
762         movdqa          $xb3,0xb0-0x100(%rcx)
763
764         pshufd          \$0x00,$xt3,$xt0        # "$xc0"
765         pshufd          \$0x55,$xt3,$xt1        # "$xc1"
766         movdqa          $xt0,0xc0-0x100(%rcx)
767         pshufd          \$0xaa,$xt3,$xt2        # "$xc2"
768         movdqa          $xt1,0xd0-0x100(%rcx)
769         pshufd          \$0xff,$xt3,$xt3        # "$xc3"
770         movdqa          $xt2,0xe0-0x100(%rcx)
771         movdqa          $xt3,0xf0-0x100(%rcx)
772
773         pshufd          \$0x00,$xd3,$xd0
774         pshufd          \$0x55,$xd3,$xd1
775         paddd           .Linc(%rip),$xd0        # don't save counters yet
776         pshufd          \$0xaa,$xd3,$xd2
777         movdqa          $xd1,0x110-0x100(%rcx)
778         pshufd          \$0xff,$xd3,$xd3
779         movdqa          $xd2,0x120-0x100(%rcx)
780         movdqa          $xd3,0x130-0x100(%rcx)
781
782         jmp             .Loop_enter4x
783
784 .align  32
785 .Loop_outer4x:
786         movdqa          0x40(%rsp),$xa0         # re-load smashed key
787         movdqa          0x50(%rsp),$xa1
788         movdqa          0x60(%rsp),$xa2
789         movdqa          0x70(%rsp),$xa3
790         movdqa          0x80-0x100(%rcx),$xb0
791         movdqa          0x90-0x100(%rcx),$xb1
792         movdqa          0xa0-0x100(%rcx),$xb2
793         movdqa          0xb0-0x100(%rcx),$xb3
794         movdqa          0xc0-0x100(%rcx),$xt0   # "$xc0"
795         movdqa          0xd0-0x100(%rcx),$xt1   # "$xc1"
796         movdqa          0xe0-0x100(%rcx),$xt2   # "$xc2"
797         movdqa          0xf0-0x100(%rcx),$xt3   # "$xc3"
798         movdqa          0x100-0x100(%rcx),$xd0
799         movdqa          0x110-0x100(%rcx),$xd1
800         movdqa          0x120-0x100(%rcx),$xd2
801         movdqa          0x130-0x100(%rcx),$xd3
802         paddd           .Lfour(%rip),$xd0       # next SIMD counters
803
804 .Loop_enter4x:
805         movdqa          $xt2,0x20(%rsp)         # SIMD equivalent of "@x[10]"
806         movdqa          $xt3,0x30(%rsp)         # SIMD equivalent of "@x[11]"
807         movdqa          (%r10),$xt3             # .Lrot16(%rip)
808         mov             \$10,%eax
809         movdqa          $xd0,0x100-0x100(%rcx)  # save SIMD counters
810         jmp             .Loop4x
811
812 .align  32
813 .Loop4x:
814 ___
815         foreach (&SSSE3_lane_ROUND(0, 4, 8,12)) { eval; }
816         foreach (&SSSE3_lane_ROUND(0, 5,10,15)) { eval; }
817 $code.=<<___;
818         dec             %eax
819         jnz             .Loop4x
820
821         paddd           0x40(%rsp),$xa0         # accumulate key material
822         paddd           0x50(%rsp),$xa1
823         paddd           0x60(%rsp),$xa2
824         paddd           0x70(%rsp),$xa3
825
826         movdqa          $xa0,$xt2               # "de-interlace" data
827         punpckldq       $xa1,$xa0
828         movdqa          $xa2,$xt3
829         punpckldq       $xa3,$xa2
830         punpckhdq       $xa1,$xt2
831         punpckhdq       $xa3,$xt3
832         movdqa          $xa0,$xa1
833         punpcklqdq      $xa2,$xa0               # "a0"
834         movdqa          $xt2,$xa3
835         punpcklqdq      $xt3,$xt2               # "a2"
836         punpckhqdq      $xa2,$xa1               # "a1"
837         punpckhqdq      $xt3,$xa3               # "a3"
838 ___
839         ($xa2,$xt2)=($xt2,$xa2);
840 $code.=<<___;
841         paddd           0x80-0x100(%rcx),$xb0
842         paddd           0x90-0x100(%rcx),$xb1
843         paddd           0xa0-0x100(%rcx),$xb2
844         paddd           0xb0-0x100(%rcx),$xb3
845
846         movdqa          $xa0,0x00(%rsp)         # offload $xaN
847         movdqa          $xa1,0x10(%rsp)
848         movdqa          0x20(%rsp),$xa0         # "xc2"
849         movdqa          0x30(%rsp),$xa1         # "xc3"
850
851         movdqa          $xb0,$xt2
852         punpckldq       $xb1,$xb0
853         movdqa          $xb2,$xt3
854         punpckldq       $xb3,$xb2
855         punpckhdq       $xb1,$xt2
856         punpckhdq       $xb3,$xt3
857         movdqa          $xb0,$xb1
858         punpcklqdq      $xb2,$xb0               # "b0"
859         movdqa          $xt2,$xb3
860         punpcklqdq      $xt3,$xt2               # "b2"
861         punpckhqdq      $xb2,$xb1               # "b1"
862         punpckhqdq      $xt3,$xb3               # "b3"
863 ___
864         ($xb2,$xt2)=($xt2,$xb2);
865         my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1);
866 $code.=<<___;
867         paddd           0xc0-0x100(%rcx),$xc0
868         paddd           0xd0-0x100(%rcx),$xc1
869         paddd           0xe0-0x100(%rcx),$xc2
870         paddd           0xf0-0x100(%rcx),$xc3
871
872         movdqa          $xa2,0x20(%rsp)         # keep offloading $xaN
873         movdqa          $xa3,0x30(%rsp)
874
875         movdqa          $xc0,$xt2
876         punpckldq       $xc1,$xc0
877         movdqa          $xc2,$xt3
878         punpckldq       $xc3,$xc2
879         punpckhdq       $xc1,$xt2
880         punpckhdq       $xc3,$xt3
881         movdqa          $xc0,$xc1
882         punpcklqdq      $xc2,$xc0               # "c0"
883         movdqa          $xt2,$xc3
884         punpcklqdq      $xt3,$xt2               # "c2"
885         punpckhqdq      $xc2,$xc1               # "c1"
886         punpckhqdq      $xt3,$xc3               # "c3"
887 ___
888         ($xc2,$xt2)=($xt2,$xc2);
889         ($xt0,$xt1)=($xa2,$xa3);                # use $xaN as temporary
890 $code.=<<___;
891         paddd           0x100-0x100(%rcx),$xd0
892         paddd           0x110-0x100(%rcx),$xd1
893         paddd           0x120-0x100(%rcx),$xd2
894         paddd           0x130-0x100(%rcx),$xd3
895
896         movdqa          $xd0,$xt2
897         punpckldq       $xd1,$xd0
898         movdqa          $xd2,$xt3
899         punpckldq       $xd3,$xd2
900         punpckhdq       $xd1,$xt2
901         punpckhdq       $xd3,$xt3
902         movdqa          $xd0,$xd1
903         punpcklqdq      $xd2,$xd0               # "d0"
904         movdqa          $xt2,$xd3
905         punpcklqdq      $xt3,$xt2               # "d2"
906         punpckhqdq      $xd2,$xd1               # "d1"
907         punpckhqdq      $xt3,$xd3               # "d3"
908 ___
909         ($xd2,$xt2)=($xt2,$xd2);
910 $code.=<<___;
911         cmp             \$64*4,$len
912         jb              .Ltail4x
913
914         movdqu          0x00($inp),$xt0         # xor with input
915         movdqu          0x10($inp),$xt1
916         movdqu          0x20($inp),$xt2
917         movdqu          0x30($inp),$xt3
918         pxor            0x00(%rsp),$xt0         # $xaN is offloaded, remember?
919         pxor            $xb0,$xt1
920         pxor            $xc0,$xt2
921         pxor            $xd0,$xt3
922
923          movdqu         $xt0,0x00($out)
924         movdqu          0x40($inp),$xt0
925          movdqu         $xt1,0x10($out)
926         movdqu          0x50($inp),$xt1
927          movdqu         $xt2,0x20($out)
928         movdqu          0x60($inp),$xt2
929          movdqu         $xt3,0x30($out)
930         movdqu          0x70($inp),$xt3
931         lea             0x80($inp),$inp         # size optimization
932         pxor            0x10(%rsp),$xt0
933         pxor            $xb1,$xt1
934         pxor            $xc1,$xt2
935         pxor            $xd1,$xt3
936
937          movdqu         $xt0,0x40($out)
938         movdqu          0x00($inp),$xt0
939          movdqu         $xt1,0x50($out)
940         movdqu          0x10($inp),$xt1
941          movdqu         $xt2,0x60($out)
942         movdqu          0x20($inp),$xt2
943          movdqu         $xt3,0x70($out)
944          lea            0x80($out),$out         # size optimization
945         movdqu          0x30($inp),$xt3
946         pxor            0x20(%rsp),$xt0
947         pxor            $xb2,$xt1
948         pxor            $xc2,$xt2
949         pxor            $xd2,$xt3
950
951          movdqu         $xt0,0x00($out)
952         movdqu          0x40($inp),$xt0
953          movdqu         $xt1,0x10($out)
954         movdqu          0x50($inp),$xt1
955          movdqu         $xt2,0x20($out)
956         movdqu          0x60($inp),$xt2
957          movdqu         $xt3,0x30($out)
958         movdqu          0x70($inp),$xt3
959         lea             0x80($inp),$inp         # inp+=64*4
960         pxor            0x30(%rsp),$xt0
961         pxor            $xb3,$xt1
962         pxor            $xc3,$xt2
963         pxor            $xd3,$xt3
964         movdqu          $xt0,0x40($out)
965         movdqu          $xt1,0x50($out)
966         movdqu          $xt2,0x60($out)
967         movdqu          $xt3,0x70($out)
968         lea             0x80($out),$out         # out+=64*4
969
970         sub             \$64*4,$len
971         jnz             .Loop_outer4x
972
973         jmp             .Ldone4x
974
975 .Ltail4x:
976         cmp             \$192,$len
977         jae             .L192_or_more4x
978         cmp             \$128,$len
979         jae             .L128_or_more4x
980         cmp             \$64,$len
981         jae             .L64_or_more4x
982
983         #movdqa         0x00(%rsp),$xt0         # $xaN is offloaded, remember?
984         xor             %r10,%r10
985         #movdqa         $xt0,0x00(%rsp)
986         movdqa          $xb0,0x10(%rsp)
987         movdqa          $xc0,0x20(%rsp)
988         movdqa          $xd0,0x30(%rsp)
989         jmp             .Loop_tail4x
990
991 .align  32
992 .L64_or_more4x:
993         movdqu          0x00($inp),$xt0         # xor with input
994         movdqu          0x10($inp),$xt1
995         movdqu          0x20($inp),$xt2
996         movdqu          0x30($inp),$xt3
997         pxor            0x00(%rsp),$xt0         # $xaxN is offloaded, remember?
998         pxor            $xb0,$xt1
999         pxor            $xc0,$xt2
1000         pxor            $xd0,$xt3
1001         movdqu          $xt0,0x00($out)
1002         movdqu          $xt1,0x10($out)
1003         movdqu          $xt2,0x20($out)
1004         movdqu          $xt3,0x30($out)
1005         je              .Ldone4x
1006
1007         movdqa          0x10(%rsp),$xt0         # $xaN is offloaded, remember?
1008         lea             0x40($inp),$inp         # inp+=64*1
1009         xor             %r10,%r10
1010         movdqa          $xt0,0x00(%rsp)
1011         movdqa          $xb1,0x10(%rsp)
1012         lea             0x40($out),$out         # out+=64*1
1013         movdqa          $xc1,0x20(%rsp)
1014         sub             \$64,$len               # len-=64*1
1015         movdqa          $xd1,0x30(%rsp)
1016         jmp             .Loop_tail4x
1017
1018 .align  32
1019 .L128_or_more4x:
1020         movdqu          0x00($inp),$xt0         # xor with input
1021         movdqu          0x10($inp),$xt1
1022         movdqu          0x20($inp),$xt2
1023         movdqu          0x30($inp),$xt3
1024         pxor            0x00(%rsp),$xt0         # $xaN is offloaded, remember?
1025         pxor            $xb0,$xt1
1026         pxor            $xc0,$xt2
1027         pxor            $xd0,$xt3
1028
1029          movdqu         $xt0,0x00($out)
1030         movdqu          0x40($inp),$xt0
1031          movdqu         $xt1,0x10($out)
1032         movdqu          0x50($inp),$xt1
1033          movdqu         $xt2,0x20($out)
1034         movdqu          0x60($inp),$xt2
1035          movdqu         $xt3,0x30($out)
1036         movdqu          0x70($inp),$xt3
1037         pxor            0x10(%rsp),$xt0
1038         pxor            $xb1,$xt1
1039         pxor            $xc1,$xt2
1040         pxor            $xd1,$xt3
1041         movdqu          $xt0,0x40($out)
1042         movdqu          $xt1,0x50($out)
1043         movdqu          $xt2,0x60($out)
1044         movdqu          $xt3,0x70($out)
1045         je              .Ldone4x
1046
1047         movdqa          0x20(%rsp),$xt0         # $xaN is offloaded, remember?
1048         lea             0x80($inp),$inp         # inp+=64*2
1049         xor             %r10,%r10
1050         movdqa          $xt0,0x00(%rsp)
1051         movdqa          $xb2,0x10(%rsp)
1052         lea             0x80($out),$out         # out+=64*2
1053         movdqa          $xc2,0x20(%rsp)
1054         sub             \$128,$len              # len-=64*2
1055         movdqa          $xd2,0x30(%rsp)
1056         jmp             .Loop_tail4x
1057
1058 .align  32
1059 .L192_or_more4x:
1060         movdqu          0x00($inp),$xt0         # xor with input
1061         movdqu          0x10($inp),$xt1
1062         movdqu          0x20($inp),$xt2
1063         movdqu          0x30($inp),$xt3
1064         pxor            0x00(%rsp),$xt0         # $xaN is offloaded, remember?
1065         pxor            $xb0,$xt1
1066         pxor            $xc0,$xt2
1067         pxor            $xd0,$xt3
1068
1069          movdqu         $xt0,0x00($out)
1070         movdqu          0x40($inp),$xt0
1071          movdqu         $xt1,0x10($out)
1072         movdqu          0x50($inp),$xt1
1073          movdqu         $xt2,0x20($out)
1074         movdqu          0x60($inp),$xt2
1075          movdqu         $xt3,0x30($out)
1076         movdqu          0x70($inp),$xt3
1077         lea             0x80($inp),$inp         # size optimization
1078         pxor            0x10(%rsp),$xt0
1079         pxor            $xb1,$xt1
1080         pxor            $xc1,$xt2
1081         pxor            $xd1,$xt3
1082
1083          movdqu         $xt0,0x40($out)
1084         movdqu          0x00($inp),$xt0
1085          movdqu         $xt1,0x50($out)
1086         movdqu          0x10($inp),$xt1
1087          movdqu         $xt2,0x60($out)
1088         movdqu          0x20($inp),$xt2
1089          movdqu         $xt3,0x70($out)
1090          lea            0x80($out),$out         # size optimization
1091         movdqu          0x30($inp),$xt3
1092         pxor            0x20(%rsp),$xt0
1093         pxor            $xb2,$xt1
1094         pxor            $xc2,$xt2
1095         pxor            $xd2,$xt3
1096         movdqu          $xt0,0x00($out)
1097         movdqu          $xt1,0x10($out)
1098         movdqu          $xt2,0x20($out)
1099         movdqu          $xt3,0x30($out)
1100         je              .Ldone4x
1101
1102         movdqa          0x30(%rsp),$xt0         # $xaN is offloaded, remember?
1103         lea             0x40($inp),$inp         # inp+=64*3
1104         xor             %r10,%r10
1105         movdqa          $xt0,0x00(%rsp)
1106         movdqa          $xb3,0x10(%rsp)
1107         lea             0x40($out),$out         # out+=64*3
1108         movdqa          $xc3,0x20(%rsp)
1109         sub             \$192,$len              # len-=64*3
1110         movdqa          $xd3,0x30(%rsp)
1111
1112 .Loop_tail4x:
1113         movzb           ($inp,%r10),%eax
1114         movzb           (%rsp,%r10),%ecx
1115         lea             1(%r10),%r10
1116         xor             %ecx,%eax
1117         mov             %al,-1($out,%r10)
1118         dec             $len
1119         jnz             .Loop_tail4x
1120
1121 .Ldone4x:
1122 ___
1123 $code.=<<___    if ($win64);
1124         lea             0x140+0x30(%rsp),%r11
1125         movaps          -0x30(%r11),%xmm6
1126         movaps          -0x20(%r11),%xmm7
1127         movaps          -0x10(%r11),%xmm8
1128         movaps          0x00(%r11),%xmm9
1129         movaps          0x10(%r11),%xmm10
1130         movaps          0x20(%r11),%xmm11
1131         movaps          0x30(%r11),%xmm12
1132         movaps          0x40(%r11),%xmm13
1133         movaps          0x50(%r11),%xmm14
1134         movaps          0x60(%r11),%xmm15
1135 ___
1136 $code.=<<___;
1137         add             \$0x148+$xframe,%rsp
1138         ret
1139 .size   ChaCha20_4x,.-ChaCha20_4x
1140 ___
1141 }
1142
1143 ########################################################################
1144 # XOP code path that handles all lengths.
1145 if ($avx) {
1146 # There is some "anomaly" observed depending on instructions' size or
1147 # alignment. If you look closely at below code you'll notice that
1148 # sometimes argument order varies. The order affects instruction
1149 # encoding by making it larger, and such fiddling gives 5% performance
1150 # improvement. This is on FX-4100...
1151
1152 my ($xb0,$xb1,$xb2,$xb3, $xd0,$xd1,$xd2,$xd3,
1153     $xa0,$xa1,$xa2,$xa3, $xt0,$xt1,$xt2,$xt3)=map("%xmm$_",(0..15));
1154 my  @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
1155          $xt0,$xt1,$xt2,$xt3, $xd0,$xd1,$xd2,$xd3);
1156
1157 sub XOP_lane_ROUND {
1158 my ($a0,$b0,$c0,$d0)=@_;
1159 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
1160 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
1161 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
1162 my @x=map("\"$_\"",@xx);
1163
1164         (
1165         "&vpaddd        (@x[$a0],@x[$a0],@x[$b0])",     # Q1
1166          "&vpaddd       (@x[$a1],@x[$a1],@x[$b1])",     # Q2
1167           "&vpaddd      (@x[$a2],@x[$a2],@x[$b2])",     # Q3
1168            "&vpaddd     (@x[$a3],@x[$a3],@x[$b3])",     # Q4
1169         "&vpxor         (@x[$d0],@x[$a0],@x[$d0])",
1170          "&vpxor        (@x[$d1],@x[$a1],@x[$d1])",
1171           "&vpxor       (@x[$d2],@x[$a2],@x[$d2])",
1172            "&vpxor      (@x[$d3],@x[$a3],@x[$d3])",
1173         "&vprotd        (@x[$d0],@x[$d0],16)",
1174          "&vprotd       (@x[$d1],@x[$d1],16)",
1175           "&vprotd      (@x[$d2],@x[$d2],16)",
1176            "&vprotd     (@x[$d3],@x[$d3],16)",
1177
1178         "&vpaddd        (@x[$c0],@x[$c0],@x[$d0])",
1179          "&vpaddd       (@x[$c1],@x[$c1],@x[$d1])",
1180           "&vpaddd      (@x[$c2],@x[$c2],@x[$d2])",
1181            "&vpaddd     (@x[$c3],@x[$c3],@x[$d3])",
1182         "&vpxor         (@x[$b0],@x[$c0],@x[$b0])",
1183          "&vpxor        (@x[$b1],@x[$c1],@x[$b1])",
1184           "&vpxor       (@x[$b2],@x[$b2],@x[$c2])",     # flip
1185            "&vpxor      (@x[$b3],@x[$b3],@x[$c3])",     # flip
1186         "&vprotd        (@x[$b0],@x[$b0],12)",
1187          "&vprotd       (@x[$b1],@x[$b1],12)",
1188           "&vprotd      (@x[$b2],@x[$b2],12)",
1189            "&vprotd     (@x[$b3],@x[$b3],12)",
1190
1191         "&vpaddd        (@x[$a0],@x[$b0],@x[$a0])",     # flip
1192          "&vpaddd       (@x[$a1],@x[$b1],@x[$a1])",     # flip
1193           "&vpaddd      (@x[$a2],@x[$a2],@x[$b2])",
1194            "&vpaddd     (@x[$a3],@x[$a3],@x[$b3])",
1195         "&vpxor         (@x[$d0],@x[$a0],@x[$d0])",
1196          "&vpxor        (@x[$d1],@x[$a1],@x[$d1])",
1197           "&vpxor       (@x[$d2],@x[$a2],@x[$d2])",
1198            "&vpxor      (@x[$d3],@x[$a3],@x[$d3])",
1199         "&vprotd        (@x[$d0],@x[$d0],8)",
1200          "&vprotd       (@x[$d1],@x[$d1],8)",
1201           "&vprotd      (@x[$d2],@x[$d2],8)",
1202            "&vprotd     (@x[$d3],@x[$d3],8)",
1203
1204         "&vpaddd        (@x[$c0],@x[$c0],@x[$d0])",
1205          "&vpaddd       (@x[$c1],@x[$c1],@x[$d1])",
1206           "&vpaddd      (@x[$c2],@x[$c2],@x[$d2])",
1207            "&vpaddd     (@x[$c3],@x[$c3],@x[$d3])",
1208         "&vpxor         (@x[$b0],@x[$c0],@x[$b0])",
1209          "&vpxor        (@x[$b1],@x[$c1],@x[$b1])",
1210           "&vpxor       (@x[$b2],@x[$b2],@x[$c2])",     # flip
1211            "&vpxor      (@x[$b3],@x[$b3],@x[$c3])",     # flip
1212         "&vprotd        (@x[$b0],@x[$b0],7)",
1213          "&vprotd       (@x[$b1],@x[$b1],7)",
1214           "&vprotd      (@x[$b2],@x[$b2],7)",
1215            "&vprotd     (@x[$b3],@x[$b3],7)"
1216         );
1217 }
1218
1219 my $xframe = $win64 ? 0xa0 : 0;
1220
1221 $code.=<<___;
1222 .type   ChaCha20_4xop,\@function,5
1223 .align  32
1224 ChaCha20_4xop:
1225 .LChaCha20_4xop:
1226         lea             -0x78(%rsp),%r11
1227         sub             \$0x148+$xframe,%rsp
1228 ___
1229         ################ stack layout
1230         # +0x00         SIMD equivalent of @x[8-12]
1231         # ...
1232         # +0x40         constant copy of key[0-2] smashed by lanes
1233         # ...
1234         # +0x100        SIMD counters (with nonce smashed by lanes)
1235         # ...
1236         # +0x140
1237 $code.=<<___    if ($win64);
1238         movaps          %xmm6,-0x30(%r11)
1239         movaps          %xmm7,-0x20(%r11)
1240         movaps          %xmm8,-0x10(%r11)
1241         movaps          %xmm9,0x00(%r11)
1242         movaps          %xmm10,0x10(%r11)
1243         movaps          %xmm11,0x20(%r11)
1244         movaps          %xmm12,0x30(%r11)
1245         movaps          %xmm13,0x40(%r11)
1246         movaps          %xmm14,0x50(%r11)
1247         movaps          %xmm15,0x60(%r11)
1248 ___
1249 $code.=<<___;
1250         vzeroupper
1251
1252         vmovdqa         .Lsigma(%rip),$xa3      # key[0]
1253         vmovdqu         ($key),$xb3             # key[1]
1254         vmovdqu         16($key),$xt3           # key[2]
1255         vmovdqu         ($counter),$xd3         # key[3]
1256         lea             0x100(%rsp),%rcx        # size optimization
1257
1258         vpshufd         \$0x00,$xa3,$xa0        # smash key by lanes...
1259         vpshufd         \$0x55,$xa3,$xa1
1260         vmovdqa         $xa0,0x40(%rsp)         # ... and offload
1261         vpshufd         \$0xaa,$xa3,$xa2
1262         vmovdqa         $xa1,0x50(%rsp)
1263         vpshufd         \$0xff,$xa3,$xa3
1264         vmovdqa         $xa2,0x60(%rsp)
1265         vmovdqa         $xa3,0x70(%rsp)
1266
1267         vpshufd         \$0x00,$xb3,$xb0
1268         vpshufd         \$0x55,$xb3,$xb1
1269         vmovdqa         $xb0,0x80-0x100(%rcx)
1270         vpshufd         \$0xaa,$xb3,$xb2
1271         vmovdqa         $xb1,0x90-0x100(%rcx)
1272         vpshufd         \$0xff,$xb3,$xb3
1273         vmovdqa         $xb2,0xa0-0x100(%rcx)
1274         vmovdqa         $xb3,0xb0-0x100(%rcx)
1275
1276         vpshufd         \$0x00,$xt3,$xt0        # "$xc0"
1277         vpshufd         \$0x55,$xt3,$xt1        # "$xc1"
1278         vmovdqa         $xt0,0xc0-0x100(%rcx)
1279         vpshufd         \$0xaa,$xt3,$xt2        # "$xc2"
1280         vmovdqa         $xt1,0xd0-0x100(%rcx)
1281         vpshufd         \$0xff,$xt3,$xt3        # "$xc3"
1282         vmovdqa         $xt2,0xe0-0x100(%rcx)
1283         vmovdqa         $xt3,0xf0-0x100(%rcx)
1284
1285         vpshufd         \$0x00,$xd3,$xd0
1286         vpshufd         \$0x55,$xd3,$xd1
1287         vpaddd          .Linc(%rip),$xd0,$xd0   # don't save counters yet
1288         vpshufd         \$0xaa,$xd3,$xd2
1289         vmovdqa         $xd1,0x110-0x100(%rcx)
1290         vpshufd         \$0xff,$xd3,$xd3
1291         vmovdqa         $xd2,0x120-0x100(%rcx)
1292         vmovdqa         $xd3,0x130-0x100(%rcx)
1293
1294         jmp             .Loop_enter4xop
1295
1296 .align  32
1297 .Loop_outer4xop:
1298         vmovdqa         0x40(%rsp),$xa0         # re-load smashed key
1299         vmovdqa         0x50(%rsp),$xa1
1300         vmovdqa         0x60(%rsp),$xa2
1301         vmovdqa         0x70(%rsp),$xa3
1302         vmovdqa         0x80-0x100(%rcx),$xb0
1303         vmovdqa         0x90-0x100(%rcx),$xb1
1304         vmovdqa         0xa0-0x100(%rcx),$xb2
1305         vmovdqa         0xb0-0x100(%rcx),$xb3
1306         vmovdqa         0xc0-0x100(%rcx),$xt0   # "$xc0"
1307         vmovdqa         0xd0-0x100(%rcx),$xt1   # "$xc1"
1308         vmovdqa         0xe0-0x100(%rcx),$xt2   # "$xc2"
1309         vmovdqa         0xf0-0x100(%rcx),$xt3   # "$xc3"
1310         vmovdqa         0x100-0x100(%rcx),$xd0
1311         vmovdqa         0x110-0x100(%rcx),$xd1
1312         vmovdqa         0x120-0x100(%rcx),$xd2
1313         vmovdqa         0x130-0x100(%rcx),$xd3
1314         vpaddd          .Lfour(%rip),$xd0,$xd0  # next SIMD counters
1315
1316 .Loop_enter4xop:
1317         mov             \$10,%eax
1318         vmovdqa         $xd0,0x100-0x100(%rcx)  # save SIMD counters
1319         jmp             .Loop4xop
1320
1321 .align  32
1322 .Loop4xop:
1323 ___
1324         foreach (&XOP_lane_ROUND(0, 4, 8,12)) { eval; }
1325         foreach (&XOP_lane_ROUND(0, 5,10,15)) { eval; }
1326 $code.=<<___;
1327         dec             %eax
1328         jnz             .Loop4xop
1329
1330         vpaddd          0x40(%rsp),$xa0,$xa0    # accumulate key material
1331         vpaddd          0x50(%rsp),$xa1,$xa1
1332         vpaddd          0x60(%rsp),$xa2,$xa2
1333         vpaddd          0x70(%rsp),$xa3,$xa3
1334
1335         vmovdqa         $xt2,0x20(%rsp)         # offload $xc2,3
1336         vmovdqa         $xt3,0x30(%rsp)
1337
1338         vpunpckldq      $xa1,$xa0,$xt2          # "de-interlace" data
1339         vpunpckldq      $xa3,$xa2,$xt3
1340         vpunpckhdq      $xa1,$xa0,$xa0
1341         vpunpckhdq      $xa3,$xa2,$xa2
1342         vpunpcklqdq     $xt3,$xt2,$xa1          # "a0"
1343         vpunpckhqdq     $xt3,$xt2,$xt2          # "a1"
1344         vpunpcklqdq     $xa2,$xa0,$xa3          # "a2"
1345         vpunpckhqdq     $xa2,$xa0,$xa0          # "a3"
1346 ___
1347         ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2);
1348 $code.=<<___;
1349         vpaddd          0x80-0x100(%rcx),$xb0,$xb0
1350         vpaddd          0x90-0x100(%rcx),$xb1,$xb1
1351         vpaddd          0xa0-0x100(%rcx),$xb2,$xb2
1352         vpaddd          0xb0-0x100(%rcx),$xb3,$xb3
1353
1354         vmovdqa         $xa0,0x00(%rsp)         # offload $xa0,1
1355         vmovdqa         $xa1,0x10(%rsp)
1356         vmovdqa         0x20(%rsp),$xa0         # "xc2"
1357         vmovdqa         0x30(%rsp),$xa1         # "xc3"
1358
1359         vpunpckldq      $xb1,$xb0,$xt2
1360         vpunpckldq      $xb3,$xb2,$xt3
1361         vpunpckhdq      $xb1,$xb0,$xb0
1362         vpunpckhdq      $xb3,$xb2,$xb2
1363         vpunpcklqdq     $xt3,$xt2,$xb1          # "b0"
1364         vpunpckhqdq     $xt3,$xt2,$xt2          # "b1"
1365         vpunpcklqdq     $xb2,$xb0,$xb3          # "b2"
1366         vpunpckhqdq     $xb2,$xb0,$xb0          # "b3"
1367 ___
1368         ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2);
1369         my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1);
1370 $code.=<<___;
1371         vpaddd          0xc0-0x100(%rcx),$xc0,$xc0
1372         vpaddd          0xd0-0x100(%rcx),$xc1,$xc1
1373         vpaddd          0xe0-0x100(%rcx),$xc2,$xc2
1374         vpaddd          0xf0-0x100(%rcx),$xc3,$xc3
1375
1376         vpunpckldq      $xc1,$xc0,$xt2
1377         vpunpckldq      $xc3,$xc2,$xt3
1378         vpunpckhdq      $xc1,$xc0,$xc0
1379         vpunpckhdq      $xc3,$xc2,$xc2
1380         vpunpcklqdq     $xt3,$xt2,$xc1          # "c0"
1381         vpunpckhqdq     $xt3,$xt2,$xt2          # "c1"
1382         vpunpcklqdq     $xc2,$xc0,$xc3          # "c2"
1383         vpunpckhqdq     $xc2,$xc0,$xc0          # "c3"
1384 ___
1385         ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2);
1386 $code.=<<___;
1387         vpaddd          0x100-0x100(%rcx),$xd0,$xd0
1388         vpaddd          0x110-0x100(%rcx),$xd1,$xd1
1389         vpaddd          0x120-0x100(%rcx),$xd2,$xd2
1390         vpaddd          0x130-0x100(%rcx),$xd3,$xd3
1391
1392         vpunpckldq      $xd1,$xd0,$xt2
1393         vpunpckldq      $xd3,$xd2,$xt3
1394         vpunpckhdq      $xd1,$xd0,$xd0
1395         vpunpckhdq      $xd3,$xd2,$xd2
1396         vpunpcklqdq     $xt3,$xt2,$xd1          # "d0"
1397         vpunpckhqdq     $xt3,$xt2,$xt2          # "d1"
1398         vpunpcklqdq     $xd2,$xd0,$xd3          # "d2"
1399         vpunpckhqdq     $xd2,$xd0,$xd0          # "d3"
1400 ___
1401         ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2);
1402         ($xa0,$xa1)=($xt2,$xt3);
1403 $code.=<<___;
1404         vmovdqa         0x00(%rsp),$xa0         # restore $xa0,1
1405         vmovdqa         0x10(%rsp),$xa1
1406
1407         cmp             \$64*4,$len
1408         jb              .Ltail4xop
1409
1410         vpxor           0x00($inp),$xa0,$xa0    # xor with input
1411         vpxor           0x10($inp),$xb0,$xb0
1412         vpxor           0x20($inp),$xc0,$xc0
1413         vpxor           0x30($inp),$xd0,$xd0
1414         vpxor           0x40($inp),$xa1,$xa1
1415         vpxor           0x50($inp),$xb1,$xb1
1416         vpxor           0x60($inp),$xc1,$xc1
1417         vpxor           0x70($inp),$xd1,$xd1
1418         lea             0x80($inp),$inp         # size optimization
1419         vpxor           0x00($inp),$xa2,$xa2
1420         vpxor           0x10($inp),$xb2,$xb2
1421         vpxor           0x20($inp),$xc2,$xc2
1422         vpxor           0x30($inp),$xd2,$xd2
1423         vpxor           0x40($inp),$xa3,$xa3
1424         vpxor           0x50($inp),$xb3,$xb3
1425         vpxor           0x60($inp),$xc3,$xc3
1426         vpxor           0x70($inp),$xd3,$xd3
1427         lea             0x80($inp),$inp         # inp+=64*4
1428
1429         vmovdqu         $xa0,0x00($out)
1430         vmovdqu         $xb0,0x10($out)
1431         vmovdqu         $xc0,0x20($out)
1432         vmovdqu         $xd0,0x30($out)
1433         vmovdqu         $xa1,0x40($out)
1434         vmovdqu         $xb1,0x50($out)
1435         vmovdqu         $xc1,0x60($out)
1436         vmovdqu         $xd1,0x70($out)
1437         lea             0x80($out),$out         # size optimization
1438         vmovdqu         $xa2,0x00($out)
1439         vmovdqu         $xb2,0x10($out)
1440         vmovdqu         $xc2,0x20($out)
1441         vmovdqu         $xd2,0x30($out)
1442         vmovdqu         $xa3,0x40($out)
1443         vmovdqu         $xb3,0x50($out)
1444         vmovdqu         $xc3,0x60($out)
1445         vmovdqu         $xd3,0x70($out)
1446         lea             0x80($out),$out         # out+=64*4
1447
1448         sub             \$64*4,$len
1449         jnz             .Loop_outer4xop
1450
1451         jmp             .Ldone4xop
1452
1453 .align  32
1454 .Ltail4xop:
1455         cmp             \$192,$len
1456         jae             .L192_or_more4xop
1457         cmp             \$128,$len
1458         jae             .L128_or_more4xop
1459         cmp             \$64,$len
1460         jae             .L64_or_more4xop
1461
1462         xor             %r10,%r10
1463         vmovdqa         $xa0,0x00(%rsp)
1464         vmovdqa         $xb0,0x10(%rsp)
1465         vmovdqa         $xc0,0x20(%rsp)
1466         vmovdqa         $xd0,0x30(%rsp)
1467         jmp             .Loop_tail4xop
1468
1469 .align  32
1470 .L64_or_more4xop:
1471         vpxor           0x00($inp),$xa0,$xa0    # xor with input
1472         vpxor           0x10($inp),$xb0,$xb0
1473         vpxor           0x20($inp),$xc0,$xc0
1474         vpxor           0x30($inp),$xd0,$xd0
1475         vmovdqu         $xa0,0x00($out)
1476         vmovdqu         $xb0,0x10($out)
1477         vmovdqu         $xc0,0x20($out)
1478         vmovdqu         $xd0,0x30($out)
1479         je              .Ldone4xop
1480
1481         lea             0x40($inp),$inp         # inp+=64*1
1482         vmovdqa         $xa1,0x00(%rsp)
1483         xor             %r10,%r10
1484         vmovdqa         $xb1,0x10(%rsp)
1485         lea             0x40($out),$out         # out+=64*1
1486         vmovdqa         $xc1,0x20(%rsp)
1487         sub             \$64,$len               # len-=64*1
1488         vmovdqa         $xd1,0x30(%rsp)
1489         jmp             .Loop_tail4xop
1490
1491 .align  32
1492 .L128_or_more4xop:
1493         vpxor           0x00($inp),$xa0,$xa0    # xor with input
1494         vpxor           0x10($inp),$xb0,$xb0
1495         vpxor           0x20($inp),$xc0,$xc0
1496         vpxor           0x30($inp),$xd0,$xd0
1497         vpxor           0x40($inp),$xa1,$xa1
1498         vpxor           0x50($inp),$xb1,$xb1
1499         vpxor           0x60($inp),$xc1,$xc1
1500         vpxor           0x70($inp),$xd1,$xd1
1501
1502         vmovdqu         $xa0,0x00($out)
1503         vmovdqu         $xb0,0x10($out)
1504         vmovdqu         $xc0,0x20($out)
1505         vmovdqu         $xd0,0x30($out)
1506         vmovdqu         $xa1,0x40($out)
1507         vmovdqu         $xb1,0x50($out)
1508         vmovdqu         $xc1,0x60($out)
1509         vmovdqu         $xd1,0x70($out)
1510         je              .Ldone4xop
1511
1512         lea             0x80($inp),$inp         # inp+=64*2
1513         vmovdqa         $xa2,0x00(%rsp)
1514         xor             %r10,%r10
1515         vmovdqa         $xb2,0x10(%rsp)
1516         lea             0x80($out),$out         # out+=64*2
1517         vmovdqa         $xc2,0x20(%rsp)
1518         sub             \$128,$len              # len-=64*2
1519         vmovdqa         $xd2,0x30(%rsp)
1520         jmp             .Loop_tail4xop
1521
1522 .align  32
1523 .L192_or_more4xop:
1524         vpxor           0x00($inp),$xa0,$xa0    # xor with input
1525         vpxor           0x10($inp),$xb0,$xb0
1526         vpxor           0x20($inp),$xc0,$xc0
1527         vpxor           0x30($inp),$xd0,$xd0
1528         vpxor           0x40($inp),$xa1,$xa1
1529         vpxor           0x50($inp),$xb1,$xb1
1530         vpxor           0x60($inp),$xc1,$xc1
1531         vpxor           0x70($inp),$xd1,$xd1
1532         lea             0x80($inp),$inp         # size optimization
1533         vpxor           0x00($inp),$xa2,$xa2
1534         vpxor           0x10($inp),$xb2,$xb2
1535         vpxor           0x20($inp),$xc2,$xc2
1536         vpxor           0x30($inp),$xd2,$xd2
1537
1538         vmovdqu         $xa0,0x00($out)
1539         vmovdqu         $xb0,0x10($out)
1540         vmovdqu         $xc0,0x20($out)
1541         vmovdqu         $xd0,0x30($out)
1542         vmovdqu         $xa1,0x40($out)
1543         vmovdqu         $xb1,0x50($out)
1544         vmovdqu         $xc1,0x60($out)
1545         vmovdqu         $xd1,0x70($out)
1546         lea             0x80($out),$out         # size optimization
1547         vmovdqu         $xa2,0x00($out)
1548         vmovdqu         $xb2,0x10($out)
1549         vmovdqu         $xc2,0x20($out)
1550         vmovdqu         $xd2,0x30($out)
1551         je              .Ldone4xop
1552
1553         lea             0x40($inp),$inp         # inp+=64*3
1554         vmovdqa         $xa3,0x00(%rsp)
1555         xor             %r10,%r10
1556         vmovdqa         $xb3,0x10(%rsp)
1557         lea             0x40($out),$out         # out+=64*3
1558         vmovdqa         $xc3,0x20(%rsp)
1559         sub             \$192,$len              # len-=64*3
1560         vmovdqa         $xd3,0x30(%rsp)
1561
1562 .Loop_tail4xop:
1563         movzb           ($inp,%r10),%eax
1564         movzb           (%rsp,%r10),%ecx
1565         lea             1(%r10),%r10
1566         xor             %ecx,%eax
1567         mov             %al,-1($out,%r10)
1568         dec             $len
1569         jnz             .Loop_tail4xop
1570
1571 .Ldone4xop:
1572         vzeroupper
1573 ___
1574 $code.=<<___    if ($win64);
1575         lea             0x140+0x30(%rsp),%r11
1576         movaps          -0x30(%r11),%xmm6
1577         movaps          -0x20(%r11),%xmm7
1578         movaps          -0x10(%r11),%xmm8
1579         movaps          0x00(%r11),%xmm9
1580         movaps          0x10(%r11),%xmm10
1581         movaps          0x20(%r11),%xmm11
1582         movaps          0x30(%r11),%xmm12
1583         movaps          0x40(%r11),%xmm13
1584         movaps          0x50(%r11),%xmm14
1585         movaps          0x60(%r11),%xmm15
1586 ___
1587 $code.=<<___;
1588         add             \$0x148+$xframe,%rsp
1589         ret
1590 .size   ChaCha20_4xop,.-ChaCha20_4xop
1591 ___
1592 }
1593
1594 ########################################################################
1595 # AVX2 code path
1596 if ($avx>1) {
1597 my ($xb0,$xb1,$xb2,$xb3, $xd0,$xd1,$xd2,$xd3,
1598     $xa0,$xa1,$xa2,$xa3, $xt0,$xt1,$xt2,$xt3)=map("%ymm$_",(0..15));
1599 my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
1600         "%nox","%nox","%nox","%nox", $xd0,$xd1,$xd2,$xd3);
1601
1602 sub AVX2_lane_ROUND {
1603 my ($a0,$b0,$c0,$d0)=@_;
1604 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
1605 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
1606 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
1607 my ($xc,$xc_,$t0,$t1)=map("\"$_\"",$xt0,$xt1,$xt2,$xt3);
1608 my @x=map("\"$_\"",@xx);
1609
1610         # Consider order in which variables are addressed by their
1611         # index:
1612         #
1613         #       a   b   c   d
1614         #
1615         #       0   4   8  12 < even round
1616         #       1   5   9  13
1617         #       2   6  10  14
1618         #       3   7  11  15
1619         #       0   5  10  15 < odd round
1620         #       1   6  11  12
1621         #       2   7   8  13
1622         #       3   4   9  14
1623         #
1624         # 'a', 'b' and 'd's are permanently allocated in registers,
1625         # @x[0..7,12..15], while 'c's are maintained in memory. If
1626         # you observe 'c' column, you'll notice that pair of 'c's is
1627         # invariant between rounds. This means that we have to reload
1628         # them once per round, in the middle. This is why you'll see
1629         # bunch of 'c' stores and loads in the middle, but none in
1630         # the beginning or end.
1631
1632         (
1633         "&vpaddd        (@x[$a0],@x[$a0],@x[$b0])",     # Q1
1634         "&vpxor         (@x[$d0],@x[$a0],@x[$d0])",
1635         "&vpshufb       (@x[$d0],@x[$d0],$t1)",
1636          "&vpaddd       (@x[$a1],@x[$a1],@x[$b1])",     # Q2
1637          "&vpxor        (@x[$d1],@x[$a1],@x[$d1])",
1638          "&vpshufb      (@x[$d1],@x[$d1],$t1)",
1639
1640         "&vpaddd        ($xc,$xc,@x[$d0])",
1641         "&vpxor         (@x[$b0],$xc,@x[$b0])",
1642         "&vpslld        ($t0,@x[$b0],12)",
1643         "&vpsrld        (@x[$b0],@x[$b0],20)",
1644         "&vpor          (@x[$b0],$t0,@x[$b0])",
1645         "&vbroadcasti128($t0,'(%r11)')",                # .Lrot24(%rip)
1646          "&vpaddd       ($xc_,$xc_,@x[$d1])",
1647          "&vpxor        (@x[$b1],$xc_,@x[$b1])",
1648          "&vpslld       ($t1,@x[$b1],12)",
1649          "&vpsrld       (@x[$b1],@x[$b1],20)",
1650          "&vpor         (@x[$b1],$t1,@x[$b1])",
1651
1652         "&vpaddd        (@x[$a0],@x[$a0],@x[$b0])",
1653         "&vpxor         (@x[$d0],@x[$a0],@x[$d0])",
1654         "&vpshufb       (@x[$d0],@x[$d0],$t0)",
1655          "&vpaddd       (@x[$a1],@x[$a1],@x[$b1])",
1656          "&vpxor        (@x[$d1],@x[$a1],@x[$d1])",
1657          "&vpshufb      (@x[$d1],@x[$d1],$t0)",
1658
1659         "&vpaddd        ($xc,$xc,@x[$d0])",
1660         "&vpxor         (@x[$b0],$xc,@x[$b0])",
1661         "&vpslld        ($t1,@x[$b0],7)",
1662         "&vpsrld        (@x[$b0],@x[$b0],25)",
1663         "&vpor          (@x[$b0],$t1,@x[$b0])",
1664         "&vbroadcasti128($t1,'(%r10)')",                # .Lrot16(%rip)
1665          "&vpaddd       ($xc_,$xc_,@x[$d1])",
1666          "&vpxor        (@x[$b1],$xc_,@x[$b1])",
1667          "&vpslld       ($t0,@x[$b1],7)",
1668          "&vpsrld       (@x[$b1],@x[$b1],25)",
1669          "&vpor         (@x[$b1],$t0,@x[$b1])",
1670
1671         "&vmovdqa       (\"`32*($c0-8)`(%rsp)\",$xc)",  # reload pair of 'c's
1672          "&vmovdqa      (\"`32*($c1-8)`(%rsp)\",$xc_)",
1673         "&vmovdqa       ($xc,\"`32*($c2-8)`(%rsp)\")",
1674          "&vmovdqa      ($xc_,\"`32*($c3-8)`(%rsp)\")",
1675
1676         "&vpaddd        (@x[$a2],@x[$a2],@x[$b2])",     # Q3
1677         "&vpxor         (@x[$d2],@x[$a2],@x[$d2])",
1678         "&vpshufb       (@x[$d2],@x[$d2],$t1)",
1679          "&vpaddd       (@x[$a3],@x[$a3],@x[$b3])",     # Q4
1680          "&vpxor        (@x[$d3],@x[$a3],@x[$d3])",
1681          "&vpshufb      (@x[$d3],@x[$d3],$t1)",
1682
1683         "&vpaddd        ($xc,$xc,@x[$d2])",
1684         "&vpxor         (@x[$b2],$xc,@x[$b2])",
1685         "&vpslld        ($t0,@x[$b2],12)",
1686         "&vpsrld        (@x[$b2],@x[$b2],20)",
1687         "&vpor          (@x[$b2],$t0,@x[$b2])",
1688         "&vbroadcasti128($t0,'(%r11)')",                # .Lrot24(%rip)
1689          "&vpaddd       ($xc_,$xc_,@x[$d3])",
1690          "&vpxor        (@x[$b3],$xc_,@x[$b3])",
1691          "&vpslld       ($t1,@x[$b3],12)",
1692          "&vpsrld       (@x[$b3],@x[$b3],20)",
1693          "&vpor         (@x[$b3],$t1,@x[$b3])",
1694
1695         "&vpaddd        (@x[$a2],@x[$a2],@x[$b2])",
1696         "&vpxor         (@x[$d2],@x[$a2],@x[$d2])",
1697         "&vpshufb       (@x[$d2],@x[$d2],$t0)",
1698          "&vpaddd       (@x[$a3],@x[$a3],@x[$b3])",
1699          "&vpxor        (@x[$d3],@x[$a3],@x[$d3])",
1700          "&vpshufb      (@x[$d3],@x[$d3],$t0)",
1701
1702         "&vpaddd        ($xc,$xc,@x[$d2])",
1703         "&vpxor         (@x[$b2],$xc,@x[$b2])",
1704         "&vpslld        ($t1,@x[$b2],7)",
1705         "&vpsrld        (@x[$b2],@x[$b2],25)",
1706         "&vpor          (@x[$b2],$t1,@x[$b2])",
1707         "&vbroadcasti128($t1,'(%r10)')",                # .Lrot16(%rip)
1708          "&vpaddd       ($xc_,$xc_,@x[$d3])",
1709          "&vpxor        (@x[$b3],$xc_,@x[$b3])",
1710          "&vpslld       ($t0,@x[$b3],7)",
1711          "&vpsrld       (@x[$b3],@x[$b3],25)",
1712          "&vpor         (@x[$b3],$t0,@x[$b3])"
1713         );
1714 }
1715
1716 my $xframe = $win64 ? 0xb0 : 8;
1717
1718 $code.=<<___;
1719 .type   ChaCha20_8x,\@function,5
1720 .align  32
1721 ChaCha20_8x:
1722 .LChaCha20_8x:
1723         mov             %rsp,%r10
1724         sub             \$0x280+$xframe,%rsp
1725         and             \$-32,%rsp
1726 ___
1727 $code.=<<___    if ($win64);
1728         lea             0x290+0x30(%rsp),%r11
1729         movaps          %xmm6,-0x30(%r11)
1730         movaps          %xmm7,-0x20(%r11)
1731         movaps          %xmm8,-0x10(%r11)
1732         movaps          %xmm9,0x00(%r11)
1733         movaps          %xmm10,0x10(%r11)
1734         movaps          %xmm11,0x20(%r11)
1735         movaps          %xmm12,0x30(%r11)
1736         movaps          %xmm13,0x40(%r11)
1737         movaps          %xmm14,0x50(%r11)
1738         movaps          %xmm15,0x60(%r11)
1739 ___
1740 $code.=<<___;
1741         vzeroupper
1742         mov             %r10,0x280(%rsp)
1743
1744         ################ stack layout
1745         # +0x00         SIMD equivalent of @x[8-12]
1746         # ...
1747         # +0x80         constant copy of key[0-2] smashed by lanes
1748         # ...
1749         # +0x200        SIMD counters (with nonce smashed by lanes)
1750         # ...
1751         # +0x280        saved %rsp
1752
1753         vbroadcasti128  .Lsigma(%rip),$xa3      # key[0]
1754         vbroadcasti128  ($key),$xb3             # key[1]
1755         vbroadcasti128  16($key),$xt3           # key[2]
1756         vbroadcasti128  ($counter),$xd3         # key[3]
1757         lea             0x100(%rsp),%rcx        # size optimization
1758         lea             0x200(%rsp),%rax        # size optimization
1759         lea             .Lrot16(%rip),%r10
1760         lea             .Lrot24(%rip),%r11
1761
1762         vpshufd         \$0x00,$xa3,$xa0        # smash key by lanes...
1763         vpshufd         \$0x55,$xa3,$xa1
1764         vmovdqa         $xa0,0x80-0x100(%rcx)   # ... and offload
1765         vpshufd         \$0xaa,$xa3,$xa2
1766         vmovdqa         $xa1,0xa0-0x100(%rcx)
1767         vpshufd         \$0xff,$xa3,$xa3
1768         vmovdqa         $xa2,0xc0-0x100(%rcx)
1769         vmovdqa         $xa3,0xe0-0x100(%rcx)
1770
1771         vpshufd         \$0x00,$xb3,$xb0
1772         vpshufd         \$0x55,$xb3,$xb1
1773         vmovdqa         $xb0,0x100-0x100(%rcx)
1774         vpshufd         \$0xaa,$xb3,$xb2
1775         vmovdqa         $xb1,0x120-0x100(%rcx)
1776         vpshufd         \$0xff,$xb3,$xb3
1777         vmovdqa         $xb2,0x140-0x100(%rcx)
1778         vmovdqa         $xb3,0x160-0x100(%rcx)
1779
1780         vpshufd         \$0x00,$xt3,$xt0        # "xc0"
1781         vpshufd         \$0x55,$xt3,$xt1        # "xc1"
1782         vmovdqa         $xt0,0x180-0x200(%rax)
1783         vpshufd         \$0xaa,$xt3,$xt2        # "xc2"
1784         vmovdqa         $xt1,0x1a0-0x200(%rax)
1785         vpshufd         \$0xff,$xt3,$xt3        # "xc3"
1786         vmovdqa         $xt2,0x1c0-0x200(%rax)
1787         vmovdqa         $xt3,0x1e0-0x200(%rax)
1788
1789         vpshufd         \$0x00,$xd3,$xd0
1790         vpshufd         \$0x55,$xd3,$xd1
1791         vpaddd          .Lincy(%rip),$xd0,$xd0  # don't save counters yet
1792         vpshufd         \$0xaa,$xd3,$xd2
1793         vmovdqa         $xd1,0x220-0x200(%rax)
1794         vpshufd         \$0xff,$xd3,$xd3
1795         vmovdqa         $xd2,0x240-0x200(%rax)
1796         vmovdqa         $xd3,0x260-0x200(%rax)
1797
1798         jmp             .Loop_enter8x
1799
1800 .align  32
1801 .Loop_outer8x:
1802         vmovdqa         0x80-0x100(%rcx),$xa0   # re-load smashed key
1803         vmovdqa         0xa0-0x100(%rcx),$xa1
1804         vmovdqa         0xc0-0x100(%rcx),$xa2
1805         vmovdqa         0xe0-0x100(%rcx),$xa3
1806         vmovdqa         0x100-0x100(%rcx),$xb0
1807         vmovdqa         0x120-0x100(%rcx),$xb1
1808         vmovdqa         0x140-0x100(%rcx),$xb2
1809         vmovdqa         0x160-0x100(%rcx),$xb3
1810         vmovdqa         0x180-0x200(%rax),$xt0  # "xc0"
1811         vmovdqa         0x1a0-0x200(%rax),$xt1  # "xc1"
1812         vmovdqa         0x1c0-0x200(%rax),$xt2  # "xc2"
1813         vmovdqa         0x1e0-0x200(%rax),$xt3  # "xc3"
1814         vmovdqa         0x200-0x200(%rax),$xd0
1815         vmovdqa         0x220-0x200(%rax),$xd1
1816         vmovdqa         0x240-0x200(%rax),$xd2
1817         vmovdqa         0x260-0x200(%rax),$xd3
1818         vpaddd          .Leight(%rip),$xd0,$xd0 # next SIMD counters
1819
1820 .Loop_enter8x:
1821         vmovdqa         $xt2,0x40(%rsp)         # SIMD equivalent of "@x[10]"
1822         vmovdqa         $xt3,0x60(%rsp)         # SIMD equivalent of "@x[11]"
1823         vbroadcasti128  (%r10),$xt3
1824         vmovdqa         $xd0,0x200-0x200(%rax)  # save SIMD counters
1825         mov             \$10,%eax
1826         jmp             .Loop8x
1827
1828 .align  32
1829 .Loop8x:
1830 ___
1831         foreach (&AVX2_lane_ROUND(0, 4, 8,12)) { eval; }
1832         foreach (&AVX2_lane_ROUND(0, 5,10,15)) { eval; }
1833 $code.=<<___;
1834         dec             %eax
1835         jnz             .Loop8x
1836
1837         lea             0x200(%rsp),%rax        # size optimization
1838         vpaddd          0x80-0x100(%rcx),$xa0,$xa0      # accumulate key
1839         vpaddd          0xa0-0x100(%rcx),$xa1,$xa1
1840         vpaddd          0xc0-0x100(%rcx),$xa2,$xa2
1841         vpaddd          0xe0-0x100(%rcx),$xa3,$xa3
1842
1843         vpunpckldq      $xa1,$xa0,$xt2          # "de-interlace" data
1844         vpunpckldq      $xa3,$xa2,$xt3
1845         vpunpckhdq      $xa1,$xa0,$xa0
1846         vpunpckhdq      $xa3,$xa2,$xa2
1847         vpunpcklqdq     $xt3,$xt2,$xa1          # "a0"
1848         vpunpckhqdq     $xt3,$xt2,$xt2          # "a1"
1849         vpunpcklqdq     $xa2,$xa0,$xa3          # "a2"
1850         vpunpckhqdq     $xa2,$xa0,$xa0          # "a3"
1851 ___
1852         ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2);
1853 $code.=<<___;
1854         vpaddd          0x100-0x100(%rcx),$xb0,$xb0
1855         vpaddd          0x120-0x100(%rcx),$xb1,$xb1
1856         vpaddd          0x140-0x100(%rcx),$xb2,$xb2
1857         vpaddd          0x160-0x100(%rcx),$xb3,$xb3
1858
1859         vpunpckldq      $xb1,$xb0,$xt2
1860         vpunpckldq      $xb3,$xb2,$xt3
1861         vpunpckhdq      $xb1,$xb0,$xb0
1862         vpunpckhdq      $xb3,$xb2,$xb2
1863         vpunpcklqdq     $xt3,$xt2,$xb1          # "b0"
1864         vpunpckhqdq     $xt3,$xt2,$xt2          # "b1"
1865         vpunpcklqdq     $xb2,$xb0,$xb3          # "b2"
1866         vpunpckhqdq     $xb2,$xb0,$xb0          # "b3"
1867 ___
1868         ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2);
1869 $code.=<<___;
1870         vperm2i128      \$0x20,$xb0,$xa0,$xt3   # "de-interlace" further
1871         vperm2i128      \$0x31,$xb0,$xa0,$xb0
1872         vperm2i128      \$0x20,$xb1,$xa1,$xa0
1873         vperm2i128      \$0x31,$xb1,$xa1,$xb1
1874         vperm2i128      \$0x20,$xb2,$xa2,$xa1
1875         vperm2i128      \$0x31,$xb2,$xa2,$xb2
1876         vperm2i128      \$0x20,$xb3,$xa3,$xa2
1877         vperm2i128      \$0x31,$xb3,$xa3,$xb3
1878 ___
1879         ($xa0,$xa1,$xa2,$xa3,$xt3)=($xt3,$xa0,$xa1,$xa2,$xa3);
1880         my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1);
1881 $code.=<<___;
1882         vmovdqa         $xa0,0x00(%rsp)         # offload $xaN
1883         vmovdqa         $xa1,0x20(%rsp)
1884         vmovdqa         0x40(%rsp),$xc2         # $xa0
1885         vmovdqa         0x60(%rsp),$xc3         # $xa1
1886
1887         vpaddd          0x180-0x200(%rax),$xc0,$xc0
1888         vpaddd          0x1a0-0x200(%rax),$xc1,$xc1
1889         vpaddd          0x1c0-0x200(%rax),$xc2,$xc2
1890         vpaddd          0x1e0-0x200(%rax),$xc3,$xc3
1891
1892         vpunpckldq      $xc1,$xc0,$xt2
1893         vpunpckldq      $xc3,$xc2,$xt3
1894         vpunpckhdq      $xc1,$xc0,$xc0
1895         vpunpckhdq      $xc3,$xc2,$xc2
1896         vpunpcklqdq     $xt3,$xt2,$xc1          # "c0"
1897         vpunpckhqdq     $xt3,$xt2,$xt2          # "c1"
1898         vpunpcklqdq     $xc2,$xc0,$xc3          # "c2"
1899         vpunpckhqdq     $xc2,$xc0,$xc0          # "c3"
1900 ___
1901         ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2);
1902 $code.=<<___;
1903         vpaddd          0x200-0x200(%rax),$xd0,$xd0
1904         vpaddd          0x220-0x200(%rax),$xd1,$xd1
1905         vpaddd          0x240-0x200(%rax),$xd2,$xd2
1906         vpaddd          0x260-0x200(%rax),$xd3,$xd3
1907
1908         vpunpckldq      $xd1,$xd0,$xt2
1909         vpunpckldq      $xd3,$xd2,$xt3
1910         vpunpckhdq      $xd1,$xd0,$xd0
1911         vpunpckhdq      $xd3,$xd2,$xd2
1912         vpunpcklqdq     $xt3,$xt2,$xd1          # "d0"
1913         vpunpckhqdq     $xt3,$xt2,$xt2          # "d1"
1914         vpunpcklqdq     $xd2,$xd0,$xd3          # "d2"
1915         vpunpckhqdq     $xd2,$xd0,$xd0          # "d3"
1916 ___
1917         ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2);
1918 $code.=<<___;
1919         vperm2i128      \$0x20,$xd0,$xc0,$xt3   # "de-interlace" further
1920         vperm2i128      \$0x31,$xd0,$xc0,$xd0
1921         vperm2i128      \$0x20,$xd1,$xc1,$xc0
1922         vperm2i128      \$0x31,$xd1,$xc1,$xd1
1923         vperm2i128      \$0x20,$xd2,$xc2,$xc1
1924         vperm2i128      \$0x31,$xd2,$xc2,$xd2
1925         vperm2i128      \$0x20,$xd3,$xc3,$xc2
1926         vperm2i128      \$0x31,$xd3,$xc3,$xd3
1927 ___
1928         ($xc0,$xc1,$xc2,$xc3,$xt3)=($xt3,$xc0,$xc1,$xc2,$xc3);
1929         ($xb0,$xb1,$xb2,$xb3,$xc0,$xc1,$xc2,$xc3)=
1930         ($xc0,$xc1,$xc2,$xc3,$xb0,$xb1,$xb2,$xb3);
1931         ($xa0,$xa1)=($xt2,$xt3);
1932 $code.=<<___;
1933         vmovdqa         0x00(%rsp),$xa0         # $xaN was offloaded, remember?
1934         vmovdqa         0x20(%rsp),$xa1
1935
1936         cmp             \$64*8,$len
1937         jb              .Ltail8x
1938
1939         vpxor           0x00($inp),$xa0,$xa0    # xor with input
1940         vpxor           0x20($inp),$xb0,$xb0
1941         vpxor           0x40($inp),$xc0,$xc0
1942         vpxor           0x60($inp),$xd0,$xd0
1943         lea             0x80($inp),$inp         # size optimization
1944         vmovdqu         $xa0,0x00($out)
1945         vmovdqu         $xb0,0x20($out)
1946         vmovdqu         $xc0,0x40($out)
1947         vmovdqu         $xd0,0x60($out)
1948         lea             0x80($out),$out         # size optimization
1949
1950         vpxor           0x00($inp),$xa1,$xa1
1951         vpxor           0x20($inp),$xb1,$xb1
1952         vpxor           0x40($inp),$xc1,$xc1
1953         vpxor           0x60($inp),$xd1,$xd1
1954         lea             0x80($inp),$inp         # size optimization
1955         vmovdqu         $xa1,0x00($out)
1956         vmovdqu         $xb1,0x20($out)
1957         vmovdqu         $xc1,0x40($out)
1958         vmovdqu         $xd1,0x60($out)
1959         lea             0x80($out),$out         # size optimization
1960
1961         vpxor           0x00($inp),$xa2,$xa2
1962         vpxor           0x20($inp),$xb2,$xb2
1963         vpxor           0x40($inp),$xc2,$xc2
1964         vpxor           0x60($inp),$xd2,$xd2
1965         lea             0x80($inp),$inp         # size optimization
1966         vmovdqu         $xa2,0x00($out)
1967         vmovdqu         $xb2,0x20($out)
1968         vmovdqu         $xc2,0x40($out)
1969         vmovdqu         $xd2,0x60($out)
1970         lea             0x80($out),$out         # size optimization
1971
1972         vpxor           0x00($inp),$xa3,$xa3
1973         vpxor           0x20($inp),$xb3,$xb3
1974         vpxor           0x40($inp),$xc3,$xc3
1975         vpxor           0x60($inp),$xd3,$xd3
1976         lea             0x80($inp),$inp         # size optimization
1977         vmovdqu         $xa3,0x00($out)
1978         vmovdqu         $xb3,0x20($out)
1979         vmovdqu         $xc3,0x40($out)
1980         vmovdqu         $xd3,0x60($out)
1981         lea             0x80($out),$out         # size optimization
1982
1983         sub             \$64*8,$len
1984         jnz             .Loop_outer8x
1985
1986         jmp             .Ldone8x
1987
1988 .Ltail8x:
1989         cmp             \$448,$len
1990         jae             .L448_or_more8x
1991         cmp             \$384,$len
1992         jae             .L384_or_more8x
1993         cmp             \$320,$len
1994         jae             .L320_or_more8x
1995         cmp             \$256,$len
1996         jae             .L256_or_more8x
1997         cmp             \$192,$len
1998         jae             .L192_or_more8x
1999         cmp             \$128,$len
2000         jae             .L128_or_more8x
2001         cmp             \$64,$len
2002         jae             .L64_or_more8x
2003
2004         xor             %r10,%r10
2005         vmovdqa         $xa0,0x00(%rsp)
2006         vmovdqa         $xb0,0x20(%rsp)
2007         jmp             .Loop_tail8x
2008
2009 .align  32
2010 .L64_or_more8x:
2011         vpxor           0x00($inp),$xa0,$xa0    # xor with input
2012         vpxor           0x20($inp),$xb0,$xb0
2013         vmovdqu         $xa0,0x00($out)
2014         vmovdqu         $xb0,0x20($out)
2015         je              .Ldone8x
2016
2017         lea             0x40($inp),$inp         # inp+=64*1
2018         xor             %r10,%r10
2019         vmovdqa         $xc0,0x00(%rsp)
2020         lea             0x40($out),$out         # out+=64*1
2021         sub             \$64,$len               # len-=64*1
2022         vmovdqa         $xd0,0x20(%rsp)
2023         jmp             .Loop_tail8x
2024
2025 .align  32
2026 .L128_or_more8x:
2027         vpxor           0x00($inp),$xa0,$xa0    # xor with input
2028         vpxor           0x20($inp),$xb0,$xb0
2029         vpxor           0x40($inp),$xc0,$xc0
2030         vpxor           0x60($inp),$xd0,$xd0
2031         vmovdqu         $xa0,0x00($out)
2032         vmovdqu         $xb0,0x20($out)
2033         vmovdqu         $xc0,0x40($out)
2034         vmovdqu         $xd0,0x60($out)
2035         je              .Ldone8x
2036
2037         lea             0x80($inp),$inp         # inp+=64*2
2038         xor             %r10,%r10
2039         vmovdqa         $xa1,0x00(%rsp)
2040         lea             0x80($out),$out         # out+=64*2
2041         sub             \$128,$len              # len-=64*2
2042         vmovdqa         $xb1,0x20(%rsp)
2043         jmp             .Loop_tail8x
2044
2045 .align  32
2046 .L192_or_more8x:
2047         vpxor           0x00($inp),$xa0,$xa0    # xor with input
2048         vpxor           0x20($inp),$xb0,$xb0
2049         vpxor           0x40($inp),$xc0,$xc0
2050         vpxor           0x60($inp),$xd0,$xd0
2051         vpxor           0x80($inp),$xa1,$xa1
2052         vpxor           0xa0($inp),$xb1,$xb1
2053         vmovdqu         $xa0,0x00($out)
2054         vmovdqu         $xb0,0x20($out)
2055         vmovdqu         $xc0,0x40($out)
2056         vmovdqu         $xd0,0x60($out)
2057         vmovdqu         $xa1,0x80($out)
2058         vmovdqu         $xb1,0xa0($out)
2059         je              .Ldone8x
2060
2061         lea             0xc0($inp),$inp         # inp+=64*3
2062         xor             %r10,%r10
2063         vmovdqa         $xc1,0x00(%rsp)
2064         lea             0xc0($out),$out         # out+=64*3
2065         sub             \$192,$len              # len-=64*3
2066         vmovdqa         $xd1,0x20(%rsp)
2067         jmp             .Loop_tail8x
2068
2069 .align  32
2070 .L256_or_more8x:
2071         vpxor           0x00($inp),$xa0,$xa0    # xor with input
2072         vpxor           0x20($inp),$xb0,$xb0
2073         vpxor           0x40($inp),$xc0,$xc0
2074         vpxor           0x60($inp),$xd0,$xd0
2075         vpxor           0x80($inp),$xa1,$xa1
2076         vpxor           0xa0($inp),$xb1,$xb1
2077         vpxor           0xc0($inp),$xc1,$xc1
2078         vpxor           0xe0($inp),$xd1,$xd1
2079         vmovdqu         $xa0,0x00($out)
2080         vmovdqu         $xb0,0x20($out)
2081         vmovdqu         $xc0,0x40($out)
2082         vmovdqu         $xd0,0x60($out)
2083         vmovdqu         $xa1,0x80($out)
2084         vmovdqu         $xb1,0xa0($out)
2085         vmovdqu         $xc1,0xc0($out)
2086         vmovdqu         $xd1,0xe0($out)
2087         je              .Ldone8x
2088
2089         lea             0x100($inp),$inp        # inp+=64*4
2090         xor             %r10,%r10
2091         vmovdqa         $xa2,0x00(%rsp)
2092         lea             0x100($out),$out        # out+=64*4
2093         sub             \$256,$len              # len-=64*4
2094         vmovdqa         $xb2,0x20(%rsp)
2095         jmp             .Loop_tail8x
2096
2097 .align  32
2098 .L320_or_more8x:
2099         vpxor           0x00($inp),$xa0,$xa0    # xor with input
2100         vpxor           0x20($inp),$xb0,$xb0
2101         vpxor           0x40($inp),$xc0,$xc0
2102         vpxor           0x60($inp),$xd0,$xd0
2103         vpxor           0x80($inp),$xa1,$xa1
2104         vpxor           0xa0($inp),$xb1,$xb1
2105         vpxor           0xc0($inp),$xc1,$xc1
2106         vpxor           0xe0($inp),$xd1,$xd1
2107         vpxor           0x100($inp),$xa2,$xa2
2108         vpxor           0x120($inp),$xb2,$xb2
2109         vmovdqu         $xa0,0x00($out)
2110         vmovdqu         $xb0,0x20($out)
2111         vmovdqu         $xc0,0x40($out)
2112         vmovdqu         $xd0,0x60($out)
2113         vmovdqu         $xa1,0x80($out)
2114         vmovdqu         $xb1,0xa0($out)
2115         vmovdqu         $xc1,0xc0($out)
2116         vmovdqu         $xd1,0xe0($out)
2117         vmovdqu         $xa2,0x100($out)
2118         vmovdqu         $xb2,0x120($out)
2119         je              .Ldone8x
2120
2121         lea             0x140($inp),$inp        # inp+=64*5
2122         xor             %r10,%r10
2123         vmovdqa         $xc2,0x00(%rsp)
2124         lea             0x140($out),$out        # out+=64*5
2125         sub             \$320,$len              # len-=64*5
2126         vmovdqa         $xd2,0x20(%rsp)
2127         jmp             .Loop_tail8x
2128
2129 .align  32
2130 .L384_or_more8x:
2131         vpxor           0x00($inp),$xa0,$xa0    # xor with input
2132         vpxor           0x20($inp),$xb0,$xb0
2133         vpxor           0x40($inp),$xc0,$xc0
2134         vpxor           0x60($inp),$xd0,$xd0
2135         vpxor           0x80($inp),$xa1,$xa1
2136         vpxor           0xa0($inp),$xb1,$xb1
2137         vpxor           0xc0($inp),$xc1,$xc1
2138         vpxor           0xe0($inp),$xd1,$xd1
2139         vpxor           0x100($inp),$xa2,$xa2
2140         vpxor           0x120($inp),$xb2,$xb2
2141         vpxor           0x140($inp),$xc2,$xc2
2142         vpxor           0x160($inp),$xd2,$xd2
2143         vmovdqu         $xa0,0x00($out)
2144         vmovdqu         $xb0,0x20($out)
2145         vmovdqu         $xc0,0x40($out)
2146         vmovdqu         $xd0,0x60($out)
2147         vmovdqu         $xa1,0x80($out)
2148         vmovdqu         $xb1,0xa0($out)
2149         vmovdqu         $xc1,0xc0($out)
2150         vmovdqu         $xd1,0xe0($out)
2151         vmovdqu         $xa2,0x100($out)
2152         vmovdqu         $xb2,0x120($out)
2153         vmovdqu         $xc2,0x140($out)
2154         vmovdqu         $xd2,0x160($out)
2155         je              .Ldone8x
2156
2157         lea             0x180($inp),$inp        # inp+=64*6
2158         xor             %r10,%r10
2159         vmovdqa         $xa3,0x00(%rsp)
2160         lea             0x180($out),$out        # out+=64*6
2161         sub             \$384,$len              # len-=64*6
2162         vmovdqa         $xb3,0x20(%rsp)
2163         jmp             .Loop_tail8x
2164
2165 .align  32
2166 .L448_or_more8x:
2167         vpxor           0x00($inp),$xa0,$xa0    # xor with input
2168         vpxor           0x20($inp),$xb0,$xb0
2169         vpxor           0x40($inp),$xc0,$xc0
2170         vpxor           0x60($inp),$xd0,$xd0
2171         vpxor           0x80($inp),$xa1,$xa1
2172         vpxor           0xa0($inp),$xb1,$xb1
2173         vpxor           0xc0($inp),$xc1,$xc1
2174         vpxor           0xe0($inp),$xd1,$xd1
2175         vpxor           0x100($inp),$xa2,$xa2
2176         vpxor           0x120($inp),$xb2,$xb2
2177         vpxor           0x140($inp),$xc2,$xc2
2178         vpxor           0x160($inp),$xd2,$xd2
2179         vpxor           0x180($inp),$xa3,$xa3
2180         vpxor           0x1a0($inp),$xb3,$xb3
2181         vmovdqu         $xa0,0x00($out)
2182         vmovdqu         $xb0,0x20($out)
2183         vmovdqu         $xc0,0x40($out)
2184         vmovdqu         $xd0,0x60($out)
2185         vmovdqu         $xa1,0x80($out)
2186         vmovdqu         $xb1,0xa0($out)
2187         vmovdqu         $xc1,0xc0($out)
2188         vmovdqu         $xd1,0xe0($out)
2189         vmovdqu         $xa2,0x100($out)
2190         vmovdqu         $xb2,0x120($out)
2191         vmovdqu         $xc2,0x140($out)
2192         vmovdqu         $xd2,0x160($out)
2193         vmovdqu         $xa3,0x180($out)
2194         vmovdqu         $xb3,0x1a0($out)
2195         je              .Ldone8x
2196
2197         lea             0x1c0($inp),$inp        # inp+=64*7
2198         xor             %r10,%r10
2199         vmovdqa         $xc3,0x00(%rsp)
2200         lea             0x1c0($out),$out        # out+=64*7
2201         sub             \$448,$len              # len-=64*7
2202         vmovdqa         $xd3,0x20(%rsp)
2203
2204 .Loop_tail8x:
2205         movzb           ($inp,%r10),%eax
2206         movzb           (%rsp,%r10),%ecx
2207         lea             1(%r10),%r10
2208         xor             %ecx,%eax
2209         mov             %al,-1($out,%r10)
2210         dec             $len
2211         jnz             .Loop_tail8x
2212
2213 .Ldone8x:
2214         vzeroall
2215 ___
2216 $code.=<<___    if ($win64);
2217         lea             0x290+0x30(%rsp),%r11
2218         movaps          -0x30(%r11),%xmm6
2219         movaps          -0x20(%r11),%xmm7
2220         movaps          -0x10(%r11),%xmm8
2221         movaps          0x00(%r11),%xmm9
2222         movaps          0x10(%r11),%xmm10
2223         movaps          0x20(%r11),%xmm11
2224         movaps          0x30(%r11),%xmm12
2225         movaps          0x40(%r11),%xmm13
2226         movaps          0x50(%r11),%xmm14
2227         movaps          0x60(%r11),%xmm15
2228 ___
2229 $code.=<<___;
2230         mov             0x280(%rsp),%rsp
2231         ret
2232 .size   ChaCha20_8x,.-ChaCha20_8x
2233 ___
2234 }
2235
2236 foreach (split("\n",$code)) {
2237         s/\`([^\`]*)\`/eval $1/geo;
2238
2239         s/%x#%y/%x/go;
2240
2241         print $_,"\n";
2242 }
2243
2244 close STDOUT;