e2019aa3b5ed65e452f7260d8c67e6a6b5f1f81b
[openssl.git] / crypto / chacha / asm / chacha-x86.pl
1 #!/usr/bin/env perl
2 #
3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9 #
10 # January 2015
11 #
12 # ChaCha20 for x86.
13 #
14 # Performance in cycles per byte out of large buffer.
15 #
16 #               1xIALU/gcc      4xSSSE3
17 # Pentium       17.5/+80%
18 # PIII          14.2/+60%
19 # P4            18.6/+84%
20 # Core2         9.56/+89%       4.83
21 # Westmere      9.50/+45%       3.35
22 # Sandy Bridge  10.5/+47%       3.20
23 # Haswell       8.15/+50%       2.83
24 # Silvermont    17.4/+36%       8.35
25 # Sledgehammer  10.2/+54%
26 # Bulldozer     13.4/+50%       4.38(*)
27 #
28 # (*)   Bulldozer actually executes 4xXOP code path that delivers 3.55;
29
30 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
31 push(@INC,"${dir}","${dir}../../perlasm");
32 require "x86asm.pl";
33
34 &asm_init($ARGV[0],"chacha-x86.pl",$ARGV[$#ARGV] eq "386");
35
36 $xmm=$ymm=0;
37 for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); }
38
39 $ymm=1 if ($xmm &&
40                 `$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
41                         =~ /GNU assembler version ([2-9]\.[0-9]+)/ &&
42                 $1>=2.19);      # first version supporting AVX
43
44 $ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32n" &&
45                 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/ &&
46                 $1>=2.03);      # first version supporting AVX
47
48 $ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32" &&
49                 `ml 2>&1` =~ /Version ([0-9]+)\./ &&
50                 $1>=10);        # first version supporting AVX
51
52 $ymm=1 if ($xmm && !$ymm &&
53                 `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/ &&
54                 $2>=3.0);       # first version supporting AVX
55
56 $a="eax";
57 ($b,$b_)=("ebx","ebp");
58 ($c,$c_)=("ecx","esi");
59 ($d,$d_)=("edx","edi");
60
61 sub QUARTERROUND {
62 my ($ai,$bi,$ci,$di,$i)=@_;
63 my ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+1)&3),($ai,$bi,$ci,$di)); # next
64 my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
65
66         #       a   b   c   d
67         #
68         #       0   4   8  12 < even round
69         #       1   5   9  13
70         #       2   6  10  14
71         #       3   7  11  15
72         #       0   5  10  15 < odd round
73         #       1   6  11  12
74         #       2   7   8  13
75         #       3   4   9  14
76
77         if ($i==0) {
78             my $j=4;
79             ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-$j--)&3),($ap,$bp,$cp,$dp));
80         } elsif ($i==3) {
81             my $j=0;
82             ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+$j++)&3),($an,$bn,$cn,$dn));
83         } elsif ($i==4) {
84             my $j=4;
85             ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_+$j--)&3),($ap,$bp,$cp,$dp));
86         } elsif ($i==7) {
87             my $j=0;
88             ($an,$bn,$cn,$dn)=map(($_&~3)+(($_-$j++)&3),($an,$bn,$cn,$dn));
89         }
90
91         #&add   ($a,$b);                        # see elsewhere
92         &xor    ($d,$a);
93          &mov   (&DWP(4*$cp,"esp"),$c_)         if ($ai>0 && $ai<3);
94         &rol    ($d,16);
95          &mov   (&DWP(4*$bp,"esp"),$b_)         if ($i!=0);
96         &add    ($c,$d);
97          &mov   ($c_,&DWP(4*$cn,"esp"))         if ($ai>0 && $ai<3);
98         &xor    ($b,$c);
99          &mov   ($d_,&DWP(4*$dn,"esp"))         if ($di!=$dn);
100         &rol    ($b,12);
101          &mov   ($b_,&DWP(4*$bn,"esp"))         if ($i<7);
102          &mov   ($b_,&DWP(128,"esp"))           if ($i==7);     # loop counter
103         &add    ($a,$b);
104         &xor    ($d,$a);
105         &mov    (&DWP(4*$ai,"esp"),$a);
106         &rol    ($d,8);
107         &mov    ($a,&DWP(4*$an,"esp"));
108         &add    ($c,$d);
109         &mov    (&DWP(4*$di,"esp"),$d)          if ($di!=$dn);
110         &mov    ($d_,$d)                        if ($di==$dn);
111         &xor    ($b,$c);
112          &add   ($a,$b_)                        if ($i<7);      # elsewhere
113         &rol    ($b,7);
114
115         ($b,$b_)=($b_,$b);
116         ($c,$c_)=($c_,$c);
117         ($d,$d_)=($d_,$d);
118 }
119
120 &static_label("ssse3_shortcut");
121 &static_label("xop_shortcut");
122 &static_label("ssse3_data");
123 &static_label("pic_point");
124
125 &function_begin("ChaCha20_ctr32");
126 if ($xmm) {
127         &call   (&label("pic_point"));
128 &set_label("pic_point");
129         &blindpop("eax");
130         &picmeup("ebp","OPENSSL_ia32cap_P","eax",&label("pic_point"));
131         &test   (&DWP(0,"ebp"),1<<24);          # test FXSR bit
132         &jz     (&label("x86"));
133         &test   (&DWP(4,"ebp"),1<<9);           # test SSSE3 bit
134         &jz     (&label("x86"));
135         &jmp    (&label("ssse3_shortcut"));
136 &set_label("x86");
137 }
138         &mov    ("esi",&wparam(3));             # key
139         &mov    ("edi",&wparam(4));             # counter and nonce
140
141         &stack_push(33);
142
143         &mov    ("eax",&DWP(4*0,"esi"));        # copy key
144         &mov    ("ebx",&DWP(4*1,"esi"));
145         &mov    ("ecx",&DWP(4*2,"esi"));
146         &mov    ("edx",&DWP(4*3,"esi"));
147         &mov    (&DWP(64+4*4,"esp"),"eax");
148         &mov    (&DWP(64+4*5,"esp"),"ebx");
149         &mov    (&DWP(64+4*6,"esp"),"ecx");
150         &mov    (&DWP(64+4*7,"esp"),"edx");
151         &mov    ("eax",&DWP(4*4,"esi"));
152         &mov    ("ebx",&DWP(4*5,"esi"));
153         &mov    ("ecx",&DWP(4*6,"esi"));
154         &mov    ("edx",&DWP(4*7,"esi"));
155         &mov    (&DWP(64+4*8,"esp"),"eax");
156         &mov    (&DWP(64+4*9,"esp"),"ebx");
157         &mov    (&DWP(64+4*10,"esp"),"ecx");
158         &mov    (&DWP(64+4*11,"esp"),"edx");
159         &mov    ("eax",&DWP(4*0,"edi"));        # copy counter and nonce
160         &mov    ("ebx",&DWP(4*1,"edi"));
161         &mov    ("ecx",&DWP(4*2,"edi"));
162         &mov    ("edx",&DWP(4*3,"edi"));
163         &sub    ("eax",1);
164         &mov    (&DWP(64+4*12,"esp"),"eax");
165         &mov    (&DWP(64+4*13,"esp"),"ebx");
166         &mov    (&DWP(64+4*14,"esp"),"ecx");
167         &mov    (&DWP(64+4*15,"esp"),"edx");
168         &jmp    (&label("entry"));
169
170 &set_label("outer_loop",16);
171         &mov    (&wparam(1),$b);                # save input
172         &mov    (&wparam(0),$a);                # save output
173         &mov    (&wparam(2),$c);                # save len
174 &set_label("entry");
175         &mov    ($a,0x61707865);
176         &mov    (&DWP(4*1,"esp"),0x3320646e);
177         &mov    (&DWP(4*2,"esp"),0x79622d32);
178         &mov    (&DWP(4*3,"esp"),0x6b206574);
179
180         &mov    ($b, &DWP(64+4*5,"esp"));       # copy key material
181         &mov    ($b_,&DWP(64+4*6,"esp"));
182         &mov    ($c, &DWP(64+4*10,"esp"));
183         &mov    ($c_,&DWP(64+4*11,"esp"));
184         &mov    ($d, &DWP(64+4*13,"esp"));
185         &mov    ($d_,&DWP(64+4*14,"esp"));
186         &mov    (&DWP(4*5,"esp"),$b);
187         &mov    (&DWP(4*6,"esp"),$b_);
188         &mov    (&DWP(4*10,"esp"),$c);
189         &mov    (&DWP(4*11,"esp"),$c_);
190         &mov    (&DWP(4*13,"esp"),$d);
191         &mov    (&DWP(4*14,"esp"),$d_);
192
193         &mov    ($b, &DWP(64+4*7,"esp"));
194         &mov    ($d_,&DWP(64+4*15,"esp"));
195         &mov    ($d, &DWP(64+4*12,"esp"));
196         &mov    ($b_,&DWP(64+4*4,"esp"));
197         &mov    ($c, &DWP(64+4*8,"esp"));
198         &mov    ($c_,&DWP(64+4*9,"esp"));
199         &add    ($d,1);                         # counter value
200         &mov    (&DWP(4*7,"esp"),$b);
201         &mov    (&DWP(4*15,"esp"),$d_);
202         &mov    (&DWP(64+4*12,"esp"),$d);       # save counter value
203
204         &mov    ($b,10);                        # loop counter
205         &jmp    (&label("loop"));
206
207 &set_label("loop",16);
208         &add    ($a,$b_);                       # elsewhere
209         &mov    (&DWP(128,"esp"),$b);           # save loop counter
210         &mov    ($b,$b_);
211         &QUARTERROUND(0, 4, 8, 12, 0);
212         &QUARTERROUND(1, 5, 9, 13, 1);
213         &QUARTERROUND(2, 6,10, 14, 2);
214         &QUARTERROUND(3, 7,11, 15, 3);
215         &QUARTERROUND(0, 5,10, 15, 4);
216         &QUARTERROUND(1, 6,11, 12, 5);
217         &QUARTERROUND(2, 7, 8, 13, 6);
218         &QUARTERROUND(3, 4, 9, 14, 7);
219         &dec    ($b);
220         &jnz    (&label("loop"));
221
222         &mov    ($b,&wparam(2));                # load len
223
224         &add    ($a,0x61707865);                # accumulate key material
225         &add    ($b_,&DWP(64+4*4,"esp"));
226         &add    ($c, &DWP(64+4*8,"esp"));
227         &add    ($c_,&DWP(64+4*9,"esp"));
228
229         &cmp    ($b,64);
230         &jb     (&label("tail"));
231
232         &mov    ($b,&wparam(1));                # load input pointer
233         &add    ($d, &DWP(64+4*12,"esp"));
234         &add    ($d_,&DWP(64+4*14,"esp"));
235
236         &xor    ($a, &DWP(4*0,$b));             # xor with input
237         &xor    ($b_,&DWP(4*4,$b));
238         &mov    (&DWP(4*0,"esp"),$a);
239         &mov    ($a,&wparam(0));                # load output pointer
240         &xor    ($c, &DWP(4*8,$b));
241         &xor    ($c_,&DWP(4*9,$b));
242         &xor    ($d, &DWP(4*12,$b));
243         &xor    ($d_,&DWP(4*14,$b));
244         &mov    (&DWP(4*4,$a),$b_);             # write output
245         &mov    (&DWP(4*8,$a),$c);
246         &mov    (&DWP(4*9,$a),$c_);
247         &mov    (&DWP(4*12,$a),$d);
248         &mov    (&DWP(4*14,$a),$d_);
249
250         &mov    ($b_,&DWP(4*1,"esp"));
251         &mov    ($c, &DWP(4*2,"esp"));
252         &mov    ($c_,&DWP(4*3,"esp"));
253         &mov    ($d, &DWP(4*5,"esp"));
254         &mov    ($d_,&DWP(4*6,"esp"));
255         &add    ($b_,0x3320646e);               # accumulate key material
256         &add    ($c, 0x79622d32);
257         &add    ($c_,0x6b206574);
258         &add    ($d, &DWP(64+4*5,"esp"));
259         &add    ($d_,&DWP(64+4*6,"esp"));
260         &xor    ($b_,&DWP(4*1,$b));
261         &xor    ($c, &DWP(4*2,$b));
262         &xor    ($c_,&DWP(4*3,$b));
263         &xor    ($d, &DWP(4*5,$b));
264         &xor    ($d_,&DWP(4*6,$b));
265         &mov    (&DWP(4*1,$a),$b_);
266         &mov    (&DWP(4*2,$a),$c);
267         &mov    (&DWP(4*3,$a),$c_);
268         &mov    (&DWP(4*5,$a),$d);
269         &mov    (&DWP(4*6,$a),$d_);
270
271         &mov    ($b_,&DWP(4*7,"esp"));
272         &mov    ($c, &DWP(4*10,"esp"));
273         &mov    ($c_,&DWP(4*11,"esp"));
274         &mov    ($d, &DWP(4*13,"esp"));
275         &mov    ($d_,&DWP(4*15,"esp"));
276         &add    ($b_,&DWP(64+4*7,"esp"));
277         &add    ($c, &DWP(64+4*10,"esp"));
278         &add    ($c_,&DWP(64+4*11,"esp"));
279         &add    ($d, &DWP(64+4*13,"esp"));
280         &add    ($d_,&DWP(64+4*15,"esp"));
281         &xor    ($b_,&DWP(4*7,$b));
282         &xor    ($c, &DWP(4*10,$b));
283         &xor    ($c_,&DWP(4*11,$b));
284         &xor    ($d, &DWP(4*13,$b));
285         &xor    ($d_,&DWP(4*15,$b));
286         &lea    ($b,&DWP(4*16,$b));
287         &mov    (&DWP(4*7,$a),$b_);
288         &mov    ($b_,&DWP(4*0,"esp"));
289         &mov    (&DWP(4*10,$a),$c);
290         &mov    ($c,&wparam(2));                # len
291         &mov    (&DWP(4*11,$a),$c_);
292         &mov    (&DWP(4*13,$a),$d);
293         &mov    (&DWP(4*15,$a),$d_);
294         &mov    (&DWP(4*0,$a),$b_);
295         &lea    ($a,&DWP(4*16,$a));
296         &sub    ($c,64);
297         &jnz    (&label("outer_loop"));
298
299         &jmp    (&label("done"));
300
301 &set_label("tail");
302         &add    ($d, &DWP(64+4*12,"esp"));
303         &add    ($d_,&DWP(64+4*14,"esp"));
304         &mov    (&DWP(4*0,"esp"),$a);
305         &mov    (&DWP(4*4,"esp"),$b_);
306         &mov    (&DWP(4*8,"esp"),$c);
307         &mov    (&DWP(4*9,"esp"),$c_);
308         &mov    (&DWP(4*12,"esp"),$d);
309         &mov    (&DWP(4*14,"esp"),$d_);
310
311         &mov    ($b_,&DWP(4*1,"esp"));
312         &mov    ($c, &DWP(4*2,"esp"));
313         &mov    ($c_,&DWP(4*3,"esp"));
314         &mov    ($d, &DWP(4*5,"esp"));
315         &mov    ($d_,&DWP(4*6,"esp"));
316         &add    ($b_,0x3320646e);               # accumulate key material
317         &add    ($c, 0x79622d32);
318         &add    ($c_,0x6b206574);
319         &add    ($d, &DWP(64+4*5,"esp"));
320         &add    ($d_,&DWP(64+4*6,"esp"));
321         &mov    (&DWP(4*1,"esp"),$b_);
322         &mov    (&DWP(4*2,"esp"),$c);
323         &mov    (&DWP(4*3,"esp"),$c_);
324         &mov    (&DWP(4*5,"esp"),$d);
325         &mov    (&DWP(4*6,"esp"),$d_);
326
327         &mov    ($b_,&DWP(4*7,"esp"));
328         &mov    ($c, &DWP(4*10,"esp"));
329         &mov    ($c_,&DWP(4*11,"esp"));
330         &mov    ($d, &DWP(4*13,"esp"));
331         &mov    ($d_,&DWP(4*15,"esp"));
332         &add    ($b_,&DWP(64+4*7,"esp"));
333         &add    ($c, &DWP(64+4*10,"esp"));
334         &add    ($c_,&DWP(64+4*11,"esp"));
335         &add    ($d, &DWP(64+4*13,"esp"));
336         &add    ($d_,&DWP(64+4*15,"esp"));
337         &mov    (&DWP(4*7,"esp"),$b_);
338         &mov    ($b_,&wparam(1));               # load input
339         &mov    (&DWP(4*10,"esp"),$c);
340         &mov    ($c,&wparam(0));                # load output
341         &mov    (&DWP(4*11,"esp"),$c_);
342         &xor    ($c_,$c_);
343         &mov    (&DWP(4*13,"esp"),$d);
344         &mov    (&DWP(4*15,"esp"),$d_);
345
346         &xor    ("eax","eax");
347         &xor    ("edx","edx");
348 &set_label("tail_loop");
349         &movb   ("al",&BP(0,$c_,$b_));
350         &movb   ("dl",&BP(0,"esp",$c_));
351         &lea    ($c_,&DWP(1,$c_));
352         &xor    ("al","dl");
353         &mov    (&BP(-1,$c,$c_),"al");
354         &dec    ($b);
355         &jnz    (&label("tail_loop"));
356
357 &set_label("done");
358         &stack_pop(33);
359 &function_end("ChaCha20_ctr32");
360
361 if ($xmm) {
362 my ($xa,$xa_,$xb,$xb_,$xc,$xc_,$xd,$xd_)=map("xmm$_",(0..7));
363 my ($out,$inp,$len)=("edi","esi","ecx");
364
365 sub QUARTERROUND_SSSE3 {
366 my ($ai,$bi,$ci,$di,$i)=@_;
367 my ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+1)&3),($ai,$bi,$ci,$di)); # next
368 my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
369
370         #       a   b   c   d
371         #
372         #       0   4   8  12 < even round
373         #       1   5   9  13
374         #       2   6  10  14
375         #       3   7  11  15
376         #       0   5  10  15 < odd round
377         #       1   6  11  12
378         #       2   7   8  13
379         #       3   4   9  14
380
381         if ($i==0) {
382             my $j=4;
383             ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-$j--)&3),($ap,$bp,$cp,$dp));
384         } elsif ($i==3) {
385             my $j=0;
386             ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+$j++)&3),($an,$bn,$cn,$dn));
387         } elsif ($i==4) {
388             my $j=4;
389             ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_+$j--)&3),($ap,$bp,$cp,$dp));
390         } elsif ($i==7) {
391             my $j=0;
392             ($an,$bn,$cn,$dn)=map(($_&~3)+(($_-$j++)&3),($an,$bn,$cn,$dn));
393         }
394
395         #&paddd ($xa,$xb);                      # see elsewhere
396         #&pxor  ($xd,$xa);                      # see elsewhere
397          &movdqa(&QWP(16*$cp-128,"ebx"),$xc_)   if ($ai>0 && $ai<3);
398         &pshufb ($xd,&QWP(0,"eax"));            # rot16
399          &movdqa(&QWP(16*$bp-128,"ebx"),$xb_)   if ($i!=0);
400         &paddd  ($xc,$xd);
401          &movdqa($xc_,&QWP(16*$cn-128,"ebx"))   if ($ai>0 && $ai<3);
402         &pxor   ($xb,$xc);
403          &movdqa($xb_,&QWP(16*$bn-128,"ebx"))   if ($i<7);
404         &movdqa ($xa_,$xb);                     # borrow as temporary
405         &pslld  ($xb,12);
406         &psrld  ($xa_,20);
407         &por    ($xb,$xa_);
408          &movdqa($xa_,&QWP(16*$an-128,"ebx"));
409         &paddd  ($xa,$xb);
410          &movdqa($xd_,&QWP(16*$dn-128,"ebx"))   if ($di!=$dn);
411         &pxor   ($xd,$xa);
412         &movdqa (&QWP(16*$ai-128,"ebx"),$xa);
413         &pshufb ($xd,&QWP(16,"eax"));           # rot8
414         &paddd  ($xc,$xd);
415         &movdqa (&QWP(16*$di-128,"ebx"),$xd)    if ($di!=$dn);
416         &movdqa ($xd_,$xd)                      if ($di==$dn);
417         &pxor   ($xb,$xc);
418          &paddd ($xa_,$xb_)                     if ($i<7);      # elsewhere
419         &movdqa ($xa,$xb);                      # borrow as temporary
420         &pslld  ($xb,7);
421         &psrld  ($xa,25);
422          &pxor  ($xd_,$xa_)                     if ($i<7);      # elsewhere
423         &por    ($xb,$xa);
424
425         ($xa,$xa_)=($xa_,$xa);
426         ($xb,$xb_)=($xb_,$xb);
427         ($xc,$xc_)=($xc_,$xc);
428         ($xd,$xd_)=($xd_,$xd);
429 }
430
431 &function_begin("ChaCha20_ssse3");
432 &set_label("ssse3_shortcut");
433         &test           (&DWP(4,"ebp"),1<<11);          # test XOP bit
434         &jnz            (&label("xop_shortcut"));
435
436         &mov            ($out,&wparam(0));
437         &mov            ($inp,&wparam(1));
438         &mov            ($len,&wparam(2));
439         &mov            ("edx",&wparam(3));             # key
440         &mov            ("ebx",&wparam(4));             # counter and nonce
441
442         &mov            ("ebp","esp");
443         &stack_push     (131);
444         &and            ("esp",-64);
445         &mov            (&DWP(512,"esp"),"ebp");
446
447         &lea            ("eax",&DWP(&label("ssse3_data")."-".
448                                     &label("pic_point"),"eax"));
449         &movdqu         ("xmm3",&QWP(0,"ebx"));         # counter and nonce
450
451         &cmp            ($len,64*4);
452         &jb             (&label("1x"));
453
454         &mov            (&DWP(512+4,"esp"),"edx");      # offload pointers
455         &mov            (&DWP(512+8,"esp"),"ebx");
456         &sub            ($len,64*4);                    # bias len
457         &lea            ("ebp",&DWP(256+128,"esp"));    # size optimization
458
459         &movdqu         ("xmm7",&QWP(0,"edx"));         # key
460         &pshufd         ("xmm0","xmm3",0x00);
461         &pshufd         ("xmm1","xmm3",0x55);
462         &pshufd         ("xmm2","xmm3",0xaa);
463         &pshufd         ("xmm3","xmm3",0xff);
464          &paddd         ("xmm0",&QWP(16*3,"eax"));      # fix counters
465         &pshufd         ("xmm4","xmm7",0x00);
466         &pshufd         ("xmm5","xmm7",0x55);
467          &psubd         ("xmm0",&QWP(16*4,"eax"));
468         &pshufd         ("xmm6","xmm7",0xaa);
469         &pshufd         ("xmm7","xmm7",0xff);
470         &movdqa         (&QWP(16*12-128,"ebp"),"xmm0");
471         &movdqa         (&QWP(16*13-128,"ebp"),"xmm1");
472         &movdqa         (&QWP(16*14-128,"ebp"),"xmm2");
473         &movdqa         (&QWP(16*15-128,"ebp"),"xmm3");
474          &movdqu        ("xmm3",&QWP(16,"edx"));        # key
475         &movdqa         (&QWP(16*4-128,"ebp"),"xmm4");
476         &movdqa         (&QWP(16*5-128,"ebp"),"xmm5");
477         &movdqa         (&QWP(16*6-128,"ebp"),"xmm6");
478         &movdqa         (&QWP(16*7-128,"ebp"),"xmm7");
479          &movdqa        ("xmm7",&QWP(16*2,"eax"));      # sigma
480          &lea           ("ebx",&DWP(128,"esp"));        # size optimization
481
482         &pshufd         ("xmm0","xmm3",0x00);
483         &pshufd         ("xmm1","xmm3",0x55);
484         &pshufd         ("xmm2","xmm3",0xaa);
485         &pshufd         ("xmm3","xmm3",0xff);
486         &pshufd         ("xmm4","xmm7",0x00);
487         &pshufd         ("xmm5","xmm7",0x55);
488         &pshufd         ("xmm6","xmm7",0xaa);
489         &pshufd         ("xmm7","xmm7",0xff);
490         &movdqa         (&QWP(16*8-128,"ebp"),"xmm0");
491         &movdqa         (&QWP(16*9-128,"ebp"),"xmm1");
492         &movdqa         (&QWP(16*10-128,"ebp"),"xmm2");
493         &movdqa         (&QWP(16*11-128,"ebp"),"xmm3");
494         &movdqa         (&QWP(16*0-128,"ebp"),"xmm4");
495         &movdqa         (&QWP(16*1-128,"ebp"),"xmm5");
496         &movdqa         (&QWP(16*2-128,"ebp"),"xmm6");
497         &movdqa         (&QWP(16*3-128,"ebp"),"xmm7");
498
499         &lea            ($inp,&DWP(128,$inp));          # size optimization
500         &lea            ($out,&DWP(128,$out));          # size optimization
501         &jmp            (&label("outer_loop"));
502
503 &set_label("outer_loop",16);
504         #&movdqa        ("xmm0",&QWP(16*0-128,"ebp"));  # copy key material
505         &movdqa         ("xmm1",&QWP(16*1-128,"ebp"));
506         &movdqa         ("xmm2",&QWP(16*2-128,"ebp"));
507         &movdqa         ("xmm3",&QWP(16*3-128,"ebp"));
508         #&movdqa        ("xmm4",&QWP(16*4-128,"ebp"));
509         &movdqa         ("xmm5",&QWP(16*5-128,"ebp"));
510         &movdqa         ("xmm6",&QWP(16*6-128,"ebp"));
511         &movdqa         ("xmm7",&QWP(16*7-128,"ebp"));
512         #&movdqa        (&QWP(16*0-128,"ebx"),"xmm0");
513         &movdqa         (&QWP(16*1-128,"ebx"),"xmm1");
514         &movdqa         (&QWP(16*2-128,"ebx"),"xmm2");
515         &movdqa         (&QWP(16*3-128,"ebx"),"xmm3");
516         #&movdqa        (&QWP(16*4-128,"ebx"),"xmm4");
517         &movdqa         (&QWP(16*5-128,"ebx"),"xmm5");
518         &movdqa         (&QWP(16*6-128,"ebx"),"xmm6");
519         &movdqa         (&QWP(16*7-128,"ebx"),"xmm7");
520         #&movdqa        ("xmm0",&QWP(16*8-128,"ebp"));
521         #&movdqa        ("xmm1",&QWP(16*9-128,"ebp"));
522         &movdqa         ("xmm2",&QWP(16*10-128,"ebp"));
523         &movdqa         ("xmm3",&QWP(16*11-128,"ebp"));
524         &movdqa         ("xmm4",&QWP(16*12-128,"ebp"));
525         &movdqa         ("xmm5",&QWP(16*13-128,"ebp"));
526         &movdqa         ("xmm6",&QWP(16*14-128,"ebp"));
527         &movdqa         ("xmm7",&QWP(16*15-128,"ebp"));
528         &paddd          ("xmm4",&QWP(16*4,"eax"));      # counter value
529         #&movdqa        (&QWP(16*8-128,"ebx"),"xmm0");
530         #&movdqa        (&QWP(16*9-128,"ebx"),"xmm1");
531         &movdqa         (&QWP(16*10-128,"ebx"),"xmm2");
532         &movdqa         (&QWP(16*11-128,"ebx"),"xmm3");
533         &movdqa         (&QWP(16*12-128,"ebx"),"xmm4");
534         &movdqa         (&QWP(16*13-128,"ebx"),"xmm5");
535         &movdqa         (&QWP(16*14-128,"ebx"),"xmm6");
536         &movdqa         (&QWP(16*15-128,"ebx"),"xmm7");
537         &movdqa         (&QWP(16*12-128,"ebp"),"xmm4"); # save counter value
538
539         &movdqa         ($xa, &QWP(16*0-128,"ebp"));
540         &movdqa         ($xd, "xmm4");
541         &movdqa         ($xb_,&QWP(16*4-128,"ebp"));
542         &movdqa         ($xc, &QWP(16*8-128,"ebp"));
543         &movdqa         ($xc_,&QWP(16*9-128,"ebp"));
544
545         &mov            ("edx",10);                     # loop counter
546         &nop            ();
547
548 &set_label("loop",16);
549         &paddd          ($xa,$xb_);                     # elsewhere
550         &movdqa         ($xb,$xb_);
551         &pxor           ($xd,$xa);                      # elsewhere
552         &QUARTERROUND_SSSE3(0, 4, 8, 12, 0);
553         &QUARTERROUND_SSSE3(1, 5, 9, 13, 1);
554         &QUARTERROUND_SSSE3(2, 6,10, 14, 2);
555         &QUARTERROUND_SSSE3(3, 7,11, 15, 3);
556         &QUARTERROUND_SSSE3(0, 5,10, 15, 4);
557         &QUARTERROUND_SSSE3(1, 6,11, 12, 5);
558         &QUARTERROUND_SSSE3(2, 7, 8, 13, 6);
559         &QUARTERROUND_SSSE3(3, 4, 9, 14, 7);
560         &dec            ("edx");
561         &jnz            (&label("loop"));
562
563         &movdqa         (&QWP(16*4-128,"ebx"),$xb_);
564         &movdqa         (&QWP(16*8-128,"ebx"),$xc);
565         &movdqa         (&QWP(16*9-128,"ebx"),$xc_);
566         &movdqa         (&QWP(16*12-128,"ebx"),$xd);
567         &movdqa         (&QWP(16*14-128,"ebx"),$xd_);
568
569     my ($xa0,$xa1,$xa2,$xa3,$xt0,$xt1,$xt2,$xt3)=map("xmm$_",(0..7));
570
571         #&movdqa        ($xa0,&QWP(16*0-128,"ebx"));    # it's there
572         &movdqa         ($xa1,&QWP(16*1-128,"ebx"));
573         &movdqa         ($xa2,&QWP(16*2-128,"ebx"));
574         &movdqa         ($xa3,&QWP(16*3-128,"ebx"));
575
576     for($i=0;$i<256;$i+=64) {
577         &paddd          ($xa0,&QWP($i+16*0-128,"ebp")); # accumulate key material
578         &paddd          ($xa1,&QWP($i+16*1-128,"ebp"));
579         &paddd          ($xa2,&QWP($i+16*2-128,"ebp"));
580         &paddd          ($xa3,&QWP($i+16*3-128,"ebp"));
581
582         &movdqa         ($xt2,$xa0);            # "de-interlace" data
583         &punpckldq      ($xa0,$xa1);
584         &movdqa         ($xt3,$xa2);
585         &punpckldq      ($xa2,$xa3);
586         &punpckhdq      ($xt2,$xa1);
587         &punpckhdq      ($xt3,$xa3);
588         &movdqa         ($xa1,$xa0);
589         &punpcklqdq     ($xa0,$xa2);            # "a0"
590         &movdqa         ($xa3,$xt2);
591         &punpcklqdq     ($xt2,$xt3);            # "a2"
592         &punpckhqdq     ($xa1,$xa2);            # "a1"
593         &punpckhqdq     ($xa3,$xt3);            # "a3"
594
595         #($xa2,$xt2)=($xt2,$xa2);
596
597         &movdqu         ($xt0,&QWP(64*0-128,$inp));     # load input
598         &movdqu         ($xt1,&QWP(64*1-128,$inp));
599         &movdqu         ($xa2,&QWP(64*2-128,$inp));
600         &movdqu         ($xt3,&QWP(64*3-128,$inp));
601         &lea            ($inp,&QWP($i<192?16:(64*4-16*3),$inp));
602         &pxor           ($xt0,$xa0);
603         &movdqa         ($xa0,&QWP($i+16*4-128,"ebx"))  if ($i<192);
604         &pxor           ($xt1,$xa1);
605         &movdqa         ($xa1,&QWP($i+16*5-128,"ebx"))  if ($i<192);
606         &pxor           ($xt2,$xa2);
607         &movdqa         ($xa2,&QWP($i+16*6-128,"ebx"))  if ($i<192);
608         &pxor           ($xt3,$xa3);
609         &movdqa         ($xa3,&QWP($i+16*7-128,"ebx"))  if ($i<192);
610         &movdqu         (&QWP(64*0-128,$out),$xt0);     # store output
611         &movdqu         (&QWP(64*1-128,$out),$xt1);
612         &movdqu         (&QWP(64*2-128,$out),$xt2);
613         &movdqu         (&QWP(64*3-128,$out),$xt3);
614         &lea            ($out,&QWP($i<192?16:(64*4-16*3),$out));
615     }
616         &sub            ($len,64*4);
617         &jnc            (&label("outer_loop"));
618
619         &add            ($len,64*4);
620         &jz             (&label("done"));
621
622         &mov            ("ebx",&DWP(512+8,"esp"));      # restore pointers
623         &lea            ($inp,&DWP(-128,$inp));
624         &mov            ("edx",&DWP(512+4,"esp"));
625         &lea            ($out,&DWP(-128,$out));
626
627         &movd           ("xmm2",&DWP(16*12-128,"ebp")); # counter value
628         &movdqu         ("xmm3",&QWP(0,"ebx"));
629         &paddd          ("xmm2",&QWP(16*6,"eax"));      # +four
630         &pand           ("xmm3",&QWP(16*7,"eax"));
631         &por            ("xmm3","xmm2");                # counter value
632 {
633 my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("xmm$_",(0..7));
634
635 sub SSSE3ROUND {        # critical path is 20 "SIMD ticks" per round
636         &paddd          ($a,$b);
637         &pxor           ($d,$a);
638         &pshufb         ($d,$rot16);
639
640         &paddd          ($c,$d);
641         &pxor           ($b,$c);
642         &movdqa         ($t,$b);
643         &psrld          ($b,20);
644         &pslld          ($t,12);
645         &por            ($b,$t);
646
647         &paddd          ($a,$b);
648         &pxor           ($d,$a);
649         &pshufb         ($d,$rot24);
650
651         &paddd          ($c,$d);
652         &pxor           ($b,$c);
653         &movdqa         ($t,$b);
654         &psrld          ($b,25);
655         &pslld          ($t,7);
656         &por            ($b,$t);
657 }
658
659 &set_label("1x");
660         &movdqa         ($a,&QWP(16*2,"eax"));          # sigma
661         &movdqu         ($b,&QWP(0,"edx"));
662         &movdqu         ($c,&QWP(16,"edx"));
663         #&movdqu        ($d,&QWP(0,"ebx"));             # already loaded
664         &movdqa         ($rot16,&QWP(0,"eax"));
665         &movdqa         ($rot24,&QWP(16,"eax"));
666         &mov            (&DWP(16*3,"esp"),"ebp");
667
668         &movdqa         (&QWP(16*0,"esp"),$a);
669         &movdqa         (&QWP(16*1,"esp"),$b);
670         &movdqa         (&QWP(16*2,"esp"),$c);
671         &movdqa         (&QWP(16*3,"esp"),$d);
672         &mov            ("edx",10);
673         &jmp            (&label("loop1x"));
674
675 &set_label("outer1x",16);
676         &movdqa         ($d,&QWP(16*5,"eax"));          # one
677         &movdqa         ($a,&QWP(16*0,"esp"));
678         &movdqa         ($b,&QWP(16*1,"esp"));
679         &movdqa         ($c,&QWP(16*2,"esp"));
680         &paddd          ($d,&QWP(16*3,"esp"));
681         &mov            ("edx",10);
682         &movdqa         (&QWP(16*3,"esp"),$d);
683         &jmp            (&label("loop1x"));
684
685 &set_label("loop1x",16);
686         &SSSE3ROUND();
687         &pshufd ($c,$c,0b01001110);
688         &pshufd ($b,$b,0b00111001);
689         &pshufd ($d,$d,0b10010011);
690         &nop    ();
691
692         &SSSE3ROUND();
693         &pshufd ($c,$c,0b01001110);
694         &pshufd ($b,$b,0b10010011);
695         &pshufd ($d,$d,0b00111001);
696
697         &dec            ("edx");
698         &jnz            (&label("loop1x"));
699
700         &paddd          ($a,&QWP(16*0,"esp"));
701         &paddd          ($b,&QWP(16*1,"esp"));
702         &paddd          ($c,&QWP(16*2,"esp"));
703         &paddd          ($d,&QWP(16*3,"esp"));
704
705         &cmp            ($len,64);
706         &jb             (&label("tail"));
707
708         &movdqu         ($t,&QWP(16*0,$inp));
709         &movdqu         ($t1,&QWP(16*1,$inp));
710         &pxor           ($a,$t);                # xor with input
711         &movdqu         ($t,&QWP(16*2,$inp));
712         &pxor           ($b,$t1);
713         &movdqu         ($t1,&QWP(16*3,$inp));
714         &pxor           ($c,$t);
715         &pxor           ($d,$t1);
716         &lea            ($inp,&DWP(16*4,$inp)); # inp+=64
717
718         &movdqu         (&QWP(16*0,$out),$a);   # write output
719         &movdqu         (&QWP(16*1,$out),$b);
720         &movdqu         (&QWP(16*2,$out),$c);
721         &movdqu         (&QWP(16*3,$out),$d);
722         &lea            ($out,&DWP(16*4,$out)); # inp+=64
723
724         &sub            ($len,64);
725         &jnz            (&label("outer1x"));
726
727         &jmp            (&label("done"));
728
729 &set_label("tail");
730         &movdqa         (&QWP(16*0,"esp"),$a);
731         &movdqa         (&QWP(16*1,"esp"),$b);
732         &movdqa         (&QWP(16*2,"esp"),$c);
733         &movdqa         (&QWP(16*3,"esp"),$d);
734
735         &xor            ("eax","eax");
736         &xor            ("edx","edx");
737         &xor            ("ebp","ebp");
738
739 &set_label("tail_loop");
740         &movb           ("al",&BP(0,"esp","ebp"));
741         &movb           ("dl",&BP(0,$inp,"ebp"));
742         &lea            ("ebp",&DWP(1,"ebp"));
743         &xor            ("al","dl");
744         &movb           (&BP(-1,$out,"ebp"),"al");
745         &dec            ($len);
746         &jnz            (&label("tail_loop"));
747 }
748 &set_label("done");
749         &mov            ("esp",&DWP(512,"esp"));
750 &function_end("ChaCha20_ssse3");
751
752 &align  (64);
753 &set_label("ssse3_data");
754 &data_byte(0x2,0x3,0x0,0x1, 0x6,0x7,0x4,0x5, 0xa,0xb,0x8,0x9, 0xe,0xf,0xc,0xd);
755 &data_byte(0x3,0x0,0x1,0x2, 0x7,0x4,0x5,0x6, 0xb,0x8,0x9,0xa, 0xf,0xc,0xd,0xe);
756 &data_word(0x61707865,0x3320646e,0x79622d32,0x6b206574);
757 &data_word(0,1,2,3);
758 &data_word(4,4,4,4);
759 &data_word(1,0,0,0);
760 &data_word(4,0,0,0);
761 &data_word(0,-1,-1,-1);
762 &align  (64);
763 }
764 &asciz  ("ChaCha20 for x86, CRYPTOGAMS by <appro\@openssl.org>");
765
766 if ($xmm) {
767 my ($xa,$xa_,$xb,$xb_,$xc,$xc_,$xd,$xd_)=map("xmm$_",(0..7));
768 my ($out,$inp,$len)=("edi","esi","ecx");
769
770 sub QUARTERROUND_XOP {
771 my ($ai,$bi,$ci,$di,$i)=@_;
772 my ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+1)&3),($ai,$bi,$ci,$di)); # next
773 my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
774
775         #       a   b   c   d
776         #
777         #       0   4   8  12 < even round
778         #       1   5   9  13
779         #       2   6  10  14
780         #       3   7  11  15
781         #       0   5  10  15 < odd round
782         #       1   6  11  12
783         #       2   7   8  13
784         #       3   4   9  14
785
786         if ($i==0) {
787             my $j=4;
788             ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-$j--)&3),($ap,$bp,$cp,$dp));
789         } elsif ($i==3) {
790             my $j=0;
791             ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+$j++)&3),($an,$bn,$cn,$dn));
792         } elsif ($i==4) {
793             my $j=4;
794             ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_+$j--)&3),($ap,$bp,$cp,$dp));
795         } elsif ($i==7) {
796             my $j=0;
797             ($an,$bn,$cn,$dn)=map(($_&~3)+(($_-$j++)&3),($an,$bn,$cn,$dn));
798         }
799
800         #&vpaddd        ($xa,$xa,$xb);                  # see elsewhere
801         #&vpxor         ($xd,$xd,$xa);                  # see elsewhere
802          &vmovdqa       (&QWP(16*$cp-128,"ebx"),$xc_)   if ($ai>0 && $ai<3);
803         &vprotd         ($xd,$xd,16);
804          &vmovdqa       (&QWP(16*$bp-128,"ebx"),$xb_)   if ($i!=0);
805         &vpaddd         ($xc,$xc,$xd);
806          &vmovdqa       ($xc_,&QWP(16*$cn-128,"ebx"))   if ($ai>0 && $ai<3);
807         &vpxor          ($xb,$i!=0?$xb:$xb_,$xc);
808          &vmovdqa       ($xa_,&QWP(16*$an-128,"ebx"));
809         &vprotd         ($xb,$xb,12);
810          &vmovdqa       ($xb_,&QWP(16*$bn-128,"ebx"))   if ($i<7);
811         &vpaddd         ($xa,$xa,$xb);
812          &vmovdqa       ($xd_,&QWP(16*$dn-128,"ebx"))   if ($di!=$dn);
813         &vpxor          ($xd,$xd,$xa);
814          &vpaddd        ($xa_,$xa_,$xb_)                if ($i<7);      # elsewhere
815         &vprotd         ($xd,$xd,8);
816         &vmovdqa        (&QWP(16*$ai-128,"ebx"),$xa);
817         &vpaddd         ($xc,$xc,$xd);
818         &vmovdqa        (&QWP(16*$di-128,"ebx"),$xd)    if ($di!=$dn);
819         &vpxor          ($xb,$xb,$xc);
820          &vpxor         ($xd_,$di==$dn?$xd:$xd_,$xa_)   if ($i<7);      # elsewhere
821         &vprotd         ($xb,$xb,7);
822
823         ($xa,$xa_)=($xa_,$xa);
824         ($xb,$xb_)=($xb_,$xb);
825         ($xc,$xc_)=($xc_,$xc);
826         ($xd,$xd_)=($xd_,$xd);
827 }
828
829 &function_begin("ChaCha20_xop");
830 &set_label("xop_shortcut");
831         &mov            ($out,&wparam(0));
832         &mov            ($inp,&wparam(1));
833         &mov            ($len,&wparam(2));
834         &mov            ("edx",&wparam(3));             # key
835         &mov            ("ebx",&wparam(4));             # counter and nonce
836         &vzeroupper     ();
837
838         &mov            ("ebp","esp");
839         &stack_push     (131);
840         &and            ("esp",-64);
841         &mov            (&DWP(512,"esp"),"ebp");
842
843         &lea            ("eax",&DWP(&label("ssse3_data")."-".
844                                     &label("pic_point"),"eax"));
845         &vmovdqu        ("xmm3",&QWP(0,"ebx"));         # counter and nonce
846
847         &cmp            ($len,64*4);
848         &jb             (&label("1x"));
849
850         &mov            (&DWP(512+4,"esp"),"edx");      # offload pointers
851         &mov            (&DWP(512+8,"esp"),"ebx");
852         &sub            ($len,64*4);                    # bias len
853         &lea            ("ebp",&DWP(256+128,"esp"));    # size optimization
854
855         &vmovdqu        ("xmm7",&QWP(0,"edx"));         # key
856         &vpshufd        ("xmm0","xmm3",0x00);
857         &vpshufd        ("xmm1","xmm3",0x55);
858         &vpshufd        ("xmm2","xmm3",0xaa);
859         &vpshufd        ("xmm3","xmm3",0xff);
860          &vpaddd        ("xmm0","xmm0",&QWP(16*3,"eax"));       # fix counters
861         &vpshufd        ("xmm4","xmm7",0x00);
862         &vpshufd        ("xmm5","xmm7",0x55);
863          &vpsubd        ("xmm0","xmm0",&QWP(16*4,"eax"));
864         &vpshufd        ("xmm6","xmm7",0xaa);
865         &vpshufd        ("xmm7","xmm7",0xff);
866         &vmovdqa        (&QWP(16*12-128,"ebp"),"xmm0");
867         &vmovdqa        (&QWP(16*13-128,"ebp"),"xmm1");
868         &vmovdqa        (&QWP(16*14-128,"ebp"),"xmm2");
869         &vmovdqa        (&QWP(16*15-128,"ebp"),"xmm3");
870          &vmovdqu       ("xmm3",&QWP(16,"edx"));        # key
871         &vmovdqa        (&QWP(16*4-128,"ebp"),"xmm4");
872         &vmovdqa        (&QWP(16*5-128,"ebp"),"xmm5");
873         &vmovdqa        (&QWP(16*6-128,"ebp"),"xmm6");
874         &vmovdqa        (&QWP(16*7-128,"ebp"),"xmm7");
875          &vmovdqa       ("xmm7",&QWP(16*2,"eax"));      # sigma
876          &lea           ("ebx",&DWP(128,"esp"));        # size optimization
877
878         &vpshufd        ("xmm0","xmm3",0x00);
879         &vpshufd        ("xmm1","xmm3",0x55);
880         &vpshufd        ("xmm2","xmm3",0xaa);
881         &vpshufd        ("xmm3","xmm3",0xff);
882         &vpshufd        ("xmm4","xmm7",0x00);
883         &vpshufd        ("xmm5","xmm7",0x55);
884         &vpshufd        ("xmm6","xmm7",0xaa);
885         &vpshufd        ("xmm7","xmm7",0xff);
886         &vmovdqa        (&QWP(16*8-128,"ebp"),"xmm0");
887         &vmovdqa        (&QWP(16*9-128,"ebp"),"xmm1");
888         &vmovdqa        (&QWP(16*10-128,"ebp"),"xmm2");
889         &vmovdqa        (&QWP(16*11-128,"ebp"),"xmm3");
890         &vmovdqa        (&QWP(16*0-128,"ebp"),"xmm4");
891         &vmovdqa        (&QWP(16*1-128,"ebp"),"xmm5");
892         &vmovdqa        (&QWP(16*2-128,"ebp"),"xmm6");
893         &vmovdqa        (&QWP(16*3-128,"ebp"),"xmm7");
894
895         &lea            ($inp,&DWP(128,$inp));          # size optimization
896         &lea            ($out,&DWP(128,$out));          # size optimization
897         &jmp            (&label("outer_loop"));
898
899 &set_label("outer_loop",32);
900         #&vmovdqa       ("xmm0",&QWP(16*0-128,"ebp"));  # copy key material
901         &vmovdqa        ("xmm1",&QWP(16*1-128,"ebp"));
902         &vmovdqa        ("xmm2",&QWP(16*2-128,"ebp"));
903         &vmovdqa        ("xmm3",&QWP(16*3-128,"ebp"));
904         #&vmovdqa       ("xmm4",&QWP(16*4-128,"ebp"));
905         &vmovdqa        ("xmm5",&QWP(16*5-128,"ebp"));
906         &vmovdqa        ("xmm6",&QWP(16*6-128,"ebp"));
907         &vmovdqa        ("xmm7",&QWP(16*7-128,"ebp"));
908         #&vmovdqa       (&QWP(16*0-128,"ebx"),"xmm0");
909         &vmovdqa        (&QWP(16*1-128,"ebx"),"xmm1");
910         &vmovdqa        (&QWP(16*2-128,"ebx"),"xmm2");
911         &vmovdqa        (&QWP(16*3-128,"ebx"),"xmm3");
912         #&vmovdqa       (&QWP(16*4-128,"ebx"),"xmm4");
913         &vmovdqa        (&QWP(16*5-128,"ebx"),"xmm5");
914         &vmovdqa        (&QWP(16*6-128,"ebx"),"xmm6");
915         &vmovdqa        (&QWP(16*7-128,"ebx"),"xmm7");
916         #&vmovdqa       ("xmm0",&QWP(16*8-128,"ebp"));
917         #&vmovdqa       ("xmm1",&QWP(16*9-128,"ebp"));
918         &vmovdqa        ("xmm2",&QWP(16*10-128,"ebp"));
919         &vmovdqa        ("xmm3",&QWP(16*11-128,"ebp"));
920         &vmovdqa        ("xmm4",&QWP(16*12-128,"ebp"));
921         &vmovdqa        ("xmm5",&QWP(16*13-128,"ebp"));
922         &vmovdqa        ("xmm6",&QWP(16*14-128,"ebp"));
923         &vmovdqa        ("xmm7",&QWP(16*15-128,"ebp"));
924         &vpaddd         ("xmm4","xmm4",&QWP(16*4,"eax"));       # counter value
925         #&vmovdqa       (&QWP(16*8-128,"ebx"),"xmm0");
926         #&vmovdqa       (&QWP(16*9-128,"ebx"),"xmm1");
927         &vmovdqa        (&QWP(16*10-128,"ebx"),"xmm2");
928         &vmovdqa        (&QWP(16*11-128,"ebx"),"xmm3");
929         &vmovdqa        (&QWP(16*12-128,"ebx"),"xmm4");
930         &vmovdqa        (&QWP(16*13-128,"ebx"),"xmm5");
931         &vmovdqa        (&QWP(16*14-128,"ebx"),"xmm6");
932         &vmovdqa        (&QWP(16*15-128,"ebx"),"xmm7");
933         &vmovdqa        (&QWP(16*12-128,"ebp"),"xmm4"); # save counter value
934
935         &vmovdqa        ($xa, &QWP(16*0-128,"ebp"));
936         &vmovdqa        ($xd, "xmm4");
937         &vmovdqa        ($xb_,&QWP(16*4-128,"ebp"));
938         &vmovdqa        ($xc, &QWP(16*8-128,"ebp"));
939         &vmovdqa        ($xc_,&QWP(16*9-128,"ebp"));
940
941         &mov            ("edx",10);                     # loop counter
942         &nop            ();
943
944 &set_label("loop",32);
945         &vpaddd         ($xa,$xa,$xb_);                 # elsewhere
946         &vpxor          ($xd,$xd,$xa);                  # elsewhere
947         &QUARTERROUND_XOP(0, 4, 8, 12, 0);
948         &QUARTERROUND_XOP(1, 5, 9, 13, 1);
949         &QUARTERROUND_XOP(2, 6,10, 14, 2);
950         &QUARTERROUND_XOP(3, 7,11, 15, 3);
951         &QUARTERROUND_XOP(0, 5,10, 15, 4);
952         &QUARTERROUND_XOP(1, 6,11, 12, 5);
953         &QUARTERROUND_XOP(2, 7, 8, 13, 6);
954         &QUARTERROUND_XOP(3, 4, 9, 14, 7);
955         &dec            ("edx");
956         &jnz            (&label("loop"));
957
958         &vmovdqa        (&QWP(16*4-128,"ebx"),$xb_);
959         &vmovdqa        (&QWP(16*8-128,"ebx"),$xc);
960         &vmovdqa        (&QWP(16*9-128,"ebx"),$xc_);
961         &vmovdqa        (&QWP(16*12-128,"ebx"),$xd);
962         &vmovdqa        (&QWP(16*14-128,"ebx"),$xd_);
963
964     my ($xa0,$xa1,$xa2,$xa3,$xt0,$xt1,$xt2,$xt3)=map("xmm$_",(0..7));
965
966         #&vmovdqa       ($xa0,&QWP(16*0-128,"ebx"));    # it's there
967         &vmovdqa        ($xa1,&QWP(16*1-128,"ebx"));
968         &vmovdqa        ($xa2,&QWP(16*2-128,"ebx"));
969         &vmovdqa        ($xa3,&QWP(16*3-128,"ebx"));
970
971     for($i=0;$i<256;$i+=64) {
972         &vpaddd         ($xa0,$xa0,&QWP($i+16*0-128,"ebp"));    # accumulate key material
973         &vpaddd         ($xa1,$xa1,&QWP($i+16*1-128,"ebp"));
974         &vpaddd         ($xa2,$xa2,&QWP($i+16*2-128,"ebp"));
975         &vpaddd         ($xa3,$xa3,&QWP($i+16*3-128,"ebp"));
976
977         &vpunpckldq     ($xt2,$xa0,$xa1);       # "de-interlace" data
978         &vpunpckldq     ($xt3,$xa2,$xa3);
979         &vpunpckhdq     ($xa0,$xa0,$xa1);
980         &vpunpckhdq     ($xa2,$xa2,$xa3);
981         &vpunpcklqdq    ($xa1,$xt2,$xt3);       # "a0"
982         &vpunpckhqdq    ($xt2,$xt2,$xt3);       # "a1"
983         &vpunpcklqdq    ($xt3,$xa0,$xa2);       # "a2"
984         &vpunpckhqdq    ($xa3,$xa0,$xa2);       # "a3"
985
986         &vpxor          ($xt0,$xa1,&QWP(64*0-128,$inp));
987         &vpxor          ($xt1,$xt2,&QWP(64*1-128,$inp));
988         &vpxor          ($xt2,$xt3,&QWP(64*2-128,$inp));
989         &vpxor          ($xt3,$xa3,&QWP(64*3-128,$inp));
990         &lea            ($inp,&QWP($i<192?16:(64*4-16*3),$inp));
991         &vmovdqa        ($xa0,&QWP($i+16*4-128,"ebx"))  if ($i<192);
992         &vmovdqa        ($xa1,&QWP($i+16*5-128,"ebx"))  if ($i<192);
993         &vmovdqa        ($xa2,&QWP($i+16*6-128,"ebx"))  if ($i<192);
994         &vmovdqa        ($xa3,&QWP($i+16*7-128,"ebx"))  if ($i<192);
995         &vmovdqu        (&QWP(64*0-128,$out),$xt0);     # store output
996         &vmovdqu        (&QWP(64*1-128,$out),$xt1);
997         &vmovdqu        (&QWP(64*2-128,$out),$xt2);
998         &vmovdqu        (&QWP(64*3-128,$out),$xt3);
999         &lea            ($out,&QWP($i<192?16:(64*4-16*3),$out));
1000     }
1001         &sub            ($len,64*4);
1002         &jnc            (&label("outer_loop"));
1003
1004         &add            ($len,64*4);
1005         &jz             (&label("done"));
1006
1007         &mov            ("ebx",&DWP(512+8,"esp"));      # restore pointers
1008         &lea            ($inp,&DWP(-128,$inp));
1009         &mov            ("edx",&DWP(512+4,"esp"));
1010         &lea            ($out,&DWP(-128,$out));
1011
1012         &vmovd          ("xmm2",&DWP(16*12-128,"ebp")); # counter value
1013         &vmovdqu        ("xmm3",&QWP(0,"ebx"));
1014         &vpaddd         ("xmm2","xmm2",&QWP(16*6,"eax"));# +four
1015         &vpand          ("xmm3","xmm3",&QWP(16*7,"eax"));
1016         &vpor           ("xmm3","xmm3","xmm2");         # counter value
1017 {
1018 my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("xmm$_",(0..7));
1019
1020 sub XOPROUND {
1021         &vpaddd         ($a,$a,$b);
1022         &vpxor          ($d,$d,$a);
1023         &vprotd         ($d,$d,16);
1024
1025         &vpaddd         ($c,$c,$d);
1026         &vpxor          ($b,$b,$c);
1027         &vprotd         ($b,$b,12);
1028
1029         &vpaddd         ($a,$a,$b);
1030         &vpxor          ($d,$d,$a);
1031         &vprotd         ($d,$d,8);
1032
1033         &vpaddd         ($c,$c,$d);
1034         &vpxor          ($b,$b,$c);
1035         &vprotd         ($b,$b,7);
1036 }
1037
1038 &set_label("1x");
1039         &vmovdqa        ($a,&QWP(16*2,"eax"));          # sigma
1040         &vmovdqu        ($b,&QWP(0,"edx"));
1041         &vmovdqu        ($c,&QWP(16,"edx"));
1042         #&vmovdqu       ($d,&QWP(0,"ebx"));             # already loaded
1043         &vmovdqa        ($rot16,&QWP(0,"eax"));
1044         &vmovdqa        ($rot24,&QWP(16,"eax"));
1045         &mov            (&DWP(16*3,"esp"),"ebp");
1046
1047         &vmovdqa        (&QWP(16*0,"esp"),$a);
1048         &vmovdqa        (&QWP(16*1,"esp"),$b);
1049         &vmovdqa        (&QWP(16*2,"esp"),$c);
1050         &vmovdqa        (&QWP(16*3,"esp"),$d);
1051         &mov            ("edx",10);
1052         &jmp            (&label("loop1x"));
1053
1054 &set_label("outer1x",16);
1055         &vmovdqa        ($d,&QWP(16*5,"eax"));          # one
1056         &vmovdqa        ($a,&QWP(16*0,"esp"));
1057         &vmovdqa        ($b,&QWP(16*1,"esp"));
1058         &vmovdqa        ($c,&QWP(16*2,"esp"));
1059         &vpaddd         ($d,$d,&QWP(16*3,"esp"));
1060         &mov            ("edx",10);
1061         &vmovdqa        (&QWP(16*3,"esp"),$d);
1062         &jmp            (&label("loop1x"));
1063
1064 &set_label("loop1x",16);
1065         &XOPROUND();
1066         &vpshufd        ($c,$c,0b01001110);
1067         &vpshufd        ($b,$b,0b00111001);
1068         &vpshufd        ($d,$d,0b10010011);
1069
1070         &XOPROUND();
1071         &vpshufd        ($c,$c,0b01001110);
1072         &vpshufd        ($b,$b,0b10010011);
1073         &vpshufd        ($d,$d,0b00111001);
1074
1075         &dec            ("edx");
1076         &jnz            (&label("loop1x"));
1077
1078         &vpaddd         ($a,$a,&QWP(16*0,"esp"));
1079         &vpaddd         ($b,$b,&QWP(16*1,"esp"));
1080         &vpaddd         ($c,$c,&QWP(16*2,"esp"));
1081         &vpaddd         ($d,$d,&QWP(16*3,"esp"));
1082
1083         &cmp            ($len,64);
1084         &jb             (&label("tail"));
1085
1086         &vpxor          ($a,$a,&QWP(16*0,$inp));        # xor with input
1087         &vpxor          ($b,$b,&QWP(16*1,$inp));
1088         &vpxor          ($c,$c,&QWP(16*2,$inp));
1089         &vpxor          ($d,$d,&QWP(16*3,$inp));
1090         &lea            ($inp,&DWP(16*4,$inp));         # inp+=64
1091
1092         &vmovdqu        (&QWP(16*0,$out),$a);           # write output
1093         &vmovdqu        (&QWP(16*1,$out),$b);
1094         &vmovdqu        (&QWP(16*2,$out),$c);
1095         &vmovdqu        (&QWP(16*3,$out),$d);
1096         &lea            ($out,&DWP(16*4,$out));         # inp+=64
1097
1098         &sub            ($len,64);
1099         &jnz            (&label("outer1x"));
1100
1101         &jmp            (&label("done"));
1102
1103 &set_label("tail");
1104         &vmovdqa        (&QWP(16*0,"esp"),$a);
1105         &vmovdqa        (&QWP(16*1,"esp"),$b);
1106         &vmovdqa        (&QWP(16*2,"esp"),$c);
1107         &vmovdqa        (&QWP(16*3,"esp"),$d);
1108
1109         &xor            ("eax","eax");
1110         &xor            ("edx","edx");
1111         &xor            ("ebp","ebp");
1112
1113 &set_label("tail_loop");
1114         &movb           ("al",&BP(0,"esp","ebp"));
1115         &movb           ("dl",&BP(0,$inp,"ebp"));
1116         &lea            ("ebp",&DWP(1,"ebp"));
1117         &xor            ("al","dl");
1118         &movb           (&BP(-1,$out,"ebp"),"al");
1119         &dec            ($len);
1120         &jnz            (&label("tail_loop"));
1121 }
1122 &set_label("done");
1123         &vzeroupper     ();
1124         &mov            ("esp",&DWP(512,"esp"));
1125 &function_end("ChaCha20_xop");
1126 }
1127
1128 &asm_finish();