3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
14 # Performance in cycles per byte out of large buffer.
20 # Core2 9.56/+89% 4.83
21 # Westmere 9.50/+45% 3.35
22 # Sandy Bridge 10.5/+47% 3.20
23 # Haswell 8.15/+50% 2.83
24 # Silvermont 17.4/+36% 8.35
25 # Sledgehammer 10.2/+54%
26 # Bulldozer 13.4/+50% 4.38(*)
28 # (*) Bulldozer actually executes 4xXOP code path that delivers 3.55;
30 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
31 push(@INC,"${dir}","${dir}../../perlasm");
34 &asm_init($ARGV[0],"chacha-x86.pl",$ARGV[$#ARGV] eq "386");
37 for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); }
40 `$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
41 =~ /GNU assembler version ([2-9]\.[0-9]+)/ &&
42 $1>=2.19); # first version supporting AVX
44 $ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32n" &&
45 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/ &&
46 $1>=2.03); # first version supporting AVX
48 $ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32" &&
49 `ml 2>&1` =~ /Version ([0-9]+)\./ &&
50 $1>=10); # first version supporting AVX
52 $ymm=1 if ($xmm && !$ymm &&
53 `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/ &&
54 $2>=3.0); # first version supporting AVX
57 ($b,$b_)=("ebx","ebp");
58 ($c,$c_)=("ecx","esi");
59 ($d,$d_)=("edx","edi");
62 my ($ai,$bi,$ci,$di,$i)=@_;
63 my ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+1)&3),($ai,$bi,$ci,$di)); # next
64 my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
68 # 0 4 8 12 < even round
72 # 0 5 10 15 < odd round
79 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-$j--)&3),($ap,$bp,$cp,$dp));
82 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+$j++)&3),($an,$bn,$cn,$dn));
85 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_+$j--)&3),($ap,$bp,$cp,$dp));
88 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_-$j++)&3),($an,$bn,$cn,$dn));
91 #&add ($a,$b); # see elsewhere
93 &mov (&DWP(4*$cp,"esp"),$c_) if ($ai>0 && $ai<3);
95 &mov (&DWP(4*$bp,"esp"),$b_) if ($i!=0);
97 &mov ($c_,&DWP(4*$cn,"esp")) if ($ai>0 && $ai<3);
99 &mov ($d_,&DWP(4*$dn,"esp")) if ($di!=$dn);
101 &mov ($b_,&DWP(4*$bn,"esp")) if ($i<7);
102 &mov ($b_,&DWP(128,"esp")) if ($i==7); # loop counter
105 &mov (&DWP(4*$ai,"esp"),$a);
107 &mov ($a,&DWP(4*$an,"esp"));
109 &mov (&DWP(4*$di,"esp"),$d) if ($di!=$dn);
110 &mov ($d_,$d) if ($di==$dn);
112 &add ($a,$b_) if ($i<7); # elsewhere
120 &static_label("ssse3_shortcut");
121 &static_label("xop_shortcut");
122 &static_label("ssse3_data");
123 &static_label("pic_point");
125 &function_begin("ChaCha20_ctr32");
127 &call (&label("pic_point"));
128 &set_label("pic_point");
130 &picmeup("ebp","OPENSSL_ia32cap_P","eax",&label("pic_point"));
131 &test (&DWP(0,"ebp"),1<<24); # test FXSR bit
133 &test (&DWP(4,"ebp"),1<<9); # test SSSE3 bit
135 &jmp (&label("ssse3_shortcut"));
138 &mov ("esi",&wparam(3)); # key
139 &mov ("edi",&wparam(4)); # counter and nonce
143 &mov ("eax",&DWP(4*0,"esi")); # copy key
144 &mov ("ebx",&DWP(4*1,"esi"));
145 &mov ("ecx",&DWP(4*2,"esi"));
146 &mov ("edx",&DWP(4*3,"esi"));
147 &mov (&DWP(64+4*4,"esp"),"eax");
148 &mov (&DWP(64+4*5,"esp"),"ebx");
149 &mov (&DWP(64+4*6,"esp"),"ecx");
150 &mov (&DWP(64+4*7,"esp"),"edx");
151 &mov ("eax",&DWP(4*4,"esi"));
152 &mov ("ebx",&DWP(4*5,"esi"));
153 &mov ("ecx",&DWP(4*6,"esi"));
154 &mov ("edx",&DWP(4*7,"esi"));
155 &mov (&DWP(64+4*8,"esp"),"eax");
156 &mov (&DWP(64+4*9,"esp"),"ebx");
157 &mov (&DWP(64+4*10,"esp"),"ecx");
158 &mov (&DWP(64+4*11,"esp"),"edx");
159 &mov ("eax",&DWP(4*0,"edi")); # copy counter and nonce
160 &mov ("ebx",&DWP(4*1,"edi"));
161 &mov ("ecx",&DWP(4*2,"edi"));
162 &mov ("edx",&DWP(4*3,"edi"));
164 &mov (&DWP(64+4*12,"esp"),"eax");
165 &mov (&DWP(64+4*13,"esp"),"ebx");
166 &mov (&DWP(64+4*14,"esp"),"ecx");
167 &mov (&DWP(64+4*15,"esp"),"edx");
168 &jmp (&label("entry"));
170 &set_label("outer_loop",16);
171 &mov (&wparam(1),$b); # save input
172 &mov (&wparam(0),$a); # save output
173 &mov (&wparam(2),$c); # save len
175 &mov ($a,0x61707865);
176 &mov (&DWP(4*1,"esp"),0x3320646e);
177 &mov (&DWP(4*2,"esp"),0x79622d32);
178 &mov (&DWP(4*3,"esp"),0x6b206574);
180 &mov ($b, &DWP(64+4*5,"esp")); # copy key material
181 &mov ($b_,&DWP(64+4*6,"esp"));
182 &mov ($c, &DWP(64+4*10,"esp"));
183 &mov ($c_,&DWP(64+4*11,"esp"));
184 &mov ($d, &DWP(64+4*13,"esp"));
185 &mov ($d_,&DWP(64+4*14,"esp"));
186 &mov (&DWP(4*5,"esp"),$b);
187 &mov (&DWP(4*6,"esp"),$b_);
188 &mov (&DWP(4*10,"esp"),$c);
189 &mov (&DWP(4*11,"esp"),$c_);
190 &mov (&DWP(4*13,"esp"),$d);
191 &mov (&DWP(4*14,"esp"),$d_);
193 &mov ($b, &DWP(64+4*7,"esp"));
194 &mov ($d_,&DWP(64+4*15,"esp"));
195 &mov ($d, &DWP(64+4*12,"esp"));
196 &mov ($b_,&DWP(64+4*4,"esp"));
197 &mov ($c, &DWP(64+4*8,"esp"));
198 &mov ($c_,&DWP(64+4*9,"esp"));
199 &add ($d,1); # counter value
200 &mov (&DWP(4*7,"esp"),$b);
201 &mov (&DWP(4*15,"esp"),$d_);
202 &mov (&DWP(64+4*12,"esp"),$d); # save counter value
204 &mov ($b,10); # loop counter
205 &jmp (&label("loop"));
207 &set_label("loop",16);
208 &add ($a,$b_); # elsewhere
209 &mov (&DWP(128,"esp"),$b); # save loop counter
211 &QUARTERROUND(0, 4, 8, 12, 0);
212 &QUARTERROUND(1, 5, 9, 13, 1);
213 &QUARTERROUND(2, 6,10, 14, 2);
214 &QUARTERROUND(3, 7,11, 15, 3);
215 &QUARTERROUND(0, 5,10, 15, 4);
216 &QUARTERROUND(1, 6,11, 12, 5);
217 &QUARTERROUND(2, 7, 8, 13, 6);
218 &QUARTERROUND(3, 4, 9, 14, 7);
220 &jnz (&label("loop"));
222 &mov ($b,&wparam(3)); # load len
224 &add ($a,0x61707865); # accumulate key material
225 &add ($b_,&DWP(64+4*4,"esp"));
226 &add ($c, &DWP(64+4*8,"esp"));
227 &add ($c_,&DWP(64+4*9,"esp"));
230 &jb (&label("tail"));
232 &mov ($b,&wparam(1)); # load input pointer
233 &add ($d, &DWP(64+4*12,"esp"));
234 &add ($d_,&DWP(64+4*14,"esp"));
236 &xor ($a, &DWP(4*0,$b)); # xor with input
237 &xor ($b_,&DWP(4*4,$b));
238 &mov (&DWP(4*0,"esp"),$a);
239 &mov ($a,&wparam(0)); # load output pointer
240 &xor ($c, &DWP(4*8,$b));
241 &xor ($c_,&DWP(4*9,$b));
242 &xor ($d, &DWP(4*12,$b));
243 &xor ($d_,&DWP(4*14,$b));
244 &mov (&DWP(4*4,$a),$b_); # write output
245 &mov (&DWP(4*8,$a),$c);
246 &mov (&DWP(4*9,$a),$c_);
247 &mov (&DWP(4*12,$a),$d);
248 &mov (&DWP(4*14,$a),$d_);
250 &mov ($b_,&DWP(4*1,"esp"));
251 &mov ($c, &DWP(4*2,"esp"));
252 &mov ($c_,&DWP(4*3,"esp"));
253 &mov ($d, &DWP(4*5,"esp"));
254 &mov ($d_,&DWP(4*6,"esp"));
255 &add ($b_,0x3320646e); # accumulate key material
256 &add ($c, 0x79622d32);
257 &add ($c_,0x6b206574);
258 &add ($d, &DWP(64+4*5,"esp"));
259 &add ($d_,&DWP(64+4*6,"esp"));
260 &xor ($b_,&DWP(4*1,$b));
261 &xor ($c, &DWP(4*2,$b));
262 &xor ($c_,&DWP(4*3,$b));
263 &xor ($d, &DWP(4*5,$b));
264 &xor ($d_,&DWP(4*6,$b));
265 &mov (&DWP(4*1,$a),$b_);
266 &mov (&DWP(4*2,$a),$c);
267 &mov (&DWP(4*3,$a),$c_);
268 &mov (&DWP(4*5,$a),$d);
269 &mov (&DWP(4*6,$a),$d_);
271 &mov ($b_,&DWP(4*7,"esp"));
272 &mov ($c, &DWP(4*10,"esp"));
273 &mov ($c_,&DWP(4*11,"esp"));
274 &mov ($d, &DWP(4*13,"esp"));
275 &mov ($d_,&DWP(4*15,"esp"));
276 &add ($b_,&DWP(64+4*7,"esp"));
277 &add ($c, &DWP(64+4*10,"esp"));
278 &add ($c_,&DWP(64+4*11,"esp"));
279 &add ($d, &DWP(64+4*13,"esp"));
280 &add ($d_,&DWP(64+4*15,"esp"));
281 &xor ($b_,&DWP(4*7,$b));
282 &xor ($c, &DWP(4*10,$b));
283 &xor ($c_,&DWP(4*11,$b));
284 &xor ($d, &DWP(4*13,$b));
285 &xor ($d_,&DWP(4*15,$b));
286 &lea ($b,&DWP(4*16,$b));
287 &mov (&DWP(4*7,$a),$b_);
288 &mov ($b_,&DWP(4*0,"esp"));
289 &mov (&DWP(4*10,$a),$c);
290 &mov ($c,&wparam(2)); # len
291 &mov (&DWP(4*11,$a),$c_);
292 &mov (&DWP(4*13,$a),$d);
293 &mov (&DWP(4*15,$a),$d_);
294 &mov (&DWP(4*0,$a),$b_);
295 &lea ($a,&DWP(4*16,$a));
297 &jnz (&label("outer_loop"));
299 &jmp (&label("done"));
302 &add ($d, &DWP(64+4*12,"esp"));
303 &add ($d_,&DWP(64+4*14,"esp"));
304 &mov (&DWP(4*0,"esp"),$a);
305 &mov (&DWP(4*4,"esp"),$b_);
306 &mov (&DWP(4*8,"esp"),$c);
307 &mov (&DWP(4*9,"esp"),$c_);
308 &mov (&DWP(4*12,"esp"),$d);
309 &mov (&DWP(4*14,"esp"),$d_);
311 &mov ($b_,&DWP(4*1,"esp"));
312 &mov ($c, &DWP(4*2,"esp"));
313 &mov ($c_,&DWP(4*3,"esp"));
314 &mov ($d, &DWP(4*5,"esp"));
315 &mov ($d_,&DWP(4*6,"esp"));
316 &add ($b_,0x3320646e); # accumulate key material
317 &add ($c, 0x79622d32);
318 &add ($c_,0x6b206574);
319 &add ($d, &DWP(64+4*5,"esp"));
320 &add ($d_,&DWP(64+4*6,"esp"));
321 &mov (&DWP(4*1,"esp"),$b_);
322 &mov (&DWP(4*2,"esp"),$c);
323 &mov (&DWP(4*3,"esp"),$c_);
324 &mov (&DWP(4*5,"esp"),$d);
325 &mov (&DWP(4*6,"esp"),$d_);
327 &mov ($b_,&DWP(4*7,"esp"));
328 &mov ($c, &DWP(4*10,"esp"));
329 &mov ($c_,&DWP(4*11,"esp"));
330 &mov ($d, &DWP(4*13,"esp"));
331 &mov ($d_,&DWP(4*15,"esp"));
332 &add ($b_,&DWP(64+4*7,"esp"));
333 &add ($c, &DWP(64+4*10,"esp"));
334 &add ($c_,&DWP(64+4*11,"esp"));
335 &add ($d, &DWP(64+4*13,"esp"));
336 &add ($d_,&DWP(64+4*15,"esp"));
337 &mov (&DWP(4*7,"esp"),$b_);
338 &mov ($b_,&wparam(1)); # load input
339 &mov (&DWP(4*10,"esp"),$c);
340 &mov ($c,&wparam(0)); # load output
341 &mov (&DWP(4*11,"esp"),$c_);
343 &mov (&DWP(4*13,"esp"),$d);
344 &mov (&DWP(4*15,"esp"),$d_);
348 &set_label("tail_loop");
349 &movb ("al",&BP(0,$c_,$b_));
350 &movb ("dl",&BP(0,"esp",$c_));
351 &lea ($c_,&DWP(1,$c_));
353 &mov (&BP(-1,$c,$c_),"al");
355 &jnz (&label("tail_loop"));
359 &function_end("ChaCha20_ctr32");
362 my ($xa,$xa_,$xb,$xb_,$xc,$xc_,$xd,$xd_)=map("xmm$_",(0..7));
363 my ($out,$inp,$len)=("edi","esi","ecx");
365 sub QUARTERROUND_SSSE3 {
366 my ($ai,$bi,$ci,$di,$i)=@_;
367 my ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+1)&3),($ai,$bi,$ci,$di)); # next
368 my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
372 # 0 4 8 12 < even round
376 # 0 5 10 15 < odd round
383 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-$j--)&3),($ap,$bp,$cp,$dp));
386 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+$j++)&3),($an,$bn,$cn,$dn));
389 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_+$j--)&3),($ap,$bp,$cp,$dp));
392 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_-$j++)&3),($an,$bn,$cn,$dn));
395 #&paddd ($xa,$xb); # see elsewhere
396 #&pxor ($xd,$xa); # see elsewhere
397 &movdqa(&QWP(16*$cp-128,"ebx"),$xc_) if ($ai>0 && $ai<3);
398 &pshufb ($xd,&QWP(0,"eax")); # rot16
399 &movdqa(&QWP(16*$bp-128,"ebx"),$xb_) if ($i!=0);
401 &movdqa($xc_,&QWP(16*$cn-128,"ebx")) if ($ai>0 && $ai<3);
403 &movdqa($xb_,&QWP(16*$bn-128,"ebx")) if ($i<7);
404 &movdqa ($xa_,$xb); # borrow as temporary
408 &movdqa($xa_,&QWP(16*$an-128,"ebx"));
410 &movdqa($xd_,&QWP(16*$dn-128,"ebx")) if ($di!=$dn);
412 &movdqa (&QWP(16*$ai-128,"ebx"),$xa);
413 &pshufb ($xd,&QWP(16,"eax")); # rot8
415 &movdqa (&QWP(16*$di-128,"ebx"),$xd) if ($di!=$dn);
416 &movdqa ($xd_,$xd) if ($di==$dn);
418 &paddd ($xa_,$xb_) if ($i<7); # elsewhere
419 &movdqa ($xa,$xb); # borrow as temporary
422 &pxor ($xd_,$xa_) if ($i<7); # elsewhere
425 ($xa,$xa_)=($xa_,$xa);
426 ($xb,$xb_)=($xb_,$xb);
427 ($xc,$xc_)=($xc_,$xc);
428 ($xd,$xd_)=($xd_,$xd);
431 &function_begin("ChaCha20_ssse3");
432 &set_label("ssse3_shortcut");
433 &test (&DWP(4,"ebp"),1<<11); # test XOP bit
434 &jnz (&label("xop_shortcut"));
436 &mov ($out,&wparam(0));
437 &mov ($inp,&wparam(1));
438 &mov ($len,&wparam(2));
439 &mov ("edx",&wparam(3)); # key
440 &mov ("ebx",&wparam(4)); # counter and nonce
445 &mov (&DWP(512,"esp"),"ebp");
447 &lea ("eax",&DWP(&label("ssse3_data")."-".
448 &label("pic_point"),"eax"));
449 &movdqu ("xmm3",&QWP(0,"ebx")); # counter and nonce
454 &mov (&DWP(512+4,"esp"),"edx"); # offload pointers
455 &mov (&DWP(512+8,"esp"),"ebx");
456 &sub ($len,64*4); # bias len
457 &lea ("ebp",&DWP(256+128,"esp")); # size optimization
459 &movdqu ("xmm7",&QWP(0,"edx")); # key
460 &pshufd ("xmm0","xmm3",0x00);
461 &pshufd ("xmm1","xmm3",0x55);
462 &pshufd ("xmm2","xmm3",0xaa);
463 &pshufd ("xmm3","xmm3",0xff);
464 &paddd ("xmm0",&QWP(16*3,"eax")); # fix counters
465 &pshufd ("xmm4","xmm7",0x00);
466 &pshufd ("xmm5","xmm7",0x55);
467 &psubd ("xmm0",&QWP(16*4,"eax"));
468 &pshufd ("xmm6","xmm7",0xaa);
469 &pshufd ("xmm7","xmm7",0xff);
470 &movdqa (&QWP(16*12-128,"ebp"),"xmm0");
471 &movdqa (&QWP(16*13-128,"ebp"),"xmm1");
472 &movdqa (&QWP(16*14-128,"ebp"),"xmm2");
473 &movdqa (&QWP(16*15-128,"ebp"),"xmm3");
474 &movdqu ("xmm3",&QWP(16,"edx")); # key
475 &movdqa (&QWP(16*4-128,"ebp"),"xmm4");
476 &movdqa (&QWP(16*5-128,"ebp"),"xmm5");
477 &movdqa (&QWP(16*6-128,"ebp"),"xmm6");
478 &movdqa (&QWP(16*7-128,"ebp"),"xmm7");
479 &movdqa ("xmm7",&QWP(16*2,"eax")); # sigma
480 &lea ("ebx",&DWP(128,"esp")); # size optimization
482 &pshufd ("xmm0","xmm3",0x00);
483 &pshufd ("xmm1","xmm3",0x55);
484 &pshufd ("xmm2","xmm3",0xaa);
485 &pshufd ("xmm3","xmm3",0xff);
486 &pshufd ("xmm4","xmm7",0x00);
487 &pshufd ("xmm5","xmm7",0x55);
488 &pshufd ("xmm6","xmm7",0xaa);
489 &pshufd ("xmm7","xmm7",0xff);
490 &movdqa (&QWP(16*8-128,"ebp"),"xmm0");
491 &movdqa (&QWP(16*9-128,"ebp"),"xmm1");
492 &movdqa (&QWP(16*10-128,"ebp"),"xmm2");
493 &movdqa (&QWP(16*11-128,"ebp"),"xmm3");
494 &movdqa (&QWP(16*0-128,"ebp"),"xmm4");
495 &movdqa (&QWP(16*1-128,"ebp"),"xmm5");
496 &movdqa (&QWP(16*2-128,"ebp"),"xmm6");
497 &movdqa (&QWP(16*3-128,"ebp"),"xmm7");
499 &lea ($inp,&DWP(128,$inp)); # size optimization
500 &lea ($out,&DWP(128,$out)); # size optimization
501 &jmp (&label("outer_loop"));
503 &set_label("outer_loop",16);
504 #&movdqa ("xmm0",&QWP(16*0-128,"ebp")); # copy key material
505 &movdqa ("xmm1",&QWP(16*1-128,"ebp"));
506 &movdqa ("xmm2",&QWP(16*2-128,"ebp"));
507 &movdqa ("xmm3",&QWP(16*3-128,"ebp"));
508 #&movdqa ("xmm4",&QWP(16*4-128,"ebp"));
509 &movdqa ("xmm5",&QWP(16*5-128,"ebp"));
510 &movdqa ("xmm6",&QWP(16*6-128,"ebp"));
511 &movdqa ("xmm7",&QWP(16*7-128,"ebp"));
512 #&movdqa (&QWP(16*0-128,"ebx"),"xmm0");
513 &movdqa (&QWP(16*1-128,"ebx"),"xmm1");
514 &movdqa (&QWP(16*2-128,"ebx"),"xmm2");
515 &movdqa (&QWP(16*3-128,"ebx"),"xmm3");
516 #&movdqa (&QWP(16*4-128,"ebx"),"xmm4");
517 &movdqa (&QWP(16*5-128,"ebx"),"xmm5");
518 &movdqa (&QWP(16*6-128,"ebx"),"xmm6");
519 &movdqa (&QWP(16*7-128,"ebx"),"xmm7");
520 #&movdqa ("xmm0",&QWP(16*8-128,"ebp"));
521 #&movdqa ("xmm1",&QWP(16*9-128,"ebp"));
522 &movdqa ("xmm2",&QWP(16*10-128,"ebp"));
523 &movdqa ("xmm3",&QWP(16*11-128,"ebp"));
524 &movdqa ("xmm4",&QWP(16*12-128,"ebp"));
525 &movdqa ("xmm5",&QWP(16*13-128,"ebp"));
526 &movdqa ("xmm6",&QWP(16*14-128,"ebp"));
527 &movdqa ("xmm7",&QWP(16*15-128,"ebp"));
528 &paddd ("xmm4",&QWP(16*4,"eax")); # counter value
529 #&movdqa (&QWP(16*8-128,"ebx"),"xmm0");
530 #&movdqa (&QWP(16*9-128,"ebx"),"xmm1");
531 &movdqa (&QWP(16*10-128,"ebx"),"xmm2");
532 &movdqa (&QWP(16*11-128,"ebx"),"xmm3");
533 &movdqa (&QWP(16*12-128,"ebx"),"xmm4");
534 &movdqa (&QWP(16*13-128,"ebx"),"xmm5");
535 &movdqa (&QWP(16*14-128,"ebx"),"xmm6");
536 &movdqa (&QWP(16*15-128,"ebx"),"xmm7");
537 &movdqa (&QWP(16*12-128,"ebp"),"xmm4"); # save counter value
539 &movdqa ($xa, &QWP(16*0-128,"ebp"));
540 &movdqa ($xd, "xmm4");
541 &movdqa ($xb_,&QWP(16*4-128,"ebp"));
542 &movdqa ($xc, &QWP(16*8-128,"ebp"));
543 &movdqa ($xc_,&QWP(16*9-128,"ebp"));
545 &mov ("edx",10); # loop counter
548 &set_label("loop",16);
549 &paddd ($xa,$xb_); # elsewhere
551 &pxor ($xd,$xa); # elsewhere
552 &QUARTERROUND_SSSE3(0, 4, 8, 12, 0);
553 &QUARTERROUND_SSSE3(1, 5, 9, 13, 1);
554 &QUARTERROUND_SSSE3(2, 6,10, 14, 2);
555 &QUARTERROUND_SSSE3(3, 7,11, 15, 3);
556 &QUARTERROUND_SSSE3(0, 5,10, 15, 4);
557 &QUARTERROUND_SSSE3(1, 6,11, 12, 5);
558 &QUARTERROUND_SSSE3(2, 7, 8, 13, 6);
559 &QUARTERROUND_SSSE3(3, 4, 9, 14, 7);
561 &jnz (&label("loop"));
563 &movdqa (&QWP(16*4-128,"ebx"),$xb_);
564 &movdqa (&QWP(16*8-128,"ebx"),$xc);
565 &movdqa (&QWP(16*9-128,"ebx"),$xc_);
566 &movdqa (&QWP(16*12-128,"ebx"),$xd);
567 &movdqa (&QWP(16*14-128,"ebx"),$xd_);
569 my ($xa0,$xa1,$xa2,$xa3,$xt0,$xt1,$xt2,$xt3)=map("xmm$_",(0..7));
571 #&movdqa ($xa0,&QWP(16*0-128,"ebx")); # it's there
572 &movdqa ($xa1,&QWP(16*1-128,"ebx"));
573 &movdqa ($xa2,&QWP(16*2-128,"ebx"));
574 &movdqa ($xa3,&QWP(16*3-128,"ebx"));
576 for($i=0;$i<256;$i+=64) {
577 &paddd ($xa0,&QWP($i+16*0-128,"ebp")); # accumulate key material
578 &paddd ($xa1,&QWP($i+16*1-128,"ebp"));
579 &paddd ($xa2,&QWP($i+16*2-128,"ebp"));
580 &paddd ($xa3,&QWP($i+16*3-128,"ebp"));
582 &movdqa ($xt2,$xa0); # "de-interlace" data
583 &punpckldq ($xa0,$xa1);
585 &punpckldq ($xa2,$xa3);
586 &punpckhdq ($xt2,$xa1);
587 &punpckhdq ($xt3,$xa3);
589 &punpcklqdq ($xa0,$xa2); # "a0"
591 &punpcklqdq ($xt2,$xt3); # "a2"
592 &punpckhqdq ($xa1,$xa2); # "a1"
593 &punpckhqdq ($xa3,$xt3); # "a3"
595 #($xa2,$xt2)=($xt2,$xa2);
597 &movdqu ($xt0,&QWP(64*0-128,$inp)); # load input
598 &movdqu ($xt1,&QWP(64*1-128,$inp));
599 &movdqu ($xa2,&QWP(64*2-128,$inp));
600 &movdqu ($xt3,&QWP(64*3-128,$inp));
601 &lea ($inp,&QWP($i<192?16:(64*4-16*3),$inp));
603 &movdqa ($xa0,&QWP($i+16*4-128,"ebx")) if ($i<192);
605 &movdqa ($xa1,&QWP($i+16*5-128,"ebx")) if ($i<192);
607 &movdqa ($xa2,&QWP($i+16*6-128,"ebx")) if ($i<192);
609 &movdqa ($xa3,&QWP($i+16*7-128,"ebx")) if ($i<192);
610 &movdqu (&QWP(64*0-128,$out),$xt0); # store output
611 &movdqu (&QWP(64*1-128,$out),$xt1);
612 &movdqu (&QWP(64*2-128,$out),$xt2);
613 &movdqu (&QWP(64*3-128,$out),$xt3);
614 &lea ($out,&QWP($i<192?16:(64*4-16*3),$out));
617 &jnc (&label("outer_loop"));
620 &jz (&label("done"));
622 &mov ("ebx",&DWP(512+8,"esp")); # restore pointers
623 &lea ($inp,&DWP(-128,$inp));
624 &mov ("edx",&DWP(512+4,"esp"));
625 &lea ($out,&DWP(-128,$out));
627 &movd ("xmm2",&DWP(16*12-128,"ebp")); # counter value
628 &movdqu ("xmm3",&QWP(0,"ebx"));
629 &paddd ("xmm2",&QWP(16*6,"eax")); # +four
630 &pand ("xmm3",&QWP(16*7,"eax"));
631 &por ("xmm3","xmm2"); # counter value
633 my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("xmm$_",(0..7));
635 sub SSSE3ROUND { # critical path is 20 "SIMD ticks" per round
660 &movdqa ($a,&QWP(16*2,"eax")); # sigma
661 &movdqu ($b,&QWP(0,"edx"));
662 &movdqu ($c,&QWP(16,"edx"));
663 #&movdqu ($d,&QWP(0,"ebx")); # already loaded
664 &movdqa ($rot16,&QWP(0,"eax"));
665 &movdqa ($rot24,&QWP(16,"eax"));
666 &mov (&DWP(16*3,"esp"),"ebp");
668 &movdqa (&QWP(16*0,"esp"),$a);
669 &movdqa (&QWP(16*1,"esp"),$b);
670 &movdqa (&QWP(16*2,"esp"),$c);
671 &movdqa (&QWP(16*3,"esp"),$d);
673 &jmp (&label("loop1x"));
675 &set_label("outer1x",16);
676 &movdqa ($d,&QWP(16*5,"eax")); # one
677 &movdqa ($a,&QWP(16*0,"esp"));
678 &movdqa ($b,&QWP(16*1,"esp"));
679 &movdqa ($c,&QWP(16*2,"esp"));
680 &paddd ($d,&QWP(16*3,"esp"));
682 &movdqa (&QWP(16*3,"esp"),$d);
683 &jmp (&label("loop1x"));
685 &set_label("loop1x",16);
687 &pshufd ($c,$c,0b01001110);
688 &pshufd ($b,$b,0b00111001);
689 &pshufd ($d,$d,0b10010011);
693 &pshufd ($c,$c,0b01001110);
694 &pshufd ($b,$b,0b10010011);
695 &pshufd ($d,$d,0b00111001);
698 &jnz (&label("loop1x"));
700 &paddd ($a,&QWP(16*0,"esp"));
701 &paddd ($b,&QWP(16*1,"esp"));
702 &paddd ($c,&QWP(16*2,"esp"));
703 &paddd ($d,&QWP(16*3,"esp"));
706 &jb (&label("tail"));
708 &movdqu ($t,&QWP(16*0,$inp));
709 &movdqu ($t1,&QWP(16*1,$inp));
710 &pxor ($a,$t); # xor with input
711 &movdqu ($t,&QWP(16*2,$inp));
713 &movdqu ($t1,&QWP(16*3,$inp));
716 &lea ($inp,&DWP(16*4,$inp)); # inp+=64
718 &movdqu (&QWP(16*0,$out),$a); # write output
719 &movdqu (&QWP(16*1,$out),$b);
720 &movdqu (&QWP(16*2,$out),$c);
721 &movdqu (&QWP(16*3,$out),$d);
722 &lea ($out,&DWP(16*4,$out)); # inp+=64
725 &jnz (&label("outer1x"));
727 &jmp (&label("done"));
730 &movdqa (&QWP(16*0,"esp"),$a);
731 &movdqa (&QWP(16*1,"esp"),$b);
732 &movdqa (&QWP(16*2,"esp"),$c);
733 &movdqa (&QWP(16*3,"esp"),$d);
739 &set_label("tail_loop");
740 &movb ("al",&BP(0,"esp","ebp"));
741 &movb ("dl",&BP(0,$inp,"ebp"));
742 &lea ("ebp",&DWP(1,"ebp"));
744 &movb (&BP(-1,$out,"ebp"),"al");
746 &jnz (&label("tail_loop"));
749 &mov ("esp",&DWP(512,"esp"));
750 &function_end("ChaCha20_ssse3");
753 &set_label("ssse3_data");
754 &data_byte(0x2,0x3,0x0,0x1, 0x6,0x7,0x4,0x5, 0xa,0xb,0x8,0x9, 0xe,0xf,0xc,0xd);
755 &data_byte(0x3,0x0,0x1,0x2, 0x7,0x4,0x5,0x6, 0xb,0x8,0x9,0xa, 0xf,0xc,0xd,0xe);
756 &data_word(0x61707865,0x3320646e,0x79622d32,0x6b206574);
761 &data_word(0,-1,-1,-1);
764 &asciz ("ChaCha20 for x86, CRYPTOGAMS by <appro\@openssl.org>");
767 my ($xa,$xa_,$xb,$xb_,$xc,$xc_,$xd,$xd_)=map("xmm$_",(0..7));
768 my ($out,$inp,$len)=("edi","esi","ecx");
770 sub QUARTERROUND_XOP {
771 my ($ai,$bi,$ci,$di,$i)=@_;
772 my ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+1)&3),($ai,$bi,$ci,$di)); # next
773 my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
777 # 0 4 8 12 < even round
781 # 0 5 10 15 < odd round
788 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-$j--)&3),($ap,$bp,$cp,$dp));
791 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+$j++)&3),($an,$bn,$cn,$dn));
794 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_+$j--)&3),($ap,$bp,$cp,$dp));
797 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_-$j++)&3),($an,$bn,$cn,$dn));
800 #&vpaddd ($xa,$xa,$xb); # see elsewhere
801 #&vpxor ($xd,$xd,$xa); # see elsewhere
802 &vmovdqa (&QWP(16*$cp-128,"ebx"),$xc_) if ($ai>0 && $ai<3);
803 &vprotd ($xd,$xd,16);
804 &vmovdqa (&QWP(16*$bp-128,"ebx"),$xb_) if ($i!=0);
805 &vpaddd ($xc,$xc,$xd);
806 &vmovdqa ($xc_,&QWP(16*$cn-128,"ebx")) if ($ai>0 && $ai<3);
807 &vpxor ($xb,$i!=0?$xb:$xb_,$xc);
808 &vmovdqa ($xa_,&QWP(16*$an-128,"ebx"));
809 &vprotd ($xb,$xb,12);
810 &vmovdqa ($xb_,&QWP(16*$bn-128,"ebx")) if ($i<7);
811 &vpaddd ($xa,$xa,$xb);
812 &vmovdqa ($xd_,&QWP(16*$dn-128,"ebx")) if ($di!=$dn);
813 &vpxor ($xd,$xd,$xa);
814 &vpaddd ($xa_,$xa_,$xb_) if ($i<7); # elsewhere
816 &vmovdqa (&QWP(16*$ai-128,"ebx"),$xa);
817 &vpaddd ($xc,$xc,$xd);
818 &vmovdqa (&QWP(16*$di-128,"ebx"),$xd) if ($di!=$dn);
819 &vpxor ($xb,$xb,$xc);
820 &vpxor ($xd_,$di==$dn?$xd:$xd_,$xa_) if ($i<7); # elsewhere
823 ($xa,$xa_)=($xa_,$xa);
824 ($xb,$xb_)=($xb_,$xb);
825 ($xc,$xc_)=($xc_,$xc);
826 ($xd,$xd_)=($xd_,$xd);
829 &function_begin("ChaCha20_xop");
830 &set_label("xop_shortcut");
831 &mov ($out,&wparam(0));
832 &mov ($inp,&wparam(1));
833 &mov ($len,&wparam(2));
834 &mov ("edx",&wparam(3)); # key
835 &mov ("ebx",&wparam(4)); # counter and nonce
841 &mov (&DWP(512,"esp"),"ebp");
843 &lea ("eax",&DWP(&label("ssse3_data")."-".
844 &label("pic_point"),"eax"));
845 &vmovdqu ("xmm3",&QWP(0,"ebx")); # counter and nonce
850 &mov (&DWP(512+4,"esp"),"edx"); # offload pointers
851 &mov (&DWP(512+8,"esp"),"ebx");
852 &sub ($len,64*4); # bias len
853 &lea ("ebp",&DWP(256+128,"esp")); # size optimization
855 &vmovdqu ("xmm7",&QWP(0,"edx")); # key
856 &vpshufd ("xmm0","xmm3",0x00);
857 &vpshufd ("xmm1","xmm3",0x55);
858 &vpshufd ("xmm2","xmm3",0xaa);
859 &vpshufd ("xmm3","xmm3",0xff);
860 &vpaddd ("xmm0","xmm0",&QWP(16*3,"eax")); # fix counters
861 &vpshufd ("xmm4","xmm7",0x00);
862 &vpshufd ("xmm5","xmm7",0x55);
863 &vpsubd ("xmm0","xmm0",&QWP(16*4,"eax"));
864 &vpshufd ("xmm6","xmm7",0xaa);
865 &vpshufd ("xmm7","xmm7",0xff);
866 &vmovdqa (&QWP(16*12-128,"ebp"),"xmm0");
867 &vmovdqa (&QWP(16*13-128,"ebp"),"xmm1");
868 &vmovdqa (&QWP(16*14-128,"ebp"),"xmm2");
869 &vmovdqa (&QWP(16*15-128,"ebp"),"xmm3");
870 &vmovdqu ("xmm3",&QWP(16,"edx")); # key
871 &vmovdqa (&QWP(16*4-128,"ebp"),"xmm4");
872 &vmovdqa (&QWP(16*5-128,"ebp"),"xmm5");
873 &vmovdqa (&QWP(16*6-128,"ebp"),"xmm6");
874 &vmovdqa (&QWP(16*7-128,"ebp"),"xmm7");
875 &vmovdqa ("xmm7",&QWP(16*2,"eax")); # sigma
876 &lea ("ebx",&DWP(128,"esp")); # size optimization
878 &vpshufd ("xmm0","xmm3",0x00);
879 &vpshufd ("xmm1","xmm3",0x55);
880 &vpshufd ("xmm2","xmm3",0xaa);
881 &vpshufd ("xmm3","xmm3",0xff);
882 &vpshufd ("xmm4","xmm7",0x00);
883 &vpshufd ("xmm5","xmm7",0x55);
884 &vpshufd ("xmm6","xmm7",0xaa);
885 &vpshufd ("xmm7","xmm7",0xff);
886 &vmovdqa (&QWP(16*8-128,"ebp"),"xmm0");
887 &vmovdqa (&QWP(16*9-128,"ebp"),"xmm1");
888 &vmovdqa (&QWP(16*10-128,"ebp"),"xmm2");
889 &vmovdqa (&QWP(16*11-128,"ebp"),"xmm3");
890 &vmovdqa (&QWP(16*0-128,"ebp"),"xmm4");
891 &vmovdqa (&QWP(16*1-128,"ebp"),"xmm5");
892 &vmovdqa (&QWP(16*2-128,"ebp"),"xmm6");
893 &vmovdqa (&QWP(16*3-128,"ebp"),"xmm7");
895 &lea ($inp,&DWP(128,$inp)); # size optimization
896 &lea ($out,&DWP(128,$out)); # size optimization
897 &jmp (&label("outer_loop"));
899 &set_label("outer_loop",32);
900 #&vmovdqa ("xmm0",&QWP(16*0-128,"ebp")); # copy key material
901 &vmovdqa ("xmm1",&QWP(16*1-128,"ebp"));
902 &vmovdqa ("xmm2",&QWP(16*2-128,"ebp"));
903 &vmovdqa ("xmm3",&QWP(16*3-128,"ebp"));
904 #&vmovdqa ("xmm4",&QWP(16*4-128,"ebp"));
905 &vmovdqa ("xmm5",&QWP(16*5-128,"ebp"));
906 &vmovdqa ("xmm6",&QWP(16*6-128,"ebp"));
907 &vmovdqa ("xmm7",&QWP(16*7-128,"ebp"));
908 #&vmovdqa (&QWP(16*0-128,"ebx"),"xmm0");
909 &vmovdqa (&QWP(16*1-128,"ebx"),"xmm1");
910 &vmovdqa (&QWP(16*2-128,"ebx"),"xmm2");
911 &vmovdqa (&QWP(16*3-128,"ebx"),"xmm3");
912 #&vmovdqa (&QWP(16*4-128,"ebx"),"xmm4");
913 &vmovdqa (&QWP(16*5-128,"ebx"),"xmm5");
914 &vmovdqa (&QWP(16*6-128,"ebx"),"xmm6");
915 &vmovdqa (&QWP(16*7-128,"ebx"),"xmm7");
916 #&vmovdqa ("xmm0",&QWP(16*8-128,"ebp"));
917 #&vmovdqa ("xmm1",&QWP(16*9-128,"ebp"));
918 &vmovdqa ("xmm2",&QWP(16*10-128,"ebp"));
919 &vmovdqa ("xmm3",&QWP(16*11-128,"ebp"));
920 &vmovdqa ("xmm4",&QWP(16*12-128,"ebp"));
921 &vmovdqa ("xmm5",&QWP(16*13-128,"ebp"));
922 &vmovdqa ("xmm6",&QWP(16*14-128,"ebp"));
923 &vmovdqa ("xmm7",&QWP(16*15-128,"ebp"));
924 &vpaddd ("xmm4","xmm4",&QWP(16*4,"eax")); # counter value
925 #&vmovdqa (&QWP(16*8-128,"ebx"),"xmm0");
926 #&vmovdqa (&QWP(16*9-128,"ebx"),"xmm1");
927 &vmovdqa (&QWP(16*10-128,"ebx"),"xmm2");
928 &vmovdqa (&QWP(16*11-128,"ebx"),"xmm3");
929 &vmovdqa (&QWP(16*12-128,"ebx"),"xmm4");
930 &vmovdqa (&QWP(16*13-128,"ebx"),"xmm5");
931 &vmovdqa (&QWP(16*14-128,"ebx"),"xmm6");
932 &vmovdqa (&QWP(16*15-128,"ebx"),"xmm7");
933 &vmovdqa (&QWP(16*12-128,"ebp"),"xmm4"); # save counter value
935 &vmovdqa ($xa, &QWP(16*0-128,"ebp"));
936 &vmovdqa ($xd, "xmm4");
937 &vmovdqa ($xb_,&QWP(16*4-128,"ebp"));
938 &vmovdqa ($xc, &QWP(16*8-128,"ebp"));
939 &vmovdqa ($xc_,&QWP(16*9-128,"ebp"));
941 &mov ("edx",10); # loop counter
944 &set_label("loop",32);
945 &vpaddd ($xa,$xa,$xb_); # elsewhere
946 &vpxor ($xd,$xd,$xa); # elsewhere
947 &QUARTERROUND_XOP(0, 4, 8, 12, 0);
948 &QUARTERROUND_XOP(1, 5, 9, 13, 1);
949 &QUARTERROUND_XOP(2, 6,10, 14, 2);
950 &QUARTERROUND_XOP(3, 7,11, 15, 3);
951 &QUARTERROUND_XOP(0, 5,10, 15, 4);
952 &QUARTERROUND_XOP(1, 6,11, 12, 5);
953 &QUARTERROUND_XOP(2, 7, 8, 13, 6);
954 &QUARTERROUND_XOP(3, 4, 9, 14, 7);
956 &jnz (&label("loop"));
958 &vmovdqa (&QWP(16*4-128,"ebx"),$xb_);
959 &vmovdqa (&QWP(16*8-128,"ebx"),$xc);
960 &vmovdqa (&QWP(16*9-128,"ebx"),$xc_);
961 &vmovdqa (&QWP(16*12-128,"ebx"),$xd);
962 &vmovdqa (&QWP(16*14-128,"ebx"),$xd_);
964 my ($xa0,$xa1,$xa2,$xa3,$xt0,$xt1,$xt2,$xt3)=map("xmm$_",(0..7));
966 #&vmovdqa ($xa0,&QWP(16*0-128,"ebx")); # it's there
967 &vmovdqa ($xa1,&QWP(16*1-128,"ebx"));
968 &vmovdqa ($xa2,&QWP(16*2-128,"ebx"));
969 &vmovdqa ($xa3,&QWP(16*3-128,"ebx"));
971 for($i=0;$i<256;$i+=64) {
972 &vpaddd ($xa0,$xa0,&QWP($i+16*0-128,"ebp")); # accumulate key material
973 &vpaddd ($xa1,$xa1,&QWP($i+16*1-128,"ebp"));
974 &vpaddd ($xa2,$xa2,&QWP($i+16*2-128,"ebp"));
975 &vpaddd ($xa3,$xa3,&QWP($i+16*3-128,"ebp"));
977 &vpunpckldq ($xt2,$xa0,$xa1); # "de-interlace" data
978 &vpunpckldq ($xt3,$xa2,$xa3);
979 &vpunpckhdq ($xa0,$xa0,$xa1);
980 &vpunpckhdq ($xa2,$xa2,$xa3);
981 &vpunpcklqdq ($xa1,$xt2,$xt3); # "a0"
982 &vpunpckhqdq ($xt2,$xt2,$xt3); # "a1"
983 &vpunpcklqdq ($xt3,$xa0,$xa2); # "a2"
984 &vpunpckhqdq ($xa3,$xa0,$xa2); # "a3"
986 &vpxor ($xt0,$xa1,&QWP(64*0-128,$inp));
987 &vpxor ($xt1,$xt2,&QWP(64*1-128,$inp));
988 &vpxor ($xt2,$xt3,&QWP(64*2-128,$inp));
989 &vpxor ($xt3,$xa3,&QWP(64*3-128,$inp));
990 &lea ($inp,&QWP($i<192?16:(64*4-16*3),$inp));
991 &vmovdqa ($xa0,&QWP($i+16*4-128,"ebx")) if ($i<192);
992 &vmovdqa ($xa1,&QWP($i+16*5-128,"ebx")) if ($i<192);
993 &vmovdqa ($xa2,&QWP($i+16*6-128,"ebx")) if ($i<192);
994 &vmovdqa ($xa3,&QWP($i+16*7-128,"ebx")) if ($i<192);
995 &vmovdqu (&QWP(64*0-128,$out),$xt0); # store output
996 &vmovdqu (&QWP(64*1-128,$out),$xt1);
997 &vmovdqu (&QWP(64*2-128,$out),$xt2);
998 &vmovdqu (&QWP(64*3-128,$out),$xt3);
999 &lea ($out,&QWP($i<192?16:(64*4-16*3),$out));
1002 &jnc (&label("outer_loop"));
1005 &jz (&label("done"));
1007 &mov ("ebx",&DWP(512+8,"esp")); # restore pointers
1008 &lea ($inp,&DWP(-128,$inp));
1009 &mov ("edx",&DWP(512+4,"esp"));
1010 &lea ($out,&DWP(-128,$out));
1012 &vmovd ("xmm2",&DWP(16*12-128,"ebp")); # counter value
1013 &vmovdqu ("xmm3",&QWP(0,"ebx"));
1014 &vpaddd ("xmm2","xmm2",&QWP(16*6,"eax"));# +four
1015 &vpand ("xmm3","xmm3",&QWP(16*7,"eax"));
1016 &vpor ("xmm3","xmm3","xmm2"); # counter value
1018 my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("xmm$_",(0..7));
1039 &vmovdqa ($a,&QWP(16*2,"eax")); # sigma
1040 &vmovdqu ($b,&QWP(0,"edx"));
1041 &vmovdqu ($c,&QWP(16,"edx"));
1042 #&vmovdqu ($d,&QWP(0,"ebx")); # already loaded
1043 &vmovdqa ($rot16,&QWP(0,"eax"));
1044 &vmovdqa ($rot24,&QWP(16,"eax"));
1045 &mov (&DWP(16*3,"esp"),"ebp");
1047 &vmovdqa (&QWP(16*0,"esp"),$a);
1048 &vmovdqa (&QWP(16*1,"esp"),$b);
1049 &vmovdqa (&QWP(16*2,"esp"),$c);
1050 &vmovdqa (&QWP(16*3,"esp"),$d);
1052 &jmp (&label("loop1x"));
1054 &set_label("outer1x",16);
1055 &vmovdqa ($d,&QWP(16*5,"eax")); # one
1056 &vmovdqa ($a,&QWP(16*0,"esp"));
1057 &vmovdqa ($b,&QWP(16*1,"esp"));
1058 &vmovdqa ($c,&QWP(16*2,"esp"));
1059 &vpaddd ($d,$d,&QWP(16*3,"esp"));
1061 &vmovdqa (&QWP(16*3,"esp"),$d);
1062 &jmp (&label("loop1x"));
1064 &set_label("loop1x",16);
1066 &vpshufd ($c,$c,0b01001110);
1067 &vpshufd ($b,$b,0b00111001);
1068 &vpshufd ($d,$d,0b10010011);
1071 &vpshufd ($c,$c,0b01001110);
1072 &vpshufd ($b,$b,0b10010011);
1073 &vpshufd ($d,$d,0b00111001);
1076 &jnz (&label("loop1x"));
1078 &vpaddd ($a,$a,&QWP(16*0,"esp"));
1079 &vpaddd ($b,$b,&QWP(16*1,"esp"));
1080 &vpaddd ($c,$c,&QWP(16*2,"esp"));
1081 &vpaddd ($d,$d,&QWP(16*3,"esp"));
1084 &jb (&label("tail"));
1086 &vpxor ($a,$a,&QWP(16*0,$inp)); # xor with input
1087 &vpxor ($b,$b,&QWP(16*1,$inp));
1088 &vpxor ($c,$c,&QWP(16*2,$inp));
1089 &vpxor ($d,$d,&QWP(16*3,$inp));
1090 &lea ($inp,&DWP(16*4,$inp)); # inp+=64
1092 &vmovdqu (&QWP(16*0,$out),$a); # write output
1093 &vmovdqu (&QWP(16*1,$out),$b);
1094 &vmovdqu (&QWP(16*2,$out),$c);
1095 &vmovdqu (&QWP(16*3,$out),$d);
1096 &lea ($out,&DWP(16*4,$out)); # inp+=64
1099 &jnz (&label("outer1x"));
1101 &jmp (&label("done"));
1104 &vmovdqa (&QWP(16*0,"esp"),$a);
1105 &vmovdqa (&QWP(16*1,"esp"),$b);
1106 &vmovdqa (&QWP(16*2,"esp"),$c);
1107 &vmovdqa (&QWP(16*3,"esp"),$d);
1113 &set_label("tail_loop");
1114 &movb ("al",&BP(0,"esp","ebp"));
1115 &movb ("dl",&BP(0,$inp,"ebp"));
1116 &lea ("ebp",&DWP(1,"ebp"));
1118 &movb (&BP(-1,$out,"ebp"),"al");
1120 &jnz (&label("tail_loop"));
1124 &mov ("esp",&DWP(512,"esp"));
1125 &function_end("ChaCha20_xop");