3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
10 # This module implements Poly1305 hash for x86.
14 # Numbers are cycles per processed byte with poly1305_blocks alone,
15 # measured with rdtsc at fixed clock frequency.
17 # IALU/gcc-3.4(*) SSE2(**) AVX2
21 # Core 2 4.85/+90% 1.80
22 # Westmere 4.58/+100% 1.43
23 # Sandy Bridge 3.90/+100% 1.36
24 # Haswell 3.88/+70% 1.18 0.72
25 # Silvermont 11.0/+40% 4.80
26 # VIA Nano 6.71/+90% 2.47
27 # Sledgehammer 3.51/+180% 4.27
28 # Bulldozer 4.53/+140% 1.31
30 # (*) gcc 4.8 for some reason generated worse code;
31 # (**) besides SSE2 there are floating-point and AVX options; FP
32 # is deemed unnecessary, because pre-SSE2 processor are too
33 # old to care about, while it's not the fastest option on
34 # SSE2-capable ones; AVX is omitted, because it doesn't give
35 # a lot of improvement, 5-10% depending on processor;
37 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
38 push(@INC,"${dir}","${dir}../../perlasm");
42 open STDOUT,">$output";
44 &asm_init($ARGV[0],"poly1305-x86.pl",$ARGV[$#ARGV] eq "386");
47 for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
50 &static_label("const_sse2");
51 &static_label("enter_blocks");
52 &static_label("enter_emit");
53 &external_label("OPENSSL_ia32cap_P");
55 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
56 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
57 $avx = ($1>=2.19) + ($1>=2.22);
60 if (!$avx && $ARGV[0] eq "win32n" &&
61 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
62 $avx = ($1>=2.09) + ($1>=2.10);
65 if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) {
66 $avx = ($2>=3.0) + ($2>3.0);
70 ########################################################################
71 # Layout of opaque area is following.
73 # unsigned __int32 h[5]; # current hash value base 2^32
74 # unsigned __int32 pad; # is_base2_26 in vector context
75 # unsigned __int32 r[4]; # key value base 2^32
78 &function_begin("poly1305_init");
79 &mov ("edi",&wparam(0)); # context
80 &mov ("esi",&wparam(1)); # key
81 &mov ("ebp",&wparam(2)); # function table
84 &mov (&DWP(4*0,"edi"),"eax"); # zero hash value
85 &mov (&DWP(4*1,"edi"),"eax");
86 &mov (&DWP(4*2,"edi"),"eax");
87 &mov (&DWP(4*3,"edi"),"eax");
88 &mov (&DWP(4*4,"edi"),"eax");
89 &mov (&DWP(4*5,"edi"),"eax"); # is_base2_26
92 &je (&label("nokey"));
95 &call (&label("pic_point"));
96 &set_label("pic_point");
99 &lea ("eax",&DWP("poly1305_blocks-".&label("pic_point"),"ebx"));
100 &lea ("edx",&DWP("poly1305_emit-".&label("pic_point"),"ebx"));
102 &picmeup("edi","OPENSSL_ia32cap_P","ebx",&label("pic_point"));
103 &mov ("ecx",&DWP(0,"edi"));
104 &and ("ecx",1<<26|1<<24);
105 &cmp ("ecx",1<<26|1<<24); # SSE2 and XMM?
106 &jne (&label("no_sse2"));
108 &lea ("eax",&DWP("_poly1305_blocks_sse2-".&label("pic_point"),"ebx"));
109 &lea ("edx",&DWP("_poly1305_emit_sse2-".&label("pic_point"),"ebx"));
112 &mov ("ecx",&DWP(8,"edi"));
113 &test ("ecx",1<<5); # AVX2?
114 &jz (&label("no_sse2"));
116 &lea ("eax",&DWP("_poly1305_blocks_avx2-".&label("pic_point"),"ebx"));
118 &set_label("no_sse2");
119 &mov ("edi",&wparam(0)); # reload context
120 &mov (&DWP(0,"ebp"),"eax"); # fill function table
121 &mov (&DWP(4,"ebp"),"edx");
124 &mov ("eax",&DWP(4*0,"esi")); # load input key
125 &mov ("ebx",&DWP(4*1,"esi"));
126 &mov ("ecx",&DWP(4*2,"esi"));
127 &mov ("edx",&DWP(4*3,"esi"));
128 &and ("eax",0x0fffffff);
129 &and ("ebx",0x0ffffffc);
130 &and ("ecx",0x0ffffffc);
131 &and ("edx",0x0ffffffc);
132 &mov (&DWP(4*6,"edi"),"eax");
133 &mov (&DWP(4*7,"edi"),"ebx");
134 &mov (&DWP(4*8,"edi"),"ecx");
135 &mov (&DWP(4*9,"edi"),"edx");
139 &function_end("poly1305_init");
141 ($h0,$h1,$h2,$h3,$h4,
144 $s1,$s2,$s3)=map(4*$_,(0..15));
146 &function_begin("poly1305_blocks");
147 &mov ("edi",&wparam(0)); # ctx
148 &mov ("esi",&wparam(1)); # inp
149 &mov ("ecx",&wparam(2)); # len
150 &set_label("enter_blocks");
152 &jz (&label("nodata"));
155 &mov ("eax",&DWP(4*6,"edi")); # r0
156 &mov ("ebx",&DWP(4*7,"edi")); # r1
157 &lea ("ebp",&DWP(0,"esi","ecx")); # end of input
158 &mov ("ecx",&DWP(4*8,"edi")); # r2
159 &mov ("edx",&DWP(4*9,"edi")); # r3
161 &mov (&wparam(2),"ebp");
164 &mov (&DWP($r0,"esp"),"eax"); # r0
167 &mov (&DWP($r1,"esp"),"ebx"); # r1
168 &add ("eax","ebx"); # s1
171 &mov (&DWP($r2,"esp"),"ecx"); # r2
172 &add ("ebx","ecx"); # s2
175 &mov (&DWP($r3,"esp"),"edx"); # r3
176 &add ("ecx","edx"); # s3
177 &mov (&DWP($s1,"esp"),"eax"); # s1
178 &mov (&DWP($s2,"esp"),"ebx"); # s2
179 &mov (&DWP($s3,"esp"),"ecx"); # s3
181 &mov ("eax",&DWP(4*0,"edi")); # load hash value
182 &mov ("ebx",&DWP(4*1,"edi"));
183 &mov ("ecx",&DWP(4*2,"edi"));
184 &mov ("esi",&DWP(4*3,"edi"));
185 &mov ("edi",&DWP(4*4,"edi"));
186 &jmp (&label("loop"));
188 &set_label("loop",32);
189 &add ("eax",&DWP(4*0,"ebp")); # accumulate input
190 &adc ("ebx",&DWP(4*1,"ebp"));
191 &adc ("ecx",&DWP(4*2,"ebp"));
192 &adc ("esi",&DWP(4*3,"ebp"));
193 &lea ("ebp",&DWP(4*4,"ebp"));
194 &adc ("edi",&wparam(3)); # padbit
196 &mov (&DWP($h0,"esp"),"eax"); # put aside hash[+inp]
197 &mov (&DWP($h3,"esp"),"esi");
199 &mul (&DWP($r0,"esp")); # h0*r0
200 &mov (&DWP($h4,"esp"),"edi");
202 &mov ("eax","ebx"); # h1
204 &mul (&DWP($s3,"esp")); # h1*s3
206 &mov ("eax","ecx"); # h2
208 &mul (&DWP($s2,"esp")); # h2*s2
210 &mov ("eax",&DWP($h3,"esp"));
212 &mul (&DWP($s1,"esp")); # h3*s1
214 &mov ("eax",&DWP($h0,"esp"));
217 &mul (&DWP($r1,"esp")); # h0*r1
218 &mov (&DWP($d0,"esp"),"edi");
221 &mov ("eax","ebx"); # h1
223 &mul (&DWP($r0,"esp")); # h1*r0
225 &mov ("eax","ecx"); # h2
227 &mul (&DWP($s3,"esp")); # h2*s3
229 &mov ("eax",&DWP($h3,"esp"));
231 &mul (&DWP($s2,"esp")); # h3*s2
233 &mov ("eax",&DWP($h4,"esp"));
235 &imul ("eax",&DWP($s1,"esp")); # h4*s1
237 &mov ("eax",&DWP($h0,"esp"));
240 &mul (&DWP($r2,"esp")); # h0*r2
241 &mov (&DWP($d1,"esp"),"esi");
244 &mov ("eax","ebx"); # h1
246 &mul (&DWP($r1,"esp")); # h1*r1
248 &mov ("eax","ecx"); # h2
250 &mul (&DWP($r0,"esp")); # h2*r0
252 &mov ("eax",&DWP($h3,"esp"));
254 &mul (&DWP($s3,"esp")); # h3*s3
256 &mov ("eax",&DWP($h4,"esp"));
258 &imul ("eax",&DWP($s2,"esp")); # h4*s2
260 &mov ("eax",&DWP($h0,"esp"));
263 &mul (&DWP($r3,"esp")); # h0*r3
264 &mov (&DWP($d2,"esp"),"edi");
267 &mov ("eax","ebx"); # h1
269 &mul (&DWP($r2,"esp")); # h1*r2
271 &mov ("eax","ecx"); # h2
273 &mul (&DWP($r1,"esp")); # h2*r1
275 &mov ("eax",&DWP($h3,"esp"));
277 &mul (&DWP($r0,"esp")); # h3*r0
279 &mov ("ecx",&DWP($h4,"esp"));
283 &imul ("ecx",&DWP($s3,"esp")); # h4*s3
285 &mov ("eax",&DWP($d0,"esp"));
288 &imul ("edx",&DWP($r0,"esp")); # h4*r0
291 &mov ("ebx",&DWP($d1,"esp"));
292 &mov ("ecx",&DWP($d2,"esp"));
294 &mov ("edi","edx"); # last reduction step
297 &lea ("edx",&DWP(0,"edx","edx",4)); # *5
303 &cmp ("ebp",&wparam(2)); # done yet?
304 &jne (&label("loop"));
306 &mov ("edx",&wparam(0)); # ctx
308 &mov (&DWP(4*0,"edx"),"eax"); # store hash value
309 &mov (&DWP(4*1,"edx"),"ebx");
310 &mov (&DWP(4*2,"edx"),"ecx");
311 &mov (&DWP(4*3,"edx"),"esi");
312 &mov (&DWP(4*4,"edx"),"edi");
313 &set_label("nodata");
314 &function_end("poly1305_blocks");
316 &function_begin("poly1305_emit");
317 &mov ("ebp",&wparam(0)); # context
318 &set_label("enter_emit");
319 &mov ("edi",&wparam(1)); # output
320 &mov ("eax",&DWP(4*0,"ebp")); # load hash value
321 &mov ("ebx",&DWP(4*1,"ebp"));
322 &mov ("ecx",&DWP(4*2,"ebp"));
323 &mov ("edx",&DWP(4*3,"ebp"));
324 &mov ("esi",&DWP(4*4,"ebp"));
326 &add ("eax",5); # compare to modulus
331 &shr ("esi",2); # did it carry/borrow?
332 &neg ("esi"); # do we choose hash-modulus?
338 &mov (&DWP(4*0,"edi"),"eax");
339 &mov (&DWP(4*1,"edi"),"ebx");
340 &mov (&DWP(4*2,"edi"),"ecx");
341 &mov (&DWP(4*3,"edi"),"edx");
343 ¬ ("esi"); # or original hash value?
344 &mov ("eax",&DWP(4*0,"ebp"));
345 &mov ("ebx",&DWP(4*1,"ebp"));
346 &mov ("ecx",&DWP(4*2,"ebp"));
347 &mov ("edx",&DWP(4*3,"ebp"));
348 &mov ("ebp",&wparam(2));
353 &or ("eax",&DWP(4*0,"edi"));
354 &or ("ebx",&DWP(4*1,"edi"));
355 &or ("ecx",&DWP(4*2,"edi"));
356 &or ("edx",&DWP(4*3,"edi"));
358 &add ("eax",&DWP(4*0,"ebp")); # accumulate key
359 &adc ("ebx",&DWP(4*1,"ebp"));
360 &adc ("ecx",&DWP(4*2,"ebp"));
361 &adc ("edx",&DWP(4*3,"ebp"));
363 &mov (&DWP(4*0,"edi"),"eax");
364 &mov (&DWP(4*1,"edi"),"ebx");
365 &mov (&DWP(4*2,"edi"),"ecx");
366 &mov (&DWP(4*3,"edi"),"edx");
367 &function_end("poly1305_emit");
370 ########################################################################
371 # Layout of opaque area is following.
373 # unsigned __int32 h[5]; # current hash value base 2^26
374 # unsigned __int32 is_base2_26;
375 # unsigned __int32 r[4]; # key value base 2^32
376 # unsigned __int32 pad[2];
377 # struct { unsigned __int32 r^4, r^3, r^2, r^1; } r[9];
379 # where r^n are base 2^26 digits of degrees of multiplier key. There are
380 # 5 digits, but last four are interleaved with multiples of 5, totalling
381 # in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.
383 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("xmm$_",(0..7));
384 my $MASK=$T2; # borrow and keep in mind
387 &function_begin_B("_poly1305_init_sse2");
388 &movdqu ($D4,&QWP(4*6,"edi")); # key base 2^32
389 &lea ("edi",&DWP(16*3,"edi")); # size optimization
391 &sub ("esp",16*(9+5));
394 #&pand ($D4,&QWP(96,"ebx")); # magic mask
395 &movq ($MASK,&QWP(64,"ebx"));
401 &pand ($D0,$MASK); # -> base 2^26
412 &lea ("edx",&DWP(16*9,"esp")); # size optimization
414 &set_label("square");
415 &movdqa (&QWP(16*0,"esp"),$D0);
416 &movdqa (&QWP(16*1,"esp"),$D1);
417 &movdqa (&QWP(16*2,"esp"),$D2);
418 &movdqa (&QWP(16*3,"esp"),$D3);
419 &movdqa (&QWP(16*4,"esp"),$D4);
425 &paddd ($T1,$D1); # *5
426 &paddd ($T0,$D2); # *5
427 &movdqa (&QWP(16*5,"esp"),$T1);
428 &movdqa (&QWP(16*6,"esp"),$T0);
433 &paddd ($T1,$D3); # *5
434 &paddd ($T0,$D4); # *5
435 &movdqa (&QWP(16*7,"esp"),$T1);
436 &movdqa (&QWP(16*8,"esp"),$T0);
438 &pshufd ($T1,$D0,0b01000100);
440 &pshufd ($D1,$D1,0b01000100);
441 &pshufd ($D2,$D2,0b01000100);
442 &pshufd ($D3,$D3,0b01000100);
443 &pshufd ($D4,$D4,0b01000100);
444 &movdqa (&QWP(16*0,"edx"),$T1);
445 &movdqa (&QWP(16*1,"edx"),$D1);
446 &movdqa (&QWP(16*2,"edx"),$D2);
447 &movdqa (&QWP(16*3,"edx"),$D3);
448 &movdqa (&QWP(16*4,"edx"),$D4);
450 ################################################################
451 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
452 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
453 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
454 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
455 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
457 &pmuludq ($D4,$D0); # h4*r0
458 &pmuludq ($D3,$D0); # h3*r0
459 &pmuludq ($D2,$D0); # h2*r0
460 &pmuludq ($D1,$D0); # h1*r0
461 &pmuludq ($D0,$T1); # h0*r0
465 my $base = shift; $base = "esp" if (!defined($base));
467 ################################################################
468 # As for choice to "rotate" $T0-$T2 in order to move paddq
469 # past next multiplication. While it makes code harder to read
470 # and doesn't have significant effect on most processors, it
471 # makes a lot of difference on Atom, up to 30% improvement.
474 &pmuludq ($T0,&QWP(16*3,$base)); # r1*h3
476 &pmuludq ($T1,&QWP(16*2,$base)); # r1*h2
479 &pmuludq ($T2,&QWP(16*1,$base)); # r1*h1
482 &pmuludq ($T0,&QWP(16*0,$base)); # r1*h0
484 &pmuludq ($T1,&QWP(16*4,$base)); # s1*h4
485 &$load ($T2,2); # r2^n
489 &pmuludq ($T2,&QWP(16*2,$base)); # r2*h2
492 &pmuludq ($T0,&QWP(16*1,$base)); # r2*h1
494 &$load ($T2,6); # s2^n
495 &pmuludq ($T1,&QWP(16*0,$base)); # r2*h0
498 &pmuludq ($T2,&QWP(16*4,$base)); # s2*h4
500 &pmuludq ($T0,&QWP(16*3,$base)); # s2*h3
501 &$load ($T1,3); # r3^n
505 &pmuludq ($T1,&QWP(16*1,$base)); # r3*h1
507 &$load ($T0,7); # s3^n
508 &pmuludq ($T2,&QWP(16*0,$base)); # r3*h0
511 &pmuludq ($T0,&QWP(16*4,$base)); # s3*h4
514 &pmuludq ($T1,&QWP(16*3,$base)); # s3*h3
516 &pmuludq ($T2,&QWP(16*2,$base)); # s3*h2
517 &$load ($T0,4); # r4^n
520 &$load ($T1,8); # s4^n
521 &pmuludq ($T0,&QWP(16*0,$base)); # r4*h0
524 &pmuludq ($T1,&QWP(16*4,$base)); # s4*h4
527 &pmuludq ($T2,&QWP(16*1,$base)); # s4*h1
530 &pmuludq ($T0,&QWP(16*2,$base)); # s4*h2
532 &pmuludq ($T1,&QWP(16*3,$base)); # s4*h3
533 &movdqa ($MASK,&QWP(64,"ebx"));
537 &pmuladd (sub { my ($reg,$i)=@_;
538 &movdqa ($reg,&QWP(16*$i,"esp"));
543 my $paddx = defined($extra) ? paddq : paddd;
545 ################################################################
546 # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
552 &$extra () if (defined($extra));
553 &paddq ($T0,$D4); # h3 -> h4
558 &paddq ($T1,$D1); # h0 -> h1
563 &paddd ($D0,$T0); # favour paddd when
568 &paddq ($T1,$D2); # h1 -> h2
569 &$paddx ($T0,$D0); # h4 -> h0
574 &paddd ($T1,$D3); # h2 -> h3
580 &paddd ($D1,$T0); # h0 -> h1
582 &paddd ($D4,$T1); # h3 -> h4
587 &jz (&label("square_break"));
589 &punpcklqdq ($D0,&QWP(16*0,"esp")); # 0:r^1:0:r^2
590 &punpcklqdq ($D1,&QWP(16*1,"esp"));
591 &punpcklqdq ($D2,&QWP(16*2,"esp"));
592 &punpcklqdq ($D3,&QWP(16*3,"esp"));
593 &punpcklqdq ($D4,&QWP(16*4,"esp"));
594 &jmp (&label("square"));
596 &set_label("square_break");
597 &psllq ($D0,32); # -> r^3:0:r^4:0
602 &por ($D0,&QWP(16*0,"esp")); # r^3:r^1:r^4:r^2
603 &por ($D1,&QWP(16*1,"esp"));
604 &por ($D2,&QWP(16*2,"esp"));
605 &por ($D3,&QWP(16*3,"esp"));
606 &por ($D4,&QWP(16*4,"esp"));
608 &pshufd ($D0,$D0,0b10001101); # -> r^1:r^2:r^3:r^4
609 &pshufd ($D1,$D1,0b10001101);
610 &pshufd ($D2,$D2,0b10001101);
611 &pshufd ($D3,$D3,0b10001101);
612 &pshufd ($D4,$D4,0b10001101);
614 &movdqu (&QWP(16*0,"edi"),$D0); # save the table
615 &movdqu (&QWP(16*1,"edi"),$D1);
616 &movdqu (&QWP(16*2,"edi"),$D2);
617 &movdqu (&QWP(16*3,"edi"),$D3);
618 &movdqu (&QWP(16*4,"edi"),$D4);
624 &paddd ($T1,$D1); # *5
625 &paddd ($T0,$D2); # *5
626 &movdqu (&QWP(16*5,"edi"),$T1);
627 &movdqu (&QWP(16*6,"edi"),$T0);
632 &paddd ($T1,$D3); # *5
633 &paddd ($T0,$D4); # *5
634 &movdqu (&QWP(16*7,"edi"),$T1);
635 &movdqu (&QWP(16*8,"edi"),$T0);
638 &lea ("edi",&DWP(-16*3,"edi")); # size de-optimization
640 &function_end_B("_poly1305_init_sse2");
643 &function_begin("_poly1305_blocks_sse2");
644 &mov ("edi",&wparam(0)); # ctx
645 &mov ("esi",&wparam(1)); # inp
646 &mov ("ecx",&wparam(2)); # len
648 &mov ("eax",&DWP(4*5,"edi")); # is_base2_26
650 &jz (&label("nodata"));
652 &jae (&label("enter_sse2"));
653 &test ("eax","eax"); # is_base2_26?
654 &jz (&label("enter_blocks"));
656 &set_label("enter_sse2",16);
657 &call (&label("pic_point"));
658 &set_label("pic_point");
660 &lea ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
662 &test ("eax","eax"); # is_base2_26?
663 &jnz (&label("base2_26"));
665 &call ("_poly1305_init_sse2");
667 ################################################# base 2^32 -> base 2^26
668 &mov ("eax",&DWP(0,"edi"));
669 &mov ("ecx",&DWP(3,"edi"));
670 &mov ("edx",&DWP(6,"edi"));
671 &mov ("esi",&DWP(9,"edi"));
672 &mov ("ebp",&DWP(13,"edi"));
673 &mov (&DWP(4*5,"edi"),1); # is_base2_26
676 &and ("eax",0x3ffffff);
678 &and ("ecx",0x3ffffff);
680 &and ("edx",0x3ffffff);
688 &mov ("esi",&wparam(1)); # [reload] inp
689 &mov ("ecx",&wparam(2)); # [reload] len
690 &jmp (&label("base2_32"));
692 &set_label("base2_26",16);
693 &movd ($D0,&DWP(4*0,"edi")); # load hash value
694 &movd ($D1,&DWP(4*1,"edi"));
695 &movd ($D2,&DWP(4*2,"edi"));
696 &movd ($D3,&DWP(4*3,"edi"));
697 &movd ($D4,&DWP(4*4,"edi"));
698 &movdqa ($MASK,&QWP(64,"ebx"));
700 &set_label("base2_32");
701 &mov ("eax",&wparam(3)); # padbit
704 &sub ("esp",16*(5+5+5+9+9));
707 &lea ("edi",&DWP(16*3,"edi")); # size optimization
708 &shl ("eax",24); # padbit
711 &jz (&label("even"));
713 ################################################################
714 # process single block, with SSE2, because it's still faster
715 # even though half of result is discarded
717 &movdqu ($T1,&QWP(0,"esi")); # input
718 &lea ("esi",&DWP(16,"esi"));
720 &movdqa ($T0,$T1); # -> base 2^26 ...
722 &paddd ($D0,$T1); # ... and accumuate
741 &movd ($T0,"eax"); # padbit
743 &movd ($T1,&DWP(16*0+12,"edi")); # r0
746 &movdqa (&QWP(16*0,"esp"),$D0);
747 &movdqa (&QWP(16*1,"esp"),$D1);
748 &movdqa (&QWP(16*2,"esp"),$D2);
749 &movdqa (&QWP(16*3,"esp"),$D3);
750 &movdqa (&QWP(16*4,"esp"),$D4);
752 ################################################################
753 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
754 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
755 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
756 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
757 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
759 &pmuludq ($D0,$T1); # h4*r0
760 &pmuludq ($D1,$T1); # h3*r0
761 &pmuludq ($D2,$T1); # h2*r0
762 &movd ($T0,&DWP(16*1+12,"edi")); # r1
763 &pmuludq ($D3,$T1); # h1*r0
764 &pmuludq ($D4,$T1); # h0*r0
766 &pmuladd (sub { my ($reg,$i)=@_;
767 &movd ($reg,&DWP(16*$i+12,"edi"));
773 &jz (&label("done"));
776 &lea ("edx",&DWP(16*(5+5+5+9),"esp"));# size optimization
777 &lea ("eax",&DWP(-16*2,"esi"));
780 ################################################################
781 # expand and copy pre-calculated table to stack
783 &movdqu ($T0,&QWP(16*0,"edi")); # r^1:r^2:r^3:r^4
784 &pshufd ($T1,$T0,0b01000100); # duplicate r^3:r^4
785 &cmovb ("esi","eax");
786 &pshufd ($T0,$T0,0b11101110); # duplicate r^1:r^2
787 &movdqa (&QWP(16*0,"edx"),$T1);
788 &lea ("eax",&DWP(16*10,"esp"));
789 &movdqu ($T1,&QWP(16*1,"edi"));
790 &movdqa (&QWP(16*(0-9),"edx"),$T0);
791 &pshufd ($T0,$T1,0b01000100);
792 &pshufd ($T1,$T1,0b11101110);
793 &movdqa (&QWP(16*1,"edx"),$T0);
794 &movdqu ($T0,&QWP(16*2,"edi"));
795 &movdqa (&QWP(16*(1-9),"edx"),$T1);
796 &pshufd ($T1,$T0,0b01000100);
797 &pshufd ($T0,$T0,0b11101110);
798 &movdqa (&QWP(16*2,"edx"),$T1);
799 &movdqu ($T1,&QWP(16*3,"edi"));
800 &movdqa (&QWP(16*(2-9),"edx"),$T0);
801 &pshufd ($T0,$T1,0b01000100);
802 &pshufd ($T1,$T1,0b11101110);
803 &movdqa (&QWP(16*3,"edx"),$T0);
804 &movdqu ($T0,&QWP(16*4,"edi"));
805 &movdqa (&QWP(16*(3-9),"edx"),$T1);
806 &pshufd ($T1,$T0,0b01000100);
807 &pshufd ($T0,$T0,0b11101110);
808 &movdqa (&QWP(16*4,"edx"),$T1);
809 &movdqu ($T1,&QWP(16*5,"edi"));
810 &movdqa (&QWP(16*(4-9),"edx"),$T0);
811 &pshufd ($T0,$T1,0b01000100);
812 &pshufd ($T1,$T1,0b11101110);
813 &movdqa (&QWP(16*5,"edx"),$T0);
814 &movdqu ($T0,&QWP(16*6,"edi"));
815 &movdqa (&QWP(16*(5-9),"edx"),$T1);
816 &pshufd ($T1,$T0,0b01000100);
817 &pshufd ($T0,$T0,0b11101110);
818 &movdqa (&QWP(16*6,"edx"),$T1);
819 &movdqu ($T1,&QWP(16*7,"edi"));
820 &movdqa (&QWP(16*(6-9),"edx"),$T0);
821 &pshufd ($T0,$T1,0b01000100);
822 &pshufd ($T1,$T1,0b11101110);
823 &movdqa (&QWP(16*7,"edx"),$T0);
824 &movdqu ($T0,&QWP(16*8,"edi"));
825 &movdqa (&QWP(16*(7-9),"edx"),$T1);
826 &pshufd ($T1,$T0,0b01000100);
827 &pshufd ($T0,$T0,0b11101110);
828 &movdqa (&QWP(16*8,"edx"),$T1);
829 &movdqa (&QWP(16*(8-9),"edx"),$T0);
832 my ($inpbase,$offbase)=@_;
834 &movdqu ($T0,&QWP($inpbase+0,"esi")); # load input
835 &movdqu ($T1,&QWP($inpbase+16,"esi"));
836 &lea ("esi",&DWP(16*2,"esi"));
838 &movdqa (&QWP($offbase+16*2,"esp"),$D2);
839 &movdqa (&QWP($offbase+16*3,"esp"),$D3);
840 &movdqa (&QWP($offbase+16*4,"esp"),$D4);
842 &movdqa ($D2,$T0); # splat input
847 &punpcklqdq ($D2,$D3); # 2:3
848 &punpckhqdq ($D4,$T1); # 4
849 &punpcklqdq ($T0,$T1); # 0:1
857 &pand ($T0,$MASK); # 0
858 &pand ($T1,$MASK); # 1
859 &pand ($D2,$MASK); # 2
860 &pand ($D3,$MASK); # 3
861 &por ($D4,&QWP(0,"ebx")); # padbit, yes, always
863 &movdqa (&QWP($offbase+16*0,"esp"),$D0) if ($offbase);
864 &movdqa (&QWP($offbase+16*1,"esp"),$D1) if ($offbase);
866 &load_input (16*2,16*5);
868 &jbe (&label("skip_loop"));
869 &jmp (&label("loop"));
871 &set_label("loop",32);
872 ################################################################
873 # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
874 # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
875 # \___________________/
876 # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
877 # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
878 # \___________________/ \____________________/
879 ################################################################
881 &movdqa ($T2,&QWP(16*(0-9),"edx")); # r0^2
882 &movdqa (&QWP(16*1,"eax"),$T1);
883 &movdqa (&QWP(16*2,"eax"),$D2);
884 &movdqa (&QWP(16*3,"eax"),$D3);
885 &movdqa (&QWP(16*4,"eax"),$D4);
887 ################################################################
888 # d4 = h4*r0 + h0*r4 + h1*r3 + h2*r2 + h3*r1
889 # d3 = h3*r0 + h0*r3 + h1*r2 + h2*r1 + h4*5*r4
890 # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
891 # d1 = h1*r0 + h0*r1 + h2*5*r4 + h3*5*r3 + h4*5*r2
892 # d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
895 &pmuludq ($T0,$T2); # h0*r0
897 &pmuludq ($T1,$T2); # h1*r0
898 &pmuludq ($D2,$T2); # h2*r0
899 &pmuludq ($D3,$T2); # h3*r0
900 &pmuludq ($D4,$T2); # h4*r0
905 &pmuludq ($D0,&$addr(8)); # h1*s4
907 &pmuludq ($D1,&$addr(1)); # h0*r1
910 &pmuludq ($T2,&$addr(2)); # h0*r2
913 &pmuludq ($T0,&$addr(3)); # h0*r3
915 &movdqa ($T2,&QWP(16*1,"eax")); # pull h1
916 &pmuludq ($T1,&$addr(4)); # h0*r4
920 &pmuludq ($T2,&$addr(1)); # h1*r1
923 &pmuludq ($T0,&$addr(2)); # h1*r2
925 &movdqa ($T2,&QWP(16*2,"eax")); # pull h2
926 &pmuludq ($T1,&$addr(3)); # h1*r3
929 &pmuludq ($T2,&$addr(7)); # h2*s3
932 &pmuludq ($T0,&$addr(8)); # h2*s4
936 &pmuludq ($T1,&$addr(1)); # h2*r1
938 &movdqa ($T0,&QWP(16*3,"eax")); # pull h3
939 &pmuludq ($T2,&$addr(2)); # h2*r2
942 &pmuludq ($T0,&$addr(6)); # h3*s2
945 &pmuludq ($T1,&$addr(7)); # h3*s3
948 &pmuludq ($T2,&$addr(8)); # h3*s4
951 &movdqa ($T1,&QWP(16*4,"eax")); # pull h4
952 &pmuludq ($T0,&$addr(1)); # h3*r1
955 &pmuludq ($T1,&$addr(8)); # h4*s4
958 &pmuludq ($T2,&$addr(5)); # h4*s1
961 &pmuludq ($T0,&$addr(6)); # h4*s2
963 &movdqa ($MASK,&QWP(64,"ebx"));
964 &pmuludq ($T1,&$addr(7)); # h4*s3
968 &pmuladd_alt (sub { my $i=shift; &QWP(16*($i-9),"edx"); });
970 &load_input (-16*2,0);
971 &lea ("eax",&DWP(-16*2,"esi"));
974 &paddd ($T0,&QWP(16*(5+0),"esp")); # add hash value
975 &paddd ($T1,&QWP(16*(5+1),"esp"));
976 &paddd ($D2,&QWP(16*(5+2),"esp"));
977 &paddd ($D3,&QWP(16*(5+3),"esp"));
978 &paddd ($D4,&QWP(16*(5+4),"esp"));
980 &cmovb ("esi","eax");
981 &lea ("eax",&DWP(16*10,"esp"));
983 &movdqa ($T2,&QWP(16*0,"edx")); # r0^4
984 &movdqa (&QWP(16*1,"esp"),$D1);
985 &movdqa (&QWP(16*1,"eax"),$T1);
986 &movdqa (&QWP(16*2,"eax"),$D2);
987 &movdqa (&QWP(16*3,"eax"),$D3);
988 &movdqa (&QWP(16*4,"eax"),$D4);
990 ################################################################
991 # d4 += h4*r0 + h0*r4 + h1*r3 + h2*r2 + h3*r1
992 # d3 += h3*r0 + h0*r3 + h1*r2 + h2*r1 + h4*5*r4
993 # d2 += h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
994 # d1 += h1*r0 + h0*r1 + h2*5*r4 + h3*5*r3 + h4*5*r2
995 # d0 += h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
998 &pmuludq ($T0,$T2); # h0*r0
1001 &pmuludq ($T1,$T2); # h1*r0
1002 &pmuludq ($D2,$T2); # h2*r0
1003 &pmuludq ($D3,$T2); # h3*r0
1004 &pmuludq ($D4,$T2); # h4*r0
1006 &paddq ($T1,&QWP(16*1,"esp"));
1007 &paddq ($D2,&QWP(16*2,"esp"));
1008 &paddq ($D3,&QWP(16*3,"esp"));
1009 &paddq ($D4,&QWP(16*4,"esp"));
1011 &pmuladd_alt (sub { my $i=shift; &QWP(16*$i,"edx"); });
1015 &load_input (16*2,16*5);
1017 &ja (&label("loop"));
1019 &set_label("skip_loop");
1020 ################################################################
1021 # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
1023 &pshufd ($T2,&QWP(16*(0-9),"edx"),0x10);# r0^n
1025 &jnz (&label("long_tail"));
1027 &paddd ($T0,$D0); # add hash value
1029 &paddd ($D2,&QWP(16*7,"esp"));
1030 &paddd ($D3,&QWP(16*8,"esp"));
1031 &paddd ($D4,&QWP(16*9,"esp"));
1033 &set_label("long_tail");
1035 &movdqa (&QWP(16*0,"eax"),$T0);
1036 &movdqa (&QWP(16*1,"eax"),$T1);
1037 &movdqa (&QWP(16*2,"eax"),$D2);
1038 &movdqa (&QWP(16*3,"eax"),$D3);
1039 &movdqa (&QWP(16*4,"eax"),$D4);
1041 ################################################################
1042 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
1043 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
1044 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
1045 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
1046 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1048 &pmuludq ($T0,$T2); # h0*r0
1049 &pmuludq ($T1,$T2); # h1*r0
1050 &pmuludq ($D2,$T2); # h2*r0
1052 &pshufd ($T0,&QWP(16*(1-9),"edx"),0x10);# r1^n
1053 &pmuludq ($D3,$T2); # h3*r0
1055 &pmuludq ($D4,$T2); # h4*r0
1057 &pmuladd (sub { my ($reg,$i)=@_;
1058 &pshufd ($reg,&QWP(16*($i-9),"edx"),0x10);
1061 &jz (&label("short_tail"));
1063 &load_input (-16*2,0);
1065 &pshufd ($T2,&QWP(16*0,"edx"),0x10); # r0^n
1066 &paddd ($T0,&QWP(16*5,"esp")); # add hash value
1067 &paddd ($T1,&QWP(16*6,"esp"));
1068 &paddd ($D2,&QWP(16*7,"esp"));
1069 &paddd ($D3,&QWP(16*8,"esp"));
1070 &paddd ($D4,&QWP(16*9,"esp"));
1072 ################################################################
1073 # multiply inp[0:1] by r^4:r^3 and accumulate
1075 &movdqa (&QWP(16*0,"esp"),$T0);
1076 &pmuludq ($T0,$T2); # h0*r0
1077 &movdqa (&QWP(16*1,"esp"),$T1);
1078 &pmuludq ($T1,$T2); # h1*r0
1081 &pmuludq ($D2,$T2); # h2*r0
1084 &pmuludq ($D3,$T2); # h3*r0
1085 &paddq ($D2,&QWP(16*2,"esp"));
1086 &movdqa (&QWP(16*2,"esp"),$T0);
1087 &pshufd ($T0,&QWP(16*1,"edx"),0x10); # r1^n
1088 &paddq ($D3,&QWP(16*3,"esp"));
1089 &movdqa (&QWP(16*3,"esp"),$T1);
1091 &pmuludq ($D4,$T2); # h4*r0
1092 &paddq ($D4,&QWP(16*4,"esp"));
1093 &movdqa (&QWP(16*4,"esp"),$T1);
1095 &pmuladd (sub { my ($reg,$i)=@_;
1096 &pshufd ($reg,&QWP(16*$i,"edx"),0x10);
1099 &set_label("short_tail");
1101 ################################################################
1102 # horizontal addition
1104 &pshufd ($T1,$D4,0b01001110);
1105 &pshufd ($T0,$D3,0b01001110);
1108 &pshufd ($T1,$D0,0b01001110);
1109 &pshufd ($T0,$D1,0b01001110);
1112 &pshufd ($T1,$D2,0b01001110);
1115 &lazy_reduction (sub { &paddq ($D2,$T1) });
1118 &movd (&DWP(-16*3+4*0,"edi"),$D0); # store hash value
1119 &movd (&DWP(-16*3+4*1,"edi"),$D1);
1120 &movd (&DWP(-16*3+4*2,"edi"),$D2);
1121 &movd (&DWP(-16*3+4*3,"edi"),$D3);
1122 &movd (&DWP(-16*3+4*4,"edi"),$D4);
1124 &set_label("nodata");
1125 &function_end("_poly1305_blocks_sse2");
1128 &function_begin("_poly1305_emit_sse2");
1129 &mov ("ebp",&wparam(0)); # context
1131 &cmp (&DWP(4*5,"ebp"),0); # is_base2_26?
1132 &je (&label("enter_emit"));
1134 &mov ("eax",&DWP(4*0,"ebp")); # load hash value
1135 &mov ("edi",&DWP(4*1,"ebp"));
1136 &mov ("ecx",&DWP(4*2,"ebp"));
1137 &mov ("edx",&DWP(4*3,"ebp"));
1138 &mov ("esi",&DWP(4*4,"ebp"));
1140 &mov ("ebx","edi"); # base 2^26 -> base 2^32
1162 &adc ("esi",0); # can be partially reduced
1164 &mov ("edi","esi"); # final reduction
1167 &lea ("ebp",&DWP(0,"edi","edi",4)); # *5
1168 &mov ("edi",&wparam(1)); # output
1170 &mov ("ebp",&wparam(2)); # key
1175 &movd ($D0,"eax"); # offload original hash value
1176 &add ("eax",5); # compare to modulus
1184 &shr ("esi",2); # did it carry/borrow?
1186 &neg ("esi"); # do we choose (hash-modulus) ...
1191 &mov (&DWP(4*0,"edi"),"eax");
1193 &mov (&DWP(4*1,"edi"),"ebx");
1195 &mov (&DWP(4*2,"edi"),"ecx");
1197 &mov (&DWP(4*3,"edi"),"edx");
1200 ¬ ("esi"); # ... or original hash value?
1203 &or ("eax",&DWP(4*0,"edi"));
1205 &or ("ebx",&DWP(4*1,"edi"));
1207 &or ("ecx",&DWP(4*2,"edi"));
1208 &or ("edx",&DWP(4*3,"edi"));
1210 &add ("eax",&DWP(4*0,"ebp")); # accumulate key
1211 &adc ("ebx",&DWP(4*1,"ebp"));
1212 &mov (&DWP(4*0,"edi"),"eax");
1213 &adc ("ecx",&DWP(4*2,"ebp"));
1214 &mov (&DWP(4*1,"edi"),"ebx");
1215 &adc ("edx",&DWP(4*3,"ebp"));
1216 &mov (&DWP(4*2,"edi"),"ecx");
1217 &mov (&DWP(4*3,"edi"),"edx");
1218 &function_end("_poly1305_emit_sse2");
1221 ########################################################################
1222 # Note that poly1305_init_avx2 operates on %xmm, I could have used
1223 # poly1305_init_sse2...
1226 &function_begin_B("_poly1305_init_avx2");
1227 &vmovdqu ($D4,&QWP(4*6,"edi")); # key base 2^32
1228 &lea ("edi",&DWP(16*3,"edi")); # size optimization
1230 &sub ("esp",16*(9+5));
1233 #&vpand ($D4,$D4,&QWP(96,"ebx")); # magic mask
1234 &vmovdqa ($MASK,&QWP(64,"ebx"));
1236 &vpand ($D0,$D4,$MASK); # -> base 2^26
1237 &vpsrlq ($D1,$D4,26);
1238 &vpsrldq ($D3,$D4,6);
1239 &vpand ($D1,$D1,$MASK);
1241 &vpsrlq ($D3,$D3,30);
1242 &vpand ($D2,$D2,$MASK);
1243 &vpand ($D3,$D3,$MASK);
1244 &vpsrldq ($D4,$D4,13);
1246 &lea ("edx",&DWP(16*9,"esp")); # size optimization
1248 &set_label("square");
1249 &vmovdqa (&QWP(16*0,"esp"),$D0);
1250 &vmovdqa (&QWP(16*1,"esp"),$D1);
1251 &vmovdqa (&QWP(16*2,"esp"),$D2);
1252 &vmovdqa (&QWP(16*3,"esp"),$D3);
1253 &vmovdqa (&QWP(16*4,"esp"),$D4);
1255 &vpslld ($T1,$D1,2);
1256 &vpslld ($T0,$D2,2);
1257 &vpaddd ($T1,$T1,$D1); # *5
1258 &vpaddd ($T0,$T0,$D2); # *5
1259 &vmovdqa (&QWP(16*5,"esp"),$T1);
1260 &vmovdqa (&QWP(16*6,"esp"),$T0);
1261 &vpslld ($T1,$D3,2);
1262 &vpslld ($T0,$D4,2);
1263 &vpaddd ($T1,$T1,$D3); # *5
1264 &vpaddd ($T0,$T0,$D4); # *5
1265 &vmovdqa (&QWP(16*7,"esp"),$T1);
1266 &vmovdqa (&QWP(16*8,"esp"),$T0);
1268 &vpshufd ($T0,$D0,0b01000100);
1270 &vpshufd ($D1,$D1,0b01000100);
1271 &vpshufd ($D2,$D2,0b01000100);
1272 &vpshufd ($D3,$D3,0b01000100);
1273 &vpshufd ($D4,$D4,0b01000100);
1274 &vmovdqa (&QWP(16*0,"edx"),$T0);
1275 &vmovdqa (&QWP(16*1,"edx"),$D1);
1276 &vmovdqa (&QWP(16*2,"edx"),$D2);
1277 &vmovdqa (&QWP(16*3,"edx"),$D3);
1278 &vmovdqa (&QWP(16*4,"edx"),$D4);
1280 ################################################################
1281 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
1282 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
1283 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
1284 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
1285 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1287 &vpmuludq ($D4,$D4,$D0); # h4*r0
1288 &vpmuludq ($D3,$D3,$D0); # h3*r0
1289 &vpmuludq ($D2,$D2,$D0); # h2*r0
1290 &vpmuludq ($D1,$D1,$D0); # h1*r0
1291 &vpmuludq ($D0,$T0,$D0); # h0*r0
1293 &vpmuludq ($T0,$T1,&QWP(16*3,"edx")); # r1*h3
1294 &vpaddq ($D4,$D4,$T0);
1295 &vpmuludq ($T2,$T1,&QWP(16*2,"edx")); # r1*h2
1296 &vpaddq ($D3,$D3,$T2);
1297 &vpmuludq ($T0,$T1,&QWP(16*1,"edx")); # r1*h1
1298 &vpaddq ($D2,$D2,$T0);
1299 &vmovdqa ($T2,&QWP(16*5,"esp")); # s1
1300 &vpmuludq ($T1,$T1,&QWP(16*0,"edx")); # r1*h0
1301 &vpaddq ($D1,$D1,$T1);
1302 &vmovdqa ($T0,&QWP(16*2,"esp")); # r2
1303 &vpmuludq ($T2,$T2,&QWP(16*4,"edx")); # s1*h4
1304 &vpaddq ($D0,$D0,$T2);
1306 &vpmuludq ($T1,$T0,&QWP(16*2,"edx")); # r2*h2
1307 &vpaddq ($D4,$D4,$T1);
1308 &vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # r2*h1
1309 &vpaddq ($D3,$D3,$T2);
1310 &vmovdqa ($T1,&QWP(16*6,"esp")); # s2
1311 &vpmuludq ($T0,$T0,&QWP(16*0,"edx")); # r2*h0
1312 &vpaddq ($D2,$D2,$T0);
1313 &vpmuludq ($T2,$T1,&QWP(16*4,"edx")); # s2*h4
1314 &vpaddq ($D1,$D1,$T2);
1315 &vmovdqa ($T0,&QWP(16*3,"esp")); # r3
1316 &vpmuludq ($T1,$T1,&QWP(16*3,"edx")); # s2*h3
1317 &vpaddq ($D0,$D0,$T1);
1319 &vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # r3*h1
1320 &vpaddq ($D4,$D4,$T2);
1321 &vmovdqa ($T1,&QWP(16*7,"esp")); # s3
1322 &vpmuludq ($T0,$T0,&QWP(16*0,"edx")); # r3*h0
1323 &vpaddq ($D3,$D3,$T0);
1324 &vpmuludq ($T2,$T1,&QWP(16*4,"edx")); # s3*h4
1325 &vpaddq ($D2,$D2,$T2);
1326 &vpmuludq ($T0,$T1,&QWP(16*3,"edx")); # s3*h3
1327 &vpaddq ($D1,$D1,$T0);
1328 &vmovdqa ($T2,&QWP(16*4,"esp")); # r4
1329 &vpmuludq ($T1,$T1,&QWP(16*2,"edx")); # s3*h2
1330 &vpaddq ($D0,$D0,$T1);
1332 &vmovdqa ($T0,&QWP(16*8,"esp")); # s4
1333 &vpmuludq ($T2,$T2,&QWP(16*0,"edx")); # r4*h0
1334 &vpaddq ($D4,$D4,$T2);
1335 &vpmuludq ($T1,$T0,&QWP(16*4,"edx")); # s4*h4
1336 &vpaddq ($D3,$D3,$T1);
1337 &vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # s4*h1
1338 &vpaddq ($D0,$D0,$T2);
1339 &vpmuludq ($T1,$T0,&QWP(16*2,"edx")); # s4*h2
1340 &vpaddq ($D1,$D1,$T1);
1341 &vmovdqa ($MASK,&QWP(64,"ebx"));
1342 &vpmuludq ($T0,$T0,&QWP(16*3,"edx")); # s4*h3
1343 &vpaddq ($D2,$D2,$T0);
1345 ################################################################
1347 &vpsrlq ($T0,$D3,26);
1348 &vpand ($D3,$D3,$MASK);
1349 &vpsrlq ($T1,$D0,26);
1350 &vpand ($D0,$D0,$MASK);
1351 &vpaddq ($D4,$D4,$T0); # h3 -> h4
1352 &vpaddq ($D1,$D1,$T1); # h0 -> h1
1353 &vpsrlq ($T0,$D4,26);
1354 &vpand ($D4,$D4,$MASK);
1355 &vpsrlq ($T1,$D1,26);
1356 &vpand ($D1,$D1,$MASK);
1357 &vpaddq ($D2,$D2,$T1); # h1 -> h2
1358 &vpaddd ($D0,$D0,$T0);
1359 &vpsllq ($T0,$T0,2);
1360 &vpsrlq ($T1,$D2,26);
1361 &vpand ($D2,$D2,$MASK);
1362 &vpaddd ($D0,$D0,$T0); # h4 -> h0
1363 &vpaddd ($D3,$D3,$T1); # h2 -> h3
1364 &vpsrlq ($T1,$D3,26);
1365 &vpsrlq ($T0,$D0,26);
1366 &vpand ($D0,$D0,$MASK);
1367 &vpand ($D3,$D3,$MASK);
1368 &vpaddd ($D1,$D1,$T0); # h0 -> h1
1369 &vpaddd ($D4,$D4,$T1); # h3 -> h4
1372 &jz (&label("square_break"));
1374 &vpunpcklqdq ($D0,$D0,&QWP(16*0,"esp")); # 0:r^1:0:r^2
1375 &vpunpcklqdq ($D1,$D1,&QWP(16*1,"esp"));
1376 &vpunpcklqdq ($D2,$D2,&QWP(16*2,"esp"));
1377 &vpunpcklqdq ($D3,$D3,&QWP(16*3,"esp"));
1378 &vpunpcklqdq ($D4,$D4,&QWP(16*4,"esp"));
1379 &jmp (&label("square"));
1381 &set_label("square_break");
1382 &vpsllq ($D0,$D0,32); # -> r^3:0:r^4:0
1383 &vpsllq ($D1,$D1,32);
1384 &vpsllq ($D2,$D2,32);
1385 &vpsllq ($D3,$D3,32);
1386 &vpsllq ($D4,$D4,32);
1387 &vpor ($D0,$D0,&QWP(16*0,"esp")); # r^3:r^1:r^4:r^2
1388 &vpor ($D1,$D1,&QWP(16*1,"esp"));
1389 &vpor ($D2,$D2,&QWP(16*2,"esp"));
1390 &vpor ($D3,$D3,&QWP(16*3,"esp"));
1391 &vpor ($D4,$D4,&QWP(16*4,"esp"));
1393 &vpshufd ($D0,$D0,0b10001101); # -> r^1:r^2:r^3:r^4
1394 &vpshufd ($D1,$D1,0b10001101);
1395 &vpshufd ($D2,$D2,0b10001101);
1396 &vpshufd ($D3,$D3,0b10001101);
1397 &vpshufd ($D4,$D4,0b10001101);
1399 &vmovdqu (&QWP(16*0,"edi"),$D0); # save the table
1400 &vmovdqu (&QWP(16*1,"edi"),$D1);
1401 &vmovdqu (&QWP(16*2,"edi"),$D2);
1402 &vmovdqu (&QWP(16*3,"edi"),$D3);
1403 &vmovdqu (&QWP(16*4,"edi"),$D4);
1405 &vpslld ($T1,$D1,2);
1406 &vpslld ($T0,$D2,2);
1407 &vpaddd ($T1,$T1,$D1); # *5
1408 &vpaddd ($T0,$T0,$D2); # *5
1409 &vmovdqu (&QWP(16*5,"edi"),$T1);
1410 &vmovdqu (&QWP(16*6,"edi"),$T0);
1411 &vpslld ($T1,$D3,2);
1412 &vpslld ($T0,$D4,2);
1413 &vpaddd ($T1,$T1,$D3); # *5
1414 &vpaddd ($T0,$T0,$D4); # *5
1415 &vmovdqu (&QWP(16*7,"edi"),$T1);
1416 &vmovdqu (&QWP(16*8,"edi"),$T0);
1419 &lea ("edi",&DWP(-16*3,"edi")); # size de-optimization
1421 &function_end_B("_poly1305_init_avx2");
1423 ########################################################################
1424 # now it's time to switch to %ymm
1426 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("ymm$_",(0..7));
1429 sub X { my $reg=shift; $reg=~s/^ymm/xmm/; $reg; }
1432 &function_begin("_poly1305_blocks_avx2");
1433 &mov ("edi",&wparam(0)); # ctx
1434 &mov ("esi",&wparam(1)); # inp
1435 &mov ("ecx",&wparam(2)); # len
1437 &mov ("eax",&DWP(4*5,"edi")); # is_base2_26
1439 &jz (&label("nodata"));
1441 &jae (&label("enter_avx2"));
1442 &test ("eax","eax"); # is_base2_26?
1443 &jz (&label("enter_blocks"));
1445 &set_label("enter_avx2");
1448 &call (&label("pic_point"));
1449 &set_label("pic_point");
1451 &lea ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
1453 &test ("eax","eax"); # is_base2_26?
1454 &jnz (&label("base2_26"));
1456 &call ("_poly1305_init_avx2");
1458 ################################################# base 2^32 -> base 2^26
1459 &mov ("eax",&DWP(0,"edi"));
1460 &mov ("ecx",&DWP(3,"edi"));
1461 &mov ("edx",&DWP(6,"edi"));
1462 &mov ("esi",&DWP(9,"edi"));
1463 &mov ("ebp",&DWP(13,"edi"));
1466 &and ("eax",0x3ffffff);
1468 &and ("ecx",0x3ffffff);
1470 &and ("edx",0x3ffffff);
1472 &mov (&DWP(4*0,"edi"),"eax");
1473 &mov (&DWP(4*1,"edi"),"ecx");
1474 &mov (&DWP(4*2,"edi"),"edx");
1475 &mov (&DWP(4*3,"edi"),"esi");
1476 &mov (&DWP(4*4,"edi"),"ebp");
1477 &mov (&DWP(4*5,"edi"),1); # is_base2_26
1479 &mov ("esi",&wparam(1)); # [reload] inp
1480 &mov ("ecx",&wparam(2)); # [reload] len
1482 &set_label("base2_26");
1483 &mov ("eax",&wparam(3)); # padbit
1486 &sub ("esp",32*(5+9));
1487 &and ("esp",-512); # ensure that frame
1488 # doesn't cross page
1489 # boundary, which is
1491 # misaligned 32-byte
1494 ################################################################
1495 # expand and copy pre-calculated table to stack
1497 &vmovdqu (&X($D0),&QWP(16*(3+0),"edi"));
1498 &lea ("edx",&DWP(32*5+128,"esp")); # +128 size optimization
1499 &vmovdqu (&X($D1),&QWP(16*(3+1),"edi"));
1500 &vmovdqu (&X($D2),&QWP(16*(3+2),"edi"));
1501 &vmovdqu (&X($D3),&QWP(16*(3+3),"edi"));
1502 &vmovdqu (&X($D4),&QWP(16*(3+4),"edi"));
1503 &lea ("edi",&DWP(16*3,"edi")); # size optimization
1504 &vpermq ($D0,$D0,0b01000000); # 00001234 -> 12343434
1505 &vpermq ($D1,$D1,0b01000000);
1506 &vpermq ($D2,$D2,0b01000000);
1507 &vpermq ($D3,$D3,0b01000000);
1508 &vpermq ($D4,$D4,0b01000000);
1509 &vpshufd ($D0,$D0,0b11001000); # 12343434 -> 14243444
1510 &vpshufd ($D1,$D1,0b11001000);
1511 &vpshufd ($D2,$D2,0b11001000);
1512 &vpshufd ($D3,$D3,0b11001000);
1513 &vpshufd ($D4,$D4,0b11001000);
1514 &vmovdqa (&QWP(32*0-128,"edx"),$D0);
1515 &vmovdqu (&X($D0),&QWP(16*5,"edi"));
1516 &vmovdqa (&QWP(32*1-128,"edx"),$D1);
1517 &vmovdqu (&X($D1),&QWP(16*6,"edi"));
1518 &vmovdqa (&QWP(32*2-128,"edx"),$D2);
1519 &vmovdqu (&X($D2),&QWP(16*7,"edi"));
1520 &vmovdqa (&QWP(32*3-128,"edx"),$D3);
1521 &vmovdqu (&X($D3),&QWP(16*8,"edi"));
1522 &vmovdqa (&QWP(32*4-128,"edx"),$D4);
1523 &vpermq ($D0,$D0,0b01000000);
1524 &vpermq ($D1,$D1,0b01000000);
1525 &vpermq ($D2,$D2,0b01000000);
1526 &vpermq ($D3,$D3,0b01000000);
1527 &vpshufd ($D0,$D0,0b11001000);
1528 &vpshufd ($D1,$D1,0b11001000);
1529 &vpshufd ($D2,$D2,0b11001000);
1530 &vpshufd ($D3,$D3,0b11001000);
1531 &vmovdqa (&QWP(32*5-128,"edx"),$D0);
1532 &vmovd (&X($D0),&DWP(-16*3+4*0,"edi"));# load hash value
1533 &vmovdqa (&QWP(32*6-128,"edx"),$D1);
1534 &vmovd (&X($D1),&DWP(-16*3+4*1,"edi"));
1535 &vmovdqa (&QWP(32*7-128,"edx"),$D2);
1536 &vmovd (&X($D2),&DWP(-16*3+4*2,"edi"));
1537 &vmovdqa (&QWP(32*8-128,"edx"),$D3);
1538 &vmovd (&X($D3),&DWP(-16*3+4*3,"edi"));
1539 &vmovd (&X($D4),&DWP(-16*3+4*4,"edi"));
1540 &vmovdqa ($MASK,&QWP(64,"ebx"));
1541 &neg ("eax"); # padbit
1544 &jz (&label("even"));
1550 &vmovdqu (&X($T0),&QWP(16*0,"esi"));
1552 &jb (&label("one"));
1554 &vmovdqu (&X($T1),&QWP(16*1,"esi"));
1555 &je (&label("two"));
1557 &vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
1558 &lea ("esi",&DWP(16*3,"esi"));
1559 &lea ("ebx",&DWP(8,"ebx")); # three padbits
1560 &lea ("edx",&DWP(32*5+128+8,"esp")); # --:r^1:r^2:r^3 (*)
1561 &jmp (&label("tail"));
1564 &lea ("esi",&DWP(16*2,"esi"));
1565 &lea ("ebx",&DWP(16,"ebx")); # two padbits
1566 &lea ("edx",&DWP(32*5+128+16,"esp"));# --:--:r^1:r^2 (*)
1567 &jmp (&label("tail"));
1570 &lea ("esi",&DWP(16*1,"esi"));
1571 &vpxor ($T1,$T1,$T1);
1572 &lea ("ebx",&DWP(32,"ebx","eax",8)); # one or no padbits
1573 &lea ("edx",&DWP(32*5+128+24,"esp"));# --:--:--:r^1 (*)
1574 &jmp (&label("tail"));
1576 # (*) spots marked with '--' are data from next table entry, but they
1577 # are multiplied by 0 and therefore rendered insignificant
1579 &set_label("even",32);
1580 &vmovdqu (&X($T0),&QWP(16*0,"esi")); # load input
1581 &vmovdqu (&X($T1),&QWP(16*1,"esi"));
1582 &vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
1583 &vinserti128 ($T1,$T1,&QWP(16*3,"esi"),1);
1584 &lea ("esi",&DWP(16*4,"esi"));
1586 &jz (&label("tail"));
1589 ################################################################
1590 # ((inp[0]*r^4+r[4])*r^4+r[8])*r^4
1591 # ((inp[1]*r^4+r[5])*r^4+r[9])*r^3
1592 # ((inp[2]*r^4+r[6])*r^4+r[10])*r^2
1593 # ((inp[3]*r^4+r[7])*r^4+r[11])*r^1
1594 # \________/ \_______/
1595 ################################################################
1598 &vmovdqa (&QWP(32*2,"esp"),$D2);
1599 &vpsrldq ($D2,$T0,6); # splat input
1600 &vmovdqa (&QWP(32*0,"esp"),$D0);
1601 &vpsrldq ($D0,$T1,6);
1602 &vmovdqa (&QWP(32*1,"esp"),$D1);
1603 &vpunpckhqdq ($D1,$T0,$T1); # 4
1604 &vpunpcklqdq ($T0,$T0,$T1); # 0:1
1605 &vpunpcklqdq ($D2,$D2,$D0); # 2:3
1607 &vpsrlq ($D0,$D2,30);
1608 &vpsrlq ($D2,$D2,4);
1609 &vpsrlq ($T1,$T0,26);
1610 &vpsrlq ($D1,$D1,40); # 4
1611 &vpand ($D2,$D2,$MASK); # 2
1612 &vpand ($T0,$T0,$MASK); # 0
1613 &vpand ($T1,$T1,$MASK); # 1
1614 &vpand ($D0,$D0,$MASK); # 3 (*)
1615 &vpor ($D1,$D1,&QWP(0,"ebx")); # padbit, yes, always
1617 # (*) note that output is counterintuitive, inp[3:4] is
1618 # returned in $D1-2, while $D3-4 are preserved;
1625 &vpaddq ($D2,$D2,&QWP(32*2,"esp")); # add hash value
1626 &vpaddq ($T0,$T0,&QWP(32*0,"esp"));
1627 &vpaddq ($T1,$T1,&QWP(32*1,"esp"));
1628 &vpaddq ($D0,$D0,$D3);
1629 &vpaddq ($D1,$D1,$D4);
1631 ################################################################
1632 # d3 = h2*r1 + h0*r3 + h1*r2 + h3*r0 + h4*5*r4
1633 # d4 = h2*r2 + h0*r4 + h1*r3 + h3*r1 + h4*r0
1634 # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1
1635 # d1 = h2*5*r4 + h0*r1 + h1*r0 + h3*5*r3 + h4*5*r2
1636 # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
1638 &vpmuludq ($D3,$D2,&$addr(1)); # d3 = h2*r1
1639 &vmovdqa (QWP(32*1,"esp"),$T1);
1640 &vpmuludq ($D4,$D2,&$addr(2)); # d4 = h2*r2
1641 &vmovdqa (QWP(32*3,"esp"),$D0);
1642 &vpmuludq ($D0,$D2,&$addr(7)); # d0 = h2*s3
1643 &vmovdqa (QWP(32*4,"esp"),$D1);
1644 &vpmuludq ($D1,$D2,&$addr(8)); # d1 = h2*s4
1645 &vpmuludq ($D2,$D2,&$addr(0)); # d2 = h2*r0
1647 &vpmuludq ($T2,$T0,&$addr(3)); # h0*r3
1648 &vpaddq ($D3,$D3,$T2); # d3 += h0*r3
1649 &vpmuludq ($T1,$T0,&$addr(4)); # h0*r4
1650 &vpaddq ($D4,$D4,$T1); # d4 + h0*r4
1651 &vpmuludq ($T2,$T0,&$addr(0)); # h0*r0
1652 &vpaddq ($D0,$D0,$T2); # d0 + h0*r0
1653 &vmovdqa ($T2,&QWP(32*1,"esp")); # h1
1654 &vpmuludq ($T1,$T0,&$addr(1)); # h0*r1
1655 &vpaddq ($D1,$D1,$T1); # d1 += h0*r1
1656 &vpmuludq ($T0,$T0,&$addr(2)); # h0*r2
1657 &vpaddq ($D2,$D2,$T0); # d2 += h0*r2
1659 &vpmuludq ($T1,$T2,&$addr(2)); # h1*r2
1660 &vpaddq ($D3,$D3,$T1); # d3 += h1*r2
1661 &vpmuludq ($T0,$T2,&$addr(3)); # h1*r3
1662 &vpaddq ($D4,$D4,$T0); # d4 += h1*r3
1663 &vpmuludq ($T1,$T2,&$addr(8)); # h1*s4
1664 &vpaddq ($D0,$D0,$T1); # d0 += h1*s4
1665 &vmovdqa ($T1,&QWP(32*3,"esp")); # h3
1666 &vpmuludq ($T0,$T2,&$addr(0)); # h1*r0
1667 &vpaddq ($D1,$D1,$T0); # d1 += h1*r0
1668 &vpmuludq ($T2,$T2,&$addr(1)); # h1*r1
1669 &vpaddq ($D2,$D2,$T2); # d2 += h1*r1
1671 &vpmuludq ($T0,$T1,&$addr(0)); # h3*r0
1672 &vpaddq ($D3,$D3,$T0); # d3 += h3*r0
1673 &vpmuludq ($T2,$T1,&$addr(1)); # h3*r1
1674 &vpaddq ($D4,$D4,$T2); # d4 += h3*r1
1675 &vpmuludq ($T0,$T1,&$addr(6)); # h3*s2
1676 &vpaddq ($D0,$D0,$T0); # d0 += h3*s2
1677 &vmovdqa ($T0,&QWP(32*4,"esp")); # h4
1678 &vpmuludq ($T2,$T1,&$addr(7)); # h3*s3
1679 &vpaddq ($D1,$D1,$T2); # d1+= h3*s3
1680 &vpmuludq ($T1,$T1,&$addr(8)); # h3*s4
1681 &vpaddq ($D2,$D2,$T1); # d2 += h3*s4
1683 &vpmuludq ($T2,$T0,&$addr(8)); # h4*s4
1684 &vpaddq ($D3,$D3,$T2); # d3 += h4*s4
1685 &vpmuludq ($T1,$T0,&$addr(5)); # h4*s1
1686 &vpaddq ($D0,$D0,$T1); # d0 += h4*s1
1687 &vpmuludq ($T2,$T0,&$addr(0)); # h4*r0
1688 &vpaddq ($D4,$D4,$T2); # d4 += h4*r0
1689 &vmovdqa ($MASK,&QWP(64,"ebx"));
1690 &vpmuludq ($T1,$T0,&$addr(6)); # h4*s2
1691 &vpaddq ($D1,$D1,$T1); # d1 += h4*s2
1692 &vpmuludq ($T0,$T0,&$addr(7)); # h4*s3
1693 &vpaddq ($D2,$D2,$T0); # d2 += h4*s3
1695 &vpmuladd (sub { my $i=shift; &QWP(32*$i-128,"edx"); });
1697 sub vlazy_reduction {
1698 ################################################################
1701 &vpsrlq ($T0,$D3,26);
1702 &vpand ($D3,$D3,$MASK);
1703 &vpsrlq ($T1,$D0,26);
1704 &vpand ($D0,$D0,$MASK);
1705 &vpaddq ($D4,$D4,$T0); # h3 -> h4
1706 &vpaddq ($D1,$D1,$T1); # h0 -> h1
1707 &vpsrlq ($T0,$D4,26);
1708 &vpand ($D4,$D4,$MASK);
1709 &vpsrlq ($T1,$D1,26);
1710 &vpand ($D1,$D1,$MASK);
1711 &vpaddq ($D2,$D2,$T1); # h1 -> h2
1712 &vpaddq ($D0,$D0,$T0);
1713 &vpsllq ($T0,$T0,2);
1714 &vpsrlq ($T1,$D2,26);
1715 &vpand ($D2,$D2,$MASK);
1716 &vpaddq ($D0,$D0,$T0); # h4 -> h0
1717 &vpaddq ($D3,$D3,$T1); # h2 -> h3
1718 &vpsrlq ($T1,$D3,26);
1719 &vpsrlq ($T0,$D0,26);
1720 &vpand ($D0,$D0,$MASK);
1721 &vpand ($D3,$D3,$MASK);
1722 &vpaddq ($D1,$D1,$T0); # h0 -> h1
1723 &vpaddq ($D4,$D4,$T1); # h3 -> h4
1727 &vmovdqu (&X($T0),&QWP(16*0,"esi")); # load input
1728 &vmovdqu (&X($T1),&QWP(16*1,"esi"));
1729 &vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
1730 &vinserti128 ($T1,$T1,&QWP(16*3,"esi"),1);
1731 &lea ("esi",&DWP(16*4,"esi"));
1733 &jnz (&label("loop"));
1737 &and ("ebx",-64); # restore pointer
1739 &vpmuladd (sub { my $i=shift; &QWP(4+32*$i-128,"edx"); });
1741 ################################################################
1742 # horizontal addition
1744 &vpsrldq ($T0,$D4,8);
1745 &vpsrldq ($T1,$D3,8);
1746 &vpaddq ($D4,$D4,$T0);
1747 &vpsrldq ($T0,$D0,8);
1748 &vpaddq ($D3,$D3,$T1);
1749 &vpsrldq ($T1,$D1,8);
1750 &vpaddq ($D0,$D0,$T0);
1751 &vpsrldq ($T0,$D2,8);
1752 &vpaddq ($D1,$D1,$T1);
1753 &vpermq ($T1,$D4,2); # keep folding
1754 &vpaddq ($D2,$D2,$T0);
1755 &vpermq ($T0,$D3,2);
1756 &vpaddq ($D4,$D4,$T1);
1757 &vpermq ($T1,$D0,2);
1758 &vpaddq ($D3,$D3,$T0);
1759 &vpermq ($T0,$D1,2);
1760 &vpaddq ($D0,$D0,$T1);
1761 &vpermq ($T1,$D2,2);
1762 &vpaddq ($D1,$D1,$T0);
1763 &vpaddq ($D2,$D2,$T1);
1768 &je (&label("done"));
1770 ################################################################
1771 # clear all but single word
1773 &vpshufd (&X($D0),&X($D0),0b11111100);
1774 &lea ("edx",&DWP(32*5+128,"esp")); # restore pointer
1775 &vpshufd (&X($D1),&X($D1),0b11111100);
1776 &vpshufd (&X($D2),&X($D2),0b11111100);
1777 &vpshufd (&X($D3),&X($D3),0b11111100);
1778 &vpshufd (&X($D4),&X($D4),0b11111100);
1779 &jmp (&label("even"));
1781 &set_label("done",16);
1782 &vmovd (&DWP(-16*3+4*0,"edi"),&X($D0));# store hash value
1783 &vmovd (&DWP(-16*3+4*1,"edi"),&X($D1));
1784 &vmovd (&DWP(-16*3+4*2,"edi"),&X($D2));
1785 &vmovd (&DWP(-16*3+4*3,"edi"),&X($D3));
1786 &vmovd (&DWP(-16*3+4*4,"edi"),&X($D4));
1789 &set_label("nodata");
1790 &function_end("_poly1305_blocks_avx2");
1792 &set_label("const_sse2",64);
1793 &data_word(1<<24,0, 1<<24,0, 1<<24,0, 1<<24,0);
1794 &data_word(0,0, 0,0, 0,0, 0,0);
1795 &data_word(0x03ffffff,0,0x03ffffff,0, 0x03ffffff,0, 0x03ffffff,0);
1796 &data_word(0x0fffffff,0x0ffffffc,0x0ffffffc,0x0ffffffc);
1798 &asciz ("Poly1305 for x86, CRYPTOGAMS by <appro\@openssl.org>");