crypto/poly1305: don't break carry chains.
[openssl.git] / crypto / poly1305 / asm / poly1305-x86.pl
1 #!/usr/bin/env perl
2 #
3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9 #
10 # This module implements Poly1305 hash for x86.
11 #
12 # April 2015
13 #
14 # Numbers are cycles per processed byte with poly1305_blocks alone,
15 # measured with rdtsc at fixed clock frequency.
16 #
17 #               IALU/gcc-3.4(*) SSE2(**)        AVX2
18 # Pentium       15.7/+80%       -
19 # PIII          6.21/+90%       -
20 # P4            19.8/+40%       3.24
21 # Core 2        4.85/+90%       1.80
22 # Westmere      4.58/+100%      1.43
23 # Sandy Bridge  3.90/+100%      1.36
24 # Haswell       3.88/+70%       1.18            0.72
25 # Silvermont    11.0/+40%       4.80
26 # VIA Nano      6.71/+90%       2.47
27 # Sledgehammer  3.51/+180%      4.27
28 # Bulldozer     4.53/+140%      1.31
29 #
30 # (*)   gcc 4.8 for some reason generated worse code;
31 # (**)  besides SSE2 there are floating-point and AVX options; FP
32 #       is deemed unnecessary, because pre-SSE2 processor are too
33 #       old to care about, while it's not the fastest option on
34 #       SSE2-capable ones; AVX is omitted, because it doesn't give
35 #       a lot of improvement, 5-10% depending on processor;
36
37 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
38 push(@INC,"${dir}","${dir}../../perlasm");
39 require "x86asm.pl";
40
41 $output=pop;
42 open STDOUT,">$output";
43
44 &asm_init($ARGV[0],"poly1305-x86.pl",$ARGV[$#ARGV] eq "386");
45
46 $sse2=$avx=0;
47 for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
48
49 if ($sse2) {
50         &static_label("const_sse2");
51         &static_label("enter_blocks");
52         &static_label("enter_emit");
53         &external_label("OPENSSL_ia32cap_P");
54
55         if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
56                         =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
57                 $avx = ($1>=2.19) + ($1>=2.22);
58         }
59
60         if (!$avx && $ARGV[0] eq "win32n" &&
61            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
62         $avx = ($1>=2.09) + ($1>=2.10);
63         }
64
65         if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) {
66                 $avx = ($2>=3.0) + ($2>3.0);
67         }
68 }
69
70 ########################################################################
71 # Layout of opaque area is following.
72 #
73 #       unsigned __int32 h[5];          # current hash value base 2^32
74 #       unsigned __int32 pad;           # is_base2_26 in vector context
75 #       unsigned __int32 r[4];          # key value base 2^32
76
77 &align(64);
78 &function_begin("poly1305_init");
79         &mov    ("edi",&wparam(0));             # context
80         &mov    ("esi",&wparam(1));             # key
81         &mov    ("ebp",&wparam(2));             # function table
82
83         &xor    ("eax","eax");
84         &mov    (&DWP(4*0,"edi"),"eax");        # zero hash value
85         &mov    (&DWP(4*1,"edi"),"eax");
86         &mov    (&DWP(4*2,"edi"),"eax");
87         &mov    (&DWP(4*3,"edi"),"eax");
88         &mov    (&DWP(4*4,"edi"),"eax");
89         &mov    (&DWP(4*5,"edi"),"eax");        # is_base2_26
90
91         &cmp    ("esi",0);
92         &je     (&label("nokey"));
93
94     if ($sse2) {
95         &call   (&label("pic_point"));
96     &set_label("pic_point");
97         &blindpop("ebx");
98
99         &lea    ("eax",&DWP("poly1305_blocks-".&label("pic_point"),"ebx"));
100         &lea    ("edx",&DWP("poly1305_emit-".&label("pic_point"),"ebx"));
101
102         &picmeup("edi","OPENSSL_ia32cap_P","ebx",&label("pic_point"));
103         &mov    ("ecx",&DWP(0,"edi"));
104         &and    ("ecx",1<<26|1<<24);
105         &cmp    ("ecx",1<<26|1<<24);            # SSE2 and XMM?
106         &jne    (&label("no_sse2"));
107
108         &lea    ("eax",&DWP("_poly1305_blocks_sse2-".&label("pic_point"),"ebx"));
109         &lea    ("edx",&DWP("_poly1305_emit_sse2-".&label("pic_point"),"ebx"));
110
111       if ($avx>1) {
112         &mov    ("ecx",&DWP(8,"edi"));
113         &test   ("ecx",1<<5);                   # AVX2?
114         &jz     (&label("no_sse2"));
115
116         &lea    ("eax",&DWP("_poly1305_blocks_avx2-".&label("pic_point"),"ebx"));
117       }
118     &set_label("no_sse2");
119         &mov    ("edi",&wparam(0));             # reload context
120         &mov    (&DWP(0,"ebp"),"eax");          # fill function table
121         &mov    (&DWP(4,"ebp"),"edx");
122     }
123
124         &mov    ("eax",&DWP(4*0,"esi"));        # load input key
125         &mov    ("ebx",&DWP(4*1,"esi"));
126         &mov    ("ecx",&DWP(4*2,"esi"));
127         &mov    ("edx",&DWP(4*3,"esi"));
128         &and    ("eax",0x0fffffff);
129         &and    ("ebx",0x0ffffffc);
130         &and    ("ecx",0x0ffffffc);
131         &and    ("edx",0x0ffffffc);
132         &mov    (&DWP(4*6,"edi"),"eax");
133         &mov    (&DWP(4*7,"edi"),"ebx");
134         &mov    (&DWP(4*8,"edi"),"ecx");
135         &mov    (&DWP(4*9,"edi"),"edx");
136
137         &mov    ("eax",$sse2);
138 &set_label("nokey");
139 &function_end("poly1305_init");
140
141 ($h0,$h1,$h2,$h3,$h4,
142  $d0,$d1,$d2,$d3,
143  $r0,$r1,$r2,$r3,
144      $s1,$s2,$s3)=map(4*$_,(0..15));
145
146 &function_begin("poly1305_blocks");
147         &mov    ("edi",&wparam(0));             # ctx
148         &mov    ("esi",&wparam(1));             # inp
149         &mov    ("ecx",&wparam(2));             # len
150 &set_label("enter_blocks");
151         &and    ("ecx",-15);
152         &jz     (&label("nodata"));
153
154         &stack_push(16);
155         &mov    ("eax",&DWP(4*6,"edi"));        # r0
156         &mov    ("ebx",&DWP(4*7,"edi"));        # r1
157          &lea   ("ebp",&DWP(0,"esi","ecx"));    # end of input
158         &mov    ("ecx",&DWP(4*8,"edi"));        # r2
159         &mov    ("edx",&DWP(4*9,"edi"));        # r3
160
161         &mov    (&wparam(2),"ebp");
162         &mov    ("ebp","esi");
163
164         &mov    (&DWP($r0,"esp"),"eax");        # r0
165         &mov    ("eax","ebx");
166         &shr    ("eax",2);
167         &mov    (&DWP($r1,"esp"),"ebx");        # r1
168         &add    ("eax","ebx");                  # s1
169         &mov    ("ebx","ecx");
170         &shr    ("ebx",2);
171         &mov    (&DWP($r2,"esp"),"ecx");        # r2
172         &add    ("ebx","ecx");                  # s2
173         &mov    ("ecx","edx");
174         &shr    ("ecx",2);
175         &mov    (&DWP($r3,"esp"),"edx");        # r3
176         &add    ("ecx","edx");                  # s3
177         &mov    (&DWP($s1,"esp"),"eax");        # s1
178         &mov    (&DWP($s2,"esp"),"ebx");        # s2
179         &mov    (&DWP($s3,"esp"),"ecx");        # s3
180
181         &mov    ("eax",&DWP(4*0,"edi"));        # load hash value
182         &mov    ("ebx",&DWP(4*1,"edi"));
183         &mov    ("ecx",&DWP(4*2,"edi"));
184         &mov    ("esi",&DWP(4*3,"edi"));
185         &mov    ("edi",&DWP(4*4,"edi"));
186         &jmp    (&label("loop"));
187
188 &set_label("loop",32);
189         &add    ("eax",&DWP(4*0,"ebp"));        # accumulate input
190         &adc    ("ebx",&DWP(4*1,"ebp"));
191         &adc    ("ecx",&DWP(4*2,"ebp"));
192         &adc    ("esi",&DWP(4*3,"ebp"));
193         &lea    ("ebp",&DWP(4*4,"ebp"));
194         &adc    ("edi",&wparam(3));             # padbit
195
196         &mov    (&DWP($h0,"esp"),"eax");        # put aside hash[+inp]
197         &mov    (&DWP($h3,"esp"),"esi");
198
199         &mul    (&DWP($r0,"esp"));              # h0*r0
200          &mov   (&DWP($h4,"esp"),"edi");
201         &mov    ("edi","eax");
202         &mov    ("eax","ebx");                  # h1
203         &mov    ("esi","edx");
204         &mul    (&DWP($s3,"esp"));              # h1*s3
205         &add    ("edi","eax");
206         &mov    ("eax","ecx");                  # h2
207         &adc    ("esi","edx");
208         &mul    (&DWP($s2,"esp"));              # h2*s2
209         &add    ("edi","eax");
210         &mov    ("eax",&DWP($h3,"esp"));
211         &adc    ("esi","edx");
212         &mul    (&DWP($s1,"esp"));              # h3*s1
213         &add    ("edi","eax");
214          &mov   ("eax",&DWP($h0,"esp"));
215         &adc    ("esi","edx");
216
217         &mul    (&DWP($r1,"esp"));              # h0*r1
218          &mov   (&DWP($d0,"esp"),"edi");
219         &xor    ("edi","edi");
220         &add    ("esi","eax");
221         &mov    ("eax","ebx");                  # h1
222         &adc    ("edi","edx");
223         &mul    (&DWP($r0,"esp"));              # h1*r0
224         &add    ("esi","eax");
225         &mov    ("eax","ecx");                  # h2
226         &adc    ("edi","edx");
227         &mul    (&DWP($s3,"esp"));              # h2*s3
228         &add    ("esi","eax");
229         &mov    ("eax",&DWP($h3,"esp"));
230         &adc    ("edi","edx");
231         &mul    (&DWP($s2,"esp"));              # h3*s2
232         &add    ("esi","eax");
233         &mov    ("eax",&DWP($h4,"esp"));
234         &adc    ("edi","edx");
235         &imul   ("eax",&DWP($s1,"esp"));        # h4*s1
236         &add    ("esi","eax");
237          &mov   ("eax",&DWP($h0,"esp"));
238         &adc    ("edi",0);
239
240         &mul    (&DWP($r2,"esp"));              # h0*r2
241          &mov   (&DWP($d1,"esp"),"esi");
242         &xor    ("esi","esi");
243         &add    ("edi","eax");
244         &mov    ("eax","ebx");                  # h1
245         &adc    ("esi","edx");
246         &mul    (&DWP($r1,"esp"));              # h1*r1
247         &add    ("edi","eax");
248         &mov    ("eax","ecx");                  # h2
249         &adc    ("esi","edx");
250         &mul    (&DWP($r0,"esp"));              # h2*r0
251         &add    ("edi","eax");
252         &mov    ("eax",&DWP($h3,"esp"));
253         &adc    ("esi","edx");
254         &mul    (&DWP($s3,"esp"));              # h3*s3
255         &add    ("edi","eax");
256         &mov    ("eax",&DWP($h4,"esp"));
257         &adc    ("esi","edx");
258         &imul   ("eax",&DWP($s2,"esp"));        # h4*s2
259         &add    ("edi","eax");
260          &mov   ("eax",&DWP($h0,"esp"));
261         &adc    ("esi",0);
262
263         &mul    (&DWP($r3,"esp"));              # h0*r3
264          &mov   (&DWP($d2,"esp"),"edi");
265         &xor    ("edi","edi");
266         &add    ("esi","eax");
267         &mov    ("eax","ebx");                  # h1
268         &adc    ("edi","edx");
269         &mul    (&DWP($r2,"esp"));              # h1*r2
270         &add    ("esi","eax");
271         &mov    ("eax","ecx");                  # h2
272         &adc    ("edi","edx");
273         &mul    (&DWP($r1,"esp"));              # h2*r1
274         &add    ("esi","eax");
275         &mov    ("eax",&DWP($h3,"esp"));
276         &adc    ("edi","edx");
277         &mul    (&DWP($r0,"esp"));              # h3*r0
278         &add    ("esi","eax");
279          &mov   ("ecx",&DWP($h4,"esp"));
280         &adc    ("edi","edx");
281
282         &mov    ("edx","ecx");
283         &imul   ("ecx",&DWP($s3,"esp"));        # h4*s3
284         &add    ("esi","ecx");
285          &mov   ("eax",&DWP($d0,"esp"));
286         &adc    ("edi",0);
287
288         &imul   ("edx",&DWP($r0,"esp"));        # h4*r0
289         &add    ("edx","edi");
290
291         &mov    ("ebx",&DWP($d1,"esp"));
292         &mov    ("ecx",&DWP($d2,"esp"));
293
294         &mov    ("edi","edx");                  # last reduction step
295         &shr    ("edx",2);
296         &and    ("edi",3);
297         &lea    ("edx",&DWP(0,"edx","edx",4));  # *5
298         &add    ("eax","edx");
299         &adc    ("ebx",0);
300         &adc    ("ecx",0);
301         &adc    ("esi",0);
302         &adc    ("edi",0);
303
304         &cmp    ("ebp",&wparam(2));             # done yet?
305         &jne    (&label("loop"));
306
307         &mov    ("edx",&wparam(0));             # ctx
308         &stack_pop(16);
309         &mov    (&DWP(4*0,"edx"),"eax");        # store hash value
310         &mov    (&DWP(4*1,"edx"),"ebx");
311         &mov    (&DWP(4*2,"edx"),"ecx");
312         &mov    (&DWP(4*3,"edx"),"esi");
313         &mov    (&DWP(4*4,"edx"),"edi");
314 &set_label("nodata");
315 &function_end("poly1305_blocks");
316
317 &function_begin("poly1305_emit");
318         &mov    ("ebp",&wparam(0));             # context
319 &set_label("enter_emit");
320         &mov    ("edi",&wparam(1));             # output
321         &mov    ("eax",&DWP(4*0,"ebp"));        # load hash value
322         &mov    ("ebx",&DWP(4*1,"ebp"));
323         &mov    ("ecx",&DWP(4*2,"ebp"));
324         &mov    ("edx",&DWP(4*3,"ebp"));
325         &mov    ("esi",&DWP(4*4,"ebp"));
326
327         &add    ("eax",5);                      # compare to modulus
328         &adc    ("ebx",0);
329         &adc    ("ecx",0);
330         &adc    ("edx",0);
331         &adc    ("esi",0);
332         &shr    ("esi",2);                      # did it carry/borrow?
333         &neg    ("esi");                        # do we choose hash-modulus?
334
335         &and    ("eax","esi");
336         &and    ("ebx","esi");
337         &and    ("ecx","esi");
338         &and    ("edx","esi");
339         &mov    (&DWP(4*0,"edi"),"eax");
340         &mov    (&DWP(4*1,"edi"),"ebx");
341         &mov    (&DWP(4*2,"edi"),"ecx");
342         &mov    (&DWP(4*3,"edi"),"edx");
343
344         &not    ("esi");                        # or original hash value?
345         &mov    ("eax",&DWP(4*0,"ebp"));
346         &mov    ("ebx",&DWP(4*1,"ebp"));
347         &mov    ("ecx",&DWP(4*2,"ebp"));
348         &mov    ("edx",&DWP(4*3,"ebp"));
349         &mov    ("ebp",&wparam(2));
350         &and    ("eax","esi");
351         &and    ("ebx","esi");
352         &and    ("ecx","esi");
353         &and    ("edx","esi");
354         &or     ("eax",&DWP(4*0,"edi"));
355         &or     ("ebx",&DWP(4*1,"edi"));
356         &or     ("ecx",&DWP(4*2,"edi"));
357         &or     ("edx",&DWP(4*3,"edi"));
358
359         &add    ("eax",&DWP(4*0,"ebp"));        # accumulate key
360         &adc    ("ebx",&DWP(4*1,"ebp"));
361         &adc    ("ecx",&DWP(4*2,"ebp"));
362         &adc    ("edx",&DWP(4*3,"ebp"));
363
364         &mov    (&DWP(4*0,"edi"),"eax");
365         &mov    (&DWP(4*1,"edi"),"ebx");
366         &mov    (&DWP(4*2,"edi"),"ecx");
367         &mov    (&DWP(4*3,"edi"),"edx");
368 &function_end("poly1305_emit");
369
370 if ($sse2) {
371 ########################################################################
372 # Layout of opaque area is following.
373 #
374 #       unsigned __int32 h[5];          # current hash value base 2^26
375 #       unsigned __int32 is_base2_26;
376 #       unsigned __int32 r[4];          # key value base 2^32
377 #       unsigned __int32 pad[2];
378 #       struct { unsigned __int32 r^4, r^3, r^2, r^1; } r[9];
379 #
380 # where r^n are base 2^26 digits of degrees of multiplier key. There are
381 # 5 digits, but last four are interleaved with multiples of 5, totalling
382 # in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.
383
384 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("xmm$_",(0..7));
385 my $MASK=$T2;   # borrow and keep in mind
386
387 &align  (32);
388 &function_begin_B("_poly1305_init_sse2");
389         &movdqu         ($D4,&QWP(4*6,"edi"));          # key base 2^32
390         &lea            ("edi",&DWP(16*3,"edi"));       # size optimization
391         &mov            ("ebp","esp");
392         &sub            ("esp",16*(9+5));
393         &and            ("esp",-16);
394
395         #&pand          ($D4,&QWP(96,"ebx"));           # magic mask
396         &movq           ($MASK,&QWP(64,"ebx"));
397
398         &movdqa         ($D0,$D4);
399         &movdqa         ($D1,$D4);
400         &movdqa         ($D2,$D4);
401
402         &pand           ($D0,$MASK);                    # -> base 2^26
403         &psrlq          ($D1,26);
404         &psrldq         ($D2,6);
405         &pand           ($D1,$MASK);
406         &movdqa         ($D3,$D2);
407         &psrlq          ($D2,4)
408         &psrlq          ($D3,30);
409         &pand           ($D2,$MASK);
410         &pand           ($D3,$MASK);
411         &psrldq         ($D4,13);
412
413         &lea            ("edx",&DWP(16*9,"esp"));       # size optimization
414         &mov            ("ecx",2);
415 &set_label("square");
416         &movdqa         (&QWP(16*0,"esp"),$D0);
417         &movdqa         (&QWP(16*1,"esp"),$D1);
418         &movdqa         (&QWP(16*2,"esp"),$D2);
419         &movdqa         (&QWP(16*3,"esp"),$D3);
420         &movdqa         (&QWP(16*4,"esp"),$D4);
421
422         &movdqa         ($T1,$D1);
423         &movdqa         ($T0,$D2);
424         &pslld          ($T1,2);
425         &pslld          ($T0,2);
426         &paddd          ($T1,$D1);                      # *5
427         &paddd          ($T0,$D2);                      # *5
428         &movdqa         (&QWP(16*5,"esp"),$T1);
429         &movdqa         (&QWP(16*6,"esp"),$T0);
430         &movdqa         ($T1,$D3);
431         &movdqa         ($T0,$D4);
432         &pslld          ($T1,2);
433         &pslld          ($T0,2);
434         &paddd          ($T1,$D3);                      # *5
435         &paddd          ($T0,$D4);                      # *5
436         &movdqa         (&QWP(16*7,"esp"),$T1);
437         &movdqa         (&QWP(16*8,"esp"),$T0);
438
439         &pshufd         ($T1,$D0,0b01000100);
440         &movdqa         ($T0,$D1);
441         &pshufd         ($D1,$D1,0b01000100);
442         &pshufd         ($D2,$D2,0b01000100);
443         &pshufd         ($D3,$D3,0b01000100);
444         &pshufd         ($D4,$D4,0b01000100);
445         &movdqa         (&QWP(16*0,"edx"),$T1);
446         &movdqa         (&QWP(16*1,"edx"),$D1);
447         &movdqa         (&QWP(16*2,"edx"),$D2);
448         &movdqa         (&QWP(16*3,"edx"),$D3);
449         &movdqa         (&QWP(16*4,"edx"),$D4);
450
451         ################################################################
452         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
453         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
454         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
455         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
456         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
457
458         &pmuludq        ($D4,$D0);                      # h4*r0
459         &pmuludq        ($D3,$D0);                      # h3*r0
460         &pmuludq        ($D2,$D0);                      # h2*r0
461         &pmuludq        ($D1,$D0);                      # h1*r0
462         &pmuludq        ($D0,$T1);                      # h0*r0
463
464 sub pmuladd {
465 my $load = shift;
466 my $base = shift; $base = "esp" if (!defined($base));
467
468         ################################################################
469         # As for choice to "rotate" $T0-$T2 in order to move paddq
470         # past next multiplication. While it makes code harder to read
471         # and doesn't have significant effect on most processors, it
472         # makes a lot of difference on Atom, up to 30% improvement.
473
474         &movdqa         ($T1,$T0);
475         &pmuludq        ($T0,&QWP(16*3,$base));         # r1*h3
476         &movdqa         ($T2,$T1);
477         &pmuludq        ($T1,&QWP(16*2,$base));         # r1*h2
478         &paddq          ($D4,$T0);
479         &movdqa         ($T0,$T2);
480         &pmuludq        ($T2,&QWP(16*1,$base));         # r1*h1
481         &paddq          ($D3,$T1);
482         &$load          ($T1,5);                        # s1
483         &pmuludq        ($T0,&QWP(16*0,$base));         # r1*h0
484         &paddq          ($D2,$T2);
485         &pmuludq        ($T1,&QWP(16*4,$base));         # s1*h4
486          &$load         ($T2,2);                        # r2^n
487         &paddq          ($D1,$T0);
488
489         &movdqa         ($T0,$T2);
490         &pmuludq        ($T2,&QWP(16*2,$base));         # r2*h2
491          &paddq         ($D0,$T1);
492         &movdqa         ($T1,$T0);
493         &pmuludq        ($T0,&QWP(16*1,$base));         # r2*h1
494         &paddq          ($D4,$T2);
495         &$load          ($T2,6);                        # s2^n
496         &pmuludq        ($T1,&QWP(16*0,$base));         # r2*h0
497         &paddq          ($D3,$T0);
498         &movdqa         ($T0,$T2);
499         &pmuludq        ($T2,&QWP(16*4,$base));         # s2*h4
500         &paddq          ($D2,$T1);
501         &pmuludq        ($T0,&QWP(16*3,$base));         # s2*h3
502          &$load         ($T1,3);                        # r3^n
503         &paddq          ($D1,$T2);
504
505         &movdqa         ($T2,$T1);
506         &pmuludq        ($T1,&QWP(16*1,$base));         # r3*h1
507          &paddq         ($D0,$T0);
508         &$load          ($T0,7);                        # s3^n
509         &pmuludq        ($T2,&QWP(16*0,$base));         # r3*h0
510         &paddq          ($D4,$T1);
511         &movdqa         ($T1,$T0);
512         &pmuludq        ($T0,&QWP(16*4,$base));         # s3*h4
513         &paddq          ($D3,$T2);
514         &movdqa         ($T2,$T1);
515         &pmuludq        ($T1,&QWP(16*3,$base));         # s3*h3
516         &paddq          ($D2,$T0);
517         &pmuludq        ($T2,&QWP(16*2,$base));         # s3*h2
518          &$load         ($T0,4);                        # r4^n
519         &paddq          ($D1,$T1);
520
521         &$load          ($T1,8);                        # s4^n
522         &pmuludq        ($T0,&QWP(16*0,$base));         # r4*h0
523          &paddq         ($D0,$T2);
524         &movdqa         ($T2,$T1);
525         &pmuludq        ($T1,&QWP(16*4,$base));         # s4*h4
526         &paddq          ($D4,$T0);
527         &movdqa         ($T0,$T2);
528         &pmuludq        ($T2,&QWP(16*1,$base));         # s4*h1
529         &paddq          ($D3,$T1);
530         &movdqa         ($T1,$T0);
531         &pmuludq        ($T0,&QWP(16*2,$base));         # s4*h2
532         &paddq          ($D0,$T2);
533         &pmuludq        ($T1,&QWP(16*3,$base));         # s4*h3
534          &movdqa        ($MASK,&QWP(64,"ebx"));
535         &paddq          ($D1,$T0);
536         &paddq          ($D2,$T1);
537 }
538         &pmuladd        (sub {  my ($reg,$i)=@_;
539                                 &movdqa ($reg,&QWP(16*$i,"esp"));
540                              },"edx");
541
542 sub lazy_reduction {
543 my $extra = shift;
544 my $paddx = defined($extra) ? paddq : paddd;
545
546         ################################################################
547         # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
548         # and P. Schwabe
549
550          &movdqa        ($T0,$D3);
551          &pand          ($D3,$MASK);
552          &psrlq         ($T0,26);
553          &$extra        ()                              if (defined($extra));
554          &paddq         ($T0,$D4);                      # h3 -> h4
555         &movdqa         ($T1,$D0);
556         &pand           ($D0,$MASK);
557         &psrlq          ($T1,26);
558          &movdqa        ($D4,$T0);
559         &paddq          ($T1,$D1);                      # h0 -> h1
560          &psrlq         ($T0,26);
561          &pand          ($D4,$MASK);
562         &movdqa         ($D1,$T1);
563         &psrlq          ($T1,26);
564          &paddd         ($D0,$T0);                      # favour paddd when
565                                                         # possible, because
566                                                         # paddq is "broken"
567                                                         # on Atom
568          &psllq         ($T0,2);
569         &paddq          ($T1,$D2);                      # h1 -> h2
570          &$paddx        ($T0,$D0);                      # h4 -> h0
571         &pand           ($D1,$MASK);
572         &movdqa         ($D2,$T1);
573         &psrlq          ($T1,26);
574         &pand           ($D2,$MASK);
575         &paddd          ($T1,$D3);                      # h2 -> h3
576          &movdqa        ($D0,$T0);
577          &psrlq         ($T0,26);
578         &movdqa         ($D3,$T1);
579         &psrlq          ($T1,26);
580          &pand          ($D0,$MASK);
581          &paddd         ($D1,$T0);                      # h0 -> h1
582         &pand           ($D3,$MASK);
583         &paddd          ($D4,$T1);                      # h3 -> h4
584 }
585         &lazy_reduction ();
586
587         &dec            ("ecx");
588         &jz             (&label("square_break"));
589
590         &punpcklqdq     ($D0,&QWP(16*0,"esp"));         # 0:r^1:0:r^2
591         &punpcklqdq     ($D1,&QWP(16*1,"esp"));
592         &punpcklqdq     ($D2,&QWP(16*2,"esp"));
593         &punpcklqdq     ($D3,&QWP(16*3,"esp"));
594         &punpcklqdq     ($D4,&QWP(16*4,"esp"));
595         &jmp            (&label("square"));
596
597 &set_label("square_break");
598         &psllq          ($D0,32);                       # -> r^3:0:r^4:0
599         &psllq          ($D1,32);
600         &psllq          ($D2,32);
601         &psllq          ($D3,32);
602         &psllq          ($D4,32);
603         &por            ($D0,&QWP(16*0,"esp"));         # r^3:r^1:r^4:r^2
604         &por            ($D1,&QWP(16*1,"esp"));
605         &por            ($D2,&QWP(16*2,"esp"));
606         &por            ($D3,&QWP(16*3,"esp"));
607         &por            ($D4,&QWP(16*4,"esp"));
608
609         &pshufd         ($D0,$D0,0b10001101);           # -> r^1:r^2:r^3:r^4
610         &pshufd         ($D1,$D1,0b10001101);
611         &pshufd         ($D2,$D2,0b10001101);
612         &pshufd         ($D3,$D3,0b10001101);
613         &pshufd         ($D4,$D4,0b10001101);
614
615         &movdqu         (&QWP(16*0,"edi"),$D0);         # save the table
616         &movdqu         (&QWP(16*1,"edi"),$D1);
617         &movdqu         (&QWP(16*2,"edi"),$D2);
618         &movdqu         (&QWP(16*3,"edi"),$D3);
619         &movdqu         (&QWP(16*4,"edi"),$D4);
620
621         &movdqa         ($T1,$D1);
622         &movdqa         ($T0,$D2);
623         &pslld          ($T1,2);
624         &pslld          ($T0,2);
625         &paddd          ($T1,$D1);                      # *5
626         &paddd          ($T0,$D2);                      # *5
627         &movdqu         (&QWP(16*5,"edi"),$T1);
628         &movdqu         (&QWP(16*6,"edi"),$T0);
629         &movdqa         ($T1,$D3);
630         &movdqa         ($T0,$D4);
631         &pslld          ($T1,2);
632         &pslld          ($T0,2);
633         &paddd          ($T1,$D3);                      # *5
634         &paddd          ($T0,$D4);                      # *5
635         &movdqu         (&QWP(16*7,"edi"),$T1);
636         &movdqu         (&QWP(16*8,"edi"),$T0);
637
638         &mov            ("esp","ebp");
639         &lea            ("edi",&DWP(-16*3,"edi"));      # size de-optimization
640         &ret            ();
641 &function_end_B("_poly1305_init_sse2");
642
643 &align  (32);
644 &function_begin("_poly1305_blocks_sse2");
645         &mov    ("edi",&wparam(0));                     # ctx
646         &mov    ("esi",&wparam(1));                     # inp
647         &mov    ("ecx",&wparam(2));                     # len
648
649         &mov    ("eax",&DWP(4*5,"edi"));                # is_base2_26
650         &and    ("ecx",-16);
651         &jz     (&label("nodata"));
652         &cmp    ("ecx",64);
653         &jae    (&label("enter_sse2"));
654         &test   ("eax","eax");                          # is_base2_26?
655         &jz     (&label("enter_blocks"));
656
657 &set_label("enter_sse2",16);
658         &call   (&label("pic_point"));
659 &set_label("pic_point");
660         &blindpop("ebx");
661         &lea    ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
662
663         &test   ("eax","eax");                          # is_base2_26?
664         &jnz    (&label("base2_26"));
665
666         &call   ("_poly1305_init_sse2");
667
668         ################################################# base 2^32 -> base 2^26
669         &mov    ("eax",&DWP(0,"edi"));
670         &mov    ("ecx",&DWP(3,"edi"));
671         &mov    ("edx",&DWP(6,"edi"));
672         &mov    ("esi",&DWP(9,"edi"));
673         &mov    ("ebp",&DWP(13,"edi"));
674         &mov    (&DWP(4*5,"edi"),1);                    # is_base2_26
675
676         &shr    ("ecx",2);
677         &and    ("eax",0x3ffffff);
678         &shr    ("edx",4);
679         &and    ("ecx",0x3ffffff);
680         &shr    ("esi",6);
681         &and    ("edx",0x3ffffff);
682
683         &movd   ($D0,"eax");
684         &movd   ($D1,"ecx");
685         &movd   ($D2,"edx");
686         &movd   ($D3,"esi");
687         &movd   ($D4,"ebp");
688
689         &mov    ("esi",&wparam(1));                     # [reload] inp
690         &mov    ("ecx",&wparam(2));                     # [reload] len
691         &jmp    (&label("base2_32"));
692
693 &set_label("base2_26",16);
694         &movd   ($D0,&DWP(4*0,"edi"));                  # load hash value
695         &movd   ($D1,&DWP(4*1,"edi"));
696         &movd   ($D2,&DWP(4*2,"edi"));
697         &movd   ($D3,&DWP(4*3,"edi"));
698         &movd   ($D4,&DWP(4*4,"edi"));
699         &movdqa ($MASK,&QWP(64,"ebx"));
700
701 &set_label("base2_32");
702         &mov    ("eax",&wparam(3));                     # padbit
703         &mov    ("ebp","esp");
704
705         &sub    ("esp",16*(5+5+5+9+9));
706         &and    ("esp",-16);
707
708         &lea    ("edi",&DWP(16*3,"edi"));               # size optimization
709         &shl    ("eax",24);                             # padbit
710
711         &test   ("ecx",31);
712         &jz     (&label("even"));
713
714         ################################################################
715         # process single block, with SSE2, because it's still faster
716         # even though half of result is discarded
717
718         &movdqu         ($T1,&QWP(0,"esi"));            # input
719         &lea            ("esi",&DWP(16,"esi"));
720
721         &movdqa         ($T0,$T1);                      # -> base 2^26 ...
722         &pand           ($T1,$MASK);
723         &paddd          ($D0,$T1);                      # ... and accumuate
724
725         &movdqa         ($T1,$T0);
726         &psrlq          ($T0,26);
727         &psrldq         ($T1,6);
728         &pand           ($T0,$MASK);
729         &paddd          ($D1,$T0);
730
731         &movdqa         ($T0,$T1);
732         &psrlq          ($T1,4);
733         &pand           ($T1,$MASK);
734         &paddd          ($D2,$T1);
735
736         &movdqa         ($T1,$T0);
737         &psrlq          ($T0,30);
738         &pand           ($T0,$MASK);
739         &psrldq         ($T1,7);
740         &paddd          ($D3,$T0);
741
742         &movd           ($T0,"eax");                    # padbit
743         &paddd          ($D4,$T1);
744          &movd          ($T1,&DWP(16*0+12,"edi"));      # r0
745         &paddd          ($D4,$T0);
746
747         &movdqa         (&QWP(16*0,"esp"),$D0);
748         &movdqa         (&QWP(16*1,"esp"),$D1);
749         &movdqa         (&QWP(16*2,"esp"),$D2);
750         &movdqa         (&QWP(16*3,"esp"),$D3);
751         &movdqa         (&QWP(16*4,"esp"),$D4);
752
753         ################################################################
754         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
755         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
756         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
757         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
758         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
759
760         &pmuludq        ($D0,$T1);                      # h4*r0
761         &pmuludq        ($D1,$T1);                      # h3*r0
762         &pmuludq        ($D2,$T1);                      # h2*r0
763          &movd          ($T0,&DWP(16*1+12,"edi"));      # r1
764         &pmuludq        ($D3,$T1);                      # h1*r0
765         &pmuludq        ($D4,$T1);                      # h0*r0
766
767         &pmuladd        (sub {  my ($reg,$i)=@_;
768                                 &movd ($reg,&DWP(16*$i+12,"edi"));
769                              });
770
771         &lazy_reduction ();
772
773         &sub            ("ecx",16);
774         &jz             (&label("done"));
775
776 &set_label("even");
777         &lea            ("edx",&DWP(16*(5+5+5+9),"esp"));# size optimization
778         &lea            ("eax",&DWP(-16*2,"esi"));
779         &sub            ("ecx",64);
780
781         ################################################################
782         # expand and copy pre-calculated table to stack
783
784         &movdqu         ($T0,&QWP(16*0,"edi"));         # r^1:r^2:r^3:r^4
785         &pshufd         ($T1,$T0,0b01000100);           # duplicate r^3:r^4
786         &cmovb          ("esi","eax");
787         &pshufd         ($T0,$T0,0b11101110);           # duplicate r^1:r^2
788         &movdqa         (&QWP(16*0,"edx"),$T1);
789         &lea            ("eax",&DWP(16*10,"esp"));
790         &movdqu         ($T1,&QWP(16*1,"edi"));
791         &movdqa         (&QWP(16*(0-9),"edx"),$T0);
792         &pshufd         ($T0,$T1,0b01000100);
793         &pshufd         ($T1,$T1,0b11101110);
794         &movdqa         (&QWP(16*1,"edx"),$T0);
795         &movdqu         ($T0,&QWP(16*2,"edi"));
796         &movdqa         (&QWP(16*(1-9),"edx"),$T1);
797         &pshufd         ($T1,$T0,0b01000100);
798         &pshufd         ($T0,$T0,0b11101110);
799         &movdqa         (&QWP(16*2,"edx"),$T1);
800         &movdqu         ($T1,&QWP(16*3,"edi"));
801         &movdqa         (&QWP(16*(2-9),"edx"),$T0);
802         &pshufd         ($T0,$T1,0b01000100);
803         &pshufd         ($T1,$T1,0b11101110);
804         &movdqa         (&QWP(16*3,"edx"),$T0);
805         &movdqu         ($T0,&QWP(16*4,"edi"));
806         &movdqa         (&QWP(16*(3-9),"edx"),$T1);
807         &pshufd         ($T1,$T0,0b01000100);
808         &pshufd         ($T0,$T0,0b11101110);
809         &movdqa         (&QWP(16*4,"edx"),$T1);
810         &movdqu         ($T1,&QWP(16*5,"edi"));
811         &movdqa         (&QWP(16*(4-9),"edx"),$T0);
812         &pshufd         ($T0,$T1,0b01000100);
813         &pshufd         ($T1,$T1,0b11101110);
814         &movdqa         (&QWP(16*5,"edx"),$T0);
815         &movdqu         ($T0,&QWP(16*6,"edi"));
816         &movdqa         (&QWP(16*(5-9),"edx"),$T1);
817         &pshufd         ($T1,$T0,0b01000100);
818         &pshufd         ($T0,$T0,0b11101110);
819         &movdqa         (&QWP(16*6,"edx"),$T1);
820         &movdqu         ($T1,&QWP(16*7,"edi"));
821         &movdqa         (&QWP(16*(6-9),"edx"),$T0);
822         &pshufd         ($T0,$T1,0b01000100);
823         &pshufd         ($T1,$T1,0b11101110);
824         &movdqa         (&QWP(16*7,"edx"),$T0);
825         &movdqu         ($T0,&QWP(16*8,"edi"));
826         &movdqa         (&QWP(16*(7-9),"edx"),$T1);
827         &pshufd         ($T1,$T0,0b01000100);
828         &pshufd         ($T0,$T0,0b11101110);
829         &movdqa         (&QWP(16*8,"edx"),$T1);
830         &movdqa         (&QWP(16*(8-9),"edx"),$T0);
831
832 sub load_input {
833 my ($inpbase,$offbase)=@_;
834
835         &movdqu         ($T0,&QWP($inpbase+0,"esi"));   # load input
836         &movdqu         ($T1,&QWP($inpbase+16,"esi"));
837         &lea            ("esi",&DWP(16*2,"esi"));
838
839         &movdqa         (&QWP($offbase+16*2,"esp"),$D2);
840         &movdqa         (&QWP($offbase+16*3,"esp"),$D3);
841         &movdqa         (&QWP($offbase+16*4,"esp"),$D4);
842
843         &movdqa         ($D2,$T0);                      # splat input
844         &movdqa         ($D3,$T1);
845         &psrldq         ($D2,6);
846         &psrldq         ($D3,6);
847         &movdqa         ($D4,$T0);
848         &punpcklqdq     ($D2,$D3);                      # 2:3
849         &punpckhqdq     ($D4,$T1);                      # 4
850         &punpcklqdq     ($T0,$T1);                      # 0:1
851
852         &movdqa         ($D3,$D2);
853         &psrlq          ($D2,4);
854         &psrlq          ($D3,30);
855         &movdqa         ($T1,$T0);
856         &psrlq          ($D4,40);                       # 4
857         &psrlq          ($T1,26);
858         &pand           ($T0,$MASK);                    # 0
859         &pand           ($T1,$MASK);                    # 1
860         &pand           ($D2,$MASK);                    # 2
861         &pand           ($D3,$MASK);                    # 3
862         &por            ($D4,&QWP(0,"ebx"));            # padbit, yes, always
863
864         &movdqa         (&QWP($offbase+16*0,"esp"),$D0) if ($offbase);
865         &movdqa         (&QWP($offbase+16*1,"esp"),$D1) if ($offbase);
866 }
867         &load_input     (16*2,16*5);
868
869         &jbe            (&label("skip_loop"));
870         &jmp            (&label("loop"));
871
872 &set_label("loop",32);
873         ################################################################
874         # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
875         # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
876         #   \___________________/
877         # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
878         # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
879         #   \___________________/ \____________________/
880         ################################################################
881
882         &movdqa         ($T2,&QWP(16*(0-9),"edx"));     # r0^2
883         &movdqa         (&QWP(16*1,"eax"),$T1);
884         &movdqa         (&QWP(16*2,"eax"),$D2);
885         &movdqa         (&QWP(16*3,"eax"),$D3);
886         &movdqa         (&QWP(16*4,"eax"),$D4);
887
888         ################################################################
889         # d4 = h4*r0 + h0*r4   + h1*r3   + h2*r2   + h3*r1
890         # d3 = h3*r0 + h0*r3   + h1*r2   + h2*r1   + h4*5*r4
891         # d2 = h2*r0 + h0*r2   + h1*r1   + h3*5*r4 + h4*5*r3
892         # d1 = h1*r0 + h0*r1   + h2*5*r4 + h3*5*r3 + h4*5*r2
893         # d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
894
895         &movdqa         ($D1,$T0);
896         &pmuludq        ($T0,$T2);                      # h0*r0
897         &movdqa         ($D0,$T1);
898         &pmuludq        ($T1,$T2);                      # h1*r0
899         &pmuludq        ($D2,$T2);                      # h2*r0
900         &pmuludq        ($D3,$T2);                      # h3*r0
901         &pmuludq        ($D4,$T2);                      # h4*r0
902
903 sub pmuladd_alt {
904 my $addr = shift;
905
906         &pmuludq        ($D0,&$addr(8));                # h1*s4
907         &movdqa         ($T2,$D1);
908         &pmuludq        ($D1,&$addr(1));                # h0*r1
909         &paddq          ($D0,$T0);
910         &movdqa         ($T0,$T2);
911         &pmuludq        ($T2,&$addr(2));                # h0*r2
912         &paddq          ($D1,$T1);
913         &movdqa         ($T1,$T0);
914         &pmuludq        ($T0,&$addr(3));                # h0*r3
915         &paddq          ($D2,$T2);
916          &movdqa        ($T2,&QWP(16*1,"eax"));         # pull h1
917         &pmuludq        ($T1,&$addr(4));                # h0*r4
918         &paddq          ($D3,$T0);
919
920         &movdqa         ($T0,$T2);
921         &pmuludq        ($T2,&$addr(1));                # h1*r1
922          &paddq         ($D4,$T1);
923         &movdqa         ($T1,$T0);
924         &pmuludq        ($T0,&$addr(2));                # h1*r2
925         &paddq          ($D2,$T2);
926         &movdqa         ($T2,&QWP(16*2,"eax"));         # pull h2
927         &pmuludq        ($T1,&$addr(3));                # h1*r3
928         &paddq          ($D3,$T0);
929         &movdqa         ($T0,$T2);
930         &pmuludq        ($T2,&$addr(7));                # h2*s3
931         &paddq          ($D4,$T1);
932         &movdqa         ($T1,$T0);
933         &pmuludq        ($T0,&$addr(8));                # h2*s4
934         &paddq          ($D0,$T2);
935
936         &movdqa         ($T2,$T1);
937         &pmuludq        ($T1,&$addr(1));                # h2*r1
938          &paddq         ($D1,$T0);
939         &movdqa         ($T0,&QWP(16*3,"eax"));         # pull h3
940         &pmuludq        ($T2,&$addr(2));                # h2*r2
941         &paddq          ($D3,$T1);
942         &movdqa         ($T1,$T0);
943         &pmuludq        ($T0,&$addr(6));                # h3*s2
944         &paddq          ($D4,$T2);
945         &movdqa         ($T2,$T1);
946         &pmuludq        ($T1,&$addr(7));                # h3*s3
947         &paddq          ($D0,$T0);
948         &movdqa         ($T0,$T2);
949         &pmuludq        ($T2,&$addr(8));                # h3*s4
950         &paddq          ($D1,$T1);
951
952         &movdqa         ($T1,&QWP(16*4,"eax"));         # pull h4
953         &pmuludq        ($T0,&$addr(1));                # h3*r1
954          &paddq         ($D2,$T2);
955         &movdqa         ($T2,$T1);
956         &pmuludq        ($T1,&$addr(8));                # h4*s4
957         &paddq          ($D4,$T0);
958         &movdqa         ($T0,$T2);
959         &pmuludq        ($T2,&$addr(5));                # h4*s1
960         &paddq          ($D3,$T1);
961         &movdqa         ($T1,$T0);
962         &pmuludq        ($T0,&$addr(6));                # h4*s2
963         &paddq          ($D0,$T2);
964          &movdqa        ($MASK,&QWP(64,"ebx"));
965         &pmuludq        ($T1,&$addr(7));                # h4*s3
966         &paddq          ($D1,$T0);
967         &paddq          ($D2,$T1);
968 }
969         &pmuladd_alt    (sub {  my $i=shift; &QWP(16*($i-9),"edx");     });
970
971         &load_input     (-16*2,0);
972         &lea            ("eax",&DWP(-16*2,"esi"));
973         &sub            ("ecx",64);
974
975         &paddd          ($T0,&QWP(16*(5+0),"esp"));     # add hash value
976         &paddd          ($T1,&QWP(16*(5+1),"esp"));
977         &paddd          ($D2,&QWP(16*(5+2),"esp"));
978         &paddd          ($D3,&QWP(16*(5+3),"esp"));
979         &paddd          ($D4,&QWP(16*(5+4),"esp"));
980
981         &cmovb          ("esi","eax");
982         &lea            ("eax",&DWP(16*10,"esp"));
983
984         &movdqa         ($T2,&QWP(16*0,"edx"));         # r0^4
985         &movdqa         (&QWP(16*1,"esp"),$D1);
986         &movdqa         (&QWP(16*1,"eax"),$T1);
987         &movdqa         (&QWP(16*2,"eax"),$D2);
988         &movdqa         (&QWP(16*3,"eax"),$D3);
989         &movdqa         (&QWP(16*4,"eax"),$D4);
990
991         ################################################################
992         # d4 += h4*r0 + h0*r4   + h1*r3   + h2*r2   + h3*r1
993         # d3 += h3*r0 + h0*r3   + h1*r2   + h2*r1   + h4*5*r4
994         # d2 += h2*r0 + h0*r2   + h1*r1   + h3*5*r4 + h4*5*r3
995         # d1 += h1*r0 + h0*r1   + h2*5*r4 + h3*5*r3 + h4*5*r2
996         # d0 += h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
997
998         &movdqa         ($D1,$T0);
999         &pmuludq        ($T0,$T2);                      # h0*r0
1000         &paddq          ($T0,$D0);
1001         &movdqa         ($D0,$T1);
1002         &pmuludq        ($T1,$T2);                      # h1*r0
1003         &pmuludq        ($D2,$T2);                      # h2*r0
1004         &pmuludq        ($D3,$T2);                      # h3*r0
1005         &pmuludq        ($D4,$T2);                      # h4*r0
1006
1007         &paddq          ($T1,&QWP(16*1,"esp"));
1008         &paddq          ($D2,&QWP(16*2,"esp"));
1009         &paddq          ($D3,&QWP(16*3,"esp"));
1010         &paddq          ($D4,&QWP(16*4,"esp"));
1011
1012         &pmuladd_alt    (sub {  my $i=shift; &QWP(16*$i,"edx"); });
1013
1014         &lazy_reduction ();
1015
1016         &load_input     (16*2,16*5);
1017
1018         &ja             (&label("loop"));
1019
1020 &set_label("skip_loop");
1021         ################################################################
1022         # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
1023
1024          &pshufd        ($T2,&QWP(16*(0-9),"edx"),0x10);# r0^n
1025         &add            ("ecx",32);
1026         &jnz            (&label("long_tail"));
1027
1028         &paddd          ($T0,$D0);                      # add hash value
1029         &paddd          ($T1,$D1);
1030         &paddd          ($D2,&QWP(16*7,"esp"));
1031         &paddd          ($D3,&QWP(16*8,"esp"));
1032         &paddd          ($D4,&QWP(16*9,"esp"));
1033
1034 &set_label("long_tail");
1035
1036         &movdqa         (&QWP(16*0,"eax"),$T0);
1037         &movdqa         (&QWP(16*1,"eax"),$T1);
1038         &movdqa         (&QWP(16*2,"eax"),$D2);
1039         &movdqa         (&QWP(16*3,"eax"),$D3);
1040         &movdqa         (&QWP(16*4,"eax"),$D4);
1041
1042         ################################################################
1043         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
1044         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
1045         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
1046         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
1047         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1048
1049         &pmuludq        ($T0,$T2);                      # h0*r0
1050         &pmuludq        ($T1,$T2);                      # h1*r0
1051         &pmuludq        ($D2,$T2);                      # h2*r0
1052         &movdqa         ($D0,$T0);
1053          &pshufd        ($T0,&QWP(16*(1-9),"edx"),0x10);# r1^n
1054         &pmuludq        ($D3,$T2);                      # h3*r0
1055         &movdqa         ($D1,$T1);
1056         &pmuludq        ($D4,$T2);                      # h4*r0
1057
1058         &pmuladd        (sub {  my ($reg,$i)=@_;
1059                                 &pshufd ($reg,&QWP(16*($i-9),"edx"),0x10);
1060                              },"eax");
1061
1062         &jz             (&label("short_tail"));
1063
1064         &load_input     (-16*2,0);
1065
1066          &pshufd        ($T2,&QWP(16*0,"edx"),0x10);    # r0^n
1067         &paddd          ($T0,&QWP(16*5,"esp"));         # add hash value
1068         &paddd          ($T1,&QWP(16*6,"esp"));
1069         &paddd          ($D2,&QWP(16*7,"esp"));
1070         &paddd          ($D3,&QWP(16*8,"esp"));
1071         &paddd          ($D4,&QWP(16*9,"esp"));
1072
1073         ################################################################
1074         # multiply inp[0:1] by r^4:r^3 and accumulate
1075
1076         &movdqa         (&QWP(16*0,"esp"),$T0);
1077         &pmuludq        ($T0,$T2);                      # h0*r0
1078         &movdqa         (&QWP(16*1,"esp"),$T1);
1079         &pmuludq        ($T1,$T2);                      # h1*r0
1080         &paddq          ($D0,$T0);
1081         &movdqa         ($T0,$D2);
1082         &pmuludq        ($D2,$T2);                      # h2*r0
1083         &paddq          ($D1,$T1);
1084         &movdqa         ($T1,$D3);
1085         &pmuludq        ($D3,$T2);                      # h3*r0
1086         &paddq          ($D2,&QWP(16*2,"esp"));
1087         &movdqa         (&QWP(16*2,"esp"),$T0);
1088          &pshufd        ($T0,&QWP(16*1,"edx"),0x10);    # r1^n
1089         &paddq          ($D3,&QWP(16*3,"esp"));
1090         &movdqa         (&QWP(16*3,"esp"),$T1);
1091         &movdqa         ($T1,$D4);
1092         &pmuludq        ($D4,$T2);                      # h4*r0
1093         &paddq          ($D4,&QWP(16*4,"esp"));
1094         &movdqa         (&QWP(16*4,"esp"),$T1);
1095
1096         &pmuladd        (sub {  my ($reg,$i)=@_;
1097                                 &pshufd ($reg,&QWP(16*$i,"edx"),0x10);
1098                              });
1099
1100 &set_label("short_tail");
1101
1102         ################################################################
1103         # horizontal addition
1104
1105         &pshufd         ($T1,$D4,0b01001110);
1106         &pshufd         ($T0,$D3,0b01001110);
1107         &paddq          ($D4,$T1);
1108         &paddq          ($D3,$T0);
1109         &pshufd         ($T1,$D0,0b01001110);
1110         &pshufd         ($T0,$D1,0b01001110);
1111         &paddq          ($D0,$T1);
1112         &paddq          ($D1,$T0);
1113         &pshufd         ($T1,$D2,0b01001110);
1114         #&paddq         ($D2,$T1);
1115
1116         &lazy_reduction (sub { &paddq ($D2,$T1) });
1117
1118 &set_label("done");
1119         &movd           (&DWP(-16*3+4*0,"edi"),$D0);    # store hash value
1120         &movd           (&DWP(-16*3+4*1,"edi"),$D1);
1121         &movd           (&DWP(-16*3+4*2,"edi"),$D2);
1122         &movd           (&DWP(-16*3+4*3,"edi"),$D3);
1123         &movd           (&DWP(-16*3+4*4,"edi"),$D4);
1124         &mov    ("esp","ebp");
1125 &set_label("nodata");
1126 &function_end("_poly1305_blocks_sse2");
1127
1128 &align  (32);
1129 &function_begin("_poly1305_emit_sse2");
1130         &mov    ("ebp",&wparam(0));             # context
1131
1132         &cmp    (&DWP(4*5,"ebp"),0);            # is_base2_26?
1133         &je     (&label("enter_emit"));
1134
1135         &mov    ("eax",&DWP(4*0,"ebp"));        # load hash value
1136         &mov    ("edi",&DWP(4*1,"ebp"));
1137         &mov    ("ecx",&DWP(4*2,"ebp"));
1138         &mov    ("edx",&DWP(4*3,"ebp"));
1139         &mov    ("esi",&DWP(4*4,"ebp"));
1140
1141         &mov    ("ebx","edi");                  # base 2^26 -> base 2^32
1142         &shl    ("edi",26);
1143         &shr    ("ebx",6);
1144         &add    ("eax","edi");
1145         &mov    ("edi","ecx");
1146         &adc    ("ebx",0);
1147
1148         &shl    ("edi",20);
1149         &shr    ("ecx",12);
1150         &add    ("ebx","edi");
1151         &mov    ("edi","edx");
1152         &adc    ("ecx",0);
1153
1154         &shl    ("edi",14);
1155         &shr    ("edx",18);
1156         &add    ("ecx","edi");
1157         &mov    ("edi","esi");
1158         &adc    ("edx",0);
1159
1160         &shl    ("edi",8);
1161         &shr    ("esi",24);
1162         &add    ("edx","edi");
1163         &adc    ("esi",0);                      # can be partially reduced
1164
1165         &mov    ("edi","esi");                  # final reduction
1166         &and    ("esi",3);
1167         &shr    ("edi",2);
1168         &lea    ("ebp",&DWP(0,"edi","edi",4));  # *5
1169          &mov   ("edi",&wparam(1));             # output
1170         &add    ("eax","ebp");
1171          &mov   ("ebp",&wparam(2));             # key
1172         &adc    ("ebx",0);
1173         &adc    ("ecx",0);
1174         &adc    ("edx",0);
1175         &adc    ("esi",0);
1176
1177         &movd   ($D0,"eax");                    # offload original hash value
1178         &add    ("eax",5);                      # compare to modulus
1179         &movd   ($D1,"ebx");
1180         &adc    ("ebx",0);
1181         &movd   ($D2,"ecx");
1182         &adc    ("ecx",0);
1183         &movd   ($D3,"edx");
1184         &adc    ("edx",0);
1185         &adc    ("esi",0);
1186         &shr    ("esi",2);                      # did it carry/borrow?
1187
1188         &neg    ("esi");                        # do we choose (hash-modulus) ...
1189         &and    ("eax","esi");
1190         &and    ("ebx","esi");
1191         &and    ("ecx","esi");
1192         &and    ("edx","esi");
1193         &mov    (&DWP(4*0,"edi"),"eax");
1194         &movd   ("eax",$D0);
1195         &mov    (&DWP(4*1,"edi"),"ebx");
1196         &movd   ("ebx",$D1);
1197         &mov    (&DWP(4*2,"edi"),"ecx");
1198         &movd   ("ecx",$D2);
1199         &mov    (&DWP(4*3,"edi"),"edx");
1200         &movd   ("edx",$D3);
1201
1202         &not    ("esi");                        # ... or original hash value?
1203         &and    ("eax","esi");
1204         &and    ("ebx","esi");
1205         &or     ("eax",&DWP(4*0,"edi"));
1206         &and    ("ecx","esi");
1207         &or     ("ebx",&DWP(4*1,"edi"));
1208         &and    ("edx","esi");
1209         &or     ("ecx",&DWP(4*2,"edi"));
1210         &or     ("edx",&DWP(4*3,"edi"));
1211
1212         &add    ("eax",&DWP(4*0,"ebp"));        # accumulate key
1213         &adc    ("ebx",&DWP(4*1,"ebp"));
1214         &mov    (&DWP(4*0,"edi"),"eax");
1215         &adc    ("ecx",&DWP(4*2,"ebp"));
1216         &mov    (&DWP(4*1,"edi"),"ebx");
1217         &adc    ("edx",&DWP(4*3,"ebp"));
1218         &mov    (&DWP(4*2,"edi"),"ecx");
1219         &mov    (&DWP(4*3,"edi"),"edx");
1220 &function_end("_poly1305_emit_sse2");
1221
1222 if ($avx>1) {
1223 ########################################################################
1224 # Note that poly1305_init_avx2 operates on %xmm, I could have used
1225 # poly1305_init_sse2...
1226
1227 &align  (32);
1228 &function_begin_B("_poly1305_init_avx2");
1229         &vmovdqu        ($D4,&QWP(4*6,"edi"));          # key base 2^32
1230         &lea            ("edi",&DWP(16*3,"edi"));       # size optimization
1231         &mov            ("ebp","esp");
1232         &sub            ("esp",16*(9+5));
1233         &and            ("esp",-16);
1234
1235         #&vpand         ($D4,$D4,&QWP(96,"ebx"));       # magic mask
1236         &vmovdqa        ($MASK,&QWP(64,"ebx"));
1237
1238         &vpand          ($D0,$D4,$MASK);                # -> base 2^26
1239         &vpsrlq         ($D1,$D4,26);
1240         &vpsrldq        ($D3,$D4,6);
1241         &vpand          ($D1,$D1,$MASK);
1242         &vpsrlq         ($D2,$D3,4)
1243         &vpsrlq         ($D3,$D3,30);
1244         &vpand          ($D2,$D2,$MASK);
1245         &vpand          ($D3,$D3,$MASK);
1246         &vpsrldq        ($D4,$D4,13);
1247
1248         &lea            ("edx",&DWP(16*9,"esp"));       # size optimization
1249         &mov            ("ecx",2);
1250 &set_label("square");
1251         &vmovdqa        (&QWP(16*0,"esp"),$D0);
1252         &vmovdqa        (&QWP(16*1,"esp"),$D1);
1253         &vmovdqa        (&QWP(16*2,"esp"),$D2);
1254         &vmovdqa        (&QWP(16*3,"esp"),$D3);
1255         &vmovdqa        (&QWP(16*4,"esp"),$D4);
1256
1257         &vpslld         ($T1,$D1,2);
1258         &vpslld         ($T0,$D2,2);
1259         &vpaddd         ($T1,$T1,$D1);                  # *5
1260         &vpaddd         ($T0,$T0,$D2);                  # *5
1261         &vmovdqa        (&QWP(16*5,"esp"),$T1);
1262         &vmovdqa        (&QWP(16*6,"esp"),$T0);
1263         &vpslld         ($T1,$D3,2);
1264         &vpslld         ($T0,$D4,2);
1265         &vpaddd         ($T1,$T1,$D3);                  # *5
1266         &vpaddd         ($T0,$T0,$D4);                  # *5
1267         &vmovdqa        (&QWP(16*7,"esp"),$T1);
1268         &vmovdqa        (&QWP(16*8,"esp"),$T0);
1269
1270         &vpshufd        ($T0,$D0,0b01000100);
1271         &vmovdqa        ($T1,$D1);
1272         &vpshufd        ($D1,$D1,0b01000100);
1273         &vpshufd        ($D2,$D2,0b01000100);
1274         &vpshufd        ($D3,$D3,0b01000100);
1275         &vpshufd        ($D4,$D4,0b01000100);
1276         &vmovdqa        (&QWP(16*0,"edx"),$T0);
1277         &vmovdqa        (&QWP(16*1,"edx"),$D1);
1278         &vmovdqa        (&QWP(16*2,"edx"),$D2);
1279         &vmovdqa        (&QWP(16*3,"edx"),$D3);
1280         &vmovdqa        (&QWP(16*4,"edx"),$D4);
1281
1282         ################################################################
1283         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
1284         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
1285         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
1286         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
1287         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1288
1289         &vpmuludq       ($D4,$D4,$D0);                  # h4*r0
1290         &vpmuludq       ($D3,$D3,$D0);                  # h3*r0
1291         &vpmuludq       ($D2,$D2,$D0);                  # h2*r0
1292         &vpmuludq       ($D1,$D1,$D0);                  # h1*r0
1293         &vpmuludq       ($D0,$T0,$D0);                  # h0*r0
1294
1295         &vpmuludq       ($T0,$T1,&QWP(16*3,"edx"));     # r1*h3
1296         &vpaddq         ($D4,$D4,$T0);
1297         &vpmuludq       ($T2,$T1,&QWP(16*2,"edx"));     # r1*h2
1298         &vpaddq         ($D3,$D3,$T2);
1299         &vpmuludq       ($T0,$T1,&QWP(16*1,"edx"));     # r1*h1
1300         &vpaddq         ($D2,$D2,$T0);
1301         &vmovdqa        ($T2,&QWP(16*5,"esp"));         # s1
1302         &vpmuludq       ($T1,$T1,&QWP(16*0,"edx"));     # r1*h0
1303         &vpaddq         ($D1,$D1,$T1);
1304          &vmovdqa       ($T0,&QWP(16*2,"esp"));         # r2
1305         &vpmuludq       ($T2,$T2,&QWP(16*4,"edx"));     # s1*h4
1306         &vpaddq         ($D0,$D0,$T2);
1307
1308         &vpmuludq       ($T1,$T0,&QWP(16*2,"edx"));     # r2*h2
1309         &vpaddq         ($D4,$D4,$T1);
1310         &vpmuludq       ($T2,$T0,&QWP(16*1,"edx"));     # r2*h1
1311         &vpaddq         ($D3,$D3,$T2);
1312         &vmovdqa        ($T1,&QWP(16*6,"esp"));         # s2
1313         &vpmuludq       ($T0,$T0,&QWP(16*0,"edx"));     # r2*h0
1314         &vpaddq         ($D2,$D2,$T0);
1315         &vpmuludq       ($T2,$T1,&QWP(16*4,"edx"));     # s2*h4
1316         &vpaddq         ($D1,$D1,$T2);
1317          &vmovdqa       ($T0,&QWP(16*3,"esp"));         # r3
1318         &vpmuludq       ($T1,$T1,&QWP(16*3,"edx"));     # s2*h3
1319         &vpaddq         ($D0,$D0,$T1);
1320
1321         &vpmuludq       ($T2,$T0,&QWP(16*1,"edx"));     # r3*h1
1322         &vpaddq         ($D4,$D4,$T2);
1323         &vmovdqa        ($T1,&QWP(16*7,"esp"));         # s3
1324         &vpmuludq       ($T0,$T0,&QWP(16*0,"edx"));     # r3*h0
1325         &vpaddq         ($D3,$D3,$T0);
1326         &vpmuludq       ($T2,$T1,&QWP(16*4,"edx"));     # s3*h4
1327         &vpaddq         ($D2,$D2,$T2);
1328         &vpmuludq       ($T0,$T1,&QWP(16*3,"edx"));     # s3*h3
1329         &vpaddq         ($D1,$D1,$T0);
1330          &vmovdqa       ($T2,&QWP(16*4,"esp"));         # r4
1331         &vpmuludq       ($T1,$T1,&QWP(16*2,"edx"));     # s3*h2
1332         &vpaddq         ($D0,$D0,$T1);
1333
1334         &vmovdqa        ($T0,&QWP(16*8,"esp"));         # s4
1335         &vpmuludq       ($T2,$T2,&QWP(16*0,"edx"));     # r4*h0
1336         &vpaddq         ($D4,$D4,$T2);
1337         &vpmuludq       ($T1,$T0,&QWP(16*4,"edx"));     # s4*h4
1338         &vpaddq         ($D3,$D3,$T1);
1339         &vpmuludq       ($T2,$T0,&QWP(16*1,"edx"));     # s4*h1
1340         &vpaddq         ($D0,$D0,$T2);
1341         &vpmuludq       ($T1,$T0,&QWP(16*2,"edx"));     # s4*h2
1342         &vpaddq         ($D1,$D1,$T1);
1343          &vmovdqa       ($MASK,&QWP(64,"ebx"));
1344         &vpmuludq       ($T0,$T0,&QWP(16*3,"edx"));     # s4*h3
1345         &vpaddq         ($D2,$D2,$T0);
1346
1347         ################################################################
1348         # lazy reduction
1349          &vpsrlq        ($T0,$D3,26);
1350          &vpand         ($D3,$D3,$MASK);
1351         &vpsrlq         ($T1,$D0,26);
1352         &vpand          ($D0,$D0,$MASK);
1353          &vpaddq        ($D4,$D4,$T0);                  # h3 -> h4
1354         &vpaddq         ($D1,$D1,$T1);                  # h0 -> h1
1355          &vpsrlq        ($T0,$D4,26);
1356          &vpand         ($D4,$D4,$MASK);
1357         &vpsrlq         ($T1,$D1,26);
1358         &vpand          ($D1,$D1,$MASK);
1359         &vpaddq         ($D2,$D2,$T1);                  # h1 -> h2
1360          &vpaddd        ($D0,$D0,$T0);
1361          &vpsllq        ($T0,$T0,2);
1362         &vpsrlq         ($T1,$D2,26);
1363         &vpand          ($D2,$D2,$MASK);
1364          &vpaddd        ($D0,$D0,$T0);                  # h4 -> h0
1365         &vpaddd         ($D3,$D3,$T1);                  # h2 -> h3
1366         &vpsrlq         ($T1,$D3,26);
1367          &vpsrlq        ($T0,$D0,26);
1368          &vpand         ($D0,$D0,$MASK);
1369         &vpand          ($D3,$D3,$MASK);
1370          &vpaddd        ($D1,$D1,$T0);                  # h0 -> h1
1371         &vpaddd         ($D4,$D4,$T1);                  # h3 -> h4
1372
1373         &dec            ("ecx");
1374         &jz             (&label("square_break"));
1375
1376         &vpunpcklqdq    ($D0,$D0,&QWP(16*0,"esp"));     # 0:r^1:0:r^2
1377         &vpunpcklqdq    ($D1,$D1,&QWP(16*1,"esp"));
1378         &vpunpcklqdq    ($D2,$D2,&QWP(16*2,"esp"));
1379         &vpunpcklqdq    ($D3,$D3,&QWP(16*3,"esp"));
1380         &vpunpcklqdq    ($D4,$D4,&QWP(16*4,"esp"));
1381         &jmp            (&label("square"));
1382
1383 &set_label("square_break");
1384         &vpsllq         ($D0,$D0,32);                   # -> r^3:0:r^4:0
1385         &vpsllq         ($D1,$D1,32);
1386         &vpsllq         ($D2,$D2,32);
1387         &vpsllq         ($D3,$D3,32);
1388         &vpsllq         ($D4,$D4,32);
1389         &vpor           ($D0,$D0,&QWP(16*0,"esp"));     # r^3:r^1:r^4:r^2
1390         &vpor           ($D1,$D1,&QWP(16*1,"esp"));
1391         &vpor           ($D2,$D2,&QWP(16*2,"esp"));
1392         &vpor           ($D3,$D3,&QWP(16*3,"esp"));
1393         &vpor           ($D4,$D4,&QWP(16*4,"esp"));
1394
1395         &vpshufd        ($D0,$D0,0b10001101);           # -> r^1:r^2:r^3:r^4
1396         &vpshufd        ($D1,$D1,0b10001101);
1397         &vpshufd        ($D2,$D2,0b10001101);
1398         &vpshufd        ($D3,$D3,0b10001101);
1399         &vpshufd        ($D4,$D4,0b10001101);
1400
1401         &vmovdqu        (&QWP(16*0,"edi"),$D0);         # save the table
1402         &vmovdqu        (&QWP(16*1,"edi"),$D1);
1403         &vmovdqu        (&QWP(16*2,"edi"),$D2);
1404         &vmovdqu        (&QWP(16*3,"edi"),$D3);
1405         &vmovdqu        (&QWP(16*4,"edi"),$D4);
1406
1407         &vpslld         ($T1,$D1,2);
1408         &vpslld         ($T0,$D2,2);
1409         &vpaddd         ($T1,$T1,$D1);                  # *5
1410         &vpaddd         ($T0,$T0,$D2);                  # *5
1411         &vmovdqu        (&QWP(16*5,"edi"),$T1);
1412         &vmovdqu        (&QWP(16*6,"edi"),$T0);
1413         &vpslld         ($T1,$D3,2);
1414         &vpslld         ($T0,$D4,2);
1415         &vpaddd         ($T1,$T1,$D3);                  # *5
1416         &vpaddd         ($T0,$T0,$D4);                  # *5
1417         &vmovdqu        (&QWP(16*7,"edi"),$T1);
1418         &vmovdqu        (&QWP(16*8,"edi"),$T0);
1419
1420         &mov            ("esp","ebp");
1421         &lea            ("edi",&DWP(-16*3,"edi"));      # size de-optimization
1422         &ret            ();
1423 &function_end_B("_poly1305_init_avx2");
1424
1425 ########################################################################
1426 # now it's time to switch to %ymm
1427
1428 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("ymm$_",(0..7));
1429 my $MASK=$T2;
1430
1431 sub X { my $reg=shift; $reg=~s/^ymm/xmm/; $reg; }
1432
1433 &align  (32);
1434 &function_begin("_poly1305_blocks_avx2");
1435         &mov    ("edi",&wparam(0));                     # ctx
1436         &mov    ("esi",&wparam(1));                     # inp
1437         &mov    ("ecx",&wparam(2));                     # len
1438
1439         &mov    ("eax",&DWP(4*5,"edi"));                # is_base2_26
1440         &and    ("ecx",-16);
1441         &jz     (&label("nodata"));
1442         &cmp    ("ecx",64);
1443         &jae    (&label("enter_avx2"));
1444         &test   ("eax","eax");                          # is_base2_26?
1445         &jz     (&label("enter_blocks"));
1446
1447 &set_label("enter_avx2");
1448         &vzeroupper     ();
1449
1450         &call   (&label("pic_point"));
1451 &set_label("pic_point");
1452         &blindpop("ebx");
1453         &lea    ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
1454
1455         &test   ("eax","eax");                          # is_base2_26?
1456         &jnz    (&label("base2_26"));
1457
1458         &call   ("_poly1305_init_avx2");
1459
1460         ################################################# base 2^32 -> base 2^26
1461         &mov    ("eax",&DWP(0,"edi"));
1462         &mov    ("ecx",&DWP(3,"edi"));
1463         &mov    ("edx",&DWP(6,"edi"));
1464         &mov    ("esi",&DWP(9,"edi"));
1465         &mov    ("ebp",&DWP(13,"edi"));
1466
1467         &shr    ("ecx",2);
1468         &and    ("eax",0x3ffffff);
1469         &shr    ("edx",4);
1470         &and    ("ecx",0x3ffffff);
1471         &shr    ("esi",6);
1472         &and    ("edx",0x3ffffff);
1473
1474         &mov    (&DWP(4*0,"edi"),"eax");
1475         &mov    (&DWP(4*1,"edi"),"ecx");
1476         &mov    (&DWP(4*2,"edi"),"edx");
1477         &mov    (&DWP(4*3,"edi"),"esi");
1478         &mov    (&DWP(4*4,"edi"),"ebp");
1479         &mov    (&DWP(4*5,"edi"),1);                    # is_base2_26
1480
1481         &mov    ("esi",&wparam(1));                     # [reload] inp
1482         &mov    ("ecx",&wparam(2));                     # [reload] len
1483
1484 &set_label("base2_26");
1485         &mov    ("eax",&wparam(3));                     # padbit
1486         &mov    ("ebp","esp");
1487
1488         &sub    ("esp",32*(5+9));
1489         &and    ("esp",-512);                           # ensure that frame
1490                                                         # doesn't cross page
1491                                                         # boundary, which is
1492                                                         # essential for
1493                                                         # misaligned 32-byte
1494                                                         # loads
1495
1496         ################################################################
1497         # expand and copy pre-calculated table to stack
1498
1499         &vmovdqu        (&X($D0),&QWP(16*(3+0),"edi"));
1500         &lea            ("edx",&DWP(32*5+128,"esp"));   # +128 size optimization
1501         &vmovdqu        (&X($D1),&QWP(16*(3+1),"edi"));
1502         &vmovdqu        (&X($D2),&QWP(16*(3+2),"edi"));
1503         &vmovdqu        (&X($D3),&QWP(16*(3+3),"edi"));
1504         &vmovdqu        (&X($D4),&QWP(16*(3+4),"edi"));
1505         &lea            ("edi",&DWP(16*3,"edi"));       # size optimization
1506         &vpermq         ($D0,$D0,0b01000000);           # 00001234 -> 12343434
1507         &vpermq         ($D1,$D1,0b01000000);
1508         &vpermq         ($D2,$D2,0b01000000);
1509         &vpermq         ($D3,$D3,0b01000000);
1510         &vpermq         ($D4,$D4,0b01000000);
1511         &vpshufd        ($D0,$D0,0b11001000);           # 12343434 -> 14243444
1512         &vpshufd        ($D1,$D1,0b11001000);
1513         &vpshufd        ($D2,$D2,0b11001000);
1514         &vpshufd        ($D3,$D3,0b11001000);
1515         &vpshufd        ($D4,$D4,0b11001000);
1516         &vmovdqa        (&QWP(32*0-128,"edx"),$D0);
1517         &vmovdqu        (&X($D0),&QWP(16*5,"edi"));
1518         &vmovdqa        (&QWP(32*1-128,"edx"),$D1);
1519         &vmovdqu        (&X($D1),&QWP(16*6,"edi"));
1520         &vmovdqa        (&QWP(32*2-128,"edx"),$D2);
1521         &vmovdqu        (&X($D2),&QWP(16*7,"edi"));
1522         &vmovdqa        (&QWP(32*3-128,"edx"),$D3);
1523         &vmovdqu        (&X($D3),&QWP(16*8,"edi"));
1524         &vmovdqa        (&QWP(32*4-128,"edx"),$D4);
1525         &vpermq         ($D0,$D0,0b01000000);
1526         &vpermq         ($D1,$D1,0b01000000);
1527         &vpermq         ($D2,$D2,0b01000000);
1528         &vpermq         ($D3,$D3,0b01000000);
1529         &vpshufd        ($D0,$D0,0b11001000);
1530         &vpshufd        ($D1,$D1,0b11001000);
1531         &vpshufd        ($D2,$D2,0b11001000);
1532         &vpshufd        ($D3,$D3,0b11001000);
1533         &vmovdqa        (&QWP(32*5-128,"edx"),$D0);
1534         &vmovd          (&X($D0),&DWP(-16*3+4*0,"edi"));# load hash value
1535         &vmovdqa        (&QWP(32*6-128,"edx"),$D1);
1536         &vmovd          (&X($D1),&DWP(-16*3+4*1,"edi"));
1537         &vmovdqa        (&QWP(32*7-128,"edx"),$D2);
1538         &vmovd          (&X($D2),&DWP(-16*3+4*2,"edi"));
1539         &vmovdqa        (&QWP(32*8-128,"edx"),$D3);
1540         &vmovd          (&X($D3),&DWP(-16*3+4*3,"edi"));
1541         &vmovd          (&X($D4),&DWP(-16*3+4*4,"edi"));
1542         &vmovdqa        ($MASK,&QWP(64,"ebx"));
1543         &neg            ("eax");                        # padbit
1544
1545         &test           ("ecx",63);
1546         &jz             (&label("even"));
1547
1548         &mov            ("edx","ecx");
1549         &and            ("ecx",-64);
1550         &and            ("edx",63);
1551
1552         &vmovdqu        (&X($T0),&QWP(16*0,"esi"));
1553         &cmp            ("edx",32);
1554         &jb             (&label("one"));
1555
1556         &vmovdqu        (&X($T1),&QWP(16*1,"esi"));
1557         &je             (&label("two"));
1558
1559         &vinserti128    ($T0,$T0,&QWP(16*2,"esi"),1);
1560         &lea            ("esi",&DWP(16*3,"esi"));
1561         &lea            ("ebx",&DWP(8,"ebx"));          # three padbits
1562         &lea            ("edx",&DWP(32*5+128+8,"esp")); # --:r^1:r^2:r^3 (*)
1563         &jmp            (&label("tail"));
1564
1565 &set_label("two");
1566         &lea            ("esi",&DWP(16*2,"esi"));
1567         &lea            ("ebx",&DWP(16,"ebx"));         # two padbits
1568         &lea            ("edx",&DWP(32*5+128+16,"esp"));# --:--:r^1:r^2 (*)
1569         &jmp            (&label("tail"));
1570
1571 &set_label("one");
1572         &lea            ("esi",&DWP(16*1,"esi"));
1573         &vpxor          ($T1,$T1,$T1);
1574         &lea            ("ebx",&DWP(32,"ebx","eax",8)); # one or no padbits
1575         &lea            ("edx",&DWP(32*5+128+24,"esp"));# --:--:--:r^1 (*)
1576         &jmp            (&label("tail"));
1577
1578 # (*)   spots marked with '--' are data from next table entry, but they
1579 #       are multiplied by 0 and therefore rendered insignificant
1580
1581 &set_label("even",32);
1582         &vmovdqu        (&X($T0),&QWP(16*0,"esi"));     # load input
1583         &vmovdqu        (&X($T1),&QWP(16*1,"esi"));
1584         &vinserti128    ($T0,$T0,&QWP(16*2,"esi"),1);
1585         &vinserti128    ($T1,$T1,&QWP(16*3,"esi"),1);
1586         &lea            ("esi",&DWP(16*4,"esi"));
1587         &sub            ("ecx",64);
1588         &jz             (&label("tail"));
1589
1590 &set_label("loop");
1591         ################################################################
1592         # ((inp[0]*r^4+r[4])*r^4+r[8])*r^4
1593         # ((inp[1]*r^4+r[5])*r^4+r[9])*r^3
1594         # ((inp[2]*r^4+r[6])*r^4+r[10])*r^2
1595         # ((inp[3]*r^4+r[7])*r^4+r[11])*r^1
1596         #   \________/ \_______/
1597         ################################################################
1598
1599 sub vsplat_input {
1600         &vmovdqa        (&QWP(32*2,"esp"),$D2);
1601         &vpsrldq        ($D2,$T0,6);                    # splat input
1602         &vmovdqa        (&QWP(32*0,"esp"),$D0);
1603         &vpsrldq        ($D0,$T1,6);
1604         &vmovdqa        (&QWP(32*1,"esp"),$D1);
1605         &vpunpckhqdq    ($D1,$T0,$T1);                  # 4
1606         &vpunpcklqdq    ($T0,$T0,$T1);                  # 0:1
1607         &vpunpcklqdq    ($D2,$D2,$D0);                  # 2:3
1608
1609         &vpsrlq         ($D0,$D2,30);
1610         &vpsrlq         ($D2,$D2,4);
1611         &vpsrlq         ($T1,$T0,26);
1612         &vpsrlq         ($D1,$D1,40);                   # 4
1613         &vpand          ($D2,$D2,$MASK);                # 2
1614         &vpand          ($T0,$T0,$MASK);                # 0
1615         &vpand          ($T1,$T1,$MASK);                # 1
1616         &vpand          ($D0,$D0,$MASK);                # 3 (*)
1617         &vpor           ($D1,$D1,&QWP(0,"ebx"));        # padbit, yes, always
1618
1619         # (*)   note that output is counterintuitive, inp[3:4] is
1620         #       returned in $D1-2, while $D3-4 are preserved;
1621 }
1622         &vsplat_input   ();
1623
1624 sub vpmuladd {
1625 my $addr = shift;
1626
1627         &vpaddq         ($D2,$D2,&QWP(32*2,"esp"));     # add hash value
1628         &vpaddq         ($T0,$T0,&QWP(32*0,"esp"));
1629         &vpaddq         ($T1,$T1,&QWP(32*1,"esp"));
1630         &vpaddq         ($D0,$D0,$D3);
1631         &vpaddq         ($D1,$D1,$D4);
1632
1633         ################################################################
1634         # d3 = h2*r1   + h0*r3 + h1*r2   + h3*r0   + h4*5*r4
1635         # d4 = h2*r2   + h0*r4 + h1*r3   + h3*r1   + h4*r0
1636         # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1
1637         # d1 = h2*5*r4 + h0*r1 + h1*r0   + h3*5*r3 + h4*5*r2
1638         # d2 = h2*r0   + h0*r2 + h1*r1   + h3*5*r4 + h4*5*r3
1639
1640         &vpmuludq       ($D3,$D2,&$addr(1));            # d3 = h2*r1
1641          &vmovdqa       (QWP(32*1,"esp"),$T1);
1642         &vpmuludq       ($D4,$D2,&$addr(2));            # d4 = h2*r2
1643          &vmovdqa       (QWP(32*3,"esp"),$D0);
1644         &vpmuludq       ($D0,$D2,&$addr(7));            # d0 = h2*s3
1645          &vmovdqa       (QWP(32*4,"esp"),$D1);
1646         &vpmuludq       ($D1,$D2,&$addr(8));            # d1 = h2*s4
1647         &vpmuludq       ($D2,$D2,&$addr(0));            # d2 = h2*r0
1648
1649         &vpmuludq       ($T2,$T0,&$addr(3));            # h0*r3
1650         &vpaddq         ($D3,$D3,$T2);                  # d3 += h0*r3
1651         &vpmuludq       ($T1,$T0,&$addr(4));            # h0*r4
1652         &vpaddq         ($D4,$D4,$T1);                  # d4 + h0*r4
1653         &vpmuludq       ($T2,$T0,&$addr(0));            # h0*r0
1654         &vpaddq         ($D0,$D0,$T2);                  # d0 + h0*r0
1655          &vmovdqa       ($T2,&QWP(32*1,"esp"));         # h1
1656         &vpmuludq       ($T1,$T0,&$addr(1));            # h0*r1
1657         &vpaddq         ($D1,$D1,$T1);                  # d1 += h0*r1
1658         &vpmuludq       ($T0,$T0,&$addr(2));            # h0*r2
1659         &vpaddq         ($D2,$D2,$T0);                  # d2 += h0*r2
1660
1661         &vpmuludq       ($T1,$T2,&$addr(2));            # h1*r2
1662         &vpaddq         ($D3,$D3,$T1);                  # d3 += h1*r2
1663         &vpmuludq       ($T0,$T2,&$addr(3));            # h1*r3
1664         &vpaddq         ($D4,$D4,$T0);                  # d4 += h1*r3
1665         &vpmuludq       ($T1,$T2,&$addr(8));            # h1*s4
1666         &vpaddq         ($D0,$D0,$T1);                  # d0 += h1*s4
1667          &vmovdqa       ($T1,&QWP(32*3,"esp"));         # h3
1668         &vpmuludq       ($T0,$T2,&$addr(0));            # h1*r0
1669         &vpaddq         ($D1,$D1,$T0);                  # d1 += h1*r0
1670         &vpmuludq       ($T2,$T2,&$addr(1));            # h1*r1
1671         &vpaddq         ($D2,$D2,$T2);                  # d2 += h1*r1
1672
1673         &vpmuludq       ($T0,$T1,&$addr(0));            # h3*r0
1674         &vpaddq         ($D3,$D3,$T0);                  # d3 += h3*r0
1675         &vpmuludq       ($T2,$T1,&$addr(1));            # h3*r1
1676         &vpaddq         ($D4,$D4,$T2);                  # d4 += h3*r1
1677         &vpmuludq       ($T0,$T1,&$addr(6));            # h3*s2
1678         &vpaddq         ($D0,$D0,$T0);                  # d0 += h3*s2
1679          &vmovdqa       ($T0,&QWP(32*4,"esp"));         # h4
1680         &vpmuludq       ($T2,$T1,&$addr(7));            # h3*s3
1681         &vpaddq         ($D1,$D1,$T2);                  # d1+= h3*s3
1682         &vpmuludq       ($T1,$T1,&$addr(8));            # h3*s4
1683         &vpaddq         ($D2,$D2,$T1);                  # d2 += h3*s4
1684
1685         &vpmuludq       ($T2,$T0,&$addr(8));            # h4*s4
1686         &vpaddq         ($D3,$D3,$T2);                  # d3 += h4*s4
1687         &vpmuludq       ($T1,$T0,&$addr(5));            # h4*s1
1688         &vpaddq         ($D0,$D0,$T1);                  # d0 += h4*s1
1689         &vpmuludq       ($T2,$T0,&$addr(0));            # h4*r0
1690         &vpaddq         ($D4,$D4,$T2);                  # d4 += h4*r0
1691          &vmovdqa       ($MASK,&QWP(64,"ebx"));
1692         &vpmuludq       ($T1,$T0,&$addr(6));            # h4*s2
1693         &vpaddq         ($D1,$D1,$T1);                  # d1 += h4*s2
1694         &vpmuludq       ($T0,$T0,&$addr(7));            # h4*s3
1695         &vpaddq         ($D2,$D2,$T0);                  # d2 += h4*s3
1696 }
1697         &vpmuladd       (sub {  my $i=shift; &QWP(32*$i-128,"edx");     });
1698
1699 sub vlazy_reduction {
1700         ################################################################
1701         # lazy reduction
1702
1703          &vpsrlq        ($T0,$D3,26);
1704          &vpand         ($D3,$D3,$MASK);
1705         &vpsrlq         ($T1,$D0,26);
1706         &vpand          ($D0,$D0,$MASK);
1707          &vpaddq        ($D4,$D4,$T0);                  # h3 -> h4
1708         &vpaddq         ($D1,$D1,$T1);                  # h0 -> h1
1709          &vpsrlq        ($T0,$D4,26);
1710          &vpand         ($D4,$D4,$MASK);
1711         &vpsrlq         ($T1,$D1,26);
1712         &vpand          ($D1,$D1,$MASK);
1713         &vpaddq         ($D2,$D2,$T1);                  # h1 -> h2
1714          &vpaddq        ($D0,$D0,$T0);
1715          &vpsllq        ($T0,$T0,2);
1716         &vpsrlq         ($T1,$D2,26);
1717         &vpand          ($D2,$D2,$MASK);
1718          &vpaddq        ($D0,$D0,$T0);                  # h4 -> h0
1719         &vpaddq         ($D3,$D3,$T1);                  # h2 -> h3
1720         &vpsrlq         ($T1,$D3,26);
1721          &vpsrlq        ($T0,$D0,26);
1722          &vpand         ($D0,$D0,$MASK);
1723         &vpand          ($D3,$D3,$MASK);
1724          &vpaddq        ($D1,$D1,$T0);                  # h0 -> h1
1725         &vpaddq         ($D4,$D4,$T1);                  # h3 -> h4
1726 }
1727         &vlazy_reduction();
1728
1729         &vmovdqu        (&X($T0),&QWP(16*0,"esi"));     # load input
1730         &vmovdqu        (&X($T1),&QWP(16*1,"esi"));
1731         &vinserti128    ($T0,$T0,&QWP(16*2,"esi"),1);
1732         &vinserti128    ($T1,$T1,&QWP(16*3,"esi"),1);
1733         &lea            ("esi",&DWP(16*4,"esi"));
1734         &sub            ("ecx",64);
1735         &jnz            (&label("loop"));
1736
1737 &set_label("tail");
1738         &vsplat_input   ();
1739         &and            ("ebx",-64);                    # restore pointer
1740
1741         &vpmuladd       (sub {  my $i=shift; &QWP(4+32*$i-128,"edx");   });
1742
1743         ################################################################
1744         # horizontal addition
1745
1746         &vpsrldq        ($T0,$D4,8);
1747         &vpsrldq        ($T1,$D3,8);
1748         &vpaddq         ($D4,$D4,$T0);
1749         &vpsrldq        ($T0,$D0,8);
1750         &vpaddq         ($D3,$D3,$T1);
1751         &vpsrldq        ($T1,$D1,8);
1752         &vpaddq         ($D0,$D0,$T0);
1753         &vpsrldq        ($T0,$D2,8);
1754         &vpaddq         ($D1,$D1,$T1);
1755         &vpermq         ($T1,$D4,2);                    # keep folding
1756         &vpaddq         ($D2,$D2,$T0);
1757         &vpermq         ($T0,$D3,2);
1758         &vpaddq         ($D4,$D4,$T1);
1759         &vpermq         ($T1,$D0,2);
1760         &vpaddq         ($D3,$D3,$T0);
1761         &vpermq         ($T0,$D1,2);
1762         &vpaddq         ($D0,$D0,$T1);
1763         &vpermq         ($T1,$D2,2);
1764         &vpaddq         ($D1,$D1,$T0);
1765         &vpaddq         ($D2,$D2,$T1);
1766
1767         &vlazy_reduction();
1768
1769         &cmp            ("ecx",0);
1770         &je             (&label("done"));
1771
1772         ################################################################
1773         # clear all but single word
1774
1775         &vpshufd        (&X($D0),&X($D0),0b11111100);
1776         &lea            ("edx",&DWP(32*5+128,"esp"));   # restore pointer
1777         &vpshufd        (&X($D1),&X($D1),0b11111100);
1778         &vpshufd        (&X($D2),&X($D2),0b11111100);
1779         &vpshufd        (&X($D3),&X($D3),0b11111100);
1780         &vpshufd        (&X($D4),&X($D4),0b11111100);
1781         &jmp            (&label("even"));
1782
1783 &set_label("done",16);
1784         &vmovd          (&DWP(-16*3+4*0,"edi"),&X($D0));# store hash value
1785         &vmovd          (&DWP(-16*3+4*1,"edi"),&X($D1));
1786         &vmovd          (&DWP(-16*3+4*2,"edi"),&X($D2));
1787         &vmovd          (&DWP(-16*3+4*3,"edi"),&X($D3));
1788         &vmovd          (&DWP(-16*3+4*4,"edi"),&X($D4));
1789         &vzeroupper     ();
1790         &mov    ("esp","ebp");
1791 &set_label("nodata");
1792 &function_end("_poly1305_blocks_avx2");
1793 }
1794 &set_label("const_sse2",64);
1795         &data_word(1<<24,0,     1<<24,0,        1<<24,0,        1<<24,0);
1796         &data_word(0,0,         0,0,            0,0,            0,0);
1797         &data_word(0x03ffffff,0,0x03ffffff,0,   0x03ffffff,0,   0x03ffffff,0);
1798         &data_word(0x0fffffff,0x0ffffffc,0x0ffffffc,0x0ffffffc);
1799 }
1800 &asciz  ("Poly1305 for x86, CRYPTOGAMS by <appro\@openssl.org>");
1801 &align  (4);
1802
1803 &asm_finish();
1804
1805 close STDOUT;