For all assembler scripts where it matters, recognise clang > 9.x
[openssl.git] / crypto / poly1305 / asm / poly1305-x86.pl
1 #! /usr/bin/env perl
2 # Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
3 #
4 # Licensed under the Apache License 2.0 (the "License").  You may not use
5 # this file except in compliance with the License.  You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
8
9 #
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
16 #
17 # This module implements Poly1305 hash for x86.
18 #
19 # April 2015
20 #
21 # Numbers are cycles per processed byte with poly1305_blocks alone,
22 # measured with rdtsc at fixed clock frequency.
23 #
24 #               IALU/gcc-3.4(*) SSE2(**)        AVX2
25 # Pentium       15.7/+80%       -
26 # PIII          6.21/+90%       -
27 # P4            19.8/+40%       3.24
28 # Core 2        4.85/+90%       1.80
29 # Westmere      4.58/+100%      1.43
30 # Sandy Bridge  3.90/+100%      1.36
31 # Haswell       3.88/+70%       1.18            0.72
32 # Skylake       3.10/+60%       1.14            0.62
33 # Silvermont    11.0/+40%       4.80
34 # Goldmont      4.10/+200%      2.10
35 # VIA Nano      6.71/+90%       2.47
36 # Sledgehammer  3.51/+180%      4.27
37 # Bulldozer     4.53/+140%      1.31
38 #
39 # (*)   gcc 4.8 for some reason generated worse code;
40 # (**)  besides SSE2 there are floating-point and AVX options; FP
41 #       is deemed unnecessary, because pre-SSE2 processor are too
42 #       old to care about, while it's not the fastest option on
43 #       SSE2-capable ones; AVX is omitted, because it doesn't give
44 #       a lot of improvement, 5-10% depending on processor;
45
46 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
47 push(@INC,"${dir}","${dir}../../perlasm");
48 require "x86asm.pl";
49
50 $output=pop and open STDOUT,">$output";
51
52 &asm_init($ARGV[0],$ARGV[$#ARGV] eq "386");
53
54 $sse2=$avx=0;
55 for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
56
57 if ($sse2) {
58         &static_label("const_sse2");
59         &static_label("enter_blocks");
60         &static_label("enter_emit");
61         &external_label("OPENSSL_ia32cap_P");
62
63         if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
64                         =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
65                 $avx = ($1>=2.19) + ($1>=2.22);
66         }
67
68         if (!$avx && $ARGV[0] eq "win32n" &&
69            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
70         $avx = ($1>=2.09) + ($1>=2.10);
71         }
72
73         if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|based on LLVM) ([0-9]+\.[0-9]+)/) {
74                 $avx = ($2>=3.0) + ($2>3.0);
75         }
76 }
77
78 ########################################################################
79 # Layout of opaque area is following.
80 #
81 #       unsigned __int32 h[5];          # current hash value base 2^32
82 #       unsigned __int32 pad;           # is_base2_26 in vector context
83 #       unsigned __int32 r[4];          # key value base 2^32
84
85 &align(64);
86 &function_begin("poly1305_init");
87         &mov    ("edi",&wparam(0));             # context
88         &mov    ("esi",&wparam(1));             # key
89         &mov    ("ebp",&wparam(2));             # function table
90
91         &xor    ("eax","eax");
92         &mov    (&DWP(4*0,"edi"),"eax");        # zero hash value
93         &mov    (&DWP(4*1,"edi"),"eax");
94         &mov    (&DWP(4*2,"edi"),"eax");
95         &mov    (&DWP(4*3,"edi"),"eax");
96         &mov    (&DWP(4*4,"edi"),"eax");
97         &mov    (&DWP(4*5,"edi"),"eax");        # is_base2_26
98
99         &cmp    ("esi",0);
100         &je     (&label("nokey"));
101
102     if ($sse2) {
103         &call   (&label("pic_point"));
104     &set_label("pic_point");
105         &blindpop("ebx");
106
107         &lea    ("eax",&DWP("poly1305_blocks-".&label("pic_point"),"ebx"));
108         &lea    ("edx",&DWP("poly1305_emit-".&label("pic_point"),"ebx"));
109
110         &picmeup("edi","OPENSSL_ia32cap_P","ebx",&label("pic_point"));
111         &mov    ("ecx",&DWP(0,"edi"));
112         &and    ("ecx",1<<26|1<<24);
113         &cmp    ("ecx",1<<26|1<<24);            # SSE2 and XMM?
114         &jne    (&label("no_sse2"));
115
116         &lea    ("eax",&DWP("_poly1305_blocks_sse2-".&label("pic_point"),"ebx"));
117         &lea    ("edx",&DWP("_poly1305_emit_sse2-".&label("pic_point"),"ebx"));
118
119       if ($avx>1) {
120         &mov    ("ecx",&DWP(8,"edi"));
121         &test   ("ecx",1<<5);                   # AVX2?
122         &jz     (&label("no_sse2"));
123
124         &lea    ("eax",&DWP("_poly1305_blocks_avx2-".&label("pic_point"),"ebx"));
125       }
126     &set_label("no_sse2");
127         &mov    ("edi",&wparam(0));             # reload context
128         &mov    (&DWP(0,"ebp"),"eax");          # fill function table
129         &mov    (&DWP(4,"ebp"),"edx");
130     }
131
132         &mov    ("eax",&DWP(4*0,"esi"));        # load input key
133         &mov    ("ebx",&DWP(4*1,"esi"));
134         &mov    ("ecx",&DWP(4*2,"esi"));
135         &mov    ("edx",&DWP(4*3,"esi"));
136         &and    ("eax",0x0fffffff);
137         &and    ("ebx",0x0ffffffc);
138         &and    ("ecx",0x0ffffffc);
139         &and    ("edx",0x0ffffffc);
140         &mov    (&DWP(4*6,"edi"),"eax");
141         &mov    (&DWP(4*7,"edi"),"ebx");
142         &mov    (&DWP(4*8,"edi"),"ecx");
143         &mov    (&DWP(4*9,"edi"),"edx");
144
145         &mov    ("eax",$sse2);
146 &set_label("nokey");
147 &function_end("poly1305_init");
148
149 ($h0,$h1,$h2,$h3,$h4,
150  $d0,$d1,$d2,$d3,
151  $r0,$r1,$r2,$r3,
152      $s1,$s2,$s3)=map(4*$_,(0..15));
153
154 &function_begin("poly1305_blocks");
155         &mov    ("edi",&wparam(0));             # ctx
156         &mov    ("esi",&wparam(1));             # inp
157         &mov    ("ecx",&wparam(2));             # len
158 &set_label("enter_blocks");
159         &and    ("ecx",-15);
160         &jz     (&label("nodata"));
161
162         &stack_push(16);
163         &mov    ("eax",&DWP(4*6,"edi"));        # r0
164         &mov    ("ebx",&DWP(4*7,"edi"));        # r1
165          &lea   ("ebp",&DWP(0,"esi","ecx"));    # end of input
166         &mov    ("ecx",&DWP(4*8,"edi"));        # r2
167         &mov    ("edx",&DWP(4*9,"edi"));        # r3
168
169         &mov    (&wparam(2),"ebp");
170         &mov    ("ebp","esi");
171
172         &mov    (&DWP($r0,"esp"),"eax");        # r0
173         &mov    ("eax","ebx");
174         &shr    ("eax",2);
175         &mov    (&DWP($r1,"esp"),"ebx");        # r1
176         &add    ("eax","ebx");                  # s1
177         &mov    ("ebx","ecx");
178         &shr    ("ebx",2);
179         &mov    (&DWP($r2,"esp"),"ecx");        # r2
180         &add    ("ebx","ecx");                  # s2
181         &mov    ("ecx","edx");
182         &shr    ("ecx",2);
183         &mov    (&DWP($r3,"esp"),"edx");        # r3
184         &add    ("ecx","edx");                  # s3
185         &mov    (&DWP($s1,"esp"),"eax");        # s1
186         &mov    (&DWP($s2,"esp"),"ebx");        # s2
187         &mov    (&DWP($s3,"esp"),"ecx");        # s3
188
189         &mov    ("eax",&DWP(4*0,"edi"));        # load hash value
190         &mov    ("ebx",&DWP(4*1,"edi"));
191         &mov    ("ecx",&DWP(4*2,"edi"));
192         &mov    ("esi",&DWP(4*3,"edi"));
193         &mov    ("edi",&DWP(4*4,"edi"));
194         &jmp    (&label("loop"));
195
196 &set_label("loop",32);
197         &add    ("eax",&DWP(4*0,"ebp"));        # accumulate input
198         &adc    ("ebx",&DWP(4*1,"ebp"));
199         &adc    ("ecx",&DWP(4*2,"ebp"));
200         &adc    ("esi",&DWP(4*3,"ebp"));
201         &lea    ("ebp",&DWP(4*4,"ebp"));
202         &adc    ("edi",&wparam(3));             # padbit
203
204         &mov    (&DWP($h0,"esp"),"eax");        # put aside hash[+inp]
205         &mov    (&DWP($h3,"esp"),"esi");
206
207         &mul    (&DWP($r0,"esp"));              # h0*r0
208          &mov   (&DWP($h4,"esp"),"edi");
209         &mov    ("edi","eax");
210         &mov    ("eax","ebx");                  # h1
211         &mov    ("esi","edx");
212         &mul    (&DWP($s3,"esp"));              # h1*s3
213         &add    ("edi","eax");
214         &mov    ("eax","ecx");                  # h2
215         &adc    ("esi","edx");
216         &mul    (&DWP($s2,"esp"));              # h2*s2
217         &add    ("edi","eax");
218         &mov    ("eax",&DWP($h3,"esp"));
219         &adc    ("esi","edx");
220         &mul    (&DWP($s1,"esp"));              # h3*s1
221         &add    ("edi","eax");
222          &mov   ("eax",&DWP($h0,"esp"));
223         &adc    ("esi","edx");
224
225         &mul    (&DWP($r1,"esp"));              # h0*r1
226          &mov   (&DWP($d0,"esp"),"edi");
227         &xor    ("edi","edi");
228         &add    ("esi","eax");
229         &mov    ("eax","ebx");                  # h1
230         &adc    ("edi","edx");
231         &mul    (&DWP($r0,"esp"));              # h1*r0
232         &add    ("esi","eax");
233         &mov    ("eax","ecx");                  # h2
234         &adc    ("edi","edx");
235         &mul    (&DWP($s3,"esp"));              # h2*s3
236         &add    ("esi","eax");
237         &mov    ("eax",&DWP($h3,"esp"));
238         &adc    ("edi","edx");
239         &mul    (&DWP($s2,"esp"));              # h3*s2
240         &add    ("esi","eax");
241         &mov    ("eax",&DWP($h4,"esp"));
242         &adc    ("edi","edx");
243         &imul   ("eax",&DWP($s1,"esp"));        # h4*s1
244         &add    ("esi","eax");
245          &mov   ("eax",&DWP($h0,"esp"));
246         &adc    ("edi",0);
247
248         &mul    (&DWP($r2,"esp"));              # h0*r2
249          &mov   (&DWP($d1,"esp"),"esi");
250         &xor    ("esi","esi");
251         &add    ("edi","eax");
252         &mov    ("eax","ebx");                  # h1
253         &adc    ("esi","edx");
254         &mul    (&DWP($r1,"esp"));              # h1*r1
255         &add    ("edi","eax");
256         &mov    ("eax","ecx");                  # h2
257         &adc    ("esi","edx");
258         &mul    (&DWP($r0,"esp"));              # h2*r0
259         &add    ("edi","eax");
260         &mov    ("eax",&DWP($h3,"esp"));
261         &adc    ("esi","edx");
262         &mul    (&DWP($s3,"esp"));              # h3*s3
263         &add    ("edi","eax");
264         &mov    ("eax",&DWP($h4,"esp"));
265         &adc    ("esi","edx");
266         &imul   ("eax",&DWP($s2,"esp"));        # h4*s2
267         &add    ("edi","eax");
268          &mov   ("eax",&DWP($h0,"esp"));
269         &adc    ("esi",0);
270
271         &mul    (&DWP($r3,"esp"));              # h0*r3
272          &mov   (&DWP($d2,"esp"),"edi");
273         &xor    ("edi","edi");
274         &add    ("esi","eax");
275         &mov    ("eax","ebx");                  # h1
276         &adc    ("edi","edx");
277         &mul    (&DWP($r2,"esp"));              # h1*r2
278         &add    ("esi","eax");
279         &mov    ("eax","ecx");                  # h2
280         &adc    ("edi","edx");
281         &mul    (&DWP($r1,"esp"));              # h2*r1
282         &add    ("esi","eax");
283         &mov    ("eax",&DWP($h3,"esp"));
284         &adc    ("edi","edx");
285         &mul    (&DWP($r0,"esp"));              # h3*r0
286         &add    ("esi","eax");
287          &mov   ("ecx",&DWP($h4,"esp"));
288         &adc    ("edi","edx");
289
290         &mov    ("edx","ecx");
291         &imul   ("ecx",&DWP($s3,"esp"));        # h4*s3
292         &add    ("esi","ecx");
293          &mov   ("eax",&DWP($d0,"esp"));
294         &adc    ("edi",0);
295
296         &imul   ("edx",&DWP($r0,"esp"));        # h4*r0
297         &add    ("edx","edi");
298
299         &mov    ("ebx",&DWP($d1,"esp"));
300         &mov    ("ecx",&DWP($d2,"esp"));
301
302         &mov    ("edi","edx");                  # last reduction step
303         &shr    ("edx",2);
304         &and    ("edi",3);
305         &lea    ("edx",&DWP(0,"edx","edx",4));  # *5
306         &add    ("eax","edx");
307         &adc    ("ebx",0);
308         &adc    ("ecx",0);
309         &adc    ("esi",0);
310         &adc    ("edi",0);
311
312         &cmp    ("ebp",&wparam(2));             # done yet?
313         &jne    (&label("loop"));
314
315         &mov    ("edx",&wparam(0));             # ctx
316         &stack_pop(16);
317         &mov    (&DWP(4*0,"edx"),"eax");        # store hash value
318         &mov    (&DWP(4*1,"edx"),"ebx");
319         &mov    (&DWP(4*2,"edx"),"ecx");
320         &mov    (&DWP(4*3,"edx"),"esi");
321         &mov    (&DWP(4*4,"edx"),"edi");
322 &set_label("nodata");
323 &function_end("poly1305_blocks");
324
325 &function_begin("poly1305_emit");
326         &mov    ("ebp",&wparam(0));             # context
327 &set_label("enter_emit");
328         &mov    ("edi",&wparam(1));             # output
329         &mov    ("eax",&DWP(4*0,"ebp"));        # load hash value
330         &mov    ("ebx",&DWP(4*1,"ebp"));
331         &mov    ("ecx",&DWP(4*2,"ebp"));
332         &mov    ("edx",&DWP(4*3,"ebp"));
333         &mov    ("esi",&DWP(4*4,"ebp"));
334
335         &add    ("eax",5);                      # compare to modulus
336         &adc    ("ebx",0);
337         &adc    ("ecx",0);
338         &adc    ("edx",0);
339         &adc    ("esi",0);
340         &shr    ("esi",2);                      # did it carry/borrow?
341         &neg    ("esi");                        # do we choose hash-modulus?
342
343         &and    ("eax","esi");
344         &and    ("ebx","esi");
345         &and    ("ecx","esi");
346         &and    ("edx","esi");
347         &mov    (&DWP(4*0,"edi"),"eax");
348         &mov    (&DWP(4*1,"edi"),"ebx");
349         &mov    (&DWP(4*2,"edi"),"ecx");
350         &mov    (&DWP(4*3,"edi"),"edx");
351
352         &not    ("esi");                        # or original hash value?
353         &mov    ("eax",&DWP(4*0,"ebp"));
354         &mov    ("ebx",&DWP(4*1,"ebp"));
355         &mov    ("ecx",&DWP(4*2,"ebp"));
356         &mov    ("edx",&DWP(4*3,"ebp"));
357         &mov    ("ebp",&wparam(2));
358         &and    ("eax","esi");
359         &and    ("ebx","esi");
360         &and    ("ecx","esi");
361         &and    ("edx","esi");
362         &or     ("eax",&DWP(4*0,"edi"));
363         &or     ("ebx",&DWP(4*1,"edi"));
364         &or     ("ecx",&DWP(4*2,"edi"));
365         &or     ("edx",&DWP(4*3,"edi"));
366
367         &add    ("eax",&DWP(4*0,"ebp"));        # accumulate key
368         &adc    ("ebx",&DWP(4*1,"ebp"));
369         &adc    ("ecx",&DWP(4*2,"ebp"));
370         &adc    ("edx",&DWP(4*3,"ebp"));
371
372         &mov    (&DWP(4*0,"edi"),"eax");
373         &mov    (&DWP(4*1,"edi"),"ebx");
374         &mov    (&DWP(4*2,"edi"),"ecx");
375         &mov    (&DWP(4*3,"edi"),"edx");
376 &function_end("poly1305_emit");
377
378 if ($sse2) {
379 ########################################################################
380 # Layout of opaque area is following.
381 #
382 #       unsigned __int32 h[5];          # current hash value base 2^26
383 #       unsigned __int32 is_base2_26;
384 #       unsigned __int32 r[4];          # key value base 2^32
385 #       unsigned __int32 pad[2];
386 #       struct { unsigned __int32 r^4, r^3, r^2, r^1; } r[9];
387 #
388 # where r^n are base 2^26 digits of degrees of multiplier key. There are
389 # 5 digits, but last four are interleaved with multiples of 5, totalling
390 # in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.
391
392 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("xmm$_",(0..7));
393 my $MASK=$T2;   # borrow and keep in mind
394
395 &align  (32);
396 &function_begin_B("_poly1305_init_sse2");
397         &movdqu         ($D4,&QWP(4*6,"edi"));          # key base 2^32
398         &lea            ("edi",&DWP(16*3,"edi"));       # size optimization
399         &mov            ("ebp","esp");
400         &sub            ("esp",16*(9+5));
401         &and            ("esp",-16);
402
403         #&pand          ($D4,&QWP(96,"ebx"));           # magic mask
404         &movq           ($MASK,&QWP(64,"ebx"));
405
406         &movdqa         ($D0,$D4);
407         &movdqa         ($D1,$D4);
408         &movdqa         ($D2,$D4);
409
410         &pand           ($D0,$MASK);                    # -> base 2^26
411         &psrlq          ($D1,26);
412         &psrldq         ($D2,6);
413         &pand           ($D1,$MASK);
414         &movdqa         ($D3,$D2);
415         &psrlq          ($D2,4)
416         &psrlq          ($D3,30);
417         &pand           ($D2,$MASK);
418         &pand           ($D3,$MASK);
419         &psrldq         ($D4,13);
420
421         &lea            ("edx",&DWP(16*9,"esp"));       # size optimization
422         &mov            ("ecx",2);
423 &set_label("square");
424         &movdqa         (&QWP(16*0,"esp"),$D0);
425         &movdqa         (&QWP(16*1,"esp"),$D1);
426         &movdqa         (&QWP(16*2,"esp"),$D2);
427         &movdqa         (&QWP(16*3,"esp"),$D3);
428         &movdqa         (&QWP(16*4,"esp"),$D4);
429
430         &movdqa         ($T1,$D1);
431         &movdqa         ($T0,$D2);
432         &pslld          ($T1,2);
433         &pslld          ($T0,2);
434         &paddd          ($T1,$D1);                      # *5
435         &paddd          ($T0,$D2);                      # *5
436         &movdqa         (&QWP(16*5,"esp"),$T1);
437         &movdqa         (&QWP(16*6,"esp"),$T0);
438         &movdqa         ($T1,$D3);
439         &movdqa         ($T0,$D4);
440         &pslld          ($T1,2);
441         &pslld          ($T0,2);
442         &paddd          ($T1,$D3);                      # *5
443         &paddd          ($T0,$D4);                      # *5
444         &movdqa         (&QWP(16*7,"esp"),$T1);
445         &movdqa         (&QWP(16*8,"esp"),$T0);
446
447         &pshufd         ($T1,$D0,0b01000100);
448         &movdqa         ($T0,$D1);
449         &pshufd         ($D1,$D1,0b01000100);
450         &pshufd         ($D2,$D2,0b01000100);
451         &pshufd         ($D3,$D3,0b01000100);
452         &pshufd         ($D4,$D4,0b01000100);
453         &movdqa         (&QWP(16*0,"edx"),$T1);
454         &movdqa         (&QWP(16*1,"edx"),$D1);
455         &movdqa         (&QWP(16*2,"edx"),$D2);
456         &movdqa         (&QWP(16*3,"edx"),$D3);
457         &movdqa         (&QWP(16*4,"edx"),$D4);
458
459         ################################################################
460         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
461         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
462         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
463         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
464         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
465
466         &pmuludq        ($D4,$D0);                      # h4*r0
467         &pmuludq        ($D3,$D0);                      # h3*r0
468         &pmuludq        ($D2,$D0);                      # h2*r0
469         &pmuludq        ($D1,$D0);                      # h1*r0
470         &pmuludq        ($D0,$T1);                      # h0*r0
471
472 sub pmuladd {
473 my $load = shift;
474 my $base = shift; $base = "esp" if (!defined($base));
475
476         ################################################################
477         # As for choice to "rotate" $T0-$T2 in order to move paddq
478         # past next multiplication. While it makes code harder to read
479         # and doesn't have significant effect on most processors, it
480         # makes a lot of difference on Atom, up to 30% improvement.
481
482         &movdqa         ($T1,$T0);
483         &pmuludq        ($T0,&QWP(16*3,$base));         # r1*h3
484         &movdqa         ($T2,$T1);
485         &pmuludq        ($T1,&QWP(16*2,$base));         # r1*h2
486         &paddq          ($D4,$T0);
487         &movdqa         ($T0,$T2);
488         &pmuludq        ($T2,&QWP(16*1,$base));         # r1*h1
489         &paddq          ($D3,$T1);
490         &$load          ($T1,5);                        # s1
491         &pmuludq        ($T0,&QWP(16*0,$base));         # r1*h0
492         &paddq          ($D2,$T2);
493         &pmuludq        ($T1,&QWP(16*4,$base));         # s1*h4
494          &$load         ($T2,2);                        # r2^n
495         &paddq          ($D1,$T0);
496
497         &movdqa         ($T0,$T2);
498         &pmuludq        ($T2,&QWP(16*2,$base));         # r2*h2
499          &paddq         ($D0,$T1);
500         &movdqa         ($T1,$T0);
501         &pmuludq        ($T0,&QWP(16*1,$base));         # r2*h1
502         &paddq          ($D4,$T2);
503         &$load          ($T2,6);                        # s2^n
504         &pmuludq        ($T1,&QWP(16*0,$base));         # r2*h0
505         &paddq          ($D3,$T0);
506         &movdqa         ($T0,$T2);
507         &pmuludq        ($T2,&QWP(16*4,$base));         # s2*h4
508         &paddq          ($D2,$T1);
509         &pmuludq        ($T0,&QWP(16*3,$base));         # s2*h3
510          &$load         ($T1,3);                        # r3^n
511         &paddq          ($D1,$T2);
512
513         &movdqa         ($T2,$T1);
514         &pmuludq        ($T1,&QWP(16*1,$base));         # r3*h1
515          &paddq         ($D0,$T0);
516         &$load          ($T0,7);                        # s3^n
517         &pmuludq        ($T2,&QWP(16*0,$base));         # r3*h0
518         &paddq          ($D4,$T1);
519         &movdqa         ($T1,$T0);
520         &pmuludq        ($T0,&QWP(16*4,$base));         # s3*h4
521         &paddq          ($D3,$T2);
522         &movdqa         ($T2,$T1);
523         &pmuludq        ($T1,&QWP(16*3,$base));         # s3*h3
524         &paddq          ($D2,$T0);
525         &pmuludq        ($T2,&QWP(16*2,$base));         # s3*h2
526          &$load         ($T0,4);                        # r4^n
527         &paddq          ($D1,$T1);
528
529         &$load          ($T1,8);                        # s4^n
530         &pmuludq        ($T0,&QWP(16*0,$base));         # r4*h0
531          &paddq         ($D0,$T2);
532         &movdqa         ($T2,$T1);
533         &pmuludq        ($T1,&QWP(16*4,$base));         # s4*h4
534         &paddq          ($D4,$T0);
535         &movdqa         ($T0,$T2);
536         &pmuludq        ($T2,&QWP(16*1,$base));         # s4*h1
537         &paddq          ($D3,$T1);
538         &movdqa         ($T1,$T0);
539         &pmuludq        ($T0,&QWP(16*2,$base));         # s4*h2
540         &paddq          ($D0,$T2);
541         &pmuludq        ($T1,&QWP(16*3,$base));         # s4*h3
542          &movdqa        ($MASK,&QWP(64,"ebx"));
543         &paddq          ($D1,$T0);
544         &paddq          ($D2,$T1);
545 }
546         &pmuladd        (sub {  my ($reg,$i)=@_;
547                                 &movdqa ($reg,&QWP(16*$i,"esp"));
548                              },"edx");
549
550 sub lazy_reduction {
551 my $extra = shift;
552
553         ################################################################
554         # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
555         # and P. Schwabe
556         #
557         # [(*) see discussion in poly1305-armv4 module]
558
559          &movdqa        ($T0,$D3);
560          &pand          ($D3,$MASK);
561          &psrlq         ($T0,26);
562          &$extra        ()                              if (defined($extra));
563          &paddq         ($T0,$D4);                      # h3 -> h4
564         &movdqa         ($T1,$D0);
565         &pand           ($D0,$MASK);
566         &psrlq          ($T1,26);
567          &movdqa        ($D4,$T0);
568         &paddq          ($T1,$D1);                      # h0 -> h1
569          &psrlq         ($T0,26);
570          &pand          ($D4,$MASK);
571         &movdqa         ($D1,$T1);
572         &psrlq          ($T1,26);
573          &paddd         ($D0,$T0);                      # favour paddd when
574                                                         # possible, because
575                                                         # paddq is "broken"
576                                                         # on Atom
577          &psllq         ($T0,2);
578         &paddq          ($T1,$D2);                      # h1 -> h2
579          &paddq         ($T0,$D0);                      # h4 -> h0 (*)
580         &pand           ($D1,$MASK);
581         &movdqa         ($D2,$T1);
582         &psrlq          ($T1,26);
583         &pand           ($D2,$MASK);
584         &paddd          ($T1,$D3);                      # h2 -> h3
585          &movdqa        ($D0,$T0);
586          &psrlq         ($T0,26);
587         &movdqa         ($D3,$T1);
588         &psrlq          ($T1,26);
589          &pand          ($D0,$MASK);
590          &paddd         ($D1,$T0);                      # h0 -> h1
591         &pand           ($D3,$MASK);
592         &paddd          ($D4,$T1);                      # h3 -> h4
593 }
594         &lazy_reduction ();
595
596         &dec            ("ecx");
597         &jz             (&label("square_break"));
598
599         &punpcklqdq     ($D0,&QWP(16*0,"esp"));         # 0:r^1:0:r^2
600         &punpcklqdq     ($D1,&QWP(16*1,"esp"));
601         &punpcklqdq     ($D2,&QWP(16*2,"esp"));
602         &punpcklqdq     ($D3,&QWP(16*3,"esp"));
603         &punpcklqdq     ($D4,&QWP(16*4,"esp"));
604         &jmp            (&label("square"));
605
606 &set_label("square_break");
607         &psllq          ($D0,32);                       # -> r^3:0:r^4:0
608         &psllq          ($D1,32);
609         &psllq          ($D2,32);
610         &psllq          ($D3,32);
611         &psllq          ($D4,32);
612         &por            ($D0,&QWP(16*0,"esp"));         # r^3:r^1:r^4:r^2
613         &por            ($D1,&QWP(16*1,"esp"));
614         &por            ($D2,&QWP(16*2,"esp"));
615         &por            ($D3,&QWP(16*3,"esp"));
616         &por            ($D4,&QWP(16*4,"esp"));
617
618         &pshufd         ($D0,$D0,0b10001101);           # -> r^1:r^2:r^3:r^4
619         &pshufd         ($D1,$D1,0b10001101);
620         &pshufd         ($D2,$D2,0b10001101);
621         &pshufd         ($D3,$D3,0b10001101);
622         &pshufd         ($D4,$D4,0b10001101);
623
624         &movdqu         (&QWP(16*0,"edi"),$D0);         # save the table
625         &movdqu         (&QWP(16*1,"edi"),$D1);
626         &movdqu         (&QWP(16*2,"edi"),$D2);
627         &movdqu         (&QWP(16*3,"edi"),$D3);
628         &movdqu         (&QWP(16*4,"edi"),$D4);
629
630         &movdqa         ($T1,$D1);
631         &movdqa         ($T0,$D2);
632         &pslld          ($T1,2);
633         &pslld          ($T0,2);
634         &paddd          ($T1,$D1);                      # *5
635         &paddd          ($T0,$D2);                      # *5
636         &movdqu         (&QWP(16*5,"edi"),$T1);
637         &movdqu         (&QWP(16*6,"edi"),$T0);
638         &movdqa         ($T1,$D3);
639         &movdqa         ($T0,$D4);
640         &pslld          ($T1,2);
641         &pslld          ($T0,2);
642         &paddd          ($T1,$D3);                      # *5
643         &paddd          ($T0,$D4);                      # *5
644         &movdqu         (&QWP(16*7,"edi"),$T1);
645         &movdqu         (&QWP(16*8,"edi"),$T0);
646
647         &mov            ("esp","ebp");
648         &lea            ("edi",&DWP(-16*3,"edi"));      # size de-optimization
649         &ret            ();
650 &function_end_B("_poly1305_init_sse2");
651
652 &align  (32);
653 &function_begin("_poly1305_blocks_sse2");
654         &mov    ("edi",&wparam(0));                     # ctx
655         &mov    ("esi",&wparam(1));                     # inp
656         &mov    ("ecx",&wparam(2));                     # len
657
658         &mov    ("eax",&DWP(4*5,"edi"));                # is_base2_26
659         &and    ("ecx",-16);
660         &jz     (&label("nodata"));
661         &cmp    ("ecx",64);
662         &jae    (&label("enter_sse2"));
663         &test   ("eax","eax");                          # is_base2_26?
664         &jz     (&label("enter_blocks"));
665
666 &set_label("enter_sse2",16);
667         &call   (&label("pic_point"));
668 &set_label("pic_point");
669         &blindpop("ebx");
670         &lea    ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
671
672         &test   ("eax","eax");                          # is_base2_26?
673         &jnz    (&label("base2_26"));
674
675         &call   ("_poly1305_init_sse2");
676
677         ################################################# base 2^32 -> base 2^26
678         &mov    ("eax",&DWP(0,"edi"));
679         &mov    ("ecx",&DWP(3,"edi"));
680         &mov    ("edx",&DWP(6,"edi"));
681         &mov    ("esi",&DWP(9,"edi"));
682         &mov    ("ebp",&DWP(13,"edi"));
683         &mov    (&DWP(4*5,"edi"),1);                    # is_base2_26
684
685         &shr    ("ecx",2);
686         &and    ("eax",0x3ffffff);
687         &shr    ("edx",4);
688         &and    ("ecx",0x3ffffff);
689         &shr    ("esi",6);
690         &and    ("edx",0x3ffffff);
691
692         &movd   ($D0,"eax");
693         &movd   ($D1,"ecx");
694         &movd   ($D2,"edx");
695         &movd   ($D3,"esi");
696         &movd   ($D4,"ebp");
697
698         &mov    ("esi",&wparam(1));                     # [reload] inp
699         &mov    ("ecx",&wparam(2));                     # [reload] len
700         &jmp    (&label("base2_32"));
701
702 &set_label("base2_26",16);
703         &movd   ($D0,&DWP(4*0,"edi"));                  # load hash value
704         &movd   ($D1,&DWP(4*1,"edi"));
705         &movd   ($D2,&DWP(4*2,"edi"));
706         &movd   ($D3,&DWP(4*3,"edi"));
707         &movd   ($D4,&DWP(4*4,"edi"));
708         &movdqa ($MASK,&QWP(64,"ebx"));
709
710 &set_label("base2_32");
711         &mov    ("eax",&wparam(3));                     # padbit
712         &mov    ("ebp","esp");
713
714         &sub    ("esp",16*(5+5+5+9+9));
715         &and    ("esp",-16);
716
717         &lea    ("edi",&DWP(16*3,"edi"));               # size optimization
718         &shl    ("eax",24);                             # padbit
719
720         &test   ("ecx",31);
721         &jz     (&label("even"));
722
723         ################################################################
724         # process single block, with SSE2, because it's still faster
725         # even though half of result is discarded
726
727         &movdqu         ($T1,&QWP(0,"esi"));            # input
728         &lea            ("esi",&DWP(16,"esi"));
729
730         &movdqa         ($T0,$T1);                      # -> base 2^26 ...
731         &pand           ($T1,$MASK);
732         &paddd          ($D0,$T1);                      # ... and accumulate
733
734         &movdqa         ($T1,$T0);
735         &psrlq          ($T0,26);
736         &psrldq         ($T1,6);
737         &pand           ($T0,$MASK);
738         &paddd          ($D1,$T0);
739
740         &movdqa         ($T0,$T1);
741         &psrlq          ($T1,4);
742         &pand           ($T1,$MASK);
743         &paddd          ($D2,$T1);
744
745         &movdqa         ($T1,$T0);
746         &psrlq          ($T0,30);
747         &pand           ($T0,$MASK);
748         &psrldq         ($T1,7);
749         &paddd          ($D3,$T0);
750
751         &movd           ($T0,"eax");                    # padbit
752         &paddd          ($D4,$T1);
753          &movd          ($T1,&DWP(16*0+12,"edi"));      # r0
754         &paddd          ($D4,$T0);
755
756         &movdqa         (&QWP(16*0,"esp"),$D0);
757         &movdqa         (&QWP(16*1,"esp"),$D1);
758         &movdqa         (&QWP(16*2,"esp"),$D2);
759         &movdqa         (&QWP(16*3,"esp"),$D3);
760         &movdqa         (&QWP(16*4,"esp"),$D4);
761
762         ################################################################
763         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
764         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
765         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
766         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
767         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
768
769         &pmuludq        ($D0,$T1);                      # h4*r0
770         &pmuludq        ($D1,$T1);                      # h3*r0
771         &pmuludq        ($D2,$T1);                      # h2*r0
772          &movd          ($T0,&DWP(16*1+12,"edi"));      # r1
773         &pmuludq        ($D3,$T1);                      # h1*r0
774         &pmuludq        ($D4,$T1);                      # h0*r0
775
776         &pmuladd        (sub {  my ($reg,$i)=@_;
777                                 &movd ($reg,&DWP(16*$i+12,"edi"));
778                              });
779
780         &lazy_reduction ();
781
782         &sub            ("ecx",16);
783         &jz             (&label("done"));
784
785 &set_label("even");
786         &lea            ("edx",&DWP(16*(5+5+5+9),"esp"));# size optimization
787         &lea            ("eax",&DWP(-16*2,"esi"));
788         &sub            ("ecx",64);
789
790         ################################################################
791         # expand and copy pre-calculated table to stack
792
793         &movdqu         ($T0,&QWP(16*0,"edi"));         # r^1:r^2:r^3:r^4
794         &pshufd         ($T1,$T0,0b01000100);           # duplicate r^3:r^4
795         &cmovb          ("esi","eax");
796         &pshufd         ($T0,$T0,0b11101110);           # duplicate r^1:r^2
797         &movdqa         (&QWP(16*0,"edx"),$T1);
798         &lea            ("eax",&DWP(16*10,"esp"));
799         &movdqu         ($T1,&QWP(16*1,"edi"));
800         &movdqa         (&QWP(16*(0-9),"edx"),$T0);
801         &pshufd         ($T0,$T1,0b01000100);
802         &pshufd         ($T1,$T1,0b11101110);
803         &movdqa         (&QWP(16*1,"edx"),$T0);
804         &movdqu         ($T0,&QWP(16*2,"edi"));
805         &movdqa         (&QWP(16*(1-9),"edx"),$T1);
806         &pshufd         ($T1,$T0,0b01000100);
807         &pshufd         ($T0,$T0,0b11101110);
808         &movdqa         (&QWP(16*2,"edx"),$T1);
809         &movdqu         ($T1,&QWP(16*3,"edi"));
810         &movdqa         (&QWP(16*(2-9),"edx"),$T0);
811         &pshufd         ($T0,$T1,0b01000100);
812         &pshufd         ($T1,$T1,0b11101110);
813         &movdqa         (&QWP(16*3,"edx"),$T0);
814         &movdqu         ($T0,&QWP(16*4,"edi"));
815         &movdqa         (&QWP(16*(3-9),"edx"),$T1);
816         &pshufd         ($T1,$T0,0b01000100);
817         &pshufd         ($T0,$T0,0b11101110);
818         &movdqa         (&QWP(16*4,"edx"),$T1);
819         &movdqu         ($T1,&QWP(16*5,"edi"));
820         &movdqa         (&QWP(16*(4-9),"edx"),$T0);
821         &pshufd         ($T0,$T1,0b01000100);
822         &pshufd         ($T1,$T1,0b11101110);
823         &movdqa         (&QWP(16*5,"edx"),$T0);
824         &movdqu         ($T0,&QWP(16*6,"edi"));
825         &movdqa         (&QWP(16*(5-9),"edx"),$T1);
826         &pshufd         ($T1,$T0,0b01000100);
827         &pshufd         ($T0,$T0,0b11101110);
828         &movdqa         (&QWP(16*6,"edx"),$T1);
829         &movdqu         ($T1,&QWP(16*7,"edi"));
830         &movdqa         (&QWP(16*(6-9),"edx"),$T0);
831         &pshufd         ($T0,$T1,0b01000100);
832         &pshufd         ($T1,$T1,0b11101110);
833         &movdqa         (&QWP(16*7,"edx"),$T0);
834         &movdqu         ($T0,&QWP(16*8,"edi"));
835         &movdqa         (&QWP(16*(7-9),"edx"),$T1);
836         &pshufd         ($T1,$T0,0b01000100);
837         &pshufd         ($T0,$T0,0b11101110);
838         &movdqa         (&QWP(16*8,"edx"),$T1);
839         &movdqa         (&QWP(16*(8-9),"edx"),$T0);
840
841 sub load_input {
842 my ($inpbase,$offbase)=@_;
843
844         &movdqu         ($T0,&QWP($inpbase+0,"esi"));   # load input
845         &movdqu         ($T1,&QWP($inpbase+16,"esi"));
846         &lea            ("esi",&DWP(16*2,"esi"));
847
848         &movdqa         (&QWP($offbase+16*2,"esp"),$D2);
849         &movdqa         (&QWP($offbase+16*3,"esp"),$D3);
850         &movdqa         (&QWP($offbase+16*4,"esp"),$D4);
851
852         &movdqa         ($D2,$T0);                      # splat input
853         &movdqa         ($D3,$T1);
854         &psrldq         ($D2,6);
855         &psrldq         ($D3,6);
856         &movdqa         ($D4,$T0);
857         &punpcklqdq     ($D2,$D3);                      # 2:3
858         &punpckhqdq     ($D4,$T1);                      # 4
859         &punpcklqdq     ($T0,$T1);                      # 0:1
860
861         &movdqa         ($D3,$D2);
862         &psrlq          ($D2,4);
863         &psrlq          ($D3,30);
864         &movdqa         ($T1,$T0);
865         &psrlq          ($D4,40);                       # 4
866         &psrlq          ($T1,26);
867         &pand           ($T0,$MASK);                    # 0
868         &pand           ($T1,$MASK);                    # 1
869         &pand           ($D2,$MASK);                    # 2
870         &pand           ($D3,$MASK);                    # 3
871         &por            ($D4,&QWP(0,"ebx"));            # padbit, yes, always
872
873         &movdqa         (&QWP($offbase+16*0,"esp"),$D0) if ($offbase);
874         &movdqa         (&QWP($offbase+16*1,"esp"),$D1) if ($offbase);
875 }
876         &load_input     (16*2,16*5);
877
878         &jbe            (&label("skip_loop"));
879         &jmp            (&label("loop"));
880
881 &set_label("loop",32);
882         ################################################################
883         # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
884         # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
885         #   \___________________/
886         # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
887         # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
888         #   \___________________/ \____________________/
889         ################################################################
890
891         &movdqa         ($T2,&QWP(16*(0-9),"edx"));     # r0^2
892         &movdqa         (&QWP(16*1,"eax"),$T1);
893         &movdqa         (&QWP(16*2,"eax"),$D2);
894         &movdqa         (&QWP(16*3,"eax"),$D3);
895         &movdqa         (&QWP(16*4,"eax"),$D4);
896
897         ################################################################
898         # d4 = h4*r0 + h0*r4   + h1*r3   + h2*r2   + h3*r1
899         # d3 = h3*r0 + h0*r3   + h1*r2   + h2*r1   + h4*5*r4
900         # d2 = h2*r0 + h0*r2   + h1*r1   + h3*5*r4 + h4*5*r3
901         # d1 = h1*r0 + h0*r1   + h2*5*r4 + h3*5*r3 + h4*5*r2
902         # d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
903
904         &movdqa         ($D1,$T0);
905         &pmuludq        ($T0,$T2);                      # h0*r0
906         &movdqa         ($D0,$T1);
907         &pmuludq        ($T1,$T2);                      # h1*r0
908         &pmuludq        ($D2,$T2);                      # h2*r0
909         &pmuludq        ($D3,$T2);                      # h3*r0
910         &pmuludq        ($D4,$T2);                      # h4*r0
911
912 sub pmuladd_alt {
913 my $addr = shift;
914
915         &pmuludq        ($D0,&$addr(8));                # h1*s4
916         &movdqa         ($T2,$D1);
917         &pmuludq        ($D1,&$addr(1));                # h0*r1
918         &paddq          ($D0,$T0);
919         &movdqa         ($T0,$T2);
920         &pmuludq        ($T2,&$addr(2));                # h0*r2
921         &paddq          ($D1,$T1);
922         &movdqa         ($T1,$T0);
923         &pmuludq        ($T0,&$addr(3));                # h0*r3
924         &paddq          ($D2,$T2);
925          &movdqa        ($T2,&QWP(16*1,"eax"));         # pull h1
926         &pmuludq        ($T1,&$addr(4));                # h0*r4
927         &paddq          ($D3,$T0);
928
929         &movdqa         ($T0,$T2);
930         &pmuludq        ($T2,&$addr(1));                # h1*r1
931          &paddq         ($D4,$T1);
932         &movdqa         ($T1,$T0);
933         &pmuludq        ($T0,&$addr(2));                # h1*r2
934         &paddq          ($D2,$T2);
935         &movdqa         ($T2,&QWP(16*2,"eax"));         # pull h2
936         &pmuludq        ($T1,&$addr(3));                # h1*r3
937         &paddq          ($D3,$T0);
938         &movdqa         ($T0,$T2);
939         &pmuludq        ($T2,&$addr(7));                # h2*s3
940         &paddq          ($D4,$T1);
941         &movdqa         ($T1,$T0);
942         &pmuludq        ($T0,&$addr(8));                # h2*s4
943         &paddq          ($D0,$T2);
944
945         &movdqa         ($T2,$T1);
946         &pmuludq        ($T1,&$addr(1));                # h2*r1
947          &paddq         ($D1,$T0);
948         &movdqa         ($T0,&QWP(16*3,"eax"));         # pull h3
949         &pmuludq        ($T2,&$addr(2));                # h2*r2
950         &paddq          ($D3,$T1);
951         &movdqa         ($T1,$T0);
952         &pmuludq        ($T0,&$addr(6));                # h3*s2
953         &paddq          ($D4,$T2);
954         &movdqa         ($T2,$T1);
955         &pmuludq        ($T1,&$addr(7));                # h3*s3
956         &paddq          ($D0,$T0);
957         &movdqa         ($T0,$T2);
958         &pmuludq        ($T2,&$addr(8));                # h3*s4
959         &paddq          ($D1,$T1);
960
961         &movdqa         ($T1,&QWP(16*4,"eax"));         # pull h4
962         &pmuludq        ($T0,&$addr(1));                # h3*r1
963          &paddq         ($D2,$T2);
964         &movdqa         ($T2,$T1);
965         &pmuludq        ($T1,&$addr(8));                # h4*s4
966         &paddq          ($D4,$T0);
967         &movdqa         ($T0,$T2);
968         &pmuludq        ($T2,&$addr(5));                # h4*s1
969         &paddq          ($D3,$T1);
970         &movdqa         ($T1,$T0);
971         &pmuludq        ($T0,&$addr(6));                # h4*s2
972         &paddq          ($D0,$T2);
973          &movdqa        ($MASK,&QWP(64,"ebx"));
974         &pmuludq        ($T1,&$addr(7));                # h4*s3
975         &paddq          ($D1,$T0);
976         &paddq          ($D2,$T1);
977 }
978         &pmuladd_alt    (sub {  my $i=shift; &QWP(16*($i-9),"edx");     });
979
980         &load_input     (-16*2,0);
981         &lea            ("eax",&DWP(-16*2,"esi"));
982         &sub            ("ecx",64);
983
984         &paddd          ($T0,&QWP(16*(5+0),"esp"));     # add hash value
985         &paddd          ($T1,&QWP(16*(5+1),"esp"));
986         &paddd          ($D2,&QWP(16*(5+2),"esp"));
987         &paddd          ($D3,&QWP(16*(5+3),"esp"));
988         &paddd          ($D4,&QWP(16*(5+4),"esp"));
989
990         &cmovb          ("esi","eax");
991         &lea            ("eax",&DWP(16*10,"esp"));
992
993         &movdqa         ($T2,&QWP(16*0,"edx"));         # r0^4
994         &movdqa         (&QWP(16*1,"esp"),$D1);
995         &movdqa         (&QWP(16*1,"eax"),$T1);
996         &movdqa         (&QWP(16*2,"eax"),$D2);
997         &movdqa         (&QWP(16*3,"eax"),$D3);
998         &movdqa         (&QWP(16*4,"eax"),$D4);
999
1000         ################################################################
1001         # d4 += h4*r0 + h0*r4   + h1*r3   + h2*r2   + h3*r1
1002         # d3 += h3*r0 + h0*r3   + h1*r2   + h2*r1   + h4*5*r4
1003         # d2 += h2*r0 + h0*r2   + h1*r1   + h3*5*r4 + h4*5*r3
1004         # d1 += h1*r0 + h0*r1   + h2*5*r4 + h3*5*r3 + h4*5*r2
1005         # d0 += h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
1006
1007         &movdqa         ($D1,$T0);
1008         &pmuludq        ($T0,$T2);                      # h0*r0
1009         &paddq          ($T0,$D0);
1010         &movdqa         ($D0,$T1);
1011         &pmuludq        ($T1,$T2);                      # h1*r0
1012         &pmuludq        ($D2,$T2);                      # h2*r0
1013         &pmuludq        ($D3,$T2);                      # h3*r0
1014         &pmuludq        ($D4,$T2);                      # h4*r0
1015
1016         &paddq          ($T1,&QWP(16*1,"esp"));
1017         &paddq          ($D2,&QWP(16*2,"esp"));
1018         &paddq          ($D3,&QWP(16*3,"esp"));
1019         &paddq          ($D4,&QWP(16*4,"esp"));
1020
1021         &pmuladd_alt    (sub {  my $i=shift; &QWP(16*$i,"edx"); });
1022
1023         &lazy_reduction ();
1024
1025         &load_input     (16*2,16*5);
1026
1027         &ja             (&label("loop"));
1028
1029 &set_label("skip_loop");
1030         ################################################################
1031         # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
1032
1033          &pshufd        ($T2,&QWP(16*(0-9),"edx"),0x10);# r0^n
1034         &add            ("ecx",32);
1035         &jnz            (&label("long_tail"));
1036
1037         &paddd          ($T0,$D0);                      # add hash value
1038         &paddd          ($T1,$D1);
1039         &paddd          ($D2,&QWP(16*7,"esp"));
1040         &paddd          ($D3,&QWP(16*8,"esp"));
1041         &paddd          ($D4,&QWP(16*9,"esp"));
1042
1043 &set_label("long_tail");
1044
1045         &movdqa         (&QWP(16*0,"eax"),$T0);
1046         &movdqa         (&QWP(16*1,"eax"),$T1);
1047         &movdqa         (&QWP(16*2,"eax"),$D2);
1048         &movdqa         (&QWP(16*3,"eax"),$D3);
1049         &movdqa         (&QWP(16*4,"eax"),$D4);
1050
1051         ################################################################
1052         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
1053         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
1054         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
1055         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
1056         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1057
1058         &pmuludq        ($T0,$T2);                      # h0*r0
1059         &pmuludq        ($T1,$T2);                      # h1*r0
1060         &pmuludq        ($D2,$T2);                      # h2*r0
1061         &movdqa         ($D0,$T0);
1062          &pshufd        ($T0,&QWP(16*(1-9),"edx"),0x10);# r1^n
1063         &pmuludq        ($D3,$T2);                      # h3*r0
1064         &movdqa         ($D1,$T1);
1065         &pmuludq        ($D4,$T2);                      # h4*r0
1066
1067         &pmuladd        (sub {  my ($reg,$i)=@_;
1068                                 &pshufd ($reg,&QWP(16*($i-9),"edx"),0x10);
1069                              },"eax");
1070
1071         &jz             (&label("short_tail"));
1072
1073         &load_input     (-16*2,0);
1074
1075          &pshufd        ($T2,&QWP(16*0,"edx"),0x10);    # r0^n
1076         &paddd          ($T0,&QWP(16*5,"esp"));         # add hash value
1077         &paddd          ($T1,&QWP(16*6,"esp"));
1078         &paddd          ($D2,&QWP(16*7,"esp"));
1079         &paddd          ($D3,&QWP(16*8,"esp"));
1080         &paddd          ($D4,&QWP(16*9,"esp"));
1081
1082         ################################################################
1083         # multiply inp[0:1] by r^4:r^3 and accumulate
1084
1085         &movdqa         (&QWP(16*0,"esp"),$T0);
1086         &pmuludq        ($T0,$T2);                      # h0*r0
1087         &movdqa         (&QWP(16*1,"esp"),$T1);
1088         &pmuludq        ($T1,$T2);                      # h1*r0
1089         &paddq          ($D0,$T0);
1090         &movdqa         ($T0,$D2);
1091         &pmuludq        ($D2,$T2);                      # h2*r0
1092         &paddq          ($D1,$T1);
1093         &movdqa         ($T1,$D3);
1094         &pmuludq        ($D3,$T2);                      # h3*r0
1095         &paddq          ($D2,&QWP(16*2,"esp"));
1096         &movdqa         (&QWP(16*2,"esp"),$T0);
1097          &pshufd        ($T0,&QWP(16*1,"edx"),0x10);    # r1^n
1098         &paddq          ($D3,&QWP(16*3,"esp"));
1099         &movdqa         (&QWP(16*3,"esp"),$T1);
1100         &movdqa         ($T1,$D4);
1101         &pmuludq        ($D4,$T2);                      # h4*r0
1102         &paddq          ($D4,&QWP(16*4,"esp"));
1103         &movdqa         (&QWP(16*4,"esp"),$T1);
1104
1105         &pmuladd        (sub {  my ($reg,$i)=@_;
1106                                 &pshufd ($reg,&QWP(16*$i,"edx"),0x10);
1107                              });
1108
1109 &set_label("short_tail");
1110
1111         ################################################################
1112         # horizontal addition
1113
1114         &pshufd         ($T1,$D4,0b01001110);
1115         &pshufd         ($T0,$D3,0b01001110);
1116         &paddq          ($D4,$T1);
1117         &paddq          ($D3,$T0);
1118         &pshufd         ($T1,$D0,0b01001110);
1119         &pshufd         ($T0,$D1,0b01001110);
1120         &paddq          ($D0,$T1);
1121         &paddq          ($D1,$T0);
1122         &pshufd         ($T1,$D2,0b01001110);
1123         #&paddq         ($D2,$T1);
1124
1125         &lazy_reduction (sub { &paddq ($D2,$T1) });
1126
1127 &set_label("done");
1128         &movd           (&DWP(-16*3+4*0,"edi"),$D0);    # store hash value
1129         &movd           (&DWP(-16*3+4*1,"edi"),$D1);
1130         &movd           (&DWP(-16*3+4*2,"edi"),$D2);
1131         &movd           (&DWP(-16*3+4*3,"edi"),$D3);
1132         &movd           (&DWP(-16*3+4*4,"edi"),$D4);
1133         &mov    ("esp","ebp");
1134 &set_label("nodata");
1135 &function_end("_poly1305_blocks_sse2");
1136
1137 &align  (32);
1138 &function_begin("_poly1305_emit_sse2");
1139         &mov    ("ebp",&wparam(0));             # context
1140
1141         &cmp    (&DWP(4*5,"ebp"),0);            # is_base2_26?
1142         &je     (&label("enter_emit"));
1143
1144         &mov    ("eax",&DWP(4*0,"ebp"));        # load hash value
1145         &mov    ("edi",&DWP(4*1,"ebp"));
1146         &mov    ("ecx",&DWP(4*2,"ebp"));
1147         &mov    ("edx",&DWP(4*3,"ebp"));
1148         &mov    ("esi",&DWP(4*4,"ebp"));
1149
1150         &mov    ("ebx","edi");                  # base 2^26 -> base 2^32
1151         &shl    ("edi",26);
1152         &shr    ("ebx",6);
1153         &add    ("eax","edi");
1154         &mov    ("edi","ecx");
1155         &adc    ("ebx",0);
1156
1157         &shl    ("edi",20);
1158         &shr    ("ecx",12);
1159         &add    ("ebx","edi");
1160         &mov    ("edi","edx");
1161         &adc    ("ecx",0);
1162
1163         &shl    ("edi",14);
1164         &shr    ("edx",18);
1165         &add    ("ecx","edi");
1166         &mov    ("edi","esi");
1167         &adc    ("edx",0);
1168
1169         &shl    ("edi",8);
1170         &shr    ("esi",24);
1171         &add    ("edx","edi");
1172         &adc    ("esi",0);                      # can be partially reduced
1173
1174         &mov    ("edi","esi");                  # final reduction
1175         &and    ("esi",3);
1176         &shr    ("edi",2);
1177         &lea    ("ebp",&DWP(0,"edi","edi",4));  # *5
1178          &mov   ("edi",&wparam(1));             # output
1179         &add    ("eax","ebp");
1180          &mov   ("ebp",&wparam(2));             # key
1181         &adc    ("ebx",0);
1182         &adc    ("ecx",0);
1183         &adc    ("edx",0);
1184         &adc    ("esi",0);
1185
1186         &movd   ($D0,"eax");                    # offload original hash value
1187         &add    ("eax",5);                      # compare to modulus
1188         &movd   ($D1,"ebx");
1189         &adc    ("ebx",0);
1190         &movd   ($D2,"ecx");
1191         &adc    ("ecx",0);
1192         &movd   ($D3,"edx");
1193         &adc    ("edx",0);
1194         &adc    ("esi",0);
1195         &shr    ("esi",2);                      # did it carry/borrow?
1196
1197         &neg    ("esi");                        # do we choose (hash-modulus) ...
1198         &and    ("eax","esi");
1199         &and    ("ebx","esi");
1200         &and    ("ecx","esi");
1201         &and    ("edx","esi");
1202         &mov    (&DWP(4*0,"edi"),"eax");
1203         &movd   ("eax",$D0);
1204         &mov    (&DWP(4*1,"edi"),"ebx");
1205         &movd   ("ebx",$D1);
1206         &mov    (&DWP(4*2,"edi"),"ecx");
1207         &movd   ("ecx",$D2);
1208         &mov    (&DWP(4*3,"edi"),"edx");
1209         &movd   ("edx",$D3);
1210
1211         &not    ("esi");                        # ... or original hash value?
1212         &and    ("eax","esi");
1213         &and    ("ebx","esi");
1214         &or     ("eax",&DWP(4*0,"edi"));
1215         &and    ("ecx","esi");
1216         &or     ("ebx",&DWP(4*1,"edi"));
1217         &and    ("edx","esi");
1218         &or     ("ecx",&DWP(4*2,"edi"));
1219         &or     ("edx",&DWP(4*3,"edi"));
1220
1221         &add    ("eax",&DWP(4*0,"ebp"));        # accumulate key
1222         &adc    ("ebx",&DWP(4*1,"ebp"));
1223         &mov    (&DWP(4*0,"edi"),"eax");
1224         &adc    ("ecx",&DWP(4*2,"ebp"));
1225         &mov    (&DWP(4*1,"edi"),"ebx");
1226         &adc    ("edx",&DWP(4*3,"ebp"));
1227         &mov    (&DWP(4*2,"edi"),"ecx");
1228         &mov    (&DWP(4*3,"edi"),"edx");
1229 &function_end("_poly1305_emit_sse2");
1230
1231 if ($avx>1) {
1232 ########################################################################
1233 # Note that poly1305_init_avx2 operates on %xmm, I could have used
1234 # poly1305_init_sse2...
1235
1236 &align  (32);
1237 &function_begin_B("_poly1305_init_avx2");
1238         &vmovdqu        ($D4,&QWP(4*6,"edi"));          # key base 2^32
1239         &lea            ("edi",&DWP(16*3,"edi"));       # size optimization
1240         &mov            ("ebp","esp");
1241         &sub            ("esp",16*(9+5));
1242         &and            ("esp",-16);
1243
1244         #&vpand         ($D4,$D4,&QWP(96,"ebx"));       # magic mask
1245         &vmovdqa        ($MASK,&QWP(64,"ebx"));
1246
1247         &vpand          ($D0,$D4,$MASK);                # -> base 2^26
1248         &vpsrlq         ($D1,$D4,26);
1249         &vpsrldq        ($D3,$D4,6);
1250         &vpand          ($D1,$D1,$MASK);
1251         &vpsrlq         ($D2,$D3,4)
1252         &vpsrlq         ($D3,$D3,30);
1253         &vpand          ($D2,$D2,$MASK);
1254         &vpand          ($D3,$D3,$MASK);
1255         &vpsrldq        ($D4,$D4,13);
1256
1257         &lea            ("edx",&DWP(16*9,"esp"));       # size optimization
1258         &mov            ("ecx",2);
1259 &set_label("square");
1260         &vmovdqa        (&QWP(16*0,"esp"),$D0);
1261         &vmovdqa        (&QWP(16*1,"esp"),$D1);
1262         &vmovdqa        (&QWP(16*2,"esp"),$D2);
1263         &vmovdqa        (&QWP(16*3,"esp"),$D3);
1264         &vmovdqa        (&QWP(16*4,"esp"),$D4);
1265
1266         &vpslld         ($T1,$D1,2);
1267         &vpslld         ($T0,$D2,2);
1268         &vpaddd         ($T1,$T1,$D1);                  # *5
1269         &vpaddd         ($T0,$T0,$D2);                  # *5
1270         &vmovdqa        (&QWP(16*5,"esp"),$T1);
1271         &vmovdqa        (&QWP(16*6,"esp"),$T0);
1272         &vpslld         ($T1,$D3,2);
1273         &vpslld         ($T0,$D4,2);
1274         &vpaddd         ($T1,$T1,$D3);                  # *5
1275         &vpaddd         ($T0,$T0,$D4);                  # *5
1276         &vmovdqa        (&QWP(16*7,"esp"),$T1);
1277         &vmovdqa        (&QWP(16*8,"esp"),$T0);
1278
1279         &vpshufd        ($T0,$D0,0b01000100);
1280         &vmovdqa        ($T1,$D1);
1281         &vpshufd        ($D1,$D1,0b01000100);
1282         &vpshufd        ($D2,$D2,0b01000100);
1283         &vpshufd        ($D3,$D3,0b01000100);
1284         &vpshufd        ($D4,$D4,0b01000100);
1285         &vmovdqa        (&QWP(16*0,"edx"),$T0);
1286         &vmovdqa        (&QWP(16*1,"edx"),$D1);
1287         &vmovdqa        (&QWP(16*2,"edx"),$D2);
1288         &vmovdqa        (&QWP(16*3,"edx"),$D3);
1289         &vmovdqa        (&QWP(16*4,"edx"),$D4);
1290
1291         ################################################################
1292         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
1293         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
1294         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
1295         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
1296         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1297
1298         &vpmuludq       ($D4,$D4,$D0);                  # h4*r0
1299         &vpmuludq       ($D3,$D3,$D0);                  # h3*r0
1300         &vpmuludq       ($D2,$D2,$D0);                  # h2*r0
1301         &vpmuludq       ($D1,$D1,$D0);                  # h1*r0
1302         &vpmuludq       ($D0,$T0,$D0);                  # h0*r0
1303
1304         &vpmuludq       ($T0,$T1,&QWP(16*3,"edx"));     # r1*h3
1305         &vpaddq         ($D4,$D4,$T0);
1306         &vpmuludq       ($T2,$T1,&QWP(16*2,"edx"));     # r1*h2
1307         &vpaddq         ($D3,$D3,$T2);
1308         &vpmuludq       ($T0,$T1,&QWP(16*1,"edx"));     # r1*h1
1309         &vpaddq         ($D2,$D2,$T0);
1310         &vmovdqa        ($T2,&QWP(16*5,"esp"));         # s1
1311         &vpmuludq       ($T1,$T1,&QWP(16*0,"edx"));     # r1*h0
1312         &vpaddq         ($D1,$D1,$T1);
1313          &vmovdqa       ($T0,&QWP(16*2,"esp"));         # r2
1314         &vpmuludq       ($T2,$T2,&QWP(16*4,"edx"));     # s1*h4
1315         &vpaddq         ($D0,$D0,$T2);
1316
1317         &vpmuludq       ($T1,$T0,&QWP(16*2,"edx"));     # r2*h2
1318         &vpaddq         ($D4,$D4,$T1);
1319         &vpmuludq       ($T2,$T0,&QWP(16*1,"edx"));     # r2*h1
1320         &vpaddq         ($D3,$D3,$T2);
1321         &vmovdqa        ($T1,&QWP(16*6,"esp"));         # s2
1322         &vpmuludq       ($T0,$T0,&QWP(16*0,"edx"));     # r2*h0
1323         &vpaddq         ($D2,$D2,$T0);
1324         &vpmuludq       ($T2,$T1,&QWP(16*4,"edx"));     # s2*h4
1325         &vpaddq         ($D1,$D1,$T2);
1326          &vmovdqa       ($T0,&QWP(16*3,"esp"));         # r3
1327         &vpmuludq       ($T1,$T1,&QWP(16*3,"edx"));     # s2*h3
1328         &vpaddq         ($D0,$D0,$T1);
1329
1330         &vpmuludq       ($T2,$T0,&QWP(16*1,"edx"));     # r3*h1
1331         &vpaddq         ($D4,$D4,$T2);
1332         &vmovdqa        ($T1,&QWP(16*7,"esp"));         # s3
1333         &vpmuludq       ($T0,$T0,&QWP(16*0,"edx"));     # r3*h0
1334         &vpaddq         ($D3,$D3,$T0);
1335         &vpmuludq       ($T2,$T1,&QWP(16*4,"edx"));     # s3*h4
1336         &vpaddq         ($D2,$D2,$T2);
1337         &vpmuludq       ($T0,$T1,&QWP(16*3,"edx"));     # s3*h3
1338         &vpaddq         ($D1,$D1,$T0);
1339          &vmovdqa       ($T2,&QWP(16*4,"esp"));         # r4
1340         &vpmuludq       ($T1,$T1,&QWP(16*2,"edx"));     # s3*h2
1341         &vpaddq         ($D0,$D0,$T1);
1342
1343         &vmovdqa        ($T0,&QWP(16*8,"esp"));         # s4
1344         &vpmuludq       ($T2,$T2,&QWP(16*0,"edx"));     # r4*h0
1345         &vpaddq         ($D4,$D4,$T2);
1346         &vpmuludq       ($T1,$T0,&QWP(16*4,"edx"));     # s4*h4
1347         &vpaddq         ($D3,$D3,$T1);
1348         &vpmuludq       ($T2,$T0,&QWP(16*1,"edx"));     # s4*h1
1349         &vpaddq         ($D0,$D0,$T2);
1350         &vpmuludq       ($T1,$T0,&QWP(16*2,"edx"));     # s4*h2
1351         &vpaddq         ($D1,$D1,$T1);
1352          &vmovdqa       ($MASK,&QWP(64,"ebx"));
1353         &vpmuludq       ($T0,$T0,&QWP(16*3,"edx"));     # s4*h3
1354         &vpaddq         ($D2,$D2,$T0);
1355
1356         ################################################################
1357         # lazy reduction
1358          &vpsrlq        ($T0,$D3,26);
1359          &vpand         ($D3,$D3,$MASK);
1360         &vpsrlq         ($T1,$D0,26);
1361         &vpand          ($D0,$D0,$MASK);
1362          &vpaddq        ($D4,$D4,$T0);                  # h3 -> h4
1363         &vpaddq         ($D1,$D1,$T1);                  # h0 -> h1
1364          &vpsrlq        ($T0,$D4,26);
1365          &vpand         ($D4,$D4,$MASK);
1366         &vpsrlq         ($T1,$D1,26);
1367         &vpand          ($D1,$D1,$MASK);
1368         &vpaddq         ($D2,$D2,$T1);                  # h1 -> h2
1369          &vpaddd        ($D0,$D0,$T0);
1370          &vpsllq        ($T0,$T0,2);
1371         &vpsrlq         ($T1,$D2,26);
1372         &vpand          ($D2,$D2,$MASK);
1373          &vpaddd        ($D0,$D0,$T0);                  # h4 -> h0
1374         &vpaddd         ($D3,$D3,$T1);                  # h2 -> h3
1375         &vpsrlq         ($T1,$D3,26);
1376          &vpsrlq        ($T0,$D0,26);
1377          &vpand         ($D0,$D0,$MASK);
1378         &vpand          ($D3,$D3,$MASK);
1379          &vpaddd        ($D1,$D1,$T0);                  # h0 -> h1
1380         &vpaddd         ($D4,$D4,$T1);                  # h3 -> h4
1381
1382         &dec            ("ecx");
1383         &jz             (&label("square_break"));
1384
1385         &vpunpcklqdq    ($D0,$D0,&QWP(16*0,"esp"));     # 0:r^1:0:r^2
1386         &vpunpcklqdq    ($D1,$D1,&QWP(16*1,"esp"));
1387         &vpunpcklqdq    ($D2,$D2,&QWP(16*2,"esp"));
1388         &vpunpcklqdq    ($D3,$D3,&QWP(16*3,"esp"));
1389         &vpunpcklqdq    ($D4,$D4,&QWP(16*4,"esp"));
1390         &jmp            (&label("square"));
1391
1392 &set_label("square_break");
1393         &vpsllq         ($D0,$D0,32);                   # -> r^3:0:r^4:0
1394         &vpsllq         ($D1,$D1,32);
1395         &vpsllq         ($D2,$D2,32);
1396         &vpsllq         ($D3,$D3,32);
1397         &vpsllq         ($D4,$D4,32);
1398         &vpor           ($D0,$D0,&QWP(16*0,"esp"));     # r^3:r^1:r^4:r^2
1399         &vpor           ($D1,$D1,&QWP(16*1,"esp"));
1400         &vpor           ($D2,$D2,&QWP(16*2,"esp"));
1401         &vpor           ($D3,$D3,&QWP(16*3,"esp"));
1402         &vpor           ($D4,$D4,&QWP(16*4,"esp"));
1403
1404         &vpshufd        ($D0,$D0,0b10001101);           # -> r^1:r^2:r^3:r^4
1405         &vpshufd        ($D1,$D1,0b10001101);
1406         &vpshufd        ($D2,$D2,0b10001101);
1407         &vpshufd        ($D3,$D3,0b10001101);
1408         &vpshufd        ($D4,$D4,0b10001101);
1409
1410         &vmovdqu        (&QWP(16*0,"edi"),$D0);         # save the table
1411         &vmovdqu        (&QWP(16*1,"edi"),$D1);
1412         &vmovdqu        (&QWP(16*2,"edi"),$D2);
1413         &vmovdqu        (&QWP(16*3,"edi"),$D3);
1414         &vmovdqu        (&QWP(16*4,"edi"),$D4);
1415
1416         &vpslld         ($T1,$D1,2);
1417         &vpslld         ($T0,$D2,2);
1418         &vpaddd         ($T1,$T1,$D1);                  # *5
1419         &vpaddd         ($T0,$T0,$D2);                  # *5
1420         &vmovdqu        (&QWP(16*5,"edi"),$T1);
1421         &vmovdqu        (&QWP(16*6,"edi"),$T0);
1422         &vpslld         ($T1,$D3,2);
1423         &vpslld         ($T0,$D4,2);
1424         &vpaddd         ($T1,$T1,$D3);                  # *5
1425         &vpaddd         ($T0,$T0,$D4);                  # *5
1426         &vmovdqu        (&QWP(16*7,"edi"),$T1);
1427         &vmovdqu        (&QWP(16*8,"edi"),$T0);
1428
1429         &mov            ("esp","ebp");
1430         &lea            ("edi",&DWP(-16*3,"edi"));      # size de-optimization
1431         &ret            ();
1432 &function_end_B("_poly1305_init_avx2");
1433
1434 ########################################################################
1435 # now it's time to switch to %ymm
1436
1437 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("ymm$_",(0..7));
1438 my $MASK=$T2;
1439
1440 sub X { my $reg=shift; $reg=~s/^ymm/xmm/; $reg; }
1441
1442 &align  (32);
1443 &function_begin("_poly1305_blocks_avx2");
1444         &mov    ("edi",&wparam(0));                     # ctx
1445         &mov    ("esi",&wparam(1));                     # inp
1446         &mov    ("ecx",&wparam(2));                     # len
1447
1448         &mov    ("eax",&DWP(4*5,"edi"));                # is_base2_26
1449         &and    ("ecx",-16);
1450         &jz     (&label("nodata"));
1451         &cmp    ("ecx",64);
1452         &jae    (&label("enter_avx2"));
1453         &test   ("eax","eax");                          # is_base2_26?
1454         &jz     (&label("enter_blocks"));
1455
1456 &set_label("enter_avx2");
1457         &vzeroupper     ();
1458
1459         &call   (&label("pic_point"));
1460 &set_label("pic_point");
1461         &blindpop("ebx");
1462         &lea    ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
1463
1464         &test   ("eax","eax");                          # is_base2_26?
1465         &jnz    (&label("base2_26"));
1466
1467         &call   ("_poly1305_init_avx2");
1468
1469         ################################################# base 2^32 -> base 2^26
1470         &mov    ("eax",&DWP(0,"edi"));
1471         &mov    ("ecx",&DWP(3,"edi"));
1472         &mov    ("edx",&DWP(6,"edi"));
1473         &mov    ("esi",&DWP(9,"edi"));
1474         &mov    ("ebp",&DWP(13,"edi"));
1475
1476         &shr    ("ecx",2);
1477         &and    ("eax",0x3ffffff);
1478         &shr    ("edx",4);
1479         &and    ("ecx",0x3ffffff);
1480         &shr    ("esi",6);
1481         &and    ("edx",0x3ffffff);
1482
1483         &mov    (&DWP(4*0,"edi"),"eax");
1484         &mov    (&DWP(4*1,"edi"),"ecx");
1485         &mov    (&DWP(4*2,"edi"),"edx");
1486         &mov    (&DWP(4*3,"edi"),"esi");
1487         &mov    (&DWP(4*4,"edi"),"ebp");
1488         &mov    (&DWP(4*5,"edi"),1);                    # is_base2_26
1489
1490         &mov    ("esi",&wparam(1));                     # [reload] inp
1491         &mov    ("ecx",&wparam(2));                     # [reload] len
1492
1493 &set_label("base2_26");
1494         &mov    ("eax",&wparam(3));                     # padbit
1495         &mov    ("ebp","esp");
1496
1497         &sub    ("esp",32*(5+9));
1498         &and    ("esp",-512);                           # ensure that frame
1499                                                         # doesn't cross page
1500                                                         # boundary, which is
1501                                                         # essential for
1502                                                         # misaligned 32-byte
1503                                                         # loads
1504
1505         ################################################################
1506         # expand and copy pre-calculated table to stack
1507
1508         &vmovdqu        (&X($D0),&QWP(16*(3+0),"edi"));
1509         &lea            ("edx",&DWP(32*5+128,"esp"));   # +128 size optimization
1510         &vmovdqu        (&X($D1),&QWP(16*(3+1),"edi"));
1511         &vmovdqu        (&X($D2),&QWP(16*(3+2),"edi"));
1512         &vmovdqu        (&X($D3),&QWP(16*(3+3),"edi"));
1513         &vmovdqu        (&X($D4),&QWP(16*(3+4),"edi"));
1514         &lea            ("edi",&DWP(16*3,"edi"));       # size optimization
1515         &vpermq         ($D0,$D0,0b01000000);           # 00001234 -> 12343434
1516         &vpermq         ($D1,$D1,0b01000000);
1517         &vpermq         ($D2,$D2,0b01000000);
1518         &vpermq         ($D3,$D3,0b01000000);
1519         &vpermq         ($D4,$D4,0b01000000);
1520         &vpshufd        ($D0,$D0,0b11001000);           # 12343434 -> 14243444
1521         &vpshufd        ($D1,$D1,0b11001000);
1522         &vpshufd        ($D2,$D2,0b11001000);
1523         &vpshufd        ($D3,$D3,0b11001000);
1524         &vpshufd        ($D4,$D4,0b11001000);
1525         &vmovdqa        (&QWP(32*0-128,"edx"),$D0);
1526         &vmovdqu        (&X($D0),&QWP(16*5,"edi"));
1527         &vmovdqa        (&QWP(32*1-128,"edx"),$D1);
1528         &vmovdqu        (&X($D1),&QWP(16*6,"edi"));
1529         &vmovdqa        (&QWP(32*2-128,"edx"),$D2);
1530         &vmovdqu        (&X($D2),&QWP(16*7,"edi"));
1531         &vmovdqa        (&QWP(32*3-128,"edx"),$D3);
1532         &vmovdqu        (&X($D3),&QWP(16*8,"edi"));
1533         &vmovdqa        (&QWP(32*4-128,"edx"),$D4);
1534         &vpermq         ($D0,$D0,0b01000000);
1535         &vpermq         ($D1,$D1,0b01000000);
1536         &vpermq         ($D2,$D2,0b01000000);
1537         &vpermq         ($D3,$D3,0b01000000);
1538         &vpshufd        ($D0,$D0,0b11001000);
1539         &vpshufd        ($D1,$D1,0b11001000);
1540         &vpshufd        ($D2,$D2,0b11001000);
1541         &vpshufd        ($D3,$D3,0b11001000);
1542         &vmovdqa        (&QWP(32*5-128,"edx"),$D0);
1543         &vmovd          (&X($D0),&DWP(-16*3+4*0,"edi"));# load hash value
1544         &vmovdqa        (&QWP(32*6-128,"edx"),$D1);
1545         &vmovd          (&X($D1),&DWP(-16*3+4*1,"edi"));
1546         &vmovdqa        (&QWP(32*7-128,"edx"),$D2);
1547         &vmovd          (&X($D2),&DWP(-16*3+4*2,"edi"));
1548         &vmovdqa        (&QWP(32*8-128,"edx"),$D3);
1549         &vmovd          (&X($D3),&DWP(-16*3+4*3,"edi"));
1550         &vmovd          (&X($D4),&DWP(-16*3+4*4,"edi"));
1551         &vmovdqa        ($MASK,&QWP(64,"ebx"));
1552         &neg            ("eax");                        # padbit
1553
1554         &test           ("ecx",63);
1555         &jz             (&label("even"));
1556
1557         &mov            ("edx","ecx");
1558         &and            ("ecx",-64);
1559         &and            ("edx",63);
1560
1561         &vmovdqu        (&X($T0),&QWP(16*0,"esi"));
1562         &cmp            ("edx",32);
1563         &jb             (&label("one"));
1564
1565         &vmovdqu        (&X($T1),&QWP(16*1,"esi"));
1566         &je             (&label("two"));
1567
1568         &vinserti128    ($T0,$T0,&QWP(16*2,"esi"),1);
1569         &lea            ("esi",&DWP(16*3,"esi"));
1570         &lea            ("ebx",&DWP(8,"ebx"));          # three padbits
1571         &lea            ("edx",&DWP(32*5+128+8,"esp")); # --:r^1:r^2:r^3 (*)
1572         &jmp            (&label("tail"));
1573
1574 &set_label("two");
1575         &lea            ("esi",&DWP(16*2,"esi"));
1576         &lea            ("ebx",&DWP(16,"ebx"));         # two padbits
1577         &lea            ("edx",&DWP(32*5+128+16,"esp"));# --:--:r^1:r^2 (*)
1578         &jmp            (&label("tail"));
1579
1580 &set_label("one");
1581         &lea            ("esi",&DWP(16*1,"esi"));
1582         &vpxor          ($T1,$T1,$T1);
1583         &lea            ("ebx",&DWP(32,"ebx","eax",8)); # one or no padbits
1584         &lea            ("edx",&DWP(32*5+128+24,"esp"));# --:--:--:r^1 (*)
1585         &jmp            (&label("tail"));
1586
1587 # (*)   spots marked with '--' are data from next table entry, but they
1588 #       are multiplied by 0 and therefore rendered insignificant
1589
1590 &set_label("even",32);
1591         &vmovdqu        (&X($T0),&QWP(16*0,"esi"));     # load input
1592         &vmovdqu        (&X($T1),&QWP(16*1,"esi"));
1593         &vinserti128    ($T0,$T0,&QWP(16*2,"esi"),1);
1594         &vinserti128    ($T1,$T1,&QWP(16*3,"esi"),1);
1595         &lea            ("esi",&DWP(16*4,"esi"));
1596         &sub            ("ecx",64);
1597         &jz             (&label("tail"));
1598
1599 &set_label("loop");
1600         ################################################################
1601         # ((inp[0]*r^4+r[4])*r^4+r[8])*r^4
1602         # ((inp[1]*r^4+r[5])*r^4+r[9])*r^3
1603         # ((inp[2]*r^4+r[6])*r^4+r[10])*r^2
1604         # ((inp[3]*r^4+r[7])*r^4+r[11])*r^1
1605         #   \________/ \_______/
1606         ################################################################
1607
1608 sub vsplat_input {
1609         &vmovdqa        (&QWP(32*2,"esp"),$D2);
1610         &vpsrldq        ($D2,$T0,6);                    # splat input
1611         &vmovdqa        (&QWP(32*0,"esp"),$D0);
1612         &vpsrldq        ($D0,$T1,6);
1613         &vmovdqa        (&QWP(32*1,"esp"),$D1);
1614         &vpunpckhqdq    ($D1,$T0,$T1);                  # 4
1615         &vpunpcklqdq    ($T0,$T0,$T1);                  # 0:1
1616         &vpunpcklqdq    ($D2,$D2,$D0);                  # 2:3
1617
1618         &vpsrlq         ($D0,$D2,30);
1619         &vpsrlq         ($D2,$D2,4);
1620         &vpsrlq         ($T1,$T0,26);
1621         &vpsrlq         ($D1,$D1,40);                   # 4
1622         &vpand          ($D2,$D2,$MASK);                # 2
1623         &vpand          ($T0,$T0,$MASK);                # 0
1624         &vpand          ($T1,$T1,$MASK);                # 1
1625         &vpand          ($D0,$D0,$MASK);                # 3 (*)
1626         &vpor           ($D1,$D1,&QWP(0,"ebx"));        # padbit, yes, always
1627
1628         # (*)   note that output is counterintuitive, inp[3:4] is
1629         #       returned in $D1-2, while $D3-4 are preserved;
1630 }
1631         &vsplat_input   ();
1632
1633 sub vpmuladd {
1634 my $addr = shift;
1635
1636         &vpaddq         ($D2,$D2,&QWP(32*2,"esp"));     # add hash value
1637         &vpaddq         ($T0,$T0,&QWP(32*0,"esp"));
1638         &vpaddq         ($T1,$T1,&QWP(32*1,"esp"));
1639         &vpaddq         ($D0,$D0,$D3);
1640         &vpaddq         ($D1,$D1,$D4);
1641
1642         ################################################################
1643         # d3 = h2*r1   + h0*r3 + h1*r2   + h3*r0   + h4*5*r4
1644         # d4 = h2*r2   + h0*r4 + h1*r3   + h3*r1   + h4*r0
1645         # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1
1646         # d1 = h2*5*r4 + h0*r1 + h1*r0   + h3*5*r3 + h4*5*r2
1647         # d2 = h2*r0   + h0*r2 + h1*r1   + h3*5*r4 + h4*5*r3
1648
1649         &vpmuludq       ($D3,$D2,&$addr(1));            # d3 = h2*r1
1650          &vmovdqa       (QWP(32*1,"esp"),$T1);
1651         &vpmuludq       ($D4,$D2,&$addr(2));            # d4 = h2*r2
1652          &vmovdqa       (QWP(32*3,"esp"),$D0);
1653         &vpmuludq       ($D0,$D2,&$addr(7));            # d0 = h2*s3
1654          &vmovdqa       (QWP(32*4,"esp"),$D1);
1655         &vpmuludq       ($D1,$D2,&$addr(8));            # d1 = h2*s4
1656         &vpmuludq       ($D2,$D2,&$addr(0));            # d2 = h2*r0
1657
1658         &vpmuludq       ($T2,$T0,&$addr(3));            # h0*r3
1659         &vpaddq         ($D3,$D3,$T2);                  # d3 += h0*r3
1660         &vpmuludq       ($T1,$T0,&$addr(4));            # h0*r4
1661         &vpaddq         ($D4,$D4,$T1);                  # d4 + h0*r4
1662         &vpmuludq       ($T2,$T0,&$addr(0));            # h0*r0
1663         &vpaddq         ($D0,$D0,$T2);                  # d0 + h0*r0
1664          &vmovdqa       ($T2,&QWP(32*1,"esp"));         # h1
1665         &vpmuludq       ($T1,$T0,&$addr(1));            # h0*r1
1666         &vpaddq         ($D1,$D1,$T1);                  # d1 += h0*r1
1667         &vpmuludq       ($T0,$T0,&$addr(2));            # h0*r2
1668         &vpaddq         ($D2,$D2,$T0);                  # d2 += h0*r2
1669
1670         &vpmuludq       ($T1,$T2,&$addr(2));            # h1*r2
1671         &vpaddq         ($D3,$D3,$T1);                  # d3 += h1*r2
1672         &vpmuludq       ($T0,$T2,&$addr(3));            # h1*r3
1673         &vpaddq         ($D4,$D4,$T0);                  # d4 += h1*r3
1674         &vpmuludq       ($T1,$T2,&$addr(8));            # h1*s4
1675         &vpaddq         ($D0,$D0,$T1);                  # d0 += h1*s4
1676          &vmovdqa       ($T1,&QWP(32*3,"esp"));         # h3
1677         &vpmuludq       ($T0,$T2,&$addr(0));            # h1*r0
1678         &vpaddq         ($D1,$D1,$T0);                  # d1 += h1*r0
1679         &vpmuludq       ($T2,$T2,&$addr(1));            # h1*r1
1680         &vpaddq         ($D2,$D2,$T2);                  # d2 += h1*r1
1681
1682         &vpmuludq       ($T0,$T1,&$addr(0));            # h3*r0
1683         &vpaddq         ($D3,$D3,$T0);                  # d3 += h3*r0
1684         &vpmuludq       ($T2,$T1,&$addr(1));            # h3*r1
1685         &vpaddq         ($D4,$D4,$T2);                  # d4 += h3*r1
1686         &vpmuludq       ($T0,$T1,&$addr(6));            # h3*s2
1687         &vpaddq         ($D0,$D0,$T0);                  # d0 += h3*s2
1688          &vmovdqa       ($T0,&QWP(32*4,"esp"));         # h4
1689         &vpmuludq       ($T2,$T1,&$addr(7));            # h3*s3
1690         &vpaddq         ($D1,$D1,$T2);                  # d1+= h3*s3
1691         &vpmuludq       ($T1,$T1,&$addr(8));            # h3*s4
1692         &vpaddq         ($D2,$D2,$T1);                  # d2 += h3*s4
1693
1694         &vpmuludq       ($T2,$T0,&$addr(8));            # h4*s4
1695         &vpaddq         ($D3,$D3,$T2);                  # d3 += h4*s4
1696         &vpmuludq       ($T1,$T0,&$addr(5));            # h4*s1
1697         &vpaddq         ($D0,$D0,$T1);                  # d0 += h4*s1
1698         &vpmuludq       ($T2,$T0,&$addr(0));            # h4*r0
1699         &vpaddq         ($D4,$D4,$T2);                  # d4 += h4*r0
1700          &vmovdqa       ($MASK,&QWP(64,"ebx"));
1701         &vpmuludq       ($T1,$T0,&$addr(6));            # h4*s2
1702         &vpaddq         ($D1,$D1,$T1);                  # d1 += h4*s2
1703         &vpmuludq       ($T0,$T0,&$addr(7));            # h4*s3
1704         &vpaddq         ($D2,$D2,$T0);                  # d2 += h4*s3
1705 }
1706         &vpmuladd       (sub {  my $i=shift; &QWP(32*$i-128,"edx");     });
1707
1708 sub vlazy_reduction {
1709         ################################################################
1710         # lazy reduction
1711
1712          &vpsrlq        ($T0,$D3,26);
1713          &vpand         ($D3,$D3,$MASK);
1714         &vpsrlq         ($T1,$D0,26);
1715         &vpand          ($D0,$D0,$MASK);
1716          &vpaddq        ($D4,$D4,$T0);                  # h3 -> h4
1717         &vpaddq         ($D1,$D1,$T1);                  # h0 -> h1
1718          &vpsrlq        ($T0,$D4,26);
1719          &vpand         ($D4,$D4,$MASK);
1720         &vpsrlq         ($T1,$D1,26);
1721         &vpand          ($D1,$D1,$MASK);
1722         &vpaddq         ($D2,$D2,$T1);                  # h1 -> h2
1723          &vpaddq        ($D0,$D0,$T0);
1724          &vpsllq        ($T0,$T0,2);
1725         &vpsrlq         ($T1,$D2,26);
1726         &vpand          ($D2,$D2,$MASK);
1727          &vpaddq        ($D0,$D0,$T0);                  # h4 -> h0
1728         &vpaddq         ($D3,$D3,$T1);                  # h2 -> h3
1729         &vpsrlq         ($T1,$D3,26);
1730          &vpsrlq        ($T0,$D0,26);
1731          &vpand         ($D0,$D0,$MASK);
1732         &vpand          ($D3,$D3,$MASK);
1733          &vpaddq        ($D1,$D1,$T0);                  # h0 -> h1
1734         &vpaddq         ($D4,$D4,$T1);                  # h3 -> h4
1735 }
1736         &vlazy_reduction();
1737
1738         &vmovdqu        (&X($T0),&QWP(16*0,"esi"));     # load input
1739         &vmovdqu        (&X($T1),&QWP(16*1,"esi"));
1740         &vinserti128    ($T0,$T0,&QWP(16*2,"esi"),1);
1741         &vinserti128    ($T1,$T1,&QWP(16*3,"esi"),1);
1742         &lea            ("esi",&DWP(16*4,"esi"));
1743         &sub            ("ecx",64);
1744         &jnz            (&label("loop"));
1745
1746 &set_label("tail");
1747         &vsplat_input   ();
1748         &and            ("ebx",-64);                    # restore pointer
1749
1750         &vpmuladd       (sub {  my $i=shift; &QWP(4+32*$i-128,"edx");   });
1751
1752         ################################################################
1753         # horizontal addition
1754
1755         &vpsrldq        ($T0,$D4,8);
1756         &vpsrldq        ($T1,$D3,8);
1757         &vpaddq         ($D4,$D4,$T0);
1758         &vpsrldq        ($T0,$D0,8);
1759         &vpaddq         ($D3,$D3,$T1);
1760         &vpsrldq        ($T1,$D1,8);
1761         &vpaddq         ($D0,$D0,$T0);
1762         &vpsrldq        ($T0,$D2,8);
1763         &vpaddq         ($D1,$D1,$T1);
1764         &vpermq         ($T1,$D4,2);                    # keep folding
1765         &vpaddq         ($D2,$D2,$T0);
1766         &vpermq         ($T0,$D3,2);
1767         &vpaddq         ($D4,$D4,$T1);
1768         &vpermq         ($T1,$D0,2);
1769         &vpaddq         ($D3,$D3,$T0);
1770         &vpermq         ($T0,$D1,2);
1771         &vpaddq         ($D0,$D0,$T1);
1772         &vpermq         ($T1,$D2,2);
1773         &vpaddq         ($D1,$D1,$T0);
1774         &vpaddq         ($D2,$D2,$T1);
1775
1776         &vlazy_reduction();
1777
1778         &cmp            ("ecx",0);
1779         &je             (&label("done"));
1780
1781         ################################################################
1782         # clear all but single word
1783
1784         &vpshufd        (&X($D0),&X($D0),0b11111100);
1785         &lea            ("edx",&DWP(32*5+128,"esp"));   # restore pointer
1786         &vpshufd        (&X($D1),&X($D1),0b11111100);
1787         &vpshufd        (&X($D2),&X($D2),0b11111100);
1788         &vpshufd        (&X($D3),&X($D3),0b11111100);
1789         &vpshufd        (&X($D4),&X($D4),0b11111100);
1790         &jmp            (&label("even"));
1791
1792 &set_label("done",16);
1793         &vmovd          (&DWP(-16*3+4*0,"edi"),&X($D0));# store hash value
1794         &vmovd          (&DWP(-16*3+4*1,"edi"),&X($D1));
1795         &vmovd          (&DWP(-16*3+4*2,"edi"),&X($D2));
1796         &vmovd          (&DWP(-16*3+4*3,"edi"),&X($D3));
1797         &vmovd          (&DWP(-16*3+4*4,"edi"),&X($D4));
1798         &vzeroupper     ();
1799         &mov    ("esp","ebp");
1800 &set_label("nodata");
1801 &function_end("_poly1305_blocks_avx2");
1802 }
1803 &set_label("const_sse2",64);
1804         &data_word(1<<24,0,     1<<24,0,        1<<24,0,        1<<24,0);
1805         &data_word(0,0,         0,0,            0,0,            0,0);
1806         &data_word(0x03ffffff,0,0x03ffffff,0,   0x03ffffff,0,   0x03ffffff,0);
1807         &data_word(0x0fffffff,0x0ffffffc,0x0ffffffc,0x0ffffffc);
1808 }
1809 &asciz  ("Poly1305 for x86, CRYPTOGAMS by <appro\@openssl.org>");
1810 &align  (4);
1811
1812 &asm_finish();
1813
1814 close STDOUT;