crypto/poly1305/asm/poly1305-x86.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the OpenSSL license (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9 #
  10 # ====================================================================
  11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  12 # project. The module is, however, dual licensed under OpenSSL and
  13 # CRYPTOGAMS licenses depending on where you obtain it. For further
  14 # details see http://www.openssl.org/~appro/cryptogams/.
  15 # ====================================================================
  16 #
  17 # This module implements Poly1305 hash for x86.
  18 #
  19 # April 2015
  20 #
  21 # Numbers are cycles per processed byte with poly1305_blocks alone,
  22 # measured with rdtsc at fixed clock frequency.
  23 #
  24 #               IALU/gcc-3.4(*) SSE2(**)        AVX2
  25 # Pentium       15.7/+80%       -
  26 # PIII          6.21/+90%       -
  27 # P4            19.8/+40%       3.24
  28 # Core 2        4.85/+90%       1.80
  29 # Westmere      4.58/+100%      1.43
  30 # Sandy Bridge  3.90/+100%      1.36
  31 # Haswell       3.88/+70%       1.18            0.72
  32 # Silvermont    11.0/+40%       4.80
  33 # VIA Nano      6.71/+90%       2.47
  34 # Sledgehammer  3.51/+180%      4.27
  35 # Bulldozer     4.53/+140%      1.31
  36 #
  37 # (*)   gcc 4.8 for some reason generated worse code;
  38 # (**)  besides SSE2 there are floating-point and AVX options; FP
  39 #       is deemed unnecessary, because pre-SSE2 processor are too
  40 #       old to care about, while it's not the fastest option on
  41 #       SSE2-capable ones; AVX is omitted, because it doesn't give
  42 #       a lot of improvement, 5-10% depending on processor;
  43
  44 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  45 push(@INC,"${dir}","${dir}../../perlasm");
  46 require "x86asm.pl";
  47
  48 $output=pop;
  49 open STDOUT,">$output";
  50
  51 &asm_init($ARGV[0],"poly1305-x86.pl",$ARGV[$#ARGV] eq "386");
  52
  53 $sse2=$avx=0;
  54 for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
  55
  56 if ($sse2) {
  57         &static_label("const_sse2");
  58         &static_label("enter_blocks");
  59         &static_label("enter_emit");
  60         &external_label("OPENSSL_ia32cap_P");
  61
  62         if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  63                         =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  64                 $avx = ($1>=2.19) + ($1>=2.22);
  65         }
  66
  67         if (!$avx && $ARGV[0] eq "win32n" &&
  68            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
  69         $avx = ($1>=2.09) + ($1>=2.10);
  70         }
  71
  72         if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) {
  73                 $avx = ($2>=3.0) + ($2>3.0);
  74         }
  75 }
  76
  77 ########################################################################
  78 # Layout of opaque area is following.
  79 #
  80 #       unsigned __int32 h[5];          # current hash value base 2^32
  81 #       unsigned __int32 pad;           # is_base2_26 in vector context
  82 #       unsigned __int32 r[4];          # key value base 2^32
  83
  84 &align(64);
  85 &function_begin("poly1305_init");
  86         &mov    ("edi",&wparam(0));             # context
  87         &mov    ("esi",&wparam(1));             # key
  88         &mov    ("ebp",&wparam(2));             # function table
  89
  90         &xor    ("eax","eax");
  91         &mov    (&DWP(4*0,"edi"),"eax");        # zero hash value
  92         &mov    (&DWP(4*1,"edi"),"eax");
  93         &mov    (&DWP(4*2,"edi"),"eax");
  94         &mov    (&DWP(4*3,"edi"),"eax");
  95         &mov    (&DWP(4*4,"edi"),"eax");
  96         &mov    (&DWP(4*5,"edi"),"eax");        # is_base2_26
  97
  98         &cmp    ("esi",0);
  99         &je     (&label("nokey"));
 100
 101     if ($sse2) {
 102         &call   (&label("pic_point"));
 103     &set_label("pic_point");
 104         &blindpop("ebx");
 105
 106         &lea    ("eax",&DWP("poly1305_blocks-".&label("pic_point"),"ebx"));
 107         &lea    ("edx",&DWP("poly1305_emit-".&label("pic_point"),"ebx"));
 108
 109         &picmeup("edi","OPENSSL_ia32cap_P","ebx",&label("pic_point"));
 110         &mov    ("ecx",&DWP(0,"edi"));
 111         &and    ("ecx",1<<26|1<<24);
 112         &cmp    ("ecx",1<<26|1<<24);            # SSE2 and XMM?
 113         &jne    (&label("no_sse2"));
 114
 115         &lea    ("eax",&DWP("_poly1305_blocks_sse2-".&label("pic_point"),"ebx"));
 116         &lea    ("edx",&DWP("_poly1305_emit_sse2-".&label("pic_point"),"ebx"));
 117
 118       if ($avx>1) {
 119         &mov    ("ecx",&DWP(8,"edi"));
 120         &test   ("ecx",1<<5);                   # AVX2?
 121         &jz     (&label("no_sse2"));
 122
 123         &lea    ("eax",&DWP("_poly1305_blocks_avx2-".&label("pic_point"),"ebx"));
 124       }
 125     &set_label("no_sse2");
 126         &mov    ("edi",&wparam(0));             # reload context
 127         &mov    (&DWP(0,"ebp"),"eax");          # fill function table
 128         &mov    (&DWP(4,"ebp"),"edx");
 129     }
 130
 131         &mov    ("eax",&DWP(4*0,"esi"));        # load input key
 132         &mov    ("ebx",&DWP(4*1,"esi"));
 133         &mov    ("ecx",&DWP(4*2,"esi"));
 134         &mov    ("edx",&DWP(4*3,"esi"));
 135         &and    ("eax",0x0fffffff);
 136         &and    ("ebx",0x0ffffffc);
 137         &and    ("ecx",0x0ffffffc);
 138         &and    ("edx",0x0ffffffc);
 139         &mov    (&DWP(4*6,"edi"),"eax");
 140         &mov    (&DWP(4*7,"edi"),"ebx");
 141         &mov    (&DWP(4*8,"edi"),"ecx");
 142         &mov    (&DWP(4*9,"edi"),"edx");
 143
 144         &mov    ("eax",$sse2);
 145 &set_label("nokey");
 146 &function_end("poly1305_init");
 147
 148 ($h0,$h1,$h2,$h3,$h4,
 149  $d0,$d1,$d2,$d3,
 150  $r0,$r1,$r2,$r3,
 151      $s1,$s2,$s3)=map(4*$_,(0..15));
 152
 153 &function_begin("poly1305_blocks");
 154         &mov    ("edi",&wparam(0));             # ctx
 155         &mov    ("esi",&wparam(1));             # inp
 156         &mov    ("ecx",&wparam(2));             # len
 157 &set_label("enter_blocks");
 158         &and    ("ecx",-15);
 159         &jz     (&label("nodata"));
 160
 161         &stack_push(16);
 162         &mov    ("eax",&DWP(4*6,"edi"));        # r0
 163         &mov    ("ebx",&DWP(4*7,"edi"));        # r1
 164          &lea   ("ebp",&DWP(0,"esi","ecx"));    # end of input
 165         &mov    ("ecx",&DWP(4*8,"edi"));        # r2
 166         &mov    ("edx",&DWP(4*9,"edi"));        # r3
 167
 168         &mov    (&wparam(2),"ebp");
 169         &mov    ("ebp","esi");
 170
 171         &mov    (&DWP($r0,"esp"),"eax");        # r0
 172         &mov    ("eax","ebx");
 173         &shr    ("eax",2);
 174         &mov    (&DWP($r1,"esp"),"ebx");        # r1
 175         &add    ("eax","ebx");                  # s1
 176         &mov    ("ebx","ecx");
 177         &shr    ("ebx",2);
 178         &mov    (&DWP($r2,"esp"),"ecx");        # r2
 179         &add    ("ebx","ecx");                  # s2
 180         &mov    ("ecx","edx");
 181         &shr    ("ecx",2);
 182         &mov    (&DWP($r3,"esp"),"edx");        # r3
 183         &add    ("ecx","edx");                  # s3
 184         &mov    (&DWP($s1,"esp"),"eax");        # s1
 185         &mov    (&DWP($s2,"esp"),"ebx");        # s2
 186         &mov    (&DWP($s3,"esp"),"ecx");        # s3
 187
 188         &mov    ("eax",&DWP(4*0,"edi"));        # load hash value
 189         &mov    ("ebx",&DWP(4*1,"edi"));
 190         &mov    ("ecx",&DWP(4*2,"edi"));
 191         &mov    ("esi",&DWP(4*3,"edi"));
 192         &mov    ("edi",&DWP(4*4,"edi"));
 193         &jmp    (&label("loop"));
 194
 195 &set_label("loop",32);
 196         &add    ("eax",&DWP(4*0,"ebp"));        # accumulate input
 197         &adc    ("ebx",&DWP(4*1,"ebp"));
 198         &adc    ("ecx",&DWP(4*2,"ebp"));
 199         &adc    ("esi",&DWP(4*3,"ebp"));
 200         &lea    ("ebp",&DWP(4*4,"ebp"));
 201         &adc    ("edi",&wparam(3));             # padbit
 202
 203         &mov    (&DWP($h0,"esp"),"eax");        # put aside hash[+inp]
 204         &mov    (&DWP($h3,"esp"),"esi");
 205
 206         &mul    (&DWP($r0,"esp"));              # h0*r0
 207          &mov   (&DWP($h4,"esp"),"edi");
 208         &mov    ("edi","eax");
 209         &mov    ("eax","ebx");                  # h1
 210         &mov    ("esi","edx");
 211         &mul    (&DWP($s3,"esp"));              # h1*s3
 212         &add    ("edi","eax");
 213         &mov    ("eax","ecx");                  # h2
 214         &adc    ("esi","edx");
 215         &mul    (&DWP($s2,"esp"));              # h2*s2
 216         &add    ("edi","eax");
 217         &mov    ("eax",&DWP($h3,"esp"));
 218         &adc    ("esi","edx");
 219         &mul    (&DWP($s1,"esp"));              # h3*s1
 220         &add    ("edi","eax");
 221          &mov   ("eax",&DWP($h0,"esp"));
 222         &adc    ("esi","edx");
 223
 224         &mul    (&DWP($r1,"esp"));              # h0*r1
 225          &mov   (&DWP($d0,"esp"),"edi");
 226         &xor    ("edi","edi");
 227         &add    ("esi","eax");
 228         &mov    ("eax","ebx");                  # h1
 229         &adc    ("edi","edx");
 230         &mul    (&DWP($r0,"esp"));              # h1*r0
 231         &add    ("esi","eax");
 232         &mov    ("eax","ecx");                  # h2
 233         &adc    ("edi","edx");
 234         &mul    (&DWP($s3,"esp"));              # h2*s3
 235         &add    ("esi","eax");
 236         &mov    ("eax",&DWP($h3,"esp"));
 237         &adc    ("edi","edx");
 238         &mul    (&DWP($s2,"esp"));              # h3*s2
 239         &add    ("esi","eax");
 240         &mov    ("eax",&DWP($h4,"esp"));
 241         &adc    ("edi","edx");
 242         &imul   ("eax",&DWP($s1,"esp"));        # h4*s1
 243         &add    ("esi","eax");
 244          &mov   ("eax",&DWP($h0,"esp"));
 245         &adc    ("edi",0);
 246
 247         &mul    (&DWP($r2,"esp"));              # h0*r2
 248          &mov   (&DWP($d1,"esp"),"esi");
 249         &xor    ("esi","esi");
 250         &add    ("edi","eax");
 251         &mov    ("eax","ebx");                  # h1
 252         &adc    ("esi","edx");
 253         &mul    (&DWP($r1,"esp"));              # h1*r1
 254         &add    ("edi","eax");
 255         &mov    ("eax","ecx");                  # h2
 256         &adc    ("esi","edx");
 257         &mul    (&DWP($r0,"esp"));              # h2*r0
 258         &add    ("edi","eax");
 259         &mov    ("eax",&DWP($h3,"esp"));
 260         &adc    ("esi","edx");
 261         &mul    (&DWP($s3,"esp"));              # h3*s3
 262         &add    ("edi","eax");
 263         &mov    ("eax",&DWP($h4,"esp"));
 264         &adc    ("esi","edx");
 265         &imul   ("eax",&DWP($s2,"esp"));        # h4*s2
 266         &add    ("edi","eax");
 267          &mov   ("eax",&DWP($h0,"esp"));
 268         &adc    ("esi",0);
 269
 270         &mul    (&DWP($r3,"esp"));              # h0*r3
 271          &mov   (&DWP($d2,"esp"),"edi");
 272         &xor    ("edi","edi");
 273         &add    ("esi","eax");
 274         &mov    ("eax","ebx");                  # h1
 275         &adc    ("edi","edx");
 276         &mul    (&DWP($r2,"esp"));              # h1*r2
 277         &add    ("esi","eax");
 278         &mov    ("eax","ecx");                  # h2
 279         &adc    ("edi","edx");
 280         &mul    (&DWP($r1,"esp"));              # h2*r1
 281         &add    ("esi","eax");
 282         &mov    ("eax",&DWP($h3,"esp"));
 283         &adc    ("edi","edx");
 284         &mul    (&DWP($r0,"esp"));              # h3*r0
 285         &add    ("esi","eax");
 286          &mov   ("ecx",&DWP($h4,"esp"));
 287         &adc    ("edi","edx");
 288
 289         &mov    ("edx","ecx");
 290         &imul   ("ecx",&DWP($s3,"esp"));        # h4*s3
 291         &add    ("esi","ecx");
 292          &mov   ("eax",&DWP($d0,"esp"));
 293         &adc    ("edi",0);
 294
 295         &imul   ("edx",&DWP($r0,"esp"));        # h4*r0
 296         &add    ("edx","edi");
 297
 298         &mov    ("ebx",&DWP($d1,"esp"));
 299         &mov    ("ecx",&DWP($d2,"esp"));
 300
 301         &mov    ("edi","edx");                  # last reduction step
 302         &shr    ("edx",2);
 303         &and    ("edi",3);
 304         &lea    ("edx",&DWP(0,"edx","edx",4));  # *5
 305         &add    ("eax","edx");
 306         &adc    ("ebx",0);
 307         &adc    ("ecx",0);
 308         &adc    ("esi",0);
 309         &adc    ("edi",0);
 310
 311         &cmp    ("ebp",&wparam(2));             # done yet?
 312         &jne    (&label("loop"));
 313
 314         &mov    ("edx",&wparam(0));             # ctx
 315         &stack_pop(16);
 316         &mov    (&DWP(4*0,"edx"),"eax");        # store hash value
 317         &mov    (&DWP(4*1,"edx"),"ebx");
 318         &mov    (&DWP(4*2,"edx"),"ecx");
 319         &mov    (&DWP(4*3,"edx"),"esi");
 320         &mov    (&DWP(4*4,"edx"),"edi");
 321 &set_label("nodata");
 322 &function_end("poly1305_blocks");
 323
 324 &function_begin("poly1305_emit");
 325         &mov    ("ebp",&wparam(0));             # context
 326 &set_label("enter_emit");
 327         &mov    ("edi",&wparam(1));             # output
 328         &mov    ("eax",&DWP(4*0,"ebp"));        # load hash value
 329         &mov    ("ebx",&DWP(4*1,"ebp"));
 330         &mov    ("ecx",&DWP(4*2,"ebp"));
 331         &mov    ("edx",&DWP(4*3,"ebp"));
 332         &mov    ("esi",&DWP(4*4,"ebp"));
 333
 334         &add    ("eax",5);                      # compare to modulus
 335         &adc    ("ebx",0);
 336         &adc    ("ecx",0);
 337         &adc    ("edx",0);
 338         &adc    ("esi",0);
 339         &shr    ("esi",2);                      # did it carry/borrow?
 340         &neg    ("esi");                        # do we choose hash-modulus?
 341
 342         &and    ("eax","esi");
 343         &and    ("ebx","esi");
 344         &and    ("ecx","esi");
 345         &and    ("edx","esi");
 346         &mov    (&DWP(4*0,"edi"),"eax");
 347         &mov    (&DWP(4*1,"edi"),"ebx");
 348         &mov    (&DWP(4*2,"edi"),"ecx");
 349         &mov    (&DWP(4*3,"edi"),"edx");
 350
 351         &not    ("esi");                        # or original hash value?
 352         &mov    ("eax",&DWP(4*0,"ebp"));
 353         &mov    ("ebx",&DWP(4*1,"ebp"));
 354         &mov    ("ecx",&DWP(4*2,"ebp"));
 355         &mov    ("edx",&DWP(4*3,"ebp"));
 356         &mov    ("ebp",&wparam(2));
 357         &and    ("eax","esi");
 358         &and    ("ebx","esi");
 359         &and    ("ecx","esi");
 360         &and    ("edx","esi");
 361         &or     ("eax",&DWP(4*0,"edi"));
 362         &or     ("ebx",&DWP(4*1,"edi"));
 363         &or     ("ecx",&DWP(4*2,"edi"));
 364         &or     ("edx",&DWP(4*3,"edi"));
 365
 366         &add    ("eax",&DWP(4*0,"ebp"));        # accumulate key
 367         &adc    ("ebx",&DWP(4*1,"ebp"));
 368         &adc    ("ecx",&DWP(4*2,"ebp"));
 369         &adc    ("edx",&DWP(4*3,"ebp"));
 370
 371         &mov    (&DWP(4*0,"edi"),"eax");
 372         &mov    (&DWP(4*1,"edi"),"ebx");
 373         &mov    (&DWP(4*2,"edi"),"ecx");
 374         &mov    (&DWP(4*3,"edi"),"edx");
 375 &function_end("poly1305_emit");
 376
 377 if ($sse2) {
 378 ########################################################################
 379 # Layout of opaque area is following.
 380 #
 381 #       unsigned __int32 h[5];          # current hash value base 2^26
 382 #       unsigned __int32 is_base2_26;
 383 #       unsigned __int32 r[4];          # key value base 2^32
 384 #       unsigned __int32 pad[2];
 385 #       struct { unsigned __int32 r^4, r^3, r^2, r^1; } r[9];
 386 #
 387 # where r^n are base 2^26 digits of degrees of multiplier key. There are
 388 # 5 digits, but last four are interleaved with multiples of 5, totalling
 389 # in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.
 390
 391 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("xmm$_",(0..7));
 392 my $MASK=$T2;   # borrow and keep in mind
 393
 394 &align  (32);
 395 &function_begin_B("_poly1305_init_sse2");
 396         &movdqu         ($D4,&QWP(4*6,"edi"));          # key base 2^32
 397         &lea            ("edi",&DWP(16*3,"edi"));       # size optimization
 398         &mov            ("ebp","esp");
 399         &sub            ("esp",16*(9+5));
 400         &and            ("esp",-16);
 401
 402         #&pand          ($D4,&QWP(96,"ebx"));           # magic mask
 403         &movq           ($MASK,&QWP(64,"ebx"));
 404
 405         &movdqa         ($D0,$D4);
 406         &movdqa         ($D1,$D4);
 407         &movdqa         ($D2,$D4);
 408
 409         &pand           ($D0,$MASK);                    # -> base 2^26
 410         &psrlq          ($D1,26);
 411         &psrldq         ($D2,6);
 412         &pand           ($D1,$MASK);
 413         &movdqa         ($D3,$D2);
 414         &psrlq          ($D2,4)
 415         &psrlq          ($D3,30);
 416         &pand           ($D2,$MASK);
 417         &pand           ($D3,$MASK);
 418         &psrldq         ($D4,13);
 419
 420         &lea            ("edx",&DWP(16*9,"esp"));       # size optimization
 421         &mov            ("ecx",2);
 422 &set_label("square");
 423         &movdqa         (&QWP(16*0,"esp"),$D0);
 424         &movdqa         (&QWP(16*1,"esp"),$D1);
 425         &movdqa         (&QWP(16*2,"esp"),$D2);
 426         &movdqa         (&QWP(16*3,"esp"),$D3);
 427         &movdqa         (&QWP(16*4,"esp"),$D4);
 428
 429         &movdqa         ($T1,$D1);
 430         &movdqa         ($T0,$D2);
 431         &pslld          ($T1,2);
 432         &pslld          ($T0,2);
 433         &paddd          ($T1,$D1);                      # *5
 434         &paddd          ($T0,$D2);                      # *5
 435         &movdqa         (&QWP(16*5,"esp"),$T1);
 436         &movdqa         (&QWP(16*6,"esp"),$T0);
 437         &movdqa         ($T1,$D3);
 438         &movdqa         ($T0,$D4);
 439         &pslld          ($T1,2);
 440         &pslld          ($T0,2);
 441         &paddd          ($T1,$D3);                      # *5
 442         &paddd          ($T0,$D4);                      # *5
 443         &movdqa         (&QWP(16*7,"esp"),$T1);
 444         &movdqa         (&QWP(16*8,"esp"),$T0);
 445
 446         &pshufd         ($T1,$D0,0b01000100);
 447         &movdqa         ($T0,$D1);
 448         &pshufd         ($D1,$D1,0b01000100);
 449         &pshufd         ($D2,$D2,0b01000100);
 450         &pshufd         ($D3,$D3,0b01000100);
 451         &pshufd         ($D4,$D4,0b01000100);
 452         &movdqa         (&QWP(16*0,"edx"),$T1);
 453         &movdqa         (&QWP(16*1,"edx"),$D1);
 454         &movdqa         (&QWP(16*2,"edx"),$D2);
 455         &movdqa         (&QWP(16*3,"edx"),$D3);
 456         &movdqa         (&QWP(16*4,"edx"),$D4);
 457
 458         ################################################################
 459         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
 460         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
 461         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
 462         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
 463         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
 464
 465         &pmuludq        ($D4,$D0);                      # h4*r0
 466         &pmuludq        ($D3,$D0);                      # h3*r0
 467         &pmuludq        ($D2,$D0);                      # h2*r0
 468         &pmuludq        ($D1,$D0);                      # h1*r0
 469         &pmuludq        ($D0,$T1);                      # h0*r0
 470
 471 sub pmuladd {
 472 my $load = shift;
 473 my $base = shift; $base = "esp" if (!defined($base));
 474
 475         ################################################################
 476         # As for choice to "rotate" $T0-$T2 in order to move paddq
 477         # past next multiplication. While it makes code harder to read
 478         # and doesn't have significant effect on most processors, it
 479         # makes a lot of difference on Atom, up to 30% improvement.
 480
 481         &movdqa         ($T1,$T0);
 482         &pmuludq        ($T0,&QWP(16*3,$base));         # r1*h3
 483         &movdqa         ($T2,$T1);
 484         &pmuludq        ($T1,&QWP(16*2,$base));         # r1*h2
 485         &paddq          ($D4,$T0);
 486         &movdqa         ($T0,$T2);
 487         &pmuludq        ($T2,&QWP(16*1,$base));         # r1*h1
 488         &paddq          ($D3,$T1);
 489         &$load          ($T1,5);                        # s1
 490         &pmuludq        ($T0,&QWP(16*0,$base));         # r1*h0
 491         &paddq          ($D2,$T2);
 492         &pmuludq        ($T1,&QWP(16*4,$base));         # s1*h4
 493          &$load         ($T2,2);                        # r2^n
 494         &paddq          ($D1,$T0);
 495
 496         &movdqa         ($T0,$T2);
 497         &pmuludq        ($T2,&QWP(16*2,$base));         # r2*h2
 498          &paddq         ($D0,$T1);
 499         &movdqa         ($T1,$T0);
 500         &pmuludq        ($T0,&QWP(16*1,$base));         # r2*h1
 501         &paddq          ($D4,$T2);
 502         &$load          ($T2,6);                        # s2^n
 503         &pmuludq        ($T1,&QWP(16*0,$base));         # r2*h0
 504         &paddq          ($D3,$T0);
 505         &movdqa         ($T0,$T2);
 506         &pmuludq        ($T2,&QWP(16*4,$base));         # s2*h4
 507         &paddq          ($D2,$T1);
 508         &pmuludq        ($T0,&QWP(16*3,$base));         # s2*h3
 509          &$load         ($T1,3);                        # r3^n
 510         &paddq          ($D1,$T2);
 511
 512         &movdqa         ($T2,$T1);
 513         &pmuludq        ($T1,&QWP(16*1,$base));         # r3*h1
 514          &paddq         ($D0,$T0);
 515         &$load          ($T0,7);                        # s3^n
 516         &pmuludq        ($T2,&QWP(16*0,$base));         # r3*h0
 517         &paddq          ($D4,$T1);
 518         &movdqa         ($T1,$T0);
 519         &pmuludq        ($T0,&QWP(16*4,$base));         # s3*h4
 520         &paddq          ($D3,$T2);
 521         &movdqa         ($T2,$T1);
 522         &pmuludq        ($T1,&QWP(16*3,$base));         # s3*h3
 523         &paddq          ($D2,$T0);
 524         &pmuludq        ($T2,&QWP(16*2,$base));         # s3*h2
 525          &$load         ($T0,4);                        # r4^n
 526         &paddq          ($D1,$T1);
 527
 528         &$load          ($T1,8);                        # s4^n
 529         &pmuludq        ($T0,&QWP(16*0,$base));         # r4*h0
 530          &paddq         ($D0,$T2);
 531         &movdqa         ($T2,$T1);
 532         &pmuludq        ($T1,&QWP(16*4,$base));         # s4*h4
 533         &paddq          ($D4,$T0);
 534         &movdqa         ($T0,$T2);
 535         &pmuludq        ($T2,&QWP(16*1,$base));         # s4*h1
 536         &paddq          ($D3,$T1);
 537         &movdqa         ($T1,$T0);
 538         &pmuludq        ($T0,&QWP(16*2,$base));         # s4*h2
 539         &paddq          ($D0,$T2);
 540         &pmuludq        ($T1,&QWP(16*3,$base));         # s4*h3
 541          &movdqa        ($MASK,&QWP(64,"ebx"));
 542         &paddq          ($D1,$T0);
 543         &paddq          ($D2,$T1);
 544 }
 545         &pmuladd        (sub {  my ($reg,$i)=@_;
 546                                 &movdqa ($reg,&QWP(16*$i,"esp"));
 547                              },"edx");
 548
 549 sub lazy_reduction {
 550 my $extra = shift;
 551
 552         ################################################################
 553         # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
 554         # and P. Schwabe
 555         #
 556         # [(*) see discussion in poly1305-armv4 module]
 557
 558          &movdqa        ($T0,$D3);
 559          &pand          ($D3,$MASK);
 560          &psrlq         ($T0,26);
 561          &$extra        ()                              if (defined($extra));
 562          &paddq         ($T0,$D4);                      # h3 -> h4
 563         &movdqa         ($T1,$D0);
 564         &pand           ($D0,$MASK);
 565         &psrlq          ($T1,26);
 566          &movdqa        ($D4,$T0);
 567         &paddq          ($T1,$D1);                      # h0 -> h1
 568          &psrlq         ($T0,26);
 569          &pand          ($D4,$MASK);
 570         &movdqa         ($D1,$T1);
 571         &psrlq          ($T1,26);
 572          &paddd         ($D0,$T0);                      # favour paddd when
 573                                                         # possible, because
 574                                                         # paddq is "broken"
 575                                                         # on Atom
 576          &psllq         ($T0,2);
 577         &paddq          ($T1,$D2);                      # h1 -> h2
 578          &paddq         ($T0,$D0);                      # h4 -> h0 (*)
 579         &pand           ($D1,$MASK);
 580         &movdqa         ($D2,$T1);
 581         &psrlq          ($T1,26);
 582         &pand           ($D2,$MASK);
 583         &paddd          ($T1,$D3);                      # h2 -> h3
 584          &movdqa        ($D0,$T0);
 585          &psrlq         ($T0,26);
 586         &movdqa         ($D3,$T1);
 587         &psrlq          ($T1,26);
 588          &pand          ($D0,$MASK);
 589          &paddd         ($D1,$T0);                      # h0 -> h1
 590         &pand           ($D3,$MASK);
 591         &paddd          ($D4,$T1);                      # h3 -> h4
 592 }
 593         &lazy_reduction ();
 594
 595         &dec            ("ecx");
 596         &jz             (&label("square_break"));
 597
 598         &punpcklqdq     ($D0,&QWP(16*0,"esp"));         # 0:r^1:0:r^2
 599         &punpcklqdq     ($D1,&QWP(16*1,"esp"));
 600         &punpcklqdq     ($D2,&QWP(16*2,"esp"));
 601         &punpcklqdq     ($D3,&QWP(16*3,"esp"));
 602         &punpcklqdq     ($D4,&QWP(16*4,"esp"));
 603         &jmp            (&label("square"));
 604
 605 &set_label("square_break");
 606         &psllq          ($D0,32);                       # -> r^3:0:r^4:0
 607         &psllq          ($D1,32);
 608         &psllq          ($D2,32);
 609         &psllq          ($D3,32);
 610         &psllq          ($D4,32);
 611         &por            ($D0,&QWP(16*0,"esp"));         # r^3:r^1:r^4:r^2
 612         &por            ($D1,&QWP(16*1,"esp"));
 613         &por            ($D2,&QWP(16*2,"esp"));
 614         &por            ($D3,&QWP(16*3,"esp"));
 615         &por            ($D4,&QWP(16*4,"esp"));
 616
 617         &pshufd         ($D0,$D0,0b10001101);           # -> r^1:r^2:r^3:r^4
 618         &pshufd         ($D1,$D1,0b10001101);
 619         &pshufd         ($D2,$D2,0b10001101);
 620         &pshufd         ($D3,$D3,0b10001101);
 621         &pshufd         ($D4,$D4,0b10001101);
 622
 623         &movdqu         (&QWP(16*0,"edi"),$D0);         # save the table
 624         &movdqu         (&QWP(16*1,"edi"),$D1);
 625         &movdqu         (&QWP(16*2,"edi"),$D2);
 626         &movdqu         (&QWP(16*3,"edi"),$D3);
 627         &movdqu         (&QWP(16*4,"edi"),$D4);
 628
 629         &movdqa         ($T1,$D1);
 630         &movdqa         ($T0,$D2);
 631         &pslld          ($T1,2);
 632         &pslld          ($T0,2);
 633         &paddd          ($T1,$D1);                      # *5
 634         &paddd          ($T0,$D2);                      # *5
 635         &movdqu         (&QWP(16*5,"edi"),$T1);
 636         &movdqu         (&QWP(16*6,"edi"),$T0);
 637         &movdqa         ($T1,$D3);
 638         &movdqa         ($T0,$D4);
 639         &pslld          ($T1,2);
 640         &pslld          ($T0,2);
 641         &paddd          ($T1,$D3);                      # *5
 642         &paddd          ($T0,$D4);                      # *5
 643         &movdqu         (&QWP(16*7,"edi"),$T1);
 644         &movdqu         (&QWP(16*8,"edi"),$T0);
 645
 646         &mov            ("esp","ebp");
 647         &lea            ("edi",&DWP(-16*3,"edi"));      # size de-optimization
 648         &ret            ();
 649 &function_end_B("_poly1305_init_sse2");
 650
 651 &align  (32);
 652 &function_begin("_poly1305_blocks_sse2");
 653         &mov    ("edi",&wparam(0));                     # ctx
 654         &mov    ("esi",&wparam(1));                     # inp
 655         &mov    ("ecx",&wparam(2));                     # len
 656
 657         &mov    ("eax",&DWP(4*5,"edi"));                # is_base2_26
 658         &and    ("ecx",-16);
 659         &jz     (&label("nodata"));
 660         &cmp    ("ecx",64);
 661         &jae    (&label("enter_sse2"));
 662         &test   ("eax","eax");                          # is_base2_26?
 663         &jz     (&label("enter_blocks"));
 664
 665 &set_label("enter_sse2",16);
 666         &call   (&label("pic_point"));
 667 &set_label("pic_point");
 668         &blindpop("ebx");
 669         &lea    ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
 670
 671         &test   ("eax","eax");                          # is_base2_26?
 672         &jnz    (&label("base2_26"));
 673
 674         &call   ("_poly1305_init_sse2");
 675
 676         ################################################# base 2^32 -> base 2^26
 677         &mov    ("eax",&DWP(0,"edi"));
 678         &mov    ("ecx",&DWP(3,"edi"));
 679         &mov    ("edx",&DWP(6,"edi"));
 680         &mov    ("esi",&DWP(9,"edi"));
 681         &mov    ("ebp",&DWP(13,"edi"));
 682         &mov    (&DWP(4*5,"edi"),1);                    # is_base2_26
 683
 684         &shr    ("ecx",2);
 685         &and    ("eax",0x3ffffff);
 686         &shr    ("edx",4);
 687         &and    ("ecx",0x3ffffff);
 688         &shr    ("esi",6);
 689         &and    ("edx",0x3ffffff);
 690
 691         &movd   ($D0,"eax");
 692         &movd   ($D1,"ecx");
 693         &movd   ($D2,"edx");
 694         &movd   ($D3,"esi");
 695         &movd   ($D4,"ebp");
 696
 697         &mov    ("esi",&wparam(1));                     # [reload] inp
 698         &mov    ("ecx",&wparam(2));                     # [reload] len
 699         &jmp    (&label("base2_32"));
 700
 701 &set_label("base2_26",16);
 702         &movd   ($D0,&DWP(4*0,"edi"));                  # load hash value
 703         &movd   ($D1,&DWP(4*1,"edi"));
 704         &movd   ($D2,&DWP(4*2,"edi"));
 705         &movd   ($D3,&DWP(4*3,"edi"));
 706         &movd   ($D4,&DWP(4*4,"edi"));
 707         &movdqa ($MASK,&QWP(64,"ebx"));
 708
 709 &set_label("base2_32");
 710         &mov    ("eax",&wparam(3));                     # padbit
 711         &mov    ("ebp","esp");
 712
 713         &sub    ("esp",16*(5+5+5+9+9));
 714         &and    ("esp",-16);
 715
 716         &lea    ("edi",&DWP(16*3,"edi"));               # size optimization
 717         &shl    ("eax",24);                             # padbit
 718
 719         &test   ("ecx",31);
 720         &jz     (&label("even"));
 721
 722         ################################################################
 723         # process single block, with SSE2, because it's still faster
 724         # even though half of result is discarded
 725
 726         &movdqu         ($T1,&QWP(0,"esi"));            # input
 727         &lea            ("esi",&DWP(16,"esi"));
 728
 729         &movdqa         ($T0,$T1);                      # -> base 2^26 ...
 730         &pand           ($T1,$MASK);
 731         &paddd          ($D0,$T1);                      # ... and accumuate
 732
 733         &movdqa         ($T1,$T0);
 734         &psrlq          ($T0,26);
 735         &psrldq         ($T1,6);
 736         &pand           ($T0,$MASK);
 737         &paddd          ($D1,$T0);
 738
 739         &movdqa         ($T0,$T1);
 740         &psrlq          ($T1,4);
 741         &pand           ($T1,$MASK);
 742         &paddd          ($D2,$T1);
 743
 744         &movdqa         ($T1,$T0);
 745         &psrlq          ($T0,30);
 746         &pand           ($T0,$MASK);
 747         &psrldq         ($T1,7);
 748         &paddd          ($D3,$T0);
 749
 750         &movd           ($T0,"eax");                    # padbit
 751         &paddd          ($D4,$T1);
 752          &movd          ($T1,&DWP(16*0+12,"edi"));      # r0
 753         &paddd          ($D4,$T0);
 754
 755         &movdqa         (&QWP(16*0,"esp"),$D0);
 756         &movdqa         (&QWP(16*1,"esp"),$D1);
 757         &movdqa         (&QWP(16*2,"esp"),$D2);
 758         &movdqa         (&QWP(16*3,"esp"),$D3);
 759         &movdqa         (&QWP(16*4,"esp"),$D4);
 760
 761         ################################################################
 762         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
 763         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
 764         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
 765         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
 766         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
 767
 768         &pmuludq        ($D0,$T1);                      # h4*r0
 769         &pmuludq        ($D1,$T1);                      # h3*r0
 770         &pmuludq        ($D2,$T1);                      # h2*r0
 771          &movd          ($T0,&DWP(16*1+12,"edi"));      # r1
 772         &pmuludq        ($D3,$T1);                      # h1*r0
 773         &pmuludq        ($D4,$T1);                      # h0*r0
 774
 775         &pmuladd        (sub {  my ($reg,$i)=@_;
 776                                 &movd ($reg,&DWP(16*$i+12,"edi"));
 777                              });
 778
 779         &lazy_reduction ();
 780
 781         &sub            ("ecx",16);
 782         &jz             (&label("done"));
 783
 784 &set_label("even");
 785         &lea            ("edx",&DWP(16*(5+5+5+9),"esp"));# size optimization
 786         &lea            ("eax",&DWP(-16*2,"esi"));
 787         &sub            ("ecx",64);
 788
 789         ################################################################
 790         # expand and copy pre-calculated table to stack
 791
 792         &movdqu         ($T0,&QWP(16*0,"edi"));         # r^1:r^2:r^3:r^4
 793         &pshufd         ($T1,$T0,0b01000100);           # duplicate r^3:r^4
 794         &cmovb          ("esi","eax");
 795         &pshufd         ($T0,$T0,0b11101110);           # duplicate r^1:r^2
 796         &movdqa         (&QWP(16*0,"edx"),$T1);
 797         &lea            ("eax",&DWP(16*10,"esp"));
 798         &movdqu         ($T1,&QWP(16*1,"edi"));
 799         &movdqa         (&QWP(16*(0-9),"edx"),$T0);
 800         &pshufd         ($T0,$T1,0b01000100);
 801         &pshufd         ($T1,$T1,0b11101110);
 802         &movdqa         (&QWP(16*1,"edx"),$T0);
 803         &movdqu         ($T0,&QWP(16*2,"edi"));
 804         &movdqa         (&QWP(16*(1-9),"edx"),$T1);
 805         &pshufd         ($T1,$T0,0b01000100);
 806         &pshufd         ($T0,$T0,0b11101110);
 807         &movdqa         (&QWP(16*2,"edx"),$T1);
 808         &movdqu         ($T1,&QWP(16*3,"edi"));
 809         &movdqa         (&QWP(16*(2-9),"edx"),$T0);
 810         &pshufd         ($T0,$T1,0b01000100);
 811         &pshufd         ($T1,$T1,0b11101110);
 812         &movdqa         (&QWP(16*3,"edx"),$T0);
 813         &movdqu         ($T0,&QWP(16*4,"edi"));
 814         &movdqa         (&QWP(16*(3-9),"edx"),$T1);
 815         &pshufd         ($T1,$T0,0b01000100);
 816         &pshufd         ($T0,$T0,0b11101110);
 817         &movdqa         (&QWP(16*4,"edx"),$T1);
 818         &movdqu         ($T1,&QWP(16*5,"edi"));
 819         &movdqa         (&QWP(16*(4-9),"edx"),$T0);
 820         &pshufd         ($T0,$T1,0b01000100);
 821         &pshufd         ($T1,$T1,0b11101110);
 822         &movdqa         (&QWP(16*5,"edx"),$T0);
 823         &movdqu         ($T0,&QWP(16*6,"edi"));
 824         &movdqa         (&QWP(16*(5-9),"edx"),$T1);
 825         &pshufd         ($T1,$T0,0b01000100);
 826         &pshufd         ($T0,$T0,0b11101110);
 827         &movdqa         (&QWP(16*6,"edx"),$T1);
 828         &movdqu         ($T1,&QWP(16*7,"edi"));
 829         &movdqa         (&QWP(16*(6-9),"edx"),$T0);
 830         &pshufd         ($T0,$T1,0b01000100);
 831         &pshufd         ($T1,$T1,0b11101110);
 832         &movdqa         (&QWP(16*7,"edx"),$T0);
 833         &movdqu         ($T0,&QWP(16*8,"edi"));
 834         &movdqa         (&QWP(16*(7-9),"edx"),$T1);
 835         &pshufd         ($T1,$T0,0b01000100);
 836         &pshufd         ($T0,$T0,0b11101110);
 837         &movdqa         (&QWP(16*8,"edx"),$T1);
 838         &movdqa         (&QWP(16*(8-9),"edx"),$T0);
 839
 840 sub load_input {
 841 my ($inpbase,$offbase)=@_;
 842
 843         &movdqu         ($T0,&QWP($inpbase+0,"esi"));   # load input
 844         &movdqu         ($T1,&QWP($inpbase+16,"esi"));
 845         &lea            ("esi",&DWP(16*2,"esi"));
 846
 847         &movdqa         (&QWP($offbase+16*2,"esp"),$D2);
 848         &movdqa         (&QWP($offbase+16*3,"esp"),$D3);
 849         &movdqa         (&QWP($offbase+16*4,"esp"),$D4);
 850
 851         &movdqa         ($D2,$T0);                      # splat input
 852         &movdqa         ($D3,$T1);
 853         &psrldq         ($D2,6);
 854         &psrldq         ($D3,6);
 855         &movdqa         ($D4,$T0);
 856         &punpcklqdq     ($D2,$D3);                      # 2:3
 857         &punpckhqdq     ($D4,$T1);                      # 4
 858         &punpcklqdq     ($T0,$T1);                      # 0:1
 859
 860         &movdqa         ($D3,$D2);
 861         &psrlq          ($D2,4);
 862         &psrlq          ($D3,30);
 863         &movdqa         ($T1,$T0);
 864         &psrlq          ($D4,40);                       # 4
 865         &psrlq          ($T1,26);
 866         &pand           ($T0,$MASK);                    # 0
 867         &pand           ($T1,$MASK);                    # 1
 868         &pand           ($D2,$MASK);                    # 2
 869         &pand           ($D3,$MASK);                    # 3
 870         &por            ($D4,&QWP(0,"ebx"));            # padbit, yes, always
 871
 872         &movdqa         (&QWP($offbase+16*0,"esp"),$D0) if ($offbase);
 873         &movdqa         (&QWP($offbase+16*1,"esp"),$D1) if ($offbase);
 874 }
 875         &load_input     (16*2,16*5);
 876
 877         &jbe            (&label("skip_loop"));
 878         &jmp            (&label("loop"));
 879
 880 &set_label("loop",32);
 881         ################################################################
 882         # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
 883         # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
 884         #   \___________________/
 885         # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
 886         # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
 887         #   \___________________/ \____________________/
 888         ################################################################
 889
 890         &movdqa         ($T2,&QWP(16*(0-9),"edx"));     # r0^2
 891         &movdqa         (&QWP(16*1,"eax"),$T1);
 892         &movdqa         (&QWP(16*2,"eax"),$D2);
 893         &movdqa         (&QWP(16*3,"eax"),$D3);
 894         &movdqa         (&QWP(16*4,"eax"),$D4);
 895
 896         ################################################################
 897         # d4 = h4*r0 + h0*r4   + h1*r3   + h2*r2   + h3*r1
 898         # d3 = h3*r0 + h0*r3   + h1*r2   + h2*r1   + h4*5*r4
 899         # d2 = h2*r0 + h0*r2   + h1*r1   + h3*5*r4 + h4*5*r3
 900         # d1 = h1*r0 + h0*r1   + h2*5*r4 + h3*5*r3 + h4*5*r2
 901         # d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
 902
 903         &movdqa         ($D1,$T0);
 904         &pmuludq        ($T0,$T2);                      # h0*r0
 905         &movdqa         ($D0,$T1);
 906         &pmuludq        ($T1,$T2);                      # h1*r0
 907         &pmuludq        ($D2,$T2);                      # h2*r0
 908         &pmuludq        ($D3,$T2);                      # h3*r0
 909         &pmuludq        ($D4,$T2);                      # h4*r0
 910
 911 sub pmuladd_alt {
 912 my $addr = shift;
 913
 914         &pmuludq        ($D0,&$addr(8));                # h1*s4
 915         &movdqa         ($T2,$D1);
 916         &pmuludq        ($D1,&$addr(1));                # h0*r1
 917         &paddq          ($D0,$T0);
 918         &movdqa         ($T0,$T2);
 919         &pmuludq        ($T2,&$addr(2));                # h0*r2
 920         &paddq          ($D1,$T1);
 921         &movdqa         ($T1,$T0);
 922         &pmuludq        ($T0,&$addr(3));                # h0*r3
 923         &paddq          ($D2,$T2);
 924          &movdqa        ($T2,&QWP(16*1,"eax"));         # pull h1
 925         &pmuludq        ($T1,&$addr(4));                # h0*r4
 926         &paddq          ($D3,$T0);
 927
 928         &movdqa         ($T0,$T2);
 929         &pmuludq        ($T2,&$addr(1));                # h1*r1
 930          &paddq         ($D4,$T1);
 931         &movdqa         ($T1,$T0);
 932         &pmuludq        ($T0,&$addr(2));                # h1*r2
 933         &paddq          ($D2,$T2);
 934         &movdqa         ($T2,&QWP(16*2,"eax"));         # pull h2
 935         &pmuludq        ($T1,&$addr(3));                # h1*r3
 936         &paddq          ($D3,$T0);
 937         &movdqa         ($T0,$T2);
 938         &pmuludq        ($T2,&$addr(7));                # h2*s3
 939         &paddq          ($D4,$T1);
 940         &movdqa         ($T1,$T0);
 941         &pmuludq        ($T0,&$addr(8));                # h2*s4
 942         &paddq          ($D0,$T2);
 943
 944         &movdqa         ($T2,$T1);
 945         &pmuludq        ($T1,&$addr(1));                # h2*r1
 946          &paddq         ($D1,$T0);
 947         &movdqa         ($T0,&QWP(16*3,"eax"));         # pull h3
 948         &pmuludq        ($T2,&$addr(2));                # h2*r2
 949         &paddq          ($D3,$T1);
 950         &movdqa         ($T1,$T0);
 951         &pmuludq        ($T0,&$addr(6));                # h3*s2
 952         &paddq          ($D4,$T2);
 953         &movdqa         ($T2,$T1);
 954         &pmuludq        ($T1,&$addr(7));                # h3*s3
 955         &paddq          ($D0,$T0);
 956         &movdqa         ($T0,$T2);
 957         &pmuludq        ($T2,&$addr(8));                # h3*s4
 958         &paddq          ($D1,$T1);
 959
 960         &movdqa         ($T1,&QWP(16*4,"eax"));         # pull h4
 961         &pmuludq        ($T0,&$addr(1));                # h3*r1
 962          &paddq         ($D2,$T2);
 963         &movdqa         ($T2,$T1);
 964         &pmuludq        ($T1,&$addr(8));                # h4*s4
 965         &paddq          ($D4,$T0);
 966         &movdqa         ($T0,$T2);
 967         &pmuludq        ($T2,&$addr(5));                # h4*s1
 968         &paddq          ($D3,$T1);
 969         &movdqa         ($T1,$T0);
 970         &pmuludq        ($T0,&$addr(6));                # h4*s2
 971         &paddq          ($D0,$T2);
 972          &movdqa        ($MASK,&QWP(64,"ebx"));
 973         &pmuludq        ($T1,&$addr(7));                # h4*s3
 974         &paddq          ($D1,$T0);
 975         &paddq          ($D2,$T1);
 976 }
 977         &pmuladd_alt    (sub {  my $i=shift; &QWP(16*($i-9),"edx");     });
 978
 979         &load_input     (-16*2,0);
 980         &lea            ("eax",&DWP(-16*2,"esi"));
 981         &sub            ("ecx",64);
 982
 983         &paddd          ($T0,&QWP(16*(5+0),"esp"));     # add hash value
 984         &paddd          ($T1,&QWP(16*(5+1),"esp"));
 985         &paddd          ($D2,&QWP(16*(5+2),"esp"));
 986         &paddd          ($D3,&QWP(16*(5+3),"esp"));
 987         &paddd          ($D4,&QWP(16*(5+4),"esp"));
 988
 989         &cmovb          ("esi","eax");
 990         &lea            ("eax",&DWP(16*10,"esp"));
 991
 992         &movdqa         ($T2,&QWP(16*0,"edx"));         # r0^4
 993         &movdqa         (&QWP(16*1,"esp"),$D1);
 994         &movdqa         (&QWP(16*1,"eax"),$T1);
 995         &movdqa         (&QWP(16*2,"eax"),$D2);
 996         &movdqa         (&QWP(16*3,"eax"),$D3);
 997         &movdqa         (&QWP(16*4,"eax"),$D4);
 998
 999         ################################################################
1000         # d4 += h4*r0 + h0*r4   + h1*r3   + h2*r2   + h3*r1
1001         # d3 += h3*r0 + h0*r3   + h1*r2   + h2*r1   + h4*5*r4
1002         # d2 += h2*r0 + h0*r2   + h1*r1   + h3*5*r4 + h4*5*r3
1003         # d1 += h1*r0 + h0*r1   + h2*5*r4 + h3*5*r3 + h4*5*r2
1004         # d0 += h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
1005
1006         &movdqa         ($D1,$T0);
1007         &pmuludq        ($T0,$T2);                      # h0*r0
1008         &paddq          ($T0,$D0);
1009         &movdqa         ($D0,$T1);
1010         &pmuludq        ($T1,$T2);                      # h1*r0
1011         &pmuludq        ($D2,$T2);                      # h2*r0
1012         &pmuludq        ($D3,$T2);                      # h3*r0
1013         &pmuludq        ($D4,$T2);                      # h4*r0
1014
1015         &paddq          ($T1,&QWP(16*1,"esp"));
1016         &paddq          ($D2,&QWP(16*2,"esp"));
1017         &paddq          ($D3,&QWP(16*3,"esp"));
1018         &paddq          ($D4,&QWP(16*4,"esp"));
1019
1020         &pmuladd_alt    (sub {  my $i=shift; &QWP(16*$i,"edx"); });
1021
1022         &lazy_reduction ();
1023
1024         &load_input     (16*2,16*5);
1025
1026         &ja             (&label("loop"));
1027
1028 &set_label("skip_loop");
1029         ################################################################
1030         # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
1031
1032          &pshufd        ($T2,&QWP(16*(0-9),"edx"),0x10);# r0^n
1033         &add            ("ecx",32);
1034         &jnz            (&label("long_tail"));
1035
1036         &paddd          ($T0,$D0);                      # add hash value
1037         &paddd          ($T1,$D1);
1038         &paddd          ($D2,&QWP(16*7,"esp"));
1039         &paddd          ($D3,&QWP(16*8,"esp"));
1040         &paddd          ($D4,&QWP(16*9,"esp"));
1041
1042 &set_label("long_tail");
1043
1044         &movdqa         (&QWP(16*0,"eax"),$T0);
1045         &movdqa         (&QWP(16*1,"eax"),$T1);
1046         &movdqa         (&QWP(16*2,"eax"),$D2);
1047         &movdqa         (&QWP(16*3,"eax"),$D3);
1048         &movdqa         (&QWP(16*4,"eax"),$D4);
1049
1050         ################################################################
1051         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
1052         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
1053         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
1054         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
1055         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1056
1057         &pmuludq        ($T0,$T2);                      # h0*r0
1058         &pmuludq        ($T1,$T2);                      # h1*r0
1059         &pmuludq        ($D2,$T2);                      # h2*r0
1060         &movdqa         ($D0,$T0);
1061          &pshufd        ($T0,&QWP(16*(1-9),"edx"),0x10);# r1^n
1062         &pmuludq        ($D3,$T2);                      # h3*r0
1063         &movdqa         ($D1,$T1);
1064         &pmuludq        ($D4,$T2);                      # h4*r0
1065
1066         &pmuladd        (sub {  my ($reg,$i)=@_;
1067                                 &pshufd ($reg,&QWP(16*($i-9),"edx"),0x10);
1068                              },"eax");
1069
1070         &jz             (&label("short_tail"));
1071
1072         &load_input     (-16*2,0);
1073
1074          &pshufd        ($T2,&QWP(16*0,"edx"),0x10);    # r0^n
1075         &paddd          ($T0,&QWP(16*5,"esp"));         # add hash value
1076         &paddd          ($T1,&QWP(16*6,"esp"));
1077         &paddd          ($D2,&QWP(16*7,"esp"));
1078         &paddd          ($D3,&QWP(16*8,"esp"));
1079         &paddd          ($D4,&QWP(16*9,"esp"));
1080
1081         ################################################################
1082         # multiply inp[0:1] by r^4:r^3 and accumulate
1083
1084         &movdqa         (&QWP(16*0,"esp"),$T0);
1085         &pmuludq        ($T0,$T2);                      # h0*r0
1086         &movdqa         (&QWP(16*1,"esp"),$T1);
1087         &pmuludq        ($T1,$T2);                      # h1*r0
1088         &paddq          ($D0,$T0);
1089         &movdqa         ($T0,$D2);
1090         &pmuludq        ($D2,$T2);                      # h2*r0
1091         &paddq          ($D1,$T1);
1092         &movdqa         ($T1,$D3);
1093         &pmuludq        ($D3,$T2);                      # h3*r0
1094         &paddq          ($D2,&QWP(16*2,"esp"));
1095         &movdqa         (&QWP(16*2,"esp"),$T0);
1096          &pshufd        ($T0,&QWP(16*1,"edx"),0x10);    # r1^n
1097         &paddq          ($D3,&QWP(16*3,"esp"));
1098         &movdqa         (&QWP(16*3,"esp"),$T1);
1099         &movdqa         ($T1,$D4);
1100         &pmuludq        ($D4,$T2);                      # h4*r0
1101         &paddq          ($D4,&QWP(16*4,"esp"));
1102         &movdqa         (&QWP(16*4,"esp"),$T1);
1103
1104         &pmuladd        (sub {  my ($reg,$i)=@_;
1105                                 &pshufd ($reg,&QWP(16*$i,"edx"),0x10);
1106                              });
1107
1108 &set_label("short_tail");
1109
1110         ################################################################
1111         # horizontal addition
1112
1113         &pshufd         ($T1,$D4,0b01001110);
1114         &pshufd         ($T0,$D3,0b01001110);
1115         &paddq          ($D4,$T1);
1116         &paddq          ($D3,$T0);
1117         &pshufd         ($T1,$D0,0b01001110);
1118         &pshufd         ($T0,$D1,0b01001110);
1119         &paddq          ($D0,$T1);
1120         &paddq          ($D1,$T0);
1121         &pshufd         ($T1,$D2,0b01001110);
1122         #&paddq         ($D2,$T1);
1123
1124         &lazy_reduction (sub { &paddq ($D2,$T1) });
1125
1126 &set_label("done");
1127         &movd           (&DWP(-16*3+4*0,"edi"),$D0);    # store hash value
1128         &movd           (&DWP(-16*3+4*1,"edi"),$D1);
1129         &movd           (&DWP(-16*3+4*2,"edi"),$D2);
1130         &movd           (&DWP(-16*3+4*3,"edi"),$D3);
1131         &movd           (&DWP(-16*3+4*4,"edi"),$D4);
1132         &mov    ("esp","ebp");
1133 &set_label("nodata");
1134 &function_end("_poly1305_blocks_sse2");
1135
1136 &align  (32);
1137 &function_begin("_poly1305_emit_sse2");
1138         &mov    ("ebp",&wparam(0));             # context
1139
1140         &cmp    (&DWP(4*5,"ebp"),0);            # is_base2_26?
1141         &je     (&label("enter_emit"));
1142
1143         &mov    ("eax",&DWP(4*0,"ebp"));        # load hash value
1144         &mov    ("edi",&DWP(4*1,"ebp"));
1145         &mov    ("ecx",&DWP(4*2,"ebp"));
1146         &mov    ("edx",&DWP(4*3,"ebp"));
1147         &mov    ("esi",&DWP(4*4,"ebp"));
1148
1149         &mov    ("ebx","edi");                  # base 2^26 -> base 2^32
1150         &shl    ("edi",26);
1151         &shr    ("ebx",6);
1152         &add    ("eax","edi");
1153         &mov    ("edi","ecx");
1154         &adc    ("ebx",0);
1155
1156         &shl    ("edi",20);
1157         &shr    ("ecx",12);
1158         &add    ("ebx","edi");
1159         &mov    ("edi","edx");
1160         &adc    ("ecx",0);
1161
1162         &shl    ("edi",14);
1163         &shr    ("edx",18);
1164         &add    ("ecx","edi");
1165         &mov    ("edi","esi");
1166         &adc    ("edx",0);
1167
1168         &shl    ("edi",8);
1169         &shr    ("esi",24);
1170         &add    ("edx","edi");
1171         &adc    ("esi",0);                      # can be partially reduced
1172
1173         &mov    ("edi","esi");                  # final reduction
1174         &and    ("esi",3);
1175         &shr    ("edi",2);
1176         &lea    ("ebp",&DWP(0,"edi","edi",4));  # *5
1177          &mov   ("edi",&wparam(1));             # output
1178         &add    ("eax","ebp");
1179          &mov   ("ebp",&wparam(2));             # key
1180         &adc    ("ebx",0);
1181         &adc    ("ecx",0);
1182         &adc    ("edx",0);
1183         &adc    ("esi",0);
1184
1185         &movd   ($D0,"eax");                    # offload original hash value
1186         &add    ("eax",5);                      # compare to modulus
1187         &movd   ($D1,"ebx");
1188         &adc    ("ebx",0);
1189         &movd   ($D2,"ecx");
1190         &adc    ("ecx",0);
1191         &movd   ($D3,"edx");
1192         &adc    ("edx",0);
1193         &adc    ("esi",0);
1194         &shr    ("esi",2);                      # did it carry/borrow?
1195
1196         &neg    ("esi");                        # do we choose (hash-modulus) ...
1197         &and    ("eax","esi");
1198         &and    ("ebx","esi");
1199         &and    ("ecx","esi");
1200         &and    ("edx","esi");
1201         &mov    (&DWP(4*0,"edi"),"eax");
1202         &movd   ("eax",$D0);
1203         &mov    (&DWP(4*1,"edi"),"ebx");
1204         &movd   ("ebx",$D1);
1205         &mov    (&DWP(4*2,"edi"),"ecx");
1206         &movd   ("ecx",$D2);
1207         &mov    (&DWP(4*3,"edi"),"edx");
1208         &movd   ("edx",$D3);
1209
1210         &not    ("esi");                        # ... or original hash value?
1211         &and    ("eax","esi");
1212         &and    ("ebx","esi");
1213         &or     ("eax",&DWP(4*0,"edi"));
1214         &and    ("ecx","esi");
1215         &or     ("ebx",&DWP(4*1,"edi"));
1216         &and    ("edx","esi");
1217         &or     ("ecx",&DWP(4*2,"edi"));
1218         &or     ("edx",&DWP(4*3,"edi"));
1219
1220         &add    ("eax",&DWP(4*0,"ebp"));        # accumulate key
1221         &adc    ("ebx",&DWP(4*1,"ebp"));
1222         &mov    (&DWP(4*0,"edi"),"eax");
1223         &adc    ("ecx",&DWP(4*2,"ebp"));
1224         &mov    (&DWP(4*1,"edi"),"ebx");
1225         &adc    ("edx",&DWP(4*3,"ebp"));
1226         &mov    (&DWP(4*2,"edi"),"ecx");
1227         &mov    (&DWP(4*3,"edi"),"edx");
1228 &function_end("_poly1305_emit_sse2");
1229
1230 if ($avx>1) {
1231 ########################################################################
1232 # Note that poly1305_init_avx2 operates on %xmm, I could have used
1233 # poly1305_init_sse2...
1234
1235 &align  (32);
1236 &function_begin_B("_poly1305_init_avx2");
1237         &vmovdqu        ($D4,&QWP(4*6,"edi"));          # key base 2^32
1238         &lea            ("edi",&DWP(16*3,"edi"));       # size optimization
1239         &mov            ("ebp","esp");
1240         &sub            ("esp",16*(9+5));
1241         &and            ("esp",-16);
1242
1243         #&vpand         ($D4,$D4,&QWP(96,"ebx"));       # magic mask
1244         &vmovdqa        ($MASK,&QWP(64,"ebx"));
1245
1246         &vpand          ($D0,$D4,$MASK);                # -> base 2^26
1247         &vpsrlq         ($D1,$D4,26);
1248         &vpsrldq        ($D3,$D4,6);
1249         &vpand          ($D1,$D1,$MASK);
1250         &vpsrlq         ($D2,$D3,4)
1251         &vpsrlq         ($D3,$D3,30);
1252         &vpand          ($D2,$D2,$MASK);
1253         &vpand          ($D3,$D3,$MASK);
1254         &vpsrldq        ($D4,$D4,13);
1255
1256         &lea            ("edx",&DWP(16*9,"esp"));       # size optimization
1257         &mov            ("ecx",2);
1258 &set_label("square");
1259         &vmovdqa        (&QWP(16*0,"esp"),$D0);
1260         &vmovdqa        (&QWP(16*1,"esp"),$D1);
1261         &vmovdqa        (&QWP(16*2,"esp"),$D2);
1262         &vmovdqa        (&QWP(16*3,"esp"),$D3);
1263         &vmovdqa        (&QWP(16*4,"esp"),$D4);
1264
1265         &vpslld         ($T1,$D1,2);
1266         &vpslld         ($T0,$D2,2);
1267         &vpaddd         ($T1,$T1,$D1);                  # *5
1268         &vpaddd         ($T0,$T0,$D2);                  # *5
1269         &vmovdqa        (&QWP(16*5,"esp"),$T1);
1270         &vmovdqa        (&QWP(16*6,"esp"),$T0);
1271         &vpslld         ($T1,$D3,2);
1272         &vpslld         ($T0,$D4,2);
1273         &vpaddd         ($T1,$T1,$D3);                  # *5
1274         &vpaddd         ($T0,$T0,$D4);                  # *5
1275         &vmovdqa        (&QWP(16*7,"esp"),$T1);
1276         &vmovdqa        (&QWP(16*8,"esp"),$T0);
1277
1278         &vpshufd        ($T0,$D0,0b01000100);
1279         &vmovdqa        ($T1,$D1);
1280         &vpshufd        ($D1,$D1,0b01000100);
1281         &vpshufd        ($D2,$D2,0b01000100);
1282         &vpshufd        ($D3,$D3,0b01000100);
1283         &vpshufd        ($D4,$D4,0b01000100);
1284         &vmovdqa        (&QWP(16*0,"edx"),$T0);
1285         &vmovdqa        (&QWP(16*1,"edx"),$D1);
1286         &vmovdqa        (&QWP(16*2,"edx"),$D2);
1287         &vmovdqa        (&QWP(16*3,"edx"),$D3);
1288         &vmovdqa        (&QWP(16*4,"edx"),$D4);
1289
1290         ################################################################
1291         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
1292         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
1293         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
1294         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
1295         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1296
1297         &vpmuludq       ($D4,$D4,$D0);                  # h4*r0
1298         &vpmuludq       ($D3,$D3,$D0);                  # h3*r0
1299         &vpmuludq       ($D2,$D2,$D0);                  # h2*r0
1300         &vpmuludq       ($D1,$D1,$D0);                  # h1*r0
1301         &vpmuludq       ($D0,$T0,$D0);                  # h0*r0
1302
1303         &vpmuludq       ($T0,$T1,&QWP(16*3,"edx"));     # r1*h3
1304         &vpaddq         ($D4,$D4,$T0);
1305         &vpmuludq       ($T2,$T1,&QWP(16*2,"edx"));     # r1*h2
1306         &vpaddq         ($D3,$D3,$T2);
1307         &vpmuludq       ($T0,$T1,&QWP(16*1,"edx"));     # r1*h1
1308         &vpaddq         ($D2,$D2,$T0);
1309         &vmovdqa        ($T2,&QWP(16*5,"esp"));         # s1
1310         &vpmuludq       ($T1,$T1,&QWP(16*0,"edx"));     # r1*h0
1311         &vpaddq         ($D1,$D1,$T1);
1312          &vmovdqa       ($T0,&QWP(16*2,"esp"));         # r2
1313         &vpmuludq       ($T2,$T2,&QWP(16*4,"edx"));     # s1*h4
1314         &vpaddq         ($D0,$D0,$T2);
1315
1316         &vpmuludq       ($T1,$T0,&QWP(16*2,"edx"));     # r2*h2
1317         &vpaddq         ($D4,$D4,$T1);
1318         &vpmuludq       ($T2,$T0,&QWP(16*1,"edx"));     # r2*h1
1319         &vpaddq         ($D3,$D3,$T2);
1320         &vmovdqa        ($T1,&QWP(16*6,"esp"));         # s2
1321         &vpmuludq       ($T0,$T0,&QWP(16*0,"edx"));     # r2*h0
1322         &vpaddq         ($D2,$D2,$T0);
1323         &vpmuludq       ($T2,$T1,&QWP(16*4,"edx"));     # s2*h4
1324         &vpaddq         ($D1,$D1,$T2);
1325          &vmovdqa       ($T0,&QWP(16*3,"esp"));         # r3
1326         &vpmuludq       ($T1,$T1,&QWP(16*3,"edx"));     # s2*h3
1327         &vpaddq         ($D0,$D0,$T1);
1328
1329         &vpmuludq       ($T2,$T0,&QWP(16*1,"edx"));     # r3*h1
1330         &vpaddq         ($D4,$D4,$T2);
1331         &vmovdqa        ($T1,&QWP(16*7,"esp"));         # s3
1332         &vpmuludq       ($T0,$T0,&QWP(16*0,"edx"));     # r3*h0
1333         &vpaddq         ($D3,$D3,$T0);
1334         &vpmuludq       ($T2,$T1,&QWP(16*4,"edx"));     # s3*h4
1335         &vpaddq         ($D2,$D2,$T2);
1336         &vpmuludq       ($T0,$T1,&QWP(16*3,"edx"));     # s3*h3
1337         &vpaddq         ($D1,$D1,$T0);
1338          &vmovdqa       ($T2,&QWP(16*4,"esp"));         # r4
1339         &vpmuludq       ($T1,$T1,&QWP(16*2,"edx"));     # s3*h2
1340         &vpaddq         ($D0,$D0,$T1);
1341
1342         &vmovdqa        ($T0,&QWP(16*8,"esp"));         # s4
1343         &vpmuludq       ($T2,$T2,&QWP(16*0,"edx"));     # r4*h0
1344         &vpaddq         ($D4,$D4,$T2);
1345         &vpmuludq       ($T1,$T0,&QWP(16*4,"edx"));     # s4*h4
1346         &vpaddq         ($D3,$D3,$T1);
1347         &vpmuludq       ($T2,$T0,&QWP(16*1,"edx"));     # s4*h1
1348         &vpaddq         ($D0,$D0,$T2);
1349         &vpmuludq       ($T1,$T0,&QWP(16*2,"edx"));     # s4*h2
1350         &vpaddq         ($D1,$D1,$T1);
1351          &vmovdqa       ($MASK,&QWP(64,"ebx"));
1352         &vpmuludq       ($T0,$T0,&QWP(16*3,"edx"));     # s4*h3
1353         &vpaddq         ($D2,$D2,$T0);
1354
1355         ################################################################
1356         # lazy reduction
1357          &vpsrlq        ($T0,$D3,26);
1358          &vpand         ($D3,$D3,$MASK);
1359         &vpsrlq         ($T1,$D0,26);
1360         &vpand          ($D0,$D0,$MASK);
1361          &vpaddq        ($D4,$D4,$T0);                  # h3 -> h4
1362         &vpaddq         ($D1,$D1,$T1);                  # h0 -> h1
1363          &vpsrlq        ($T0,$D4,26);
1364          &vpand         ($D4,$D4,$MASK);
1365         &vpsrlq         ($T1,$D1,26);
1366         &vpand          ($D1,$D1,$MASK);
1367         &vpaddq         ($D2,$D2,$T1);                  # h1 -> h2
1368          &vpaddd        ($D0,$D0,$T0);
1369          &vpsllq        ($T0,$T0,2);
1370         &vpsrlq         ($T1,$D2,26);
1371         &vpand          ($D2,$D2,$MASK);
1372          &vpaddd        ($D0,$D0,$T0);                  # h4 -> h0
1373         &vpaddd         ($D3,$D3,$T1);                  # h2 -> h3
1374         &vpsrlq         ($T1,$D3,26);
1375          &vpsrlq        ($T0,$D0,26);
1376          &vpand         ($D0,$D0,$MASK);
1377         &vpand          ($D3,$D3,$MASK);
1378          &vpaddd        ($D1,$D1,$T0);                  # h0 -> h1
1379         &vpaddd         ($D4,$D4,$T1);                  # h3 -> h4
1380
1381         &dec            ("ecx");
1382         &jz             (&label("square_break"));
1383
1384         &vpunpcklqdq    ($D0,$D0,&QWP(16*0,"esp"));     # 0:r^1:0:r^2
1385         &vpunpcklqdq    ($D1,$D1,&QWP(16*1,"esp"));
1386         &vpunpcklqdq    ($D2,$D2,&QWP(16*2,"esp"));
1387         &vpunpcklqdq    ($D3,$D3,&QWP(16*3,"esp"));
1388         &vpunpcklqdq    ($D4,$D4,&QWP(16*4,"esp"));
1389         &jmp            (&label("square"));
1390
1391 &set_label("square_break");
1392         &vpsllq         ($D0,$D0,32);                   # -> r^3:0:r^4:0
1393         &vpsllq         ($D1,$D1,32);
1394         &vpsllq         ($D2,$D2,32);
1395         &vpsllq         ($D3,$D3,32);
1396         &vpsllq         ($D4,$D4,32);
1397         &vpor           ($D0,$D0,&QWP(16*0,"esp"));     # r^3:r^1:r^4:r^2
1398         &vpor           ($D1,$D1,&QWP(16*1,"esp"));
1399         &vpor           ($D2,$D2,&QWP(16*2,"esp"));
1400         &vpor           ($D3,$D3,&QWP(16*3,"esp"));
1401         &vpor           ($D4,$D4,&QWP(16*4,"esp"));
1402
1403         &vpshufd        ($D0,$D0,0b10001101);           # -> r^1:r^2:r^3:r^4
1404         &vpshufd        ($D1,$D1,0b10001101);
1405         &vpshufd        ($D2,$D2,0b10001101);
1406         &vpshufd        ($D3,$D3,0b10001101);
1407         &vpshufd        ($D4,$D4,0b10001101);
1408
1409         &vmovdqu        (&QWP(16*0,"edi"),$D0);         # save the table
1410         &vmovdqu        (&QWP(16*1,"edi"),$D1);
1411         &vmovdqu        (&QWP(16*2,"edi"),$D2);
1412         &vmovdqu        (&QWP(16*3,"edi"),$D3);
1413         &vmovdqu        (&QWP(16*4,"edi"),$D4);
1414
1415         &vpslld         ($T1,$D1,2);
1416         &vpslld         ($T0,$D2,2);
1417         &vpaddd         ($T1,$T1,$D1);                  # *5
1418         &vpaddd         ($T0,$T0,$D2);                  # *5
1419         &vmovdqu        (&QWP(16*5,"edi"),$T1);
1420         &vmovdqu        (&QWP(16*6,"edi"),$T0);
1421         &vpslld         ($T1,$D3,2);
1422         &vpslld         ($T0,$D4,2);
1423         &vpaddd         ($T1,$T1,$D3);                  # *5
1424         &vpaddd         ($T0,$T0,$D4);                  # *5
1425         &vmovdqu        (&QWP(16*7,"edi"),$T1);
1426         &vmovdqu        (&QWP(16*8,"edi"),$T0);
1427
1428         &mov            ("esp","ebp");
1429         &lea            ("edi",&DWP(-16*3,"edi"));      # size de-optimization
1430         &ret            ();
1431 &function_end_B("_poly1305_init_avx2");
1432
1433 ########################################################################
1434 # now it's time to switch to %ymm
1435
1436 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("ymm$_",(0..7));
1437 my $MASK=$T2;
1438
1439 sub X { my $reg=shift; $reg=~s/^ymm/xmm/; $reg; }
1440
1441 &align  (32);
1442 &function_begin("_poly1305_blocks_avx2");
1443         &mov    ("edi",&wparam(0));                     # ctx
1444         &mov    ("esi",&wparam(1));                     # inp
1445         &mov    ("ecx",&wparam(2));                     # len
1446
1447         &mov    ("eax",&DWP(4*5,"edi"));                # is_base2_26
1448         &and    ("ecx",-16);
1449         &jz     (&label("nodata"));
1450         &cmp    ("ecx",64);
1451         &jae    (&label("enter_avx2"));
1452         &test   ("eax","eax");                          # is_base2_26?
1453         &jz     (&label("enter_blocks"));
1454
1455 &set_label("enter_avx2");
1456         &vzeroupper     ();
1457
1458         &call   (&label("pic_point"));
1459 &set_label("pic_point");
1460         &blindpop("ebx");
1461         &lea    ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
1462
1463         &test   ("eax","eax");                          # is_base2_26?
1464         &jnz    (&label("base2_26"));
1465
1466         &call   ("_poly1305_init_avx2");
1467
1468         ################################################# base 2^32 -> base 2^26
1469         &mov    ("eax",&DWP(0,"edi"));
1470         &mov    ("ecx",&DWP(3,"edi"));
1471         &mov    ("edx",&DWP(6,"edi"));
1472         &mov    ("esi",&DWP(9,"edi"));
1473         &mov    ("ebp",&DWP(13,"edi"));
1474
1475         &shr    ("ecx",2);
1476         &and    ("eax",0x3ffffff);
1477         &shr    ("edx",4);
1478         &and    ("ecx",0x3ffffff);
1479         &shr    ("esi",6);
1480         &and    ("edx",0x3ffffff);
1481
1482         &mov    (&DWP(4*0,"edi"),"eax");
1483         &mov    (&DWP(4*1,"edi"),"ecx");
1484         &mov    (&DWP(4*2,"edi"),"edx");
1485         &mov    (&DWP(4*3,"edi"),"esi");
1486         &mov    (&DWP(4*4,"edi"),"ebp");
1487         &mov    (&DWP(4*5,"edi"),1);                    # is_base2_26
1488
1489         &mov    ("esi",&wparam(1));                     # [reload] inp
1490         &mov    ("ecx",&wparam(2));                     # [reload] len
1491
1492 &set_label("base2_26");
1493         &mov    ("eax",&wparam(3));                     # padbit
1494         &mov    ("ebp","esp");
1495
1496         &sub    ("esp",32*(5+9));
1497         &and    ("esp",-512);                           # ensure that frame
1498                                                         # doesn't cross page
1499                                                         # boundary, which is
1500                                                         # essential for
1501                                                         # misaligned 32-byte
1502                                                         # loads
1503
1504         ################################################################
1505         # expand and copy pre-calculated table to stack
1506
1507         &vmovdqu        (&X($D0),&QWP(16*(3+0),"edi"));
1508         &lea            ("edx",&DWP(32*5+128,"esp"));   # +128 size optimization
1509         &vmovdqu        (&X($D1),&QWP(16*(3+1),"edi"));
1510         &vmovdqu        (&X($D2),&QWP(16*(3+2),"edi"));
1511         &vmovdqu        (&X($D3),&QWP(16*(3+3),"edi"));
1512         &vmovdqu        (&X($D4),&QWP(16*(3+4),"edi"));
1513         &lea            ("edi",&DWP(16*3,"edi"));       # size optimization
1514         &vpermq         ($D0,$D0,0b01000000);           # 00001234 -> 12343434
1515         &vpermq         ($D1,$D1,0b01000000);
1516         &vpermq         ($D2,$D2,0b01000000);
1517         &vpermq         ($D3,$D3,0b01000000);
1518         &vpermq         ($D4,$D4,0b01000000);
1519         &vpshufd        ($D0,$D0,0b11001000);           # 12343434 -> 14243444
1520         &vpshufd        ($D1,$D1,0b11001000);
1521         &vpshufd        ($D2,$D2,0b11001000);
1522         &vpshufd        ($D3,$D3,0b11001000);
1523         &vpshufd        ($D4,$D4,0b11001000);
1524         &vmovdqa        (&QWP(32*0-128,"edx"),$D0);
1525         &vmovdqu        (&X($D0),&QWP(16*5,"edi"));
1526         &vmovdqa        (&QWP(32*1-128,"edx"),$D1);
1527         &vmovdqu        (&X($D1),&QWP(16*6,"edi"));
1528         &vmovdqa        (&QWP(32*2-128,"edx"),$D2);
1529         &vmovdqu        (&X($D2),&QWP(16*7,"edi"));
1530         &vmovdqa        (&QWP(32*3-128,"edx"),$D3);
1531         &vmovdqu        (&X($D3),&QWP(16*8,"edi"));
1532         &vmovdqa        (&QWP(32*4-128,"edx"),$D4);
1533         &vpermq         ($D0,$D0,0b01000000);
1534         &vpermq         ($D1,$D1,0b01000000);
1535         &vpermq         ($D2,$D2,0b01000000);
1536         &vpermq         ($D3,$D3,0b01000000);
1537         &vpshufd        ($D0,$D0,0b11001000);
1538         &vpshufd        ($D1,$D1,0b11001000);
1539         &vpshufd        ($D2,$D2,0b11001000);
1540         &vpshufd        ($D3,$D3,0b11001000);
1541         &vmovdqa        (&QWP(32*5-128,"edx"),$D0);
1542         &vmovd          (&X($D0),&DWP(-16*3+4*0,"edi"));# load hash value
1543         &vmovdqa        (&QWP(32*6-128,"edx"),$D1);
1544         &vmovd          (&X($D1),&DWP(-16*3+4*1,"edi"));
1545         &vmovdqa        (&QWP(32*7-128,"edx"),$D2);
1546         &vmovd          (&X($D2),&DWP(-16*3+4*2,"edi"));
1547         &vmovdqa        (&QWP(32*8-128,"edx"),$D3);
1548         &vmovd          (&X($D3),&DWP(-16*3+4*3,"edi"));
1549         &vmovd          (&X($D4),&DWP(-16*3+4*4,"edi"));
1550         &vmovdqa        ($MASK,&QWP(64,"ebx"));
1551         &neg            ("eax");                        # padbit
1552
1553         &test           ("ecx",63);
1554         &jz             (&label("even"));
1555
1556         &mov            ("edx","ecx");
1557         &and            ("ecx",-64);
1558         &and            ("edx",63);
1559
1560         &vmovdqu        (&X($T0),&QWP(16*0,"esi"));
1561         &cmp            ("edx",32);
1562         &jb             (&label("one"));
1563
1564         &vmovdqu        (&X($T1),&QWP(16*1,"esi"));
1565         &je             (&label("two"));
1566
1567         &vinserti128    ($T0,$T0,&QWP(16*2,"esi"),1);
1568         &lea            ("esi",&DWP(16*3,"esi"));
1569         &lea            ("ebx",&DWP(8,"ebx"));          # three padbits
1570         &lea            ("edx",&DWP(32*5+128+8,"esp")); # --:r^1:r^2:r^3 (*)
1571         &jmp            (&label("tail"));
1572
1573 &set_label("two");
1574         &lea            ("esi",&DWP(16*2,"esi"));
1575         &lea            ("ebx",&DWP(16,"ebx"));         # two padbits
1576         &lea            ("edx",&DWP(32*5+128+16,"esp"));# --:--:r^1:r^2 (*)
1577         &jmp            (&label("tail"));
1578
1579 &set_label("one");
1580         &lea            ("esi",&DWP(16*1,"esi"));
1581         &vpxor          ($T1,$T1,$T1);
1582         &lea            ("ebx",&DWP(32,"ebx","eax",8)); # one or no padbits
1583         &lea            ("edx",&DWP(32*5+128+24,"esp"));# --:--:--:r^1 (*)
1584         &jmp            (&label("tail"));
1585
1586 # (*)   spots marked with '--' are data from next table entry, but they
1587 #       are multiplied by 0 and therefore rendered insignificant
1588
1589 &set_label("even",32);
1590         &vmovdqu        (&X($T0),&QWP(16*0,"esi"));     # load input
1591         &vmovdqu        (&X($T1),&QWP(16*1,"esi"));
1592         &vinserti128    ($T0,$T0,&QWP(16*2,"esi"),1);
1593         &vinserti128    ($T1,$T1,&QWP(16*3,"esi"),1);
1594         &lea            ("esi",&DWP(16*4,"esi"));
1595         &sub            ("ecx",64);
1596         &jz             (&label("tail"));
1597
1598 &set_label("loop");
1599         ################################################################
1600         # ((inp[0]*r^4+r[4])*r^4+r[8])*r^4
1601         # ((inp[1]*r^4+r[5])*r^4+r[9])*r^3
1602         # ((inp[2]*r^4+r[6])*r^4+r[10])*r^2
1603         # ((inp[3]*r^4+r[7])*r^4+r[11])*r^1
1604         #   \________/ \_______/
1605         ################################################################
1606
1607 sub vsplat_input {
1608         &vmovdqa        (&QWP(32*2,"esp"),$D2);
1609         &vpsrldq        ($D2,$T0,6);                    # splat input
1610         &vmovdqa        (&QWP(32*0,"esp"),$D0);
1611         &vpsrldq        ($D0,$T1,6);
1612         &vmovdqa        (&QWP(32*1,"esp"),$D1);
1613         &vpunpckhqdq    ($D1,$T0,$T1);                  # 4
1614         &vpunpcklqdq    ($T0,$T0,$T1);                  # 0:1
1615         &vpunpcklqdq    ($D2,$D2,$D0);                  # 2:3
1616
1617         &vpsrlq         ($D0,$D2,30);
1618         &vpsrlq         ($D2,$D2,4);
1619         &vpsrlq         ($T1,$T0,26);
1620         &vpsrlq         ($D1,$D1,40);                   # 4
1621         &vpand          ($D2,$D2,$MASK);                # 2
1622         &vpand          ($T0,$T0,$MASK);                # 0
1623         &vpand          ($T1,$T1,$MASK);                # 1
1624         &vpand          ($D0,$D0,$MASK);                # 3 (*)
1625         &vpor           ($D1,$D1,&QWP(0,"ebx"));        # padbit, yes, always
1626
1627         # (*)   note that output is counterintuitive, inp[3:4] is
1628         #       returned in $D1-2, while $D3-4 are preserved;
1629 }
1630         &vsplat_input   ();
1631
1632 sub vpmuladd {
1633 my $addr = shift;
1634
1635         &vpaddq         ($D2,$D2,&QWP(32*2,"esp"));     # add hash value
1636         &vpaddq         ($T0,$T0,&QWP(32*0,"esp"));
1637         &vpaddq         ($T1,$T1,&QWP(32*1,"esp"));
1638         &vpaddq         ($D0,$D0,$D3);
1639         &vpaddq         ($D1,$D1,$D4);
1640
1641         ################################################################
1642         # d3 = h2*r1   + h0*r3 + h1*r2   + h3*r0   + h4*5*r4
1643         # d4 = h2*r2   + h0*r4 + h1*r3   + h3*r1   + h4*r0
1644         # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1
1645         # d1 = h2*5*r4 + h0*r1 + h1*r0   + h3*5*r3 + h4*5*r2
1646         # d2 = h2*r0   + h0*r2 + h1*r1   + h3*5*r4 + h4*5*r3
1647
1648         &vpmuludq       ($D3,$D2,&$addr(1));            # d3 = h2*r1
1649          &vmovdqa       (QWP(32*1,"esp"),$T1);
1650         &vpmuludq       ($D4,$D2,&$addr(2));            # d4 = h2*r2
1651          &vmovdqa       (QWP(32*3,"esp"),$D0);
1652         &vpmuludq       ($D0,$D2,&$addr(7));            # d0 = h2*s3
1653          &vmovdqa       (QWP(32*4,"esp"),$D1);
1654         &vpmuludq       ($D1,$D2,&$addr(8));            # d1 = h2*s4
1655         &vpmuludq       ($D2,$D2,&$addr(0));            # d2 = h2*r0
1656
1657         &vpmuludq       ($T2,$T0,&$addr(3));            # h0*r3
1658         &vpaddq         ($D3,$D3,$T2);                  # d3 += h0*r3
1659         &vpmuludq       ($T1,$T0,&$addr(4));            # h0*r4
1660         &vpaddq         ($D4,$D4,$T1);                  # d4 + h0*r4
1661         &vpmuludq       ($T2,$T0,&$addr(0));            # h0*r0
1662         &vpaddq         ($D0,$D0,$T2);                  # d0 + h0*r0
1663          &vmovdqa       ($T2,&QWP(32*1,"esp"));         # h1
1664         &vpmuludq       ($T1,$T0,&$addr(1));            # h0*r1
1665         &vpaddq         ($D1,$D1,$T1);                  # d1 += h0*r1
1666         &vpmuludq       ($T0,$T0,&$addr(2));            # h0*r2
1667         &vpaddq         ($D2,$D2,$T0);                  # d2 += h0*r2
1668
1669         &vpmuludq       ($T1,$T2,&$addr(2));            # h1*r2
1670         &vpaddq         ($D3,$D3,$T1);                  # d3 += h1*r2
1671         &vpmuludq       ($T0,$T2,&$addr(3));            # h1*r3
1672         &vpaddq         ($D4,$D4,$T0);                  # d4 += h1*r3
1673         &vpmuludq       ($T1,$T2,&$addr(8));            # h1*s4
1674         &vpaddq         ($D0,$D0,$T1);                  # d0 += h1*s4
1675          &vmovdqa       ($T1,&QWP(32*3,"esp"));         # h3
1676         &vpmuludq       ($T0,$T2,&$addr(0));            # h1*r0
1677         &vpaddq         ($D1,$D1,$T0);                  # d1 += h1*r0
1678         &vpmuludq       ($T2,$T2,&$addr(1));            # h1*r1
1679         &vpaddq         ($D2,$D2,$T2);                  # d2 += h1*r1
1680
1681         &vpmuludq       ($T0,$T1,&$addr(0));            # h3*r0
1682         &vpaddq         ($D3,$D3,$T0);                  # d3 += h3*r0
1683         &vpmuludq       ($T2,$T1,&$addr(1));            # h3*r1
1684         &vpaddq         ($D4,$D4,$T2);                  # d4 += h3*r1
1685         &vpmuludq       ($T0,$T1,&$addr(6));            # h3*s2
1686         &vpaddq         ($D0,$D0,$T0);                  # d0 += h3*s2
1687          &vmovdqa       ($T0,&QWP(32*4,"esp"));         # h4
1688         &vpmuludq       ($T2,$T1,&$addr(7));            # h3*s3
1689         &vpaddq         ($D1,$D1,$T2);                  # d1+= h3*s3
1690         &vpmuludq       ($T1,$T1,&$addr(8));            # h3*s4
1691         &vpaddq         ($D2,$D2,$T1);                  # d2 += h3*s4
1692
1693         &vpmuludq       ($T2,$T0,&$addr(8));            # h4*s4
1694         &vpaddq         ($D3,$D3,$T2);                  # d3 += h4*s4
1695         &vpmuludq       ($T1,$T0,&$addr(5));            # h4*s1
1696         &vpaddq         ($D0,$D0,$T1);                  # d0 += h4*s1
1697         &vpmuludq       ($T2,$T0,&$addr(0));            # h4*r0
1698         &vpaddq         ($D4,$D4,$T2);                  # d4 += h4*r0
1699          &vmovdqa       ($MASK,&QWP(64,"ebx"));
1700         &vpmuludq       ($T1,$T0,&$addr(6));            # h4*s2
1701         &vpaddq         ($D1,$D1,$T1);                  # d1 += h4*s2
1702         &vpmuludq       ($T0,$T0,&$addr(7));            # h4*s3
1703         &vpaddq         ($D2,$D2,$T0);                  # d2 += h4*s3
1704 }
1705         &vpmuladd       (sub {  my $i=shift; &QWP(32*$i-128,"edx");     });
1706
1707 sub vlazy_reduction {
1708         ################################################################
1709         # lazy reduction
1710
1711          &vpsrlq        ($T0,$D3,26);
1712          &vpand         ($D3,$D3,$MASK);
1713         &vpsrlq         ($T1,$D0,26);
1714         &vpand          ($D0,$D0,$MASK);
1715          &vpaddq        ($D4,$D4,$T0);                  # h3 -> h4
1716         &vpaddq         ($D1,$D1,$T1);                  # h0 -> h1
1717          &vpsrlq        ($T0,$D4,26);
1718          &vpand         ($D4,$D4,$MASK);
1719         &vpsrlq         ($T1,$D1,26);
1720         &vpand          ($D1,$D1,$MASK);
1721         &vpaddq         ($D2,$D2,$T1);                  # h1 -> h2
1722          &vpaddq        ($D0,$D0,$T0);
1723          &vpsllq        ($T0,$T0,2);
1724         &vpsrlq         ($T1,$D2,26);
1725         &vpand          ($D2,$D2,$MASK);
1726          &vpaddq        ($D0,$D0,$T0);                  # h4 -> h0
1727         &vpaddq         ($D3,$D3,$T1);                  # h2 -> h3
1728         &vpsrlq         ($T1,$D3,26);
1729          &vpsrlq        ($T0,$D0,26);
1730          &vpand         ($D0,$D0,$MASK);
1731         &vpand          ($D3,$D3,$MASK);
1732          &vpaddq        ($D1,$D1,$T0);                  # h0 -> h1
1733         &vpaddq         ($D4,$D4,$T1);                  # h3 -> h4
1734 }
1735         &vlazy_reduction();
1736
1737         &vmovdqu        (&X($T0),&QWP(16*0,"esi"));     # load input
1738         &vmovdqu        (&X($T1),&QWP(16*1,"esi"));
1739         &vinserti128    ($T0,$T0,&QWP(16*2,"esi"),1);
1740         &vinserti128    ($T1,$T1,&QWP(16*3,"esi"),1);
1741         &lea            ("esi",&DWP(16*4,"esi"));
1742         &sub            ("ecx",64);
1743         &jnz            (&label("loop"));
1744
1745 &set_label("tail");
1746         &vsplat_input   ();
1747         &and            ("ebx",-64);                    # restore pointer
1748
1749         &vpmuladd       (sub {  my $i=shift; &QWP(4+32*$i-128,"edx");   });
1750
1751         ################################################################
1752         # horizontal addition
1753
1754         &vpsrldq        ($T0,$D4,8);
1755         &vpsrldq        ($T1,$D3,8);
1756         &vpaddq         ($D4,$D4,$T0);
1757         &vpsrldq        ($T0,$D0,8);
1758         &vpaddq         ($D3,$D3,$T1);
1759         &vpsrldq        ($T1,$D1,8);
1760         &vpaddq         ($D0,$D0,$T0);
1761         &vpsrldq        ($T0,$D2,8);
1762         &vpaddq         ($D1,$D1,$T1);
1763         &vpermq         ($T1,$D4,2);                    # keep folding
1764         &vpaddq         ($D2,$D2,$T0);
1765         &vpermq         ($T0,$D3,2);
1766         &vpaddq         ($D4,$D4,$T1);
1767         &vpermq         ($T1,$D0,2);
1768         &vpaddq         ($D3,$D3,$T0);
1769         &vpermq         ($T0,$D1,2);
1770         &vpaddq         ($D0,$D0,$T1);
1771         &vpermq         ($T1,$D2,2);
1772         &vpaddq         ($D1,$D1,$T0);
1773         &vpaddq         ($D2,$D2,$T1);
1774
1775         &vlazy_reduction();
1776
1777         &cmp            ("ecx",0);
1778         &je             (&label("done"));
1779
1780         ################################################################
1781         # clear all but single word
1782
1783         &vpshufd        (&X($D0),&X($D0),0b11111100);
1784         &lea            ("edx",&DWP(32*5+128,"esp"));   # restore pointer
1785         &vpshufd        (&X($D1),&X($D1),0b11111100);
1786         &vpshufd        (&X($D2),&X($D2),0b11111100);
1787         &vpshufd        (&X($D3),&X($D3),0b11111100);
1788         &vpshufd        (&X($D4),&X($D4),0b11111100);
1789         &jmp            (&label("even"));
1790
1791 &set_label("done",16);
1792         &vmovd          (&DWP(-16*3+4*0,"edi"),&X($D0));# store hash value
1793         &vmovd          (&DWP(-16*3+4*1,"edi"),&X($D1));
1794         &vmovd          (&DWP(-16*3+4*2,"edi"),&X($D2));
1795         &vmovd          (&DWP(-16*3+4*3,"edi"),&X($D3));
1796         &vmovd          (&DWP(-16*3+4*4,"edi"),&X($D4));
1797         &vzeroupper     ();
1798         &mov    ("esp","ebp");
1799 &set_label("nodata");
1800 &function_end("_poly1305_blocks_avx2");
1801 }
1802 &set_label("const_sse2",64);
1803         &data_word(1<<24,0,     1<<24,0,        1<<24,0,        1<<24,0);
1804         &data_word(0,0,         0,0,            0,0,            0,0);
1805         &data_word(0x03ffffff,0,0x03ffffff,0,   0x03ffffff,0,   0x03ffffff,0);
1806         &data_word(0x0fffffff,0x0ffffffc,0x0ffffffc,0x0ffffffc);
1807 }
1808 &asciz  ("Poly1305 for x86, CRYPTOGAMS by <appro\@openssl.org>");
1809 &align  (4);
1810
1811 &asm_finish();
1812
1813 close STDOUT;