crypto/poly1305/asm/poly1305-x86.pl

   1 #!/usr/bin/env perl
   2 #
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9 #
  10 # This module implements Poly1305 hash for x86.
  11 #
  12 # April 2015
  13 #
  14 # Numbers are cycles per processed byte with poly1305_blocks alone,
  15 # measured with rdtsc at fixed clock frequency.
  16 #
  17 #               IALU/gcc-3.4(*) SSE2(**)        AVX2
  18 # Pentium       15.7/+80%       -
  19 # PIII          6.21/+90%       -
  20 # P4            19.8/+40%       3.24
  21 # Core 2        4.85/+90%       1.80
  22 # Westmere      4.58/+100%      1.43
  23 # Sandy Bridge  3.90/+100%      1.36
  24 # Haswell       3.88/+70%       1.18            0.72
  25 # Silvermont    11.0/+40%       4.80
  26 # VIA Nano      6.71/+90%       2.47
  27 # Sledgehammer  3.51/+180%      4.27
  28 # Bulldozer     4.53/+140%      1.31
  29 #
  30 # (*)   gcc 4.8 for some reason generated worse code;
  31 # (**)  besides SSE2 there are floating-point and AVX options; FP
  32 #       is deemed unnecessary, because pre-SSE2 processor are too
  33 #       old to care about, while it's not the fastest option on
  34 #       SSE2-capable ones; AVX is omitted, because it doesn't give
  35 #       a lot of improvement, 5-10% depending on processor;
  36
  37 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  38 push(@INC,"${dir}","${dir}../../perlasm");
  39 require "x86asm.pl";
  40
  41 &asm_init($ARGV[0],"poly1305-x86.pl",$ARGV[$#ARGV] eq "386");
  42
  43 $sse2=$avx=0;
  44 for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
  45
  46 if ($sse2) {
  47         &static_label("const_sse2");
  48         &static_label("enter_blocks");
  49         &static_label("enter_emit");
  50         &external_label("OPENSSL_ia32cap_P");
  51
  52         if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  53                         =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  54                 $avx = ($1>=2.19) + ($1>=2.22);
  55         }
  56
  57         if (!$avx && $ARGV[0] eq "win32n" &&
  58            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
  59         $avx = ($1>=2.09) + ($1>=2.10);
  60         }
  61
  62         if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) {
  63                 $avx = ($2>=3.0) + ($2>3.0);
  64         }
  65 }
  66
  67 ########################################################################
  68 # Layout of opaque area is following.
  69 #
  70 #       unsigned __int32 h[5];          # current hash value base 2^32
  71 #       unsigned __int32 pad;           # is_base2_26 in vector context
  72 #       unsigned __int32 r[4];          # key value base 2^32
  73
  74 &align(64);
  75 &function_begin("poly1305_init");
  76         &mov    ("edi",&wparam(0));             # context
  77         &mov    ("esi",&wparam(1));             # key
  78         &mov    ("ebp",&wparam(2));             # function table
  79
  80         &xor    ("eax","eax");
  81         &mov    (&DWP(4*0,"edi"),"eax");        # zero hash value
  82         &mov    (&DWP(4*1,"edi"),"eax");
  83         &mov    (&DWP(4*2,"edi"),"eax");
  84         &mov    (&DWP(4*3,"edi"),"eax");
  85         &mov    (&DWP(4*4,"edi"),"eax");
  86         &mov    (&DWP(4*5,"edi"),"eax");        # is_base2_26
  87
  88         &cmp    ("esi",0);
  89         &je     (&label("nokey"));
  90
  91     if ($sse2) {
  92         &call   (&label("pic_point"));
  93     &set_label("pic_point");
  94         &blindpop("ebx");
  95
  96         &lea    ("eax",&DWP("poly1305_blocks-".&label("pic_point"),"ebx"));
  97         &lea    ("edx",&DWP("poly1305_emit-".&label("pic_point"),"ebx"));
  98
  99         &picmeup("edi","OPENSSL_ia32cap_P","ebx",&label("pic_point"));
 100         &mov    ("ecx",&DWP(0,"edi"));
 101         &and    ("ecx",1<<26|1<<24);
 102         &cmp    ("ecx",1<<26|1<<24);            # SSE2 and XMM?
 103         &jne    (&label("no_sse2"));
 104
 105         &lea    ("eax",&DWP("_poly1305_blocks_sse2-".&label("pic_point"),"ebx"));
 106         &lea    ("edx",&DWP("_poly1305_emit_sse2-".&label("pic_point"),"ebx"));
 107
 108       if ($avx>1) {
 109         &mov    ("ecx",&DWP(8,"edi"));
 110         &test   ("ecx",1<<5);                   # AVX2?
 111         &jz     (&label("no_sse2"));
 112
 113         &lea    ("eax",&DWP("_poly1305_blocks_avx2-".&label("pic_point"),"ebx"));
 114       }
 115     &set_label("no_sse2");
 116         &mov    ("edi",&wparam(0));             # reload context
 117         &mov    (&DWP(0,"ebp"),"eax");          # fill function table
 118         &mov    (&DWP(4,"ebp"),"edx");
 119     }
 120
 121         &mov    ("eax",&DWP(4*0,"esi"));        # load input key
 122         &mov    ("ebx",&DWP(4*1,"esi"));
 123         &mov    ("ecx",&DWP(4*2,"esi"));
 124         &mov    ("edx",&DWP(4*3,"esi"));
 125         &and    ("eax",0x0fffffff);
 126         &and    ("ebx",0x0ffffffc);
 127         &and    ("ecx",0x0ffffffc);
 128         &and    ("edx",0x0ffffffc);
 129         &mov    (&DWP(4*6,"edi"),"eax");
 130         &mov    (&DWP(4*7,"edi"),"ebx");
 131         &mov    (&DWP(4*8,"edi"),"ecx");
 132         &mov    (&DWP(4*9,"edi"),"edx");
 133
 134         &mov    ("eax",$sse2);
 135 &set_label("nokey");
 136 &function_end("poly1305_init");
 137
 138 ($h0,$h1,$h2,$h3,$h4,
 139  $d0,$d1,$d2,$d3,
 140  $r0,$r1,$r2,$r3,
 141      $s1,$s2,$s3)=map(4*$_,(0..15));
 142
 143 &function_begin("poly1305_blocks");
 144         &mov    ("edi",&wparam(0));             # ctx
 145         &mov    ("esi",&wparam(1));             # inp
 146         &mov    ("ecx",&wparam(2));             # len
 147 &set_label("enter_blocks");
 148         &and    ("ecx",-15);
 149         &jz     (&label("nodata"));
 150
 151         &stack_push(16);
 152         &mov    ("eax",&DWP(4*6,"edi"));        # r0
 153         &mov    ("ebx",&DWP(4*7,"edi"));        # r1
 154          &lea   ("ebp",&DWP(0,"esi","ecx"));    # end of input
 155         &mov    ("ecx",&DWP(4*8,"edi"));        # r2
 156         &mov    ("edx",&DWP(4*9,"edi"));        # r3
 157
 158         &mov    (&wparam(2),"ebp");
 159         &mov    ("ebp","esi");
 160
 161         &mov    (&DWP($r0,"esp"),"eax");        # r0
 162         &mov    ("eax","ebx");
 163         &shr    ("eax",2);
 164         &mov    (&DWP($r1,"esp"),"ebx");        # r1
 165         &add    ("eax","ebx");                  # s1
 166         &mov    ("ebx","ecx");
 167         &shr    ("ebx",2);
 168         &mov    (&DWP($r2,"esp"),"ecx");        # r2
 169         &add    ("ebx","ecx");                  # s2
 170         &mov    ("ecx","edx");
 171         &shr    ("ecx",2);
 172         &mov    (&DWP($r3,"esp"),"edx");        # r3
 173         &add    ("ecx","edx");                  # s3
 174         &mov    (&DWP($s1,"esp"),"eax");        # s1
 175         &mov    (&DWP($s2,"esp"),"ebx");        # s2
 176         &mov    (&DWP($s3,"esp"),"ecx");        # s3
 177
 178         &mov    ("eax",&DWP(4*0,"edi"));        # load hash value
 179         &mov    ("ebx",&DWP(4*1,"edi"));
 180         &mov    ("ecx",&DWP(4*2,"edi"));
 181         &mov    ("esi",&DWP(4*3,"edi"));
 182         &mov    ("edi",&DWP(4*4,"edi"));
 183         &jmp    (&label("loop"));
 184
 185 &set_label("loop",32);
 186         &add    ("eax",&DWP(4*0,"ebp"));        # accumulate input
 187         &adc    ("ebx",&DWP(4*1,"ebp"));
 188         &adc    ("ecx",&DWP(4*2,"ebp"));
 189         &adc    ("esi",&DWP(4*3,"ebp"));
 190         &lea    ("ebp",&DWP(4*4,"ebp"));
 191         &adc    ("edi",&wparam(3));             # padbit
 192
 193         &mov    (&DWP($h0,"esp"),"eax");        # put aside hash[+inp]
 194         &mov    (&DWP($h3,"esp"),"esi");
 195
 196         &mul    (&DWP($r0,"esp"));              # h0*r0
 197          &mov   (&DWP($h4,"esp"),"edi");
 198         &mov    ("edi","eax");
 199         &mov    ("eax","ebx");                  # h1
 200         &mov    ("esi","edx");
 201         &mul    (&DWP($s3,"esp"));              # h1*s3
 202         &add    ("edi","eax");
 203         &mov    ("eax","ecx");                  # h2
 204         &adc    ("esi","edx");
 205         &mul    (&DWP($s2,"esp"));              # h2*s2
 206         &add    ("edi","eax");
 207         &mov    ("eax",&DWP($h3,"esp"));
 208         &adc    ("esi","edx");
 209         &mul    (&DWP($s1,"esp"));              # h3*s1
 210         &add    ("edi","eax");
 211          &mov   ("eax",&DWP($h0,"esp"));
 212         &adc    ("esi","edx");
 213
 214         &mul    (&DWP($r1,"esp"));              # h0*r1
 215          &mov   (&DWP($d0,"esp"),"edi");
 216         &xor    ("edi","edi");
 217         &add    ("esi","eax");
 218         &mov    ("eax","ebx");                  # h1
 219         &adc    ("edi","edx");
 220         &mul    (&DWP($r0,"esp"));              # h1*r0
 221         &add    ("esi","eax");
 222         &mov    ("eax","ecx");                  # h2
 223         &adc    ("edi","edx");
 224         &mul    (&DWP($s3,"esp"));              # h2*s3
 225         &add    ("esi","eax");
 226         &mov    ("eax",&DWP($h3,"esp"));
 227         &adc    ("edi","edx");
 228         &mul    (&DWP($s2,"esp"));              # h3*s2
 229         &add    ("esi","eax");
 230         &mov    ("eax",&DWP($h4,"esp"));
 231         &adc    ("edi","edx");
 232         &imul   ("eax",&DWP($s1,"esp"));        # h4*s1
 233         &add    ("esi","eax");
 234          &mov   ("eax",&DWP($h0,"esp"));
 235         &adc    ("edi",0);
 236
 237         &mul    (&DWP($r2,"esp"));              # h0*r2
 238          &mov   (&DWP($d1,"esp"),"esi");
 239         &xor    ("esi","esi");
 240         &add    ("edi","eax");
 241         &mov    ("eax","ebx");                  # h1
 242         &adc    ("esi","edx");
 243         &mul    (&DWP($r1,"esp"));              # h1*r1
 244         &add    ("edi","eax");
 245         &mov    ("eax","ecx");                  # h2
 246         &adc    ("esi","edx");
 247         &mul    (&DWP($r0,"esp"));              # h2*r0
 248         &add    ("edi","eax");
 249         &mov    ("eax",&DWP($h3,"esp"));
 250         &adc    ("esi","edx");
 251         &mul    (&DWP($s3,"esp"));              # h3*s3
 252         &add    ("edi","eax");
 253         &mov    ("eax",&DWP($h4,"esp"));
 254         &adc    ("esi","edx");
 255         &imul   ("eax",&DWP($s2,"esp"));        # h4*s2
 256         &add    ("edi","eax");
 257          &mov   ("eax",&DWP($h0,"esp"));
 258         &adc    ("esi",0);
 259
 260         &mul    (&DWP($r3,"esp"));              # h0*r3
 261          &mov   (&DWP($d2,"esp"),"edi");
 262         &xor    ("edi","edi");
 263         &add    ("esi","eax");
 264         &mov    ("eax","ebx");                  # h1
 265         &adc    ("edi","edx");
 266         &mul    (&DWP($r2,"esp"));              # h1*r2
 267         &add    ("esi","eax");
 268         &mov    ("eax","ecx");                  # h2
 269         &adc    ("edi","edx");
 270         &mul    (&DWP($r1,"esp"));              # h2*r1
 271         &add    ("esi","eax");
 272         &mov    ("eax",&DWP($h3,"esp"));
 273         &adc    ("edi","edx");
 274         &mul    (&DWP($r0,"esp"));              # h3*r0
 275         &add    ("esi","eax");
 276          &mov   ("ecx",&DWP($h4,"esp"));
 277         &adc    ("edi","edx");
 278
 279         &mov    ("edx","ecx");
 280         &imul   ("ecx",&DWP($s3,"esp"));        # h4*s3
 281         &add    ("esi","ecx");
 282          &mov   ("eax",&DWP($d0,"esp"));
 283         &adc    ("edi",0);
 284
 285         &imul   ("edx",&DWP($r0,"esp"));        # h4*r0
 286         &add    ("edx","edi");
 287
 288         &mov    ("ebx",&DWP($d1,"esp"));
 289         &mov    ("ecx",&DWP($d2,"esp"));
 290
 291         &mov    ("edi","edx");                  # last reduction step
 292         &shr    ("edx",2);
 293         &and    ("edi",3);
 294         &lea    ("edx",&DWP(0,"edx","edx",4));  # *5
 295         &add    ("eax","edx");
 296         &adc    ("ebx",0);
 297         &adc    ("ecx",0);
 298         &adc    ("esi",0);
 299
 300         &cmp    ("ebp",&wparam(2));             # done yet?
 301         &jne    (&label("loop"));
 302
 303         &mov    ("edx",&wparam(0));             # ctx
 304         &stack_pop(16);
 305         &mov    (&DWP(4*0,"edx"),"eax");        # store hash value
 306         &mov    (&DWP(4*1,"edx"),"ebx");
 307         &mov    (&DWP(4*2,"edx"),"ecx");
 308         &mov    (&DWP(4*3,"edx"),"esi");
 309         &mov    (&DWP(4*4,"edx"),"edi");
 310 &set_label("nodata");
 311 &function_end("poly1305_blocks");
 312
 313 &function_begin("poly1305_emit");
 314         &mov    ("ebp",&wparam(0));             # context
 315 &set_label("enter_emit");
 316         &mov    ("edi",&wparam(1));             # output
 317         &mov    ("eax",&DWP(4*0,"ebp"));        # load hash value
 318         &mov    ("ebx",&DWP(4*1,"ebp"));
 319         &mov    ("ecx",&DWP(4*2,"ebp"));
 320         &mov    ("edx",&DWP(4*3,"ebp"));
 321         &mov    ("esi",&DWP(4*4,"ebp"));
 322
 323         &add    ("eax",5);                      # compare to modulus
 324         &adc    ("ebx",0);
 325         &adc    ("ecx",0);
 326         &adc    ("edx",0);
 327         &adc    ("esi",0);
 328         &shr    ("esi",2);                      # did it carry/borrow?
 329         &neg    ("esi");                        # do we choose hash-modulus?
 330
 331         &and    ("eax","esi");
 332         &and    ("ebx","esi");
 333         &and    ("ecx","esi");
 334         &and    ("edx","esi");
 335         &mov    (&DWP(4*0,"edi"),"eax");
 336         &mov    (&DWP(4*1,"edi"),"ebx");
 337         &mov    (&DWP(4*2,"edi"),"ecx");
 338         &mov    (&DWP(4*3,"edi"),"edx");
 339
 340         &not    ("esi");                        # or original hash value?
 341         &mov    ("eax",&DWP(4*0,"ebp"));
 342         &mov    ("ebx",&DWP(4*1,"ebp"));
 343         &mov    ("ecx",&DWP(4*2,"ebp"));
 344         &mov    ("edx",&DWP(4*3,"ebp"));
 345         &mov    ("ebp",&wparam(2));
 346         &and    ("eax","esi");
 347         &and    ("ebx","esi");
 348         &and    ("ecx","esi");
 349         &and    ("edx","esi");
 350         &or     ("eax",&DWP(4*0,"edi"));
 351         &or     ("ebx",&DWP(4*1,"edi"));
 352         &or     ("ecx",&DWP(4*2,"edi"));
 353         &or     ("edx",&DWP(4*3,"edi"));
 354
 355         &add    ("eax",&DWP(4*0,"ebp"));        # accumulate key
 356         &adc    ("ebx",&DWP(4*1,"ebp"));
 357         &adc    ("ecx",&DWP(4*2,"ebp"));
 358         &adc    ("edx",&DWP(4*3,"ebp"));
 359
 360         &mov    (&DWP(4*0,"edi"),"eax");
 361         &mov    (&DWP(4*1,"edi"),"ebx");
 362         &mov    (&DWP(4*2,"edi"),"ecx");
 363         &mov    (&DWP(4*3,"edi"),"edx");
 364 &function_end("poly1305_emit");
 365
 366 if ($sse2) {
 367 ########################################################################
 368 # Layout of opaque area is following.
 369 #
 370 #       unsigned __int32 h[5];          # current hash value base 2^26
 371 #       unsigned __int32 is_base2_26;
 372 #       unsigned __int32 r[4];          # key value base 2^32
 373 #       unsigned __int32 pad[2];
 374 #       struct { unsigned __int32 r^4, r^3, r^2, r^1; } r[9];
 375 #
 376 # where r^n are base 2^26 digits of degrees of multiplier key. There are
 377 # 5 digits, but last four are interleaved with multiples of 5, totalling
 378 # in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.
 379
 380 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("xmm$_",(0..7));
 381 my $MASK=$T2;   # borrow and keep in mind
 382
 383 &align  (32);
 384 &function_begin_B("_poly1305_init_sse2");
 385         &movdqu         ($D4,&QWP(4*6,"edi"));          # key base 2^32
 386         &lea            ("edi",&DWP(16*3,"edi"));       # size optimization
 387         &mov            ("ebp","esp");
 388         &sub            ("esp",16*(9+5));
 389         &and            ("esp",-16);
 390
 391         #&pand          ($D4,&QWP(96,"ebx"));           # magic mask
 392         &movq           ($MASK,&QWP(64,"ebx"));
 393
 394         &movdqa         ($D0,$D4);
 395         &movdqa         ($D1,$D4);
 396         &movdqa         ($D2,$D4);
 397
 398         &pand           ($D0,$MASK);                    # -> base 2^26
 399         &psrlq          ($D1,26);
 400         &psrldq         ($D2,6);
 401         &pand           ($D1,$MASK);
 402         &movdqa         ($D3,$D2);
 403         &psrlq          ($D2,4)
 404         &psrlq          ($D3,30);
 405         &pand           ($D2,$MASK);
 406         &pand           ($D3,$MASK);
 407         &psrldq         ($D4,13);
 408
 409         &lea            ("edx",&DWP(16*9,"esp"));       # size optimization
 410         &mov            ("ecx",2);
 411 &set_label("square");
 412         &movdqa         (&QWP(16*0,"esp"),$D0);
 413         &movdqa         (&QWP(16*1,"esp"),$D1);
 414         &movdqa         (&QWP(16*2,"esp"),$D2);
 415         &movdqa         (&QWP(16*3,"esp"),$D3);
 416         &movdqa         (&QWP(16*4,"esp"),$D4);
 417
 418         &movdqa         ($T1,$D1);
 419         &movdqa         ($T0,$D2);
 420         &pslld          ($T1,2);
 421         &pslld          ($T0,2);
 422         &paddd          ($T1,$D1);                      # *5
 423         &paddd          ($T0,$D2);                      # *5
 424         &movdqa         (&QWP(16*5,"esp"),$T1);
 425         &movdqa         (&QWP(16*6,"esp"),$T0);
 426         &movdqa         ($T1,$D3);
 427         &movdqa         ($T0,$D4);
 428         &pslld          ($T1,2);
 429         &pslld          ($T0,2);
 430         &paddd          ($T1,$D3);                      # *5
 431         &paddd          ($T0,$D4);                      # *5
 432         &movdqa         (&QWP(16*7,"esp"),$T1);
 433         &movdqa         (&QWP(16*8,"esp"),$T0);
 434
 435         &pshufd         ($T1,$D0,0b01000100);
 436         &movdqa         ($T0,$D1);
 437         &pshufd         ($D1,$D1,0b01000100);
 438         &pshufd         ($D2,$D2,0b01000100);
 439         &pshufd         ($D3,$D3,0b01000100);
 440         &pshufd         ($D4,$D4,0b01000100);
 441         &movdqa         (&QWP(16*0,"edx"),$T1);
 442         &movdqa         (&QWP(16*1,"edx"),$D1);
 443         &movdqa         (&QWP(16*2,"edx"),$D2);
 444         &movdqa         (&QWP(16*3,"edx"),$D3);
 445         &movdqa         (&QWP(16*4,"edx"),$D4);
 446
 447         ################################################################
 448         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
 449         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
 450         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
 451         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
 452         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
 453
 454         &pmuludq        ($D4,$D0);                      # h4*r0
 455         &pmuludq        ($D3,$D0);                      # h3*r0
 456         &pmuludq        ($D2,$D0);                      # h2*r0
 457         &pmuludq        ($D1,$D0);                      # h1*r0
 458         &pmuludq        ($D0,$T1);                      # h0*r0
 459
 460 sub pmuladd {
 461 my $load = shift;
 462 my $base = shift; $base = "esp" if (!defined($base));
 463
 464         ################################################################
 465         # As for choice to "rotate" $T0-$T2 in order to move paddq
 466         # past next multiplication. While it makes code harder to read
 467         # and doesn't have significant effect on most processors, it
 468         # makes a lot of difference on Atom, up to 30% improvement.
 469
 470         &movdqa         ($T1,$T0);
 471         &pmuludq        ($T0,&QWP(16*3,$base));         # r1*h3
 472         &movdqa         ($T2,$T1);
 473         &pmuludq        ($T1,&QWP(16*2,$base));         # r1*h2
 474         &paddq          ($D4,$T0);
 475         &movdqa         ($T0,$T2);
 476         &pmuludq        ($T2,&QWP(16*1,$base));         # r1*h1
 477         &paddq          ($D3,$T1);
 478         &$load          ($T1,5);                        # s1
 479         &pmuludq        ($T0,&QWP(16*0,$base));         # r1*h0
 480         &paddq          ($D2,$T2);
 481         &pmuludq        ($T1,&QWP(16*4,$base));         # s1*h4
 482          &$load         ($T2,2);                        # r2^n
 483         &paddq          ($D1,$T0);
 484
 485         &movdqa         ($T0,$T2);
 486         &pmuludq        ($T2,&QWP(16*2,$base));         # r2*h2
 487          &paddq         ($D0,$T1);
 488         &movdqa         ($T1,$T0);
 489         &pmuludq        ($T0,&QWP(16*1,$base));         # r2*h1
 490         &paddq          ($D4,$T2);
 491         &$load          ($T2,6);                        # s2^n
 492         &pmuludq        ($T1,&QWP(16*0,$base));         # r2*h0
 493         &paddq          ($D3,$T0);
 494         &movdqa         ($T0,$T2);
 495         &pmuludq        ($T2,&QWP(16*4,$base));         # s2*h4
 496         &paddq          ($D2,$T1);
 497         &pmuludq        ($T0,&QWP(16*3,$base));         # s2*h3
 498          &$load         ($T1,3);                        # r3^n
 499         &paddq          ($D1,$T2);
 500
 501         &movdqa         ($T2,$T1);
 502         &pmuludq        ($T1,&QWP(16*1,$base));         # r3*h1
 503          &paddq         ($D0,$T0);
 504         &$load          ($T0,7);                        # s3^n
 505         &pmuludq        ($T2,&QWP(16*0,$base));         # r3*h0
 506         &paddq          ($D4,$T1);
 507         &movdqa         ($T1,$T0);
 508         &pmuludq        ($T0,&QWP(16*4,$base));         # s3*h4
 509         &paddq          ($D3,$T2);
 510         &movdqa         ($T2,$T1);
 511         &pmuludq        ($T1,&QWP(16*3,$base));         # s3*h3
 512         &paddq          ($D2,$T0);
 513         &pmuludq        ($T2,&QWP(16*2,$base));         # s3*h2
 514          &$load         ($T0,4);                        # r4^n
 515         &paddq          ($D1,$T1);
 516
 517         &$load          ($T1,8);                        # s4^n
 518         &pmuludq        ($T0,&QWP(16*0,$base));         # r4*h0
 519          &paddq         ($D0,$T2);
 520         &movdqa         ($T2,$T1);
 521         &pmuludq        ($T1,&QWP(16*4,$base));         # s4*h4
 522         &paddq          ($D4,$T0);
 523         &movdqa         ($T0,$T2);
 524         &pmuludq        ($T2,&QWP(16*1,$base));         # s4*h1
 525         &paddq          ($D3,$T1);
 526         &movdqa         ($T1,$T0);
 527         &pmuludq        ($T0,&QWP(16*2,$base));         # s4*h2
 528         &paddq          ($D0,$T2);
 529         &pmuludq        ($T1,&QWP(16*3,$base));         # s4*h3
 530          &movdqa        ($MASK,&QWP(64,"ebx"));
 531         &paddq          ($D1,$T0);
 532         &paddq          ($D2,$T1);
 533 }
 534         &pmuladd        (sub {  my ($reg,$i)=@_;
 535                                 &movdqa ($reg,&QWP(16*$i,"esp"));
 536                              },"edx");
 537
 538 sub lazy_reduction {
 539 my $extra = shift;
 540
 541         ################################################################
 542         # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
 543         # and P. Schwabe
 544
 545          &movdqa        ($T0,$D3);
 546          &pand          ($D3,$MASK);
 547          &psrlq         ($T0,26);
 548          &$extra        ()                              if (defined($extra));
 549          &paddq         ($T0,$D4);                      # h3 -> h4
 550         &movdqa         ($T1,$D0);
 551         &pand           ($D0,$MASK);
 552         &psrlq          ($T1,26);
 553          &movdqa        ($D4,$T0);
 554         &paddq          ($T1,$D1);                      # h0 -> h1
 555          &psrlq         ($T0,26);
 556          &pand          ($D4,$MASK);
 557         &movdqa         ($D1,$T1);
 558         &psrlq          ($T1,26);
 559          &paddd         ($D0,$T0);                      # favour paddd when
 560                                                         # possible, because
 561                                                         # paddq is "broken"
 562                                                         # on Atom
 563         &pand           ($D1,$MASK);
 564         &paddq          ($T1,$D2);                      # h1 -> h2
 565          &psllq         ($T0,2);
 566         &movdqa         ($D2,$T1);
 567         &psrlq          ($T1,26);
 568          &paddd         ($T0,$D0);                      # h4 -> h0
 569         &pand           ($D2,$MASK);
 570         &paddd          ($T1,$D3);                      # h2 -> h3
 571          &movdqa        ($D0,$T0);
 572          &psrlq         ($T0,26);
 573         &movdqa         ($D3,$T1);
 574         &psrlq          ($T1,26);
 575          &pand          ($D0,$MASK);
 576          &paddd         ($D1,$T0);                      # h0 -> h1
 577         &pand           ($D3,$MASK);
 578         &paddd          ($D4,$T1);                      # h3 -> h4
 579 }
 580         &lazy_reduction ();
 581
 582         &dec            ("ecx");
 583         &jz             (&label("square_break"));
 584
 585         &punpcklqdq     ($D0,&QWP(16*0,"esp"));         # 0:r^1:0:r^2
 586         &punpcklqdq     ($D1,&QWP(16*1,"esp"));
 587         &punpcklqdq     ($D2,&QWP(16*2,"esp"));
 588         &punpcklqdq     ($D3,&QWP(16*3,"esp"));
 589         &punpcklqdq     ($D4,&QWP(16*4,"esp"));
 590         &jmp            (&label("square"));
 591
 592 &set_label("square_break");
 593         &psllq          ($D0,32);                       # -> r^3:0:r^4:0
 594         &psllq          ($D1,32);
 595         &psllq          ($D2,32);
 596         &psllq          ($D3,32);
 597         &psllq          ($D4,32);
 598         &por            ($D0,&QWP(16*0,"esp"));         # r^3:r^1:r^4:r^2
 599         &por            ($D1,&QWP(16*1,"esp"));
 600         &por            ($D2,&QWP(16*2,"esp"));
 601         &por            ($D3,&QWP(16*3,"esp"));
 602         &por            ($D4,&QWP(16*4,"esp"));
 603
 604         &pshufd         ($D0,$D0,0b10001101);           # -> r^1:r^2:r^3:r^4
 605         &pshufd         ($D1,$D1,0b10001101);
 606         &pshufd         ($D2,$D2,0b10001101);
 607         &pshufd         ($D3,$D3,0b10001101);
 608         &pshufd         ($D4,$D4,0b10001101);
 609
 610         &movdqu         (&QWP(16*0,"edi"),$D0);         # save the table
 611         &movdqu         (&QWP(16*1,"edi"),$D1);
 612         &movdqu         (&QWP(16*2,"edi"),$D2);
 613         &movdqu         (&QWP(16*3,"edi"),$D3);
 614         &movdqu         (&QWP(16*4,"edi"),$D4);
 615
 616         &movdqa         ($T1,$D1);
 617         &movdqa         ($T0,$D2);
 618         &pslld          ($T1,2);
 619         &pslld          ($T0,2);
 620         &paddd          ($T1,$D1);                      # *5
 621         &paddd          ($T0,$D2);                      # *5
 622         &movdqu         (&QWP(16*5,"edi"),$T1);
 623         &movdqu         (&QWP(16*6,"edi"),$T0);
 624         &movdqa         ($T1,$D3);
 625         &movdqa         ($T0,$D4);
 626         &pslld          ($T1,2);
 627         &pslld          ($T0,2);
 628         &paddd          ($T1,$D3);                      # *5
 629         &paddd          ($T0,$D4);                      # *5
 630         &movdqu         (&QWP(16*7,"edi"),$T1);
 631         &movdqu         (&QWP(16*8,"edi"),$T0);
 632
 633         &mov            ("esp","ebp");
 634         &lea            ("edi",&DWP(-16*3,"edi"));      # size de-optimization
 635         &ret            ();
 636 &function_end_B("_poly1305_init_sse2");
 637
 638 &align  (32);
 639 &function_begin("_poly1305_blocks_sse2");
 640         &mov    ("edi",&wparam(0));                     # ctx
 641         &mov    ("esi",&wparam(1));                     # inp
 642         &mov    ("ecx",&wparam(2));                     # len
 643
 644         &mov    ("eax",&DWP(4*5,"edi"));                # is_base2_26
 645         &and    ("ecx",-16);
 646         &jz     (&label("nodata"));
 647         &cmp    ("ecx",64);
 648         &jae    (&label("enter_sse2"));
 649         &test   ("eax","eax");                          # is_base2_26?
 650         &jz     (&label("enter_blocks"));
 651
 652 &set_label("enter_sse2",16);
 653         &call   (&label("pic_point"));
 654 &set_label("pic_point");
 655         &blindpop("ebx");
 656         &lea    ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
 657
 658         &test   ("eax","eax");                          # is_base2_26?
 659         &jnz    (&label("base2_26"));
 660
 661         &call   ("_poly1305_init_sse2");
 662
 663         ################################################# base 2^32 -> base 2^26
 664         &mov    ("eax",&DWP(0,"edi"));
 665         &mov    ("ecx",&DWP(3,"edi"));
 666         &mov    ("edx",&DWP(6,"edi"));
 667         &mov    ("esi",&DWP(9,"edi"));
 668         &mov    ("ebp",&DWP(13,"edi"));
 669         &mov    (&DWP(4*5,"edi"),1);                    # is_base2_26
 670
 671         &shr    ("ecx",2);
 672         &and    ("eax",0x3ffffff);
 673         &shr    ("edx",4);
 674         &and    ("ecx",0x3ffffff);
 675         &shr    ("esi",6);
 676         &and    ("edx",0x3ffffff);
 677
 678         &movd   ($D0,"eax");
 679         &movd   ($D1,"ecx");
 680         &movd   ($D2,"edx");
 681         &movd   ($D3,"esi");
 682         &movd   ($D4,"ebp");
 683
 684         &mov    ("esi",&wparam(1));                     # [reload] inp
 685         &mov    ("ecx",&wparam(2));                     # [reload] len
 686         &jmp    (&label("base2_32"));
 687
 688 &set_label("base2_26",16);
 689         &movd   ($D0,&DWP(4*0,"edi"));                  # load hash value
 690         &movd   ($D1,&DWP(4*1,"edi"));
 691         &movd   ($D2,&DWP(4*2,"edi"));
 692         &movd   ($D3,&DWP(4*3,"edi"));
 693         &movd   ($D4,&DWP(4*4,"edi"));
 694         &movdqa ($MASK,&QWP(64,"ebx"));
 695
 696 &set_label("base2_32");
 697         &mov    ("eax",&wparam(3));                     # padbit
 698         &mov    ("ebp","esp");
 699
 700         &sub    ("esp",16*(5+5+5+9+9));
 701         &and    ("esp",-16);
 702
 703         &lea    ("edi",&DWP(16*3,"edi"));               # size optimization
 704         &shl    ("eax",24);                             # padbit
 705
 706         &test   ("ecx",31);
 707         &jz     (&label("even"));
 708
 709         ################################################################
 710         # process single block, with SSE2, because it's still faster
 711         # even though half of result is discarded
 712
 713         &movdqu         ($T1,&QWP(0,"esi"));            # input
 714         &lea            ("esi",&DWP(16,"esi"));
 715
 716         &movdqa         ($T0,$T1);                      # -> base 2^26 ...
 717         &pand           ($T1,$MASK);
 718         &paddd          ($D0,$T1);                      # ... and accumuate
 719
 720         &movdqa         ($T1,$T0);
 721         &psrlq          ($T0,26);
 722         &psrldq         ($T1,6);
 723         &pand           ($T0,$MASK);
 724         &paddd          ($D1,$T0);
 725
 726         &movdqa         ($T0,$T1);
 727         &psrlq          ($T1,4);
 728         &pand           ($T1,$MASK);
 729         &paddd          ($D2,$T1);
 730
 731         &movdqa         ($T1,$T0);
 732         &psrlq          ($T0,30);
 733         &pand           ($T0,$MASK);
 734         &psrldq         ($T1,7);
 735         &paddd          ($D3,$T0);
 736
 737         &movd           ($T0,"eax");                    # padbit
 738         &paddd          ($D4,$T1);
 739          &movd          ($T1,&DWP(16*0+12,"edi"));      # r0
 740         &paddd          ($D4,$T0);
 741
 742         &movdqa         (&QWP(16*0,"esp"),$D0);
 743         &movdqa         (&QWP(16*1,"esp"),$D1);
 744         &movdqa         (&QWP(16*2,"esp"),$D2);
 745         &movdqa         (&QWP(16*3,"esp"),$D3);
 746         &movdqa         (&QWP(16*4,"esp"),$D4);
 747
 748         ################################################################
 749         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
 750         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
 751         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
 752         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
 753         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
 754
 755         &pmuludq        ($D0,$T1);                      # h4*r0
 756         &pmuludq        ($D1,$T1);                      # h3*r0
 757         &pmuludq        ($D2,$T1);                      # h2*r0
 758          &movd          ($T0,&DWP(16*1+12,"edi"));      # r1
 759         &pmuludq        ($D3,$T1);                      # h1*r0
 760         &pmuludq        ($D4,$T1);                      # h0*r0
 761
 762         &pmuladd        (sub {  my ($reg,$i)=@_;
 763                                 &movd ($reg,&DWP(16*$i+12,"edi"));
 764                              });
 765
 766         &lazy_reduction ();
 767
 768         &sub            ("ecx",16);
 769         &jz             (&label("done"));
 770
 771 &set_label("even");
 772         &lea            ("edx",&DWP(16*(5+5+5+9),"esp"));# size optimization
 773         &lea            ("eax",&DWP(-16*2,"esi"));
 774         &sub            ("ecx",64);
 775
 776         ################################################################
 777         # expand and copy pre-calculated table to stack
 778
 779         &movdqu         ($T0,&QWP(16*0,"edi"));         # r^1:r^2:r^3:r^4
 780         &pshufd         ($T1,$T0,0b01000100);           # duplicate r^3:r^4
 781         &cmovb          ("esi","eax");
 782         &pshufd         ($T0,$T0,0b11101110);           # duplicate r^1:r^2
 783         &movdqa         (&QWP(16*0,"edx"),$T1);
 784         &lea            ("eax",&DWP(16*10,"esp"));
 785         &movdqu         ($T1,&QWP(16*1,"edi"));
 786         &movdqa         (&QWP(16*(0-9),"edx"),$T0);
 787         &pshufd         ($T0,$T1,0b01000100);
 788         &pshufd         ($T1,$T1,0b11101110);
 789         &movdqa         (&QWP(16*1,"edx"),$T0);
 790         &movdqu         ($T0,&QWP(16*2,"edi"));
 791         &movdqa         (&QWP(16*(1-9),"edx"),$T1);
 792         &pshufd         ($T1,$T0,0b01000100);
 793         &pshufd         ($T0,$T0,0b11101110);
 794         &movdqa         (&QWP(16*2,"edx"),$T1);
 795         &movdqu         ($T1,&QWP(16*3,"edi"));
 796         &movdqa         (&QWP(16*(2-9),"edx"),$T0);
 797         &pshufd         ($T0,$T1,0b01000100);
 798         &pshufd         ($T1,$T1,0b11101110);
 799         &movdqa         (&QWP(16*3,"edx"),$T0);
 800         &movdqu         ($T0,&QWP(16*4,"edi"));
 801         &movdqa         (&QWP(16*(3-9),"edx"),$T1);
 802         &pshufd         ($T1,$T0,0b01000100);
 803         &pshufd         ($T0,$T0,0b11101110);
 804         &movdqa         (&QWP(16*4,"edx"),$T1);
 805         &movdqu         ($T1,&QWP(16*5,"edi"));
 806         &movdqa         (&QWP(16*(4-9),"edx"),$T0);
 807         &pshufd         ($T0,$T1,0b01000100);
 808         &pshufd         ($T1,$T1,0b11101110);
 809         &movdqa         (&QWP(16*5,"edx"),$T0);
 810         &movdqu         ($T0,&QWP(16*6,"edi"));
 811         &movdqa         (&QWP(16*(5-9),"edx"),$T1);
 812         &pshufd         ($T1,$T0,0b01000100);
 813         &pshufd         ($T0,$T0,0b11101110);
 814         &movdqa         (&QWP(16*6,"edx"),$T1);
 815         &movdqu         ($T1,&QWP(16*7,"edi"));
 816         &movdqa         (&QWP(16*(6-9),"edx"),$T0);
 817         &pshufd         ($T0,$T1,0b01000100);
 818         &pshufd         ($T1,$T1,0b11101110);
 819         &movdqa         (&QWP(16*7,"edx"),$T0);
 820         &movdqu         ($T0,&QWP(16*8,"edi"));
 821         &movdqa         (&QWP(16*(7-9),"edx"),$T1);
 822         &pshufd         ($T1,$T0,0b01000100);
 823         &pshufd         ($T0,$T0,0b11101110);
 824         &movdqa         (&QWP(16*8,"edx"),$T1);
 825         &movdqa         (&QWP(16*(8-9),"edx"),$T0);
 826
 827 sub load_input {
 828 my ($inpbase,$offbase)=@_;
 829
 830         &movdqu         ($T0,&QWP($inpbase+0,"esi"));   # load input
 831         &movdqu         ($T1,&QWP($inpbase+16,"esi"));
 832         &lea            ("esi",&DWP(16*2,"esi"));
 833
 834         &movdqa         (&QWP($offbase+16*2,"esp"),$D2);
 835         &movdqa         (&QWP($offbase+16*3,"esp"),$D3);
 836         &movdqa         (&QWP($offbase+16*4,"esp"),$D4);
 837
 838         &movdqa         ($D2,$T0);                      # splat input
 839         &movdqa         ($D3,$T1);
 840         &psrldq         ($D2,6);
 841         &psrldq         ($D3,6);
 842         &movdqa         ($D4,$T0);
 843         &punpcklqdq     ($D2,$D3);                      # 2:3
 844         &punpckhqdq     ($D4,$T1);                      # 4
 845         &punpcklqdq     ($T0,$T1);                      # 0:1
 846
 847         &movdqa         ($D3,$D2);
 848         &psrlq          ($D2,4);
 849         &psrlq          ($D3,30);
 850         &movdqa         ($T1,$T0);
 851         &psrlq          ($D4,40);                       # 4
 852         &psrlq          ($T1,26);
 853         &pand           ($T0,$MASK);                    # 0
 854         &pand           ($T1,$MASK);                    # 1
 855         &pand           ($D2,$MASK);                    # 2
 856         &pand           ($D3,$MASK);                    # 3
 857         &por            ($D4,&QWP(0,"ebx"));            # padbit, yes, always
 858
 859         &movdqa         (&QWP($offbase+16*0,"esp"),$D0) if ($offbase);
 860         &movdqa         (&QWP($offbase+16*1,"esp"),$D1) if ($offbase);
 861 }
 862         &load_input     (16*2,16*5);
 863
 864         &jbe            (&label("skip_loop"));
 865         &jmp            (&label("loop"));
 866
 867 &set_label("loop",32);
 868         ################################################################
 869         # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
 870         # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
 871         #   \___________________/
 872         # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
 873         # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
 874         #   \___________________/ \____________________/
 875         ################################################################
 876
 877         &movdqa         ($T2,&QWP(16*(0-9),"edx"));     # r0^2
 878         &movdqa         (&QWP(16*1,"eax"),$T1);
 879         &movdqa         (&QWP(16*2,"eax"),$D2);
 880         &movdqa         (&QWP(16*3,"eax"),$D3);
 881         &movdqa         (&QWP(16*4,"eax"),$D4);
 882
 883         ################################################################
 884         # d4 = h4*r0 + h0*r4   + h1*r3   + h2*r2   + h3*r1
 885         # d3 = h3*r0 + h0*r3   + h1*r2   + h2*r1   + h4*5*r4
 886         # d2 = h2*r0 + h0*r2   + h1*r1   + h3*5*r4 + h4*5*r3
 887         # d1 = h1*r0 + h0*r1   + h2*5*r4 + h3*5*r3 + h4*5*r2
 888         # d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
 889
 890         &movdqa         ($D1,$T0);
 891         &pmuludq        ($T0,$T2);                      # h0*r0
 892         &movdqa         ($D0,$T1);
 893         &pmuludq        ($T1,$T2);                      # h1*r0
 894         &pmuludq        ($D2,$T2);                      # h2*r0
 895         &pmuludq        ($D3,$T2);                      # h3*r0
 896         &pmuludq        ($D4,$T2);                      # h4*r0
 897
 898 sub pmuladd_alt {
 899 my $addr = shift;
 900
 901         &pmuludq        ($D0,&$addr(8));                # h1*s4
 902         &movdqa         ($T2,$D1);
 903         &pmuludq        ($D1,&$addr(1));                # h0*r1
 904         &paddq          ($D0,$T0);
 905         &movdqa         ($T0,$T2);
 906         &pmuludq        ($T2,&$addr(2));                # h0*r2
 907         &paddq          ($D1,$T1);
 908         &movdqa         ($T1,$T0);
 909         &pmuludq        ($T0,&$addr(3));                # h0*r3
 910         &paddq          ($D2,$T2);
 911          &movdqa        ($T2,&QWP(16*1,"eax"));         # pull h1
 912         &pmuludq        ($T1,&$addr(4));                # h0*r4
 913         &paddq          ($D3,$T0);
 914
 915         &movdqa         ($T0,$T2);
 916         &pmuludq        ($T2,&$addr(1));                # h1*r1
 917          &paddq         ($D4,$T1);
 918         &movdqa         ($T1,$T0);
 919         &pmuludq        ($T0,&$addr(2));                # h1*r2
 920         &paddq          ($D2,$T2);
 921         &movdqa         ($T2,&QWP(16*2,"eax"));         # pull h2
 922         &pmuludq        ($T1,&$addr(3));                # h1*r3
 923         &paddq          ($D3,$T0);
 924         &movdqa         ($T0,$T2);
 925         &pmuludq        ($T2,&$addr(7));                # h2*s3
 926         &paddq          ($D4,$T1);
 927         &movdqa         ($T1,$T0);
 928         &pmuludq        ($T0,&$addr(8));                # h2*s4
 929         &paddq          ($D0,$T2);
 930
 931         &movdqa         ($T2,$T1);
 932         &pmuludq        ($T1,&$addr(1));                # h2*r1
 933          &paddq         ($D1,$T0);
 934         &movdqa         ($T0,&QWP(16*3,"eax"));         # pull h3
 935         &pmuludq        ($T2,&$addr(2));                # h2*r2
 936         &paddq          ($D3,$T1);
 937         &movdqa         ($T1,$T0);
 938         &pmuludq        ($T0,&$addr(6));                # h3*s2
 939         &paddq          ($D4,$T2);
 940         &movdqa         ($T2,$T1);
 941         &pmuludq        ($T1,&$addr(7));                # h3*s3
 942         &paddq          ($D0,$T0);
 943         &movdqa         ($T0,$T2);
 944         &pmuludq        ($T2,&$addr(8));                # h3*s4
 945         &paddq          ($D1,$T1);
 946
 947         &movdqa         ($T1,&QWP(16*4,"eax"));         # pull h4
 948         &pmuludq        ($T0,&$addr(1));                # h3*r1
 949          &paddq         ($D2,$T2);
 950         &movdqa         ($T2,$T1);
 951         &pmuludq        ($T1,&$addr(8));                # h4*s4
 952         &paddq          ($D4,$T0);
 953         &movdqa         ($T0,$T2);
 954         &pmuludq        ($T2,&$addr(5));                # h4*s1
 955         &paddq          ($D3,$T1);
 956         &movdqa         ($T1,$T0);
 957         &pmuludq        ($T0,&$addr(6));                # h4*s2
 958         &paddq          ($D0,$T2);
 959          &movdqa        ($MASK,&QWP(64,"ebx"));
 960         &pmuludq        ($T1,&$addr(7));                # h4*s3
 961         &paddq          ($D1,$T0);
 962         &paddq          ($D2,$T1);
 963 }
 964         &pmuladd_alt    (sub {  my $i=shift; &QWP(16*($i-9),"edx");     });
 965
 966         &load_input     (-16*2,0);
 967         &lea            ("eax",&DWP(-16*2,"esi"));
 968         &sub            ("ecx",64);
 969
 970         &paddd          ($T0,&QWP(16*(5+0),"esp"));     # add hash value
 971         &paddd          ($T1,&QWP(16*(5+1),"esp"));
 972         &paddd          ($D2,&QWP(16*(5+2),"esp"));
 973         &paddd          ($D3,&QWP(16*(5+3),"esp"));
 974         &paddd          ($D4,&QWP(16*(5+4),"esp"));
 975
 976         &cmovb          ("esi","eax");
 977         &lea            ("eax",&DWP(16*10,"esp"));
 978
 979         &movdqa         ($T2,&QWP(16*0,"edx"));         # r0^4
 980         &movdqa         (&QWP(16*1,"esp"),$D1);
 981         &movdqa         (&QWP(16*1,"eax"),$T1);
 982         &movdqa         (&QWP(16*2,"eax"),$D2);
 983         &movdqa         (&QWP(16*3,"eax"),$D3);
 984         &movdqa         (&QWP(16*4,"eax"),$D4);
 985
 986         ################################################################
 987         # d4 += h4*r0 + h0*r4   + h1*r3   + h2*r2   + h3*r1
 988         # d3 += h3*r0 + h0*r3   + h1*r2   + h2*r1   + h4*5*r4
 989         # d2 += h2*r0 + h0*r2   + h1*r1   + h3*5*r4 + h4*5*r3
 990         # d1 += h1*r0 + h0*r1   + h2*5*r4 + h3*5*r3 + h4*5*r2
 991         # d0 += h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
 992
 993         &movdqa         ($D1,$T0);
 994         &pmuludq        ($T0,$T2);                      # h0*r0
 995         &paddq          ($T0,$D0);
 996         &movdqa         ($D0,$T1);
 997         &pmuludq        ($T1,$T2);                      # h1*r0
 998         &pmuludq        ($D2,$T2);                      # h2*r0
 999         &pmuludq        ($D3,$T2);                      # h3*r0
1000         &pmuludq        ($D4,$T2);                      # h4*r0
1001
1002         &paddq          ($T1,&QWP(16*1,"esp"));
1003         &paddq          ($D2,&QWP(16*2,"esp"));
1004         &paddq          ($D3,&QWP(16*3,"esp"));
1005         &paddq          ($D4,&QWP(16*4,"esp"));
1006
1007         &pmuladd_alt    (sub {  my $i=shift; &QWP(16*$i,"edx"); });
1008
1009         &lazy_reduction ();
1010
1011         &load_input     (16*2,16*5);
1012
1013         &ja             (&label("loop"));
1014
1015 &set_label("skip_loop");
1016         ################################################################
1017         # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
1018
1019          &pshufd        ($T2,&QWP(16*(0-9),"edx"),0x10);# r0^n
1020         &add            ("ecx",32);
1021         &jnz            (&label("long_tail"));
1022
1023         &paddd          ($T0,$D0);                      # add hash value
1024         &paddd          ($T1,$D1);
1025         &paddd          ($D2,&QWP(16*7,"esp"));
1026         &paddd          ($D3,&QWP(16*8,"esp"));
1027         &paddd          ($D4,&QWP(16*9,"esp"));
1028
1029 &set_label("long_tail");
1030
1031         &movdqa         (&QWP(16*0,"eax"),$T0);
1032         &movdqa         (&QWP(16*1,"eax"),$T1);
1033         &movdqa         (&QWP(16*2,"eax"),$D2);
1034         &movdqa         (&QWP(16*3,"eax"),$D3);
1035         &movdqa         (&QWP(16*4,"eax"),$D4);
1036
1037         ################################################################
1038         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
1039         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
1040         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
1041         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
1042         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1043
1044         &pmuludq        ($T0,$T2);                      # h0*r0
1045         &pmuludq        ($T1,$T2);                      # h1*r0
1046         &pmuludq        ($D2,$T2);                      # h2*r0
1047         &movdqa         ($D0,$T0);
1048          &pshufd        ($T0,&QWP(16*(1-9),"edx"),0x10);# r1^n
1049         &pmuludq        ($D3,$T2);                      # h3*r0
1050         &movdqa         ($D1,$T1);
1051         &pmuludq        ($D4,$T2);                      # h4*r0
1052
1053         &pmuladd        (sub {  my ($reg,$i)=@_;
1054                                 &pshufd ($reg,&QWP(16*($i-9),"edx"),0x10);
1055                              },"eax");
1056
1057         &jz             (&label("short_tail"));
1058
1059         &load_input     (-16*2,0);
1060
1061          &pshufd        ($T2,&QWP(16*0,"edx"),0x10);    # r0^n
1062         &paddd          ($T0,&QWP(16*5,"esp"));         # add hash value
1063         &paddd          ($T1,&QWP(16*6,"esp"));
1064         &paddd          ($D2,&QWP(16*7,"esp"));
1065         &paddd          ($D3,&QWP(16*8,"esp"));
1066         &paddd          ($D4,&QWP(16*9,"esp"));
1067
1068         ################################################################
1069         # multiply inp[0:1] by r^4:r^3 and accumulate
1070
1071         &movdqa         (&QWP(16*0,"esp"),$T0);
1072         &pmuludq        ($T0,$T2);                      # h0*r0
1073         &movdqa         (&QWP(16*1,"esp"),$T1);
1074         &pmuludq        ($T1,$T2);                      # h1*r0
1075         &paddq          ($D0,$T0);
1076         &movdqa         ($T0,$D2);
1077         &pmuludq        ($D2,$T2);                      # h2*r0
1078         &paddq          ($D1,$T1);
1079         &movdqa         ($T1,$D3);
1080         &pmuludq        ($D3,$T2);                      # h3*r0
1081         &paddq          ($D2,&QWP(16*2,"esp"));
1082         &movdqa         (&QWP(16*2,"esp"),$T0);
1083          &pshufd        ($T0,&QWP(16*1,"edx"),0x10);    # r1^n
1084         &paddq          ($D3,&QWP(16*3,"esp"));
1085         &movdqa         (&QWP(16*3,"esp"),$T1);
1086         &movdqa         ($T1,$D4);
1087         &pmuludq        ($D4,$T2);                      # h4*r0
1088         &paddq          ($D4,&QWP(16*4,"esp"));
1089         &movdqa         (&QWP(16*4,"esp"),$T1);
1090
1091         &pmuladd        (sub {  my ($reg,$i)=@_;
1092                                 &pshufd ($reg,&QWP(16*$i,"edx"),0x10);
1093                              });
1094
1095 &set_label("short_tail");
1096
1097         ################################################################
1098         # horizontal addition
1099
1100         &pshufd         ($T1,$D4,0b01001110);
1101         &pshufd         ($T0,$D3,0b01001110);
1102         &paddq          ($D4,$T1);
1103         &paddq          ($D3,$T0);
1104         &pshufd         ($T1,$D0,0b01001110);
1105         &pshufd         ($T0,$D1,0b01001110);
1106         &paddq          ($D0,$T1);
1107         &paddq          ($D1,$T0);
1108         &pshufd         ($T1,$D2,0b01001110);
1109         #&paddq         ($D2,$T1);
1110
1111         &lazy_reduction (sub { &paddq ($D2,$T1) });
1112
1113 &set_label("done");
1114         &movd           (&DWP(-16*3+4*0,"edi"),$D0);    # store hash value
1115         &movd           (&DWP(-16*3+4*1,"edi"),$D1);
1116         &movd           (&DWP(-16*3+4*2,"edi"),$D2);
1117         &movd           (&DWP(-16*3+4*3,"edi"),$D3);
1118         &movd           (&DWP(-16*3+4*4,"edi"),$D4);
1119         &mov    ("esp","ebp");
1120 &set_label("nodata");
1121 &function_end("_poly1305_blocks_sse2");
1122
1123 &align  (32);
1124 &function_begin("_poly1305_emit_sse2");
1125         &mov    ("ebp",&wparam(0));             # context
1126
1127         &cmp    (&DWP(4*5,"ebp"),0);            # is_base2_26?
1128         &je     (&label("enter_emit"));
1129
1130         &mov    ("eax",&DWP(4*0,"ebp"));        # load hash value
1131         &mov    ("edi",&DWP(4*1,"ebp"));
1132         &mov    ("ecx",&DWP(4*2,"ebp"));
1133         &mov    ("edx",&DWP(4*3,"ebp"));
1134         &mov    ("esi",&DWP(4*4,"ebp"));
1135
1136         &mov    ("ebx","edi");                  # base 2^26 -> base 2^32
1137         &shl    ("edi",26);
1138         &shr    ("ebx",6);
1139         &add    ("eax","edi");
1140         &mov    ("edi","ecx");
1141         &adc    ("ebx",0);
1142
1143         &shl    ("edi",20);
1144         &shr    ("ecx",12);
1145         &add    ("ebx","edi");
1146         &mov    ("edi","edx");
1147         &adc    ("ecx",0);
1148
1149         &shl    ("edi",14);
1150         &shr    ("edx",18);
1151         &add    ("ecx","edi");
1152         &mov    ("edi","esi");
1153         &adc    ("edx",0);
1154
1155         &shl    ("edi",8);
1156         &shr    ("esi",24);
1157         &add    ("edx","edi");
1158         &adc    ("esi",0);                      # can be partially reduced
1159
1160         &mov    ("edi","esi");                  # final reduction
1161         &and    ("esi",3);
1162         &shr    ("edi",2);
1163         &lea    ("ebp",&DWP(0,"edi","edi",4));  # *5
1164          &mov   ("edi",&wparam(1));             # output
1165         add     ("eax","ebp");
1166          &mov   ("ebp",&wparam(2));             # key
1167         adc     ("ebx",0);
1168         adc     ("ecx",0);
1169         adc     ("edx",0);
1170
1171         &movd   ($D0,"eax");                    # offload original hash value
1172         &add    ("eax",5);                      # compare to modulus
1173         &movd   ($D1,"ebx");
1174         &adc    ("ebx",0);
1175         &movd   ($D2,"ecx");
1176         &adc    ("ecx",0);
1177         &movd   ($D3,"edx");
1178         &adc    ("edx",0);
1179         &adc    ("esi",0);
1180         &shr    ("esi",2);                      # did it carry/borrow?
1181
1182         &neg    ("esi");                        # do we choose (hash-modulus) ...
1183         &and    ("eax","esi");
1184         &and    ("ebx","esi");
1185         &and    ("ecx","esi");
1186         &and    ("edx","esi");
1187         &mov    (&DWP(4*0,"edi"),"eax");
1188         &movd   ("eax",$D0);
1189         &mov    (&DWP(4*1,"edi"),"ebx");
1190         &movd   ("ebx",$D1);
1191         &mov    (&DWP(4*2,"edi"),"ecx");
1192         &movd   ("ecx",$D2);
1193         &mov    (&DWP(4*3,"edi"),"edx");
1194         &movd   ("edx",$D3);
1195
1196         &not    ("esi");                        # ... or original hash value?
1197         &and    ("eax","esi");
1198         &and    ("ebx","esi");
1199         &or     ("eax",&DWP(4*0,"edi"));
1200         &and    ("ecx","esi");
1201         &or     ("ebx",&DWP(4*1,"edi"));
1202         &and    ("edx","esi");
1203         &or     ("ecx",&DWP(4*2,"edi"));
1204         &or     ("edx",&DWP(4*3,"edi"));
1205
1206         &add    ("eax",&DWP(4*0,"ebp"));        # accumulate key
1207         &adc    ("ebx",&DWP(4*1,"ebp"));
1208         &mov    (&DWP(4*0,"edi"),"eax");
1209         &adc    ("ecx",&DWP(4*2,"ebp"));
1210         &mov    (&DWP(4*1,"edi"),"ebx");
1211         &adc    ("edx",&DWP(4*3,"ebp"));
1212         &mov    (&DWP(4*2,"edi"),"ecx");
1213         &mov    (&DWP(4*3,"edi"),"edx");
1214 &function_end("_poly1305_emit_sse2");
1215
1216 if ($avx>1) {
1217 ########################################################################
1218 # Note that poly1305_init_avx2 operates on %xmm, I could have used
1219 # poly1305_init_sse2...
1220
1221 &align  (32);
1222 &function_begin_B("_poly1305_init_avx2");
1223         &vmovdqu        ($D4,&QWP(4*6,"edi"));          # key base 2^32
1224         &lea            ("edi",&DWP(16*3,"edi"));       # size optimization
1225         &mov            ("ebp","esp");
1226         &sub            ("esp",16*(9+5));
1227         &and            ("esp",-16);
1228
1229         #&vpand         ($D4,$D4,&QWP(96,"ebx"));       # magic mask
1230         &vmovdqa        ($MASK,&QWP(64,"ebx"));
1231
1232         &vpand          ($D0,$D4,$MASK);                # -> base 2^26
1233         &vpsrlq         ($D1,$D4,26);
1234         &vpsrldq        ($D3,$D4,6);
1235         &vpand          ($D1,$D1,$MASK);
1236         &vpsrlq         ($D2,$D3,4)
1237         &vpsrlq         ($D3,$D3,30);
1238         &vpand          ($D2,$D2,$MASK);
1239         &vpand          ($D3,$D3,$MASK);
1240         &vpsrldq        ($D4,$D4,13);
1241
1242         &lea            ("edx",&DWP(16*9,"esp"));       # size optimization
1243         &mov            ("ecx",2);
1244 &set_label("square");
1245         &vmovdqa        (&QWP(16*0,"esp"),$D0);
1246         &vmovdqa        (&QWP(16*1,"esp"),$D1);
1247         &vmovdqa        (&QWP(16*2,"esp"),$D2);
1248         &vmovdqa        (&QWP(16*3,"esp"),$D3);
1249         &vmovdqa        (&QWP(16*4,"esp"),$D4);
1250
1251         &vpslld         ($T1,$D1,2);
1252         &vpslld         ($T0,$D2,2);
1253         &vpaddd         ($T1,$T1,$D1);                  # *5
1254         &vpaddd         ($T0,$T0,$D2);                  # *5
1255         &vmovdqa        (&QWP(16*5,"esp"),$T1);
1256         &vmovdqa        (&QWP(16*6,"esp"),$T0);
1257         &vpslld         ($T1,$D3,2);
1258         &vpslld         ($T0,$D4,2);
1259         &vpaddd         ($T1,$T1,$D3);                  # *5
1260         &vpaddd         ($T0,$T0,$D4);                  # *5
1261         &vmovdqa        (&QWP(16*7,"esp"),$T1);
1262         &vmovdqa        (&QWP(16*8,"esp"),$T0);
1263
1264         &vpshufd        ($T0,$D0,0b01000100);
1265         &vmovdqa        ($T1,$D1);
1266         &vpshufd        ($D1,$D1,0b01000100);
1267         &vpshufd        ($D2,$D2,0b01000100);
1268         &vpshufd        ($D3,$D3,0b01000100);
1269         &vpshufd        ($D4,$D4,0b01000100);
1270         &vmovdqa        (&QWP(16*0,"edx"),$T0);
1271         &vmovdqa        (&QWP(16*1,"edx"),$D1);
1272         &vmovdqa        (&QWP(16*2,"edx"),$D2);
1273         &vmovdqa        (&QWP(16*3,"edx"),$D3);
1274         &vmovdqa        (&QWP(16*4,"edx"),$D4);
1275
1276         ################################################################
1277         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
1278         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
1279         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
1280         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
1281         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1282
1283         &vpmuludq       ($D4,$D4,$D0);                  # h4*r0
1284         &vpmuludq       ($D3,$D3,$D0);                  # h3*r0
1285         &vpmuludq       ($D2,$D2,$D0);                  # h2*r0
1286         &vpmuludq       ($D1,$D1,$D0);                  # h1*r0
1287         &vpmuludq       ($D0,$T0,$D0);                  # h0*r0
1288
1289         &vpmuludq       ($T0,$T1,&QWP(16*3,"edx"));     # r1*h3
1290         &vpaddq         ($D4,$D4,$T0);
1291         &vpmuludq       ($T2,$T1,&QWP(16*2,"edx"));     # r1*h2
1292         &vpaddq         ($D3,$D3,$T2);
1293         &vpmuludq       ($T0,$T1,&QWP(16*1,"edx"));     # r1*h1
1294         &vpaddq         ($D2,$D2,$T0);
1295         &vmovdqa        ($T2,&QWP(16*5,"esp"));         # s1
1296         &vpmuludq       ($T1,$T1,&QWP(16*0,"edx"));     # r1*h0
1297         &vpaddq         ($D1,$D1,$T1);
1298          &vmovdqa       ($T0,&QWP(16*2,"esp"));         # r2
1299         &vpmuludq       ($T2,$T2,&QWP(16*4,"edx"));     # s1*h4
1300         &vpaddq         ($D0,$D0,$T2);
1301
1302         &vpmuludq       ($T1,$T0,&QWP(16*2,"edx"));     # r2*h2
1303         &vpaddq         ($D4,$D4,$T1);
1304         &vpmuludq       ($T2,$T0,&QWP(16*1,"edx"));     # r2*h1
1305         &vpaddq         ($D3,$D3,$T2);
1306         &vmovdqa        ($T1,&QWP(16*6,"esp"));         # s2
1307         &vpmuludq       ($T0,$T0,&QWP(16*0,"edx"));     # r2*h0
1308         &vpaddq         ($D2,$D2,$T0);
1309         &vpmuludq       ($T2,$T1,&QWP(16*4,"edx"));     # s2*h4
1310         &vpaddq         ($D1,$D1,$T2);
1311          &vmovdqa       ($T0,&QWP(16*3,"esp"));         # r3
1312         &vpmuludq       ($T1,$T1,&QWP(16*3,"edx"));     # s2*h3
1313         &vpaddq         ($D0,$D0,$T1);
1314
1315         &vpmuludq       ($T2,$T0,&QWP(16*1,"edx"));     # r3*h1
1316         &vpaddq         ($D4,$D4,$T2);
1317         &vmovdqa        ($T1,&QWP(16*7,"esp"));         # s3
1318         &vpmuludq       ($T0,$T0,&QWP(16*0,"edx"));     # r3*h0
1319         &vpaddq         ($D3,$D3,$T0);
1320         &vpmuludq       ($T2,$T1,&QWP(16*4,"edx"));     # s3*h4
1321         &vpaddq         ($D2,$D2,$T2);
1322         &vpmuludq       ($T0,$T1,&QWP(16*3,"edx"));     # s3*h3
1323         &vpaddq         ($D1,$D1,$T0);
1324          &vmovdqa       ($T2,&QWP(16*4,"esp"));         # r4
1325         &vpmuludq       ($T1,$T1,&QWP(16*2,"edx"));     # s3*h2
1326         &vpaddq         ($D0,$D0,$T1);
1327
1328         &vmovdqa        ($T0,&QWP(16*8,"esp"));         # s4
1329         &vpmuludq       ($T2,$T2,&QWP(16*0,"edx"));     # r4*h0
1330         &vpaddq         ($D4,$D4,$T2);
1331         &vpmuludq       ($T1,$T0,&QWP(16*4,"edx"));     # s4*h4
1332         &vpaddq         ($D3,$D3,$T1);
1333         &vpmuludq       ($T2,$T0,&QWP(16*1,"edx"));     # s4*h1
1334         &vpaddq         ($D0,$D0,$T2);
1335         &vpmuludq       ($T1,$T0,&QWP(16*2,"edx"));     # s4*h2
1336         &vpaddq         ($D1,$D1,$T1);
1337          &vmovdqa       ($MASK,&QWP(64,"ebx"));
1338         &vpmuludq       ($T0,$T0,&QWP(16*3,"edx"));     # s4*h3
1339         &vpaddq         ($D2,$D2,$T0);
1340
1341         ################################################################
1342         # lazy reduction
1343          &vpsrlq        ($T0,$D3,26);
1344          &vpand         ($D3,$D3,$MASK);
1345         &vpsrlq         ($T1,$D0,26);
1346         &vpand          ($D0,$D0,$MASK);
1347          &vpaddq        ($D4,$D4,$T0);                  # h3 -> h4
1348         &vpaddq         ($D1,$D1,$T1);                  # h0 -> h1
1349          &vpsrlq        ($T0,$D4,26);
1350          &vpand         ($D4,$D4,$MASK);
1351         &vpsrlq         ($T1,$D1,26);
1352         &vpand          ($D1,$D1,$MASK);
1353         &vpaddq         ($D2,$D2,$T1);                  # h1 -> h2
1354          &vpaddd        ($D0,$D0,$T0);
1355          &vpsllq        ($T0,$T0,2);
1356         &vpsrlq         ($T1,$D2,26);
1357         &vpand          ($D2,$D2,$MASK);
1358          &vpaddd        ($D0,$D0,$T0);                  # h4 -> h0
1359         &vpaddd         ($D3,$D3,$T1);                  # h2 -> h3
1360         &vpsrlq         ($T1,$D3,26);
1361          &vpsrlq        ($T0,$D0,26);
1362          &vpand         ($D0,$D0,$MASK);
1363         &vpand          ($D3,$D3,$MASK);
1364          &vpaddd        ($D1,$D1,$T0);                  # h0 -> h1
1365         &vpaddd         ($D4,$D4,$T1);                  # h3 -> h4
1366
1367         &dec            ("ecx");
1368         &jz             (&label("square_break"));
1369
1370         &vpunpcklqdq    ($D0,$D0,&QWP(16*0,"esp"));     # 0:r^1:0:r^2
1371         &vpunpcklqdq    ($D1,$D1,&QWP(16*1,"esp"));
1372         &vpunpcklqdq    ($D2,$D2,&QWP(16*2,"esp"));
1373         &vpunpcklqdq    ($D3,$D3,&QWP(16*3,"esp"));
1374         &vpunpcklqdq    ($D4,$D4,&QWP(16*4,"esp"));
1375         &jmp            (&label("square"));
1376
1377 &set_label("square_break");
1378         &vpsllq         ($D0,$D0,32);                   # -> r^3:0:r^4:0
1379         &vpsllq         ($D1,$D1,32);
1380         &vpsllq         ($D2,$D2,32);
1381         &vpsllq         ($D3,$D3,32);
1382         &vpsllq         ($D4,$D4,32);
1383         &vpor           ($D0,$D0,&QWP(16*0,"esp"));     # r^3:r^1:r^4:r^2
1384         &vpor           ($D1,$D1,&QWP(16*1,"esp"));
1385         &vpor           ($D2,$D2,&QWP(16*2,"esp"));
1386         &vpor           ($D3,$D3,&QWP(16*3,"esp"));
1387         &vpor           ($D4,$D4,&QWP(16*4,"esp"));
1388
1389         &vpshufd        ($D0,$D0,0b10001101);           # -> r^1:r^2:r^3:r^4
1390         &vpshufd        ($D1,$D1,0b10001101);
1391         &vpshufd        ($D2,$D2,0b10001101);
1392         &vpshufd        ($D3,$D3,0b10001101);
1393         &vpshufd        ($D4,$D4,0b10001101);
1394
1395         &vmovdqu        (&QWP(16*0,"edi"),$D0);         # save the table
1396         &vmovdqu        (&QWP(16*1,"edi"),$D1);
1397         &vmovdqu        (&QWP(16*2,"edi"),$D2);
1398         &vmovdqu        (&QWP(16*3,"edi"),$D3);
1399         &vmovdqu        (&QWP(16*4,"edi"),$D4);
1400
1401         &vpslld         ($T1,$D1,2);
1402         &vpslld         ($T0,$D2,2);
1403         &vpaddd         ($T1,$T1,$D1);                  # *5
1404         &vpaddd         ($T0,$T0,$D2);                  # *5
1405         &vmovdqu        (&QWP(16*5,"edi"),$T1);
1406         &vmovdqu        (&QWP(16*6,"edi"),$T0);
1407         &vpslld         ($T1,$D3,2);
1408         &vpslld         ($T0,$D4,2);
1409         &vpaddd         ($T1,$T1,$D3);                  # *5
1410         &vpaddd         ($T0,$T0,$D4);                  # *5
1411         &vmovdqu        (&QWP(16*7,"edi"),$T1);
1412         &vmovdqu        (&QWP(16*8,"edi"),$T0);
1413
1414         &mov            ("esp","ebp");
1415         &lea            ("edi",&DWP(-16*3,"edi"));      # size de-optimization
1416         &ret            ();
1417 &function_end_B("_poly1305_init_avx2");
1418
1419 ########################################################################
1420 # now it's time to switch to %ymm
1421
1422 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("ymm$_",(0..7));
1423 my $MASK=$T2;
1424
1425 sub X { my $reg=shift; $reg=~s/^ymm/xmm/; $reg; }
1426
1427 &align  (32);
1428 &function_begin("_poly1305_blocks_avx2");
1429         &mov    ("edi",&wparam(0));                     # ctx
1430         &mov    ("esi",&wparam(1));                     # inp
1431         &mov    ("ecx",&wparam(2));                     # len
1432
1433         &mov    ("eax",&DWP(4*5,"edi"));                # is_base2_26
1434         &and    ("ecx",-16);
1435         &jz     (&label("nodata"));
1436         &cmp    ("ecx",64);
1437         &jae    (&label("enter_avx2"));
1438         &test   ("eax","eax");                          # is_base2_26?
1439         &jz     (&label("enter_blocks"));
1440
1441 &set_label("enter_avx2");
1442         &vzeroupper     ();
1443
1444         &call   (&label("pic_point"));
1445 &set_label("pic_point");
1446         &blindpop("ebx");
1447         &lea    ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
1448
1449         &test   ("eax","eax");                          # is_base2_26?
1450         &jnz    (&label("base2_26"));
1451
1452         &call   ("_poly1305_init_avx2");
1453
1454         ################################################# base 2^32 -> base 2^26
1455         &mov    ("eax",&DWP(0,"edi"));
1456         &mov    ("ecx",&DWP(3,"edi"));
1457         &mov    ("edx",&DWP(6,"edi"));
1458         &mov    ("esi",&DWP(9,"edi"));
1459         &mov    ("ebp",&DWP(13,"edi"));
1460
1461         &shr    ("ecx",2);
1462         &and    ("eax",0x3ffffff);
1463         &shr    ("edx",4);
1464         &and    ("ecx",0x3ffffff);
1465         &shr    ("esi",6);
1466         &and    ("edx",0x3ffffff);
1467
1468         &mov    (&DWP(4*0,"edi"),"eax");
1469         &mov    (&DWP(4*1,"edi"),"ecx");
1470         &mov    (&DWP(4*2,"edi"),"edx");
1471         &mov    (&DWP(4*3,"edi"),"esi");
1472         &mov    (&DWP(4*4,"edi"),"ebp");
1473         &mov    (&DWP(4*5,"edi"),1);                    # is_base2_26
1474
1475         &mov    ("esi",&wparam(1));                     # [reload] inp
1476         &mov    ("ecx",&wparam(2));                     # [reload] len
1477
1478 &set_label("base2_26");
1479         &mov    ("eax",&wparam(3));                     # padbit
1480         &mov    ("ebp","esp");
1481
1482         &sub    ("esp",32*(5+9));
1483         &and    ("esp",-512);                           # ensure that frame
1484                                                         # doesn't cross page
1485                                                         # boundary, which is
1486                                                         # essential for
1487                                                         # misaligned 32-byte
1488                                                         # loads
1489
1490         ################################################################
1491         # expand and copy pre-calculated table to stack
1492
1493         &vmovdqu        (&X($D0),&QWP(16*(3+0),"edi"));
1494         &lea            ("edx",&DWP(32*5+128,"esp"));   # +128 size optimization
1495         &vmovdqu        (&X($D1),&QWP(16*(3+1),"edi"));
1496         &vmovdqu        (&X($D2),&QWP(16*(3+2),"edi"));
1497         &vmovdqu        (&X($D3),&QWP(16*(3+3),"edi"));
1498         &vmovdqu        (&X($D4),&QWP(16*(3+4),"edi"));
1499         &lea            ("edi",&DWP(16*3,"edi"));       # size optimization
1500         &vpermq         ($D0,$D0,0b01000000);           # 00001234 -> 12343434
1501         &vpermq         ($D1,$D1,0b01000000);
1502         &vpermq         ($D2,$D2,0b01000000);
1503         &vpermq         ($D3,$D3,0b01000000);
1504         &vpermq         ($D4,$D4,0b01000000);
1505         &vpshufd        ($D0,$D0,0b11001000);           # 12343434 -> 14243444
1506         &vpshufd        ($D1,$D1,0b11001000);
1507         &vpshufd        ($D2,$D2,0b11001000);
1508         &vpshufd        ($D3,$D3,0b11001000);
1509         &vpshufd        ($D4,$D4,0b11001000);
1510         &vmovdqa        (&QWP(32*0-128,"edx"),$D0);
1511         &vmovdqu        (&X($D0),&QWP(16*5,"edi"));
1512         &vmovdqa        (&QWP(32*1-128,"edx"),$D1);
1513         &vmovdqu        (&X($D1),&QWP(16*6,"edi"));
1514         &vmovdqa        (&QWP(32*2-128,"edx"),$D2);
1515         &vmovdqu        (&X($D2),&QWP(16*7,"edi"));
1516         &vmovdqa        (&QWP(32*3-128,"edx"),$D3);
1517         &vmovdqu        (&X($D3),&QWP(16*8,"edi"));
1518         &vmovdqa        (&QWP(32*4-128,"edx"),$D4);
1519         &vpermq         ($D0,$D0,0b01000000);
1520         &vpermq         ($D1,$D1,0b01000000);
1521         &vpermq         ($D2,$D2,0b01000000);
1522         &vpermq         ($D3,$D3,0b01000000);
1523         &vpshufd        ($D0,$D0,0b11001000);
1524         &vpshufd        ($D1,$D1,0b11001000);
1525         &vpshufd        ($D2,$D2,0b11001000);
1526         &vpshufd        ($D3,$D3,0b11001000);
1527         &vmovdqa        (&QWP(32*5-128,"edx"),$D0);
1528         &vmovd          (&X($D0),&DWP(-16*3+4*0,"edi"));# load hash value
1529         &vmovdqa        (&QWP(32*6-128,"edx"),$D1);
1530         &vmovd          (&X($D1),&DWP(-16*3+4*1,"edi"));
1531         &vmovdqa        (&QWP(32*7-128,"edx"),$D2);
1532         &vmovd          (&X($D2),&DWP(-16*3+4*2,"edi"));
1533         &vmovdqa        (&QWP(32*8-128,"edx"),$D3);
1534         &vmovd          (&X($D3),&DWP(-16*3+4*3,"edi"));
1535         &vmovd          (&X($D4),&DWP(-16*3+4*4,"edi"));
1536         &vmovdqa        ($MASK,&QWP(64,"ebx"));
1537         &neg            ("eax");                        # padbit
1538
1539         &test           ("ecx",63);
1540         &jz             (&label("even"));
1541
1542         &mov            ("edx","ecx");
1543         &and            ("ecx",-64);
1544         &and            ("edx",63);
1545
1546         &vmovdqu        (&X($T0),&QWP(16*0,"esi"));
1547         &cmp            ("edx",32);
1548         &jb             (&label("one"));
1549
1550         &vmovdqu        (&X($T1),&QWP(16*1,"esi"));
1551         &je             (&label("two"));
1552
1553         &vinserti128    ($T0,$T0,&QWP(16*2,"esi"),1);
1554         &lea            ("esi",&DWP(16*3,"esi"));
1555         &lea            ("ebx",&DWP(8,"ebx"));          # three padbits
1556         &lea            ("edx",&DWP(32*5+128+8,"esp")); # --:r^1:r^2:r^3 (*)
1557         &jmp            (&label("tail"));
1558
1559 &set_label("two");
1560         &lea            ("esi",&DWP(16*2,"esi"));
1561         &lea            ("ebx",&DWP(16,"ebx"));         # two padbits
1562         &lea            ("edx",&DWP(32*5+128+16,"esp"));# --:--:r^1:r^2 (*)
1563         &jmp            (&label("tail"));
1564
1565 &set_label("one");
1566         &lea            ("esi",&DWP(16*1,"esi"));
1567         &vpxor          ($T1,$T1,$T1);
1568         &lea            ("ebx",&DWP(32,"ebx","eax",8)); # one or no padbits
1569         &lea            ("edx",&DWP(32*5+128+24,"esp"));# --:--:--:r^1 (*)
1570         &jmp            (&label("tail"));
1571
1572 # (*)   spots marked with '--' are data from next table entry, but they
1573 #       are multiplied by 0 and therefore rendered insignificant
1574
1575 &set_label("even",32);
1576         &vmovdqu        (&X($T0),&QWP(16*0,"esi"));     # load input
1577         &vmovdqu        (&X($T1),&QWP(16*1,"esi"));
1578         &vinserti128    ($T0,$T0,&QWP(16*2,"esi"),1);
1579         &vinserti128    ($T1,$T1,&QWP(16*3,"esi"),1);
1580         &lea            ("esi",&DWP(16*4,"esi"));
1581         &sub            ("ecx",64);
1582         &jz             (&label("tail"));
1583
1584 &set_label("loop");
1585         ################################################################
1586         # ((inp[0]*r^4+r[4])*r^4+r[8])*r^4
1587         # ((inp[1]*r^4+r[5])*r^4+r[9])*r^3
1588         # ((inp[2]*r^4+r[6])*r^4+r[10])*r^2
1589         # ((inp[3]*r^4+r[7])*r^4+r[11])*r^1
1590         #   \________/ \_______/
1591         ################################################################
1592
1593 sub vsplat_input {
1594         &vmovdqa        (&QWP(32*2,"esp"),$D2);
1595         &vpsrldq        ($D2,$T0,6);                    # splat input
1596         &vmovdqa        (&QWP(32*0,"esp"),$D0);
1597         &vpsrldq        ($D0,$T1,6);
1598         &vmovdqa        (&QWP(32*1,"esp"),$D1);
1599         &vpunpckhqdq    ($D1,$T0,$T1);                  # 4
1600         &vpunpcklqdq    ($T0,$T0,$T1);                  # 0:1
1601         &vpunpcklqdq    ($D2,$D2,$D0);                  # 2:3
1602
1603         &vpsrlq         ($D0,$D2,30);
1604         &vpsrlq         ($D2,$D2,4);
1605         &vpsrlq         ($T1,$T0,26);
1606         &vpsrlq         ($D1,$D1,40);                   # 4
1607         &vpand          ($D2,$D2,$MASK);                # 2
1608         &vpand          ($T0,$T0,$MASK);                # 0
1609         &vpand          ($T1,$T1,$MASK);                # 1
1610         &vpand          ($D0,$D0,$MASK);                # 3 (*)
1611         &vpor           ($D1,$D1,&QWP(0,"ebx"));        # padbit, yes, always
1612
1613         # (*)   note that output is counterintuitive, inp[3:4] is
1614         #       returned in $D1-2, while $D3-4 are preserved;
1615 }
1616         &vsplat_input   ();
1617
1618 sub vpmuladd {
1619 my $addr = shift;
1620
1621         &vpaddq         ($D2,$D2,&QWP(32*2,"esp"));     # add hash value
1622         &vpaddq         ($T0,$T0,&QWP(32*0,"esp"));
1623         &vpaddq         ($T1,$T1,&QWP(32*1,"esp"));
1624         &vpaddq         ($D0,$D0,$D3);
1625         &vpaddq         ($D1,$D1,$D4);
1626
1627         ################################################################
1628         # d3 = h2*r1   + h0*r3 + h1*r2   + h3*r0   + h4*5*r4
1629         # d4 = h2*r2   + h0*r4 + h1*r3   + h3*r1   + h4*r0
1630         # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1
1631         # d1 = h2*5*r4 + h0*r1 + h1*r0   + h3*5*r3 + h4*5*r2
1632         # d2 = h2*r0   + h0*r2 + h1*r1   + h3*5*r4 + h4*5*r3
1633
1634         &vpmuludq       ($D3,$D2,&$addr(1));            # d3 = h2*r1
1635          &vmovdqa       (QWP(32*1,"esp"),$T1);
1636         &vpmuludq       ($D4,$D2,&$addr(2));            # d4 = h2*r2
1637          &vmovdqa       (QWP(32*3,"esp"),$D0);
1638         &vpmuludq       ($D0,$D2,&$addr(7));            # d0 = h2*s3
1639          &vmovdqa       (QWP(32*4,"esp"),$D1);
1640         &vpmuludq       ($D1,$D2,&$addr(8));            # d1 = h2*s4
1641         &vpmuludq       ($D2,$D2,&$addr(0));            # d2 = h2*r0
1642
1643         &vpmuludq       ($T2,$T0,&$addr(3));            # h0*r3
1644         &vpaddq         ($D3,$D3,$T2);                  # d3 += h0*r3
1645         &vpmuludq       ($T1,$T0,&$addr(4));            # h0*r4
1646         &vpaddq         ($D4,$D4,$T1);                  # d4 + h0*r4
1647         &vpmuludq       ($T2,$T0,&$addr(0));            # h0*r0
1648         &vpaddq         ($D0,$D0,$T2);                  # d0 + h0*r0
1649          &vmovdqa       ($T2,&QWP(32*1,"esp"));         # h1
1650         &vpmuludq       ($T1,$T0,&$addr(1));            # h0*r1
1651         &vpaddq         ($D1,$D1,$T1);                  # d1 += h0*r1
1652         &vpmuludq       ($T0,$T0,&$addr(2));            # h0*r2
1653         &vpaddq         ($D2,$D2,$T0);                  # d2 += h0*r2
1654
1655         &vpmuludq       ($T1,$T2,&$addr(2));            # h1*r2
1656         &vpaddq         ($D3,$D3,$T1);                  # d3 += h1*r2
1657         &vpmuludq       ($T0,$T2,&$addr(3));            # h1*r3
1658         &vpaddq         ($D4,$D4,$T0);                  # d4 += h1*r3
1659         &vpmuludq       ($T1,$T2,&$addr(8));            # h1*s4
1660         &vpaddq         ($D0,$D0,$T1);                  # d0 += h1*s4
1661          &vmovdqa       ($T1,&QWP(32*3,"esp"));         # h3
1662         &vpmuludq       ($T0,$T2,&$addr(0));            # h1*r0
1663         &vpaddq         ($D1,$D1,$T0);                  # d1 += h1*r0
1664         &vpmuludq       ($T2,$T2,&$addr(1));            # h1*r1
1665         &vpaddq         ($D2,$D2,$T2);                  # d2 += h1*r1
1666
1667         &vpmuludq       ($T0,$T1,&$addr(0));            # h3*r0
1668         &vpaddq         ($D3,$D3,$T0);                  # d3 += h3*r0
1669         &vpmuludq       ($T2,$T1,&$addr(1));            # h3*r1
1670         &vpaddq         ($D4,$D4,$T2);                  # d4 += h3*r1
1671         &vpmuludq       ($T0,$T1,&$addr(6));            # h3*s2
1672         &vpaddq         ($D0,$D0,$T0);                  # d0 += h3*s2
1673          &vmovdqa       ($T0,&QWP(32*4,"esp"));         # h4
1674         &vpmuludq       ($T2,$T1,&$addr(7));            # h3*s3
1675         &vpaddq         ($D1,$D1,$T2);                  # d1+= h3*s3
1676         &vpmuludq       ($T1,$T1,&$addr(8));            # h3*s4
1677         &vpaddq         ($D2,$D2,$T1);                  # d2 += h3*s4
1678
1679         &vpmuludq       ($T2,$T0,&$addr(8));            # h4*s4
1680         &vpaddq         ($D3,$D3,$T2);                  # d3 += h4*s4
1681         &vpmuludq       ($T1,$T0,&$addr(5));            # h4*s1
1682         &vpaddq         ($D0,$D0,$T1);                  # d0 += h4*s1
1683         &vpmuludq       ($T2,$T0,&$addr(0));            # h4*r0
1684         &vpaddq         ($D4,$D4,$T2);                  # d4 += h4*r0
1685          &vmovdqa       ($MASK,&QWP(64,"ebx"));
1686         &vpmuludq       ($T1,$T0,&$addr(6));            # h4*s2
1687         &vpaddq         ($D1,$D1,$T1);                  # d1 += h4*s2
1688         &vpmuludq       ($T0,$T0,&$addr(7));            # h4*s3
1689         &vpaddq         ($D2,$D2,$T0);                  # d2 += h4*s3
1690 }
1691         &vpmuladd       (sub {  my $i=shift; &QWP(32*$i-128,"edx");     });
1692
1693 sub vlazy_reduction {
1694         ################################################################
1695         # lazy reduction
1696
1697          &vpsrlq        ($T0,$D3,26);
1698          &vpand         ($D3,$D3,$MASK);
1699         &vpsrlq         ($T1,$D0,26);
1700         &vpand          ($D0,$D0,$MASK);
1701          &vpaddq        ($D4,$D4,$T0);                  # h3 -> h4
1702         &vpaddq         ($D1,$D1,$T1);                  # h0 -> h1
1703          &vpsrlq        ($T0,$D4,26);
1704          &vpand         ($D4,$D4,$MASK);
1705         &vpsrlq         ($T1,$D1,26);
1706         &vpand          ($D1,$D1,$MASK);
1707         &vpaddq         ($D2,$D2,$T1);                  # h1 -> h2
1708          &vpaddd        ($D0,$D0,$T0);
1709          &vpsllq        ($T0,$T0,2);
1710         &vpsrlq         ($T1,$D2,26);
1711         &vpand          ($D2,$D2,$MASK);
1712          &vpaddd        ($D0,$D0,$T0);                  # h4 -> h0
1713         &vpaddd         ($D3,$D3,$T1);                  # h2 -> h3
1714         &vpsrlq         ($T1,$D3,26);
1715          &vpsrlq        ($T0,$D0,26);
1716          &vpand         ($D0,$D0,$MASK);
1717         &vpand          ($D3,$D3,$MASK);
1718          &vpaddd        ($D1,$D1,$T0);                  # h0 -> h1
1719         &vpaddd         ($D4,$D4,$T1);                  # h3 -> h4
1720 }
1721         &vlazy_reduction();
1722
1723         &vmovdqu        (&X($T0),&QWP(16*0,"esi"));     # load input
1724         &vmovdqu        (&X($T1),&QWP(16*1,"esi"));
1725         &vinserti128    ($T0,$T0,&QWP(16*2,"esi"),1);
1726         &vinserti128    ($T1,$T1,&QWP(16*3,"esi"),1);
1727         &lea            ("esi",&DWP(16*4,"esi"));
1728         &sub            ("ecx",64);
1729         &jnz            (&label("loop"));
1730
1731 &set_label("tail");
1732         &vsplat_input   ();
1733         &and            ("ebx",-64);                    # restore pointer
1734
1735         &vpmuladd       (sub {  my $i=shift; &QWP(4+32*$i-128,"edx");   });
1736
1737         ################################################################
1738         # horizontal addition
1739
1740         &vpsrldq        ($T0,$D4,8);
1741         &vpsrldq        ($T1,$D3,8);
1742         &vpaddq         ($D4,$D4,$T0);
1743         &vpsrldq        ($T0,$D0,8);
1744         &vpaddq         ($D3,$D3,$T1);
1745         &vpsrldq        ($T1,$D1,8);
1746         &vpaddq         ($D0,$D0,$T0);
1747         &vpsrldq        ($T0,$D2,8);
1748         &vpaddq         ($D1,$D1,$T1);
1749         &vpermq         ($T1,$D4,2);                    # keep folding
1750         &vpaddq         ($D2,$D2,$T0);
1751         &vpermq         ($T0,$D3,2);
1752         &vpaddq         ($D4,$D4,$T1);
1753         &vpermq         ($T1,$D0,2);
1754         &vpaddq         ($D3,$D3,$T0);
1755         &vpermq         ($T0,$D1,2);
1756         &vpaddq         ($D0,$D0,$T1);
1757         &vpermq         ($T1,$D2,2);
1758         &vpaddq         ($D1,$D1,$T0);
1759         &vpaddq         ($D2,$D2,$T1);
1760
1761         &vlazy_reduction();
1762
1763         &cmp            ("ecx",0);
1764         &je             (&label("done"));
1765
1766         ################################################################
1767         # clear all but single word
1768
1769         &vpshufd        (&X($D0),&X($D0),0b11111100);
1770         &lea            ("edx",&DWP(32*5+128,"esp"));   # restore pointer
1771         &vpshufd        (&X($D1),&X($D1),0b11111100);
1772         &vpshufd        (&X($D2),&X($D2),0b11111100);
1773         &vpshufd        (&X($D3),&X($D3),0b11111100);
1774         &vpshufd        (&X($D4),&X($D4),0b11111100);
1775         &jmp            (&label("even"));
1776
1777 &set_label("done",16);
1778         &vmovd          (&DWP(-16*3+4*0,"edi"),&X($D0));# store hash value
1779         &vmovd          (&DWP(-16*3+4*1,"edi"),&X($D1));
1780         &vmovd          (&DWP(-16*3+4*2,"edi"),&X($D2));
1781         &vmovd          (&DWP(-16*3+4*3,"edi"),&X($D3));
1782         &vmovd          (&DWP(-16*3+4*4,"edi"),&X($D4));
1783         &vzeroupper     ();
1784         &mov    ("esp","ebp");
1785 &set_label("nodata");
1786 &function_end("_poly1305_blocks_avx2");
1787 }
1788 &set_label("const_sse2",64);
1789         &data_word(1<<24,0,     1<<24,0,        1<<24,0,        1<<24,0);
1790         &data_word(0,0,         0,0,            0,0,            0,0);
1791         &data_word(0x03ffffff,0,0x03ffffff,0,   0x03ffffff,0,   0x03ffffff,0);
1792         &data_word(0x0fffffff,0x0ffffffc,0x0ffffffc,0x0ffffffc);
1793 }
1794 &asciz  ("Poly1305 for x86, CRYPTOGAMS by <appro\@openssl.org>");
1795 &align  (4);
1796
1797 &asm_finish();