crypto/poly1305/asm/poly1305-x86.pl

   1 #!/usr/bin/env perl
   2 #
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9 #
  10 # This module implements Poly1305 hash for x86.
  11 #
  12 # April 2015
  13 #
  14 # Numbers are cycles per processed byte with poly1305_blocks alone,
  15 # measured with rdtsc at fixed clock frequency.
  16 #
  17 #               IALU/gcc-3.4(*) SSE2(**)        AVX2
  18 # Pentium       15.7/+80%       -
  19 # PIII          6.21/+90%       -
  20 # P4            19.8/+40%       3.24
  21 # Core 2        4.85/+90%       1.80
  22 # Westmere      4.58/+100%      1.43
  23 # Sandy Bridge  3.90/+100%      1.36
  24 # Haswell       3.88/+70%       1.18            0.72
  25 # Silvermont    11.0/+40%       4.80
  26 # VIA Nano      6.71/+90%       2.47
  27 # Sledgehammer  3.51/+180%      4.27
  28 # Bulldozer     4.53/+140%      1.31
  29 #
  30 # (*)   gcc 4.8 for some reason generated worse code;
  31 # (**)  besides SSE2 there are floating-point and AVX options; FP
  32 #       is deemed unnecessary, because pre-SSE2 processor are too
  33 #       old to care about, while it's not the fastest option on
  34 #       SSE2-capable ones; AVX is omitted, because it doesn't give
  35 #       a lot of improvement, 5-10% depending on processor;
  36
  37 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  38 push(@INC,"${dir}","${dir}../../perlasm");
  39 require "x86asm.pl";
  40
  41 $output=pop;
  42 open STDOUT,">$output";
  43
  44 &asm_init($ARGV[0],"poly1305-x86.pl",$ARGV[$#ARGV] eq "386");
  45
  46 $sse2=$avx=0;
  47 for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
  48
  49 if ($sse2) {
  50         &static_label("const_sse2");
  51         &static_label("enter_blocks");
  52         &static_label("enter_emit");
  53         &external_label("OPENSSL_ia32cap_P");
  54
  55         if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  56                         =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  57                 $avx = ($1>=2.19) + ($1>=2.22);
  58         }
  59
  60         if (!$avx && $ARGV[0] eq "win32n" &&
  61            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
  62         $avx = ($1>=2.09) + ($1>=2.10);
  63         }
  64
  65         if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) {
  66                 $avx = ($2>=3.0) + ($2>3.0);
  67         }
  68 }
  69
  70 ########################################################################
  71 # Layout of opaque area is following.
  72 #
  73 #       unsigned __int32 h[5];          # current hash value base 2^32
  74 #       unsigned __int32 pad;           # is_base2_26 in vector context
  75 #       unsigned __int32 r[4];          # key value base 2^32
  76
  77 &align(64);
  78 &function_begin("poly1305_init");
  79         &mov    ("edi",&wparam(0));             # context
  80         &mov    ("esi",&wparam(1));             # key
  81         &mov    ("ebp",&wparam(2));             # function table
  82
  83         &xor    ("eax","eax");
  84         &mov    (&DWP(4*0,"edi"),"eax");        # zero hash value
  85         &mov    (&DWP(4*1,"edi"),"eax");
  86         &mov    (&DWP(4*2,"edi"),"eax");
  87         &mov    (&DWP(4*3,"edi"),"eax");
  88         &mov    (&DWP(4*4,"edi"),"eax");
  89         &mov    (&DWP(4*5,"edi"),"eax");        # is_base2_26
  90
  91         &cmp    ("esi",0);
  92         &je     (&label("nokey"));
  93
  94     if ($sse2) {
  95         &call   (&label("pic_point"));
  96     &set_label("pic_point");
  97         &blindpop("ebx");
  98
  99         &lea    ("eax",&DWP("poly1305_blocks-".&label("pic_point"),"ebx"));
 100         &lea    ("edx",&DWP("poly1305_emit-".&label("pic_point"),"ebx"));
 101
 102         &picmeup("edi","OPENSSL_ia32cap_P","ebx",&label("pic_point"));
 103         &mov    ("ecx",&DWP(0,"edi"));
 104         &and    ("ecx",1<<26|1<<24);
 105         &cmp    ("ecx",1<<26|1<<24);            # SSE2 and XMM?
 106         &jne    (&label("no_sse2"));
 107
 108         &lea    ("eax",&DWP("_poly1305_blocks_sse2-".&label("pic_point"),"ebx"));
 109         &lea    ("edx",&DWP("_poly1305_emit_sse2-".&label("pic_point"),"ebx"));
 110
 111       if ($avx>1) {
 112         &mov    ("ecx",&DWP(8,"edi"));
 113         &test   ("ecx",1<<5);                   # AVX2?
 114         &jz     (&label("no_sse2"));
 115
 116         &lea    ("eax",&DWP("_poly1305_blocks_avx2-".&label("pic_point"),"ebx"));
 117       }
 118     &set_label("no_sse2");
 119         &mov    ("edi",&wparam(0));             # reload context
 120         &mov    (&DWP(0,"ebp"),"eax");          # fill function table
 121         &mov    (&DWP(4,"ebp"),"edx");
 122     }
 123
 124         &mov    ("eax",&DWP(4*0,"esi"));        # load input key
 125         &mov    ("ebx",&DWP(4*1,"esi"));
 126         &mov    ("ecx",&DWP(4*2,"esi"));
 127         &mov    ("edx",&DWP(4*3,"esi"));
 128         &and    ("eax",0x0fffffff);
 129         &and    ("ebx",0x0ffffffc);
 130         &and    ("ecx",0x0ffffffc);
 131         &and    ("edx",0x0ffffffc);
 132         &mov    (&DWP(4*6,"edi"),"eax");
 133         &mov    (&DWP(4*7,"edi"),"ebx");
 134         &mov    (&DWP(4*8,"edi"),"ecx");
 135         &mov    (&DWP(4*9,"edi"),"edx");
 136
 137         &mov    ("eax",$sse2);
 138 &set_label("nokey");
 139 &function_end("poly1305_init");
 140
 141 ($h0,$h1,$h2,$h3,$h4,
 142  $d0,$d1,$d2,$d3,
 143  $r0,$r1,$r2,$r3,
 144      $s1,$s2,$s3)=map(4*$_,(0..15));
 145
 146 &function_begin("poly1305_blocks");
 147         &mov    ("edi",&wparam(0));             # ctx
 148         &mov    ("esi",&wparam(1));             # inp
 149         &mov    ("ecx",&wparam(2));             # len
 150 &set_label("enter_blocks");
 151         &and    ("ecx",-15);
 152         &jz     (&label("nodata"));
 153
 154         &stack_push(16);
 155         &mov    ("eax",&DWP(4*6,"edi"));        # r0
 156         &mov    ("ebx",&DWP(4*7,"edi"));        # r1
 157          &lea   ("ebp",&DWP(0,"esi","ecx"));    # end of input
 158         &mov    ("ecx",&DWP(4*8,"edi"));        # r2
 159         &mov    ("edx",&DWP(4*9,"edi"));        # r3
 160
 161         &mov    (&wparam(2),"ebp");
 162         &mov    ("ebp","esi");
 163
 164         &mov    (&DWP($r0,"esp"),"eax");        # r0
 165         &mov    ("eax","ebx");
 166         &shr    ("eax",2);
 167         &mov    (&DWP($r1,"esp"),"ebx");        # r1
 168         &add    ("eax","ebx");                  # s1
 169         &mov    ("ebx","ecx");
 170         &shr    ("ebx",2);
 171         &mov    (&DWP($r2,"esp"),"ecx");        # r2
 172         &add    ("ebx","ecx");                  # s2
 173         &mov    ("ecx","edx");
 174         &shr    ("ecx",2);
 175         &mov    (&DWP($r3,"esp"),"edx");        # r3
 176         &add    ("ecx","edx");                  # s3
 177         &mov    (&DWP($s1,"esp"),"eax");        # s1
 178         &mov    (&DWP($s2,"esp"),"ebx");        # s2
 179         &mov    (&DWP($s3,"esp"),"ecx");        # s3
 180
 181         &mov    ("eax",&DWP(4*0,"edi"));        # load hash value
 182         &mov    ("ebx",&DWP(4*1,"edi"));
 183         &mov    ("ecx",&DWP(4*2,"edi"));
 184         &mov    ("esi",&DWP(4*3,"edi"));
 185         &mov    ("edi",&DWP(4*4,"edi"));
 186         &jmp    (&label("loop"));
 187
 188 &set_label("loop",32);
 189         &add    ("eax",&DWP(4*0,"ebp"));        # accumulate input
 190         &adc    ("ebx",&DWP(4*1,"ebp"));
 191         &adc    ("ecx",&DWP(4*2,"ebp"));
 192         &adc    ("esi",&DWP(4*3,"ebp"));
 193         &lea    ("ebp",&DWP(4*4,"ebp"));
 194         &adc    ("edi",&wparam(3));             # padbit
 195
 196         &mov    (&DWP($h0,"esp"),"eax");        # put aside hash[+inp]
 197         &mov    (&DWP($h3,"esp"),"esi");
 198
 199         &mul    (&DWP($r0,"esp"));              # h0*r0
 200          &mov   (&DWP($h4,"esp"),"edi");
 201         &mov    ("edi","eax");
 202         &mov    ("eax","ebx");                  # h1
 203         &mov    ("esi","edx");
 204         &mul    (&DWP($s3,"esp"));              # h1*s3
 205         &add    ("edi","eax");
 206         &mov    ("eax","ecx");                  # h2
 207         &adc    ("esi","edx");
 208         &mul    (&DWP($s2,"esp"));              # h2*s2
 209         &add    ("edi","eax");
 210         &mov    ("eax",&DWP($h3,"esp"));
 211         &adc    ("esi","edx");
 212         &mul    (&DWP($s1,"esp"));              # h3*s1
 213         &add    ("edi","eax");
 214          &mov   ("eax",&DWP($h0,"esp"));
 215         &adc    ("esi","edx");
 216
 217         &mul    (&DWP($r1,"esp"));              # h0*r1
 218          &mov   (&DWP($d0,"esp"),"edi");
 219         &xor    ("edi","edi");
 220         &add    ("esi","eax");
 221         &mov    ("eax","ebx");                  # h1
 222         &adc    ("edi","edx");
 223         &mul    (&DWP($r0,"esp"));              # h1*r0
 224         &add    ("esi","eax");
 225         &mov    ("eax","ecx");                  # h2
 226         &adc    ("edi","edx");
 227         &mul    (&DWP($s3,"esp"));              # h2*s3
 228         &add    ("esi","eax");
 229         &mov    ("eax",&DWP($h3,"esp"));
 230         &adc    ("edi","edx");
 231         &mul    (&DWP($s2,"esp"));              # h3*s2
 232         &add    ("esi","eax");
 233         &mov    ("eax",&DWP($h4,"esp"));
 234         &adc    ("edi","edx");
 235         &imul   ("eax",&DWP($s1,"esp"));        # h4*s1
 236         &add    ("esi","eax");
 237          &mov   ("eax",&DWP($h0,"esp"));
 238         &adc    ("edi",0);
 239
 240         &mul    (&DWP($r2,"esp"));              # h0*r2
 241          &mov   (&DWP($d1,"esp"),"esi");
 242         &xor    ("esi","esi");
 243         &add    ("edi","eax");
 244         &mov    ("eax","ebx");                  # h1
 245         &adc    ("esi","edx");
 246         &mul    (&DWP($r1,"esp"));              # h1*r1
 247         &add    ("edi","eax");
 248         &mov    ("eax","ecx");                  # h2
 249         &adc    ("esi","edx");
 250         &mul    (&DWP($r0,"esp"));              # h2*r0
 251         &add    ("edi","eax");
 252         &mov    ("eax",&DWP($h3,"esp"));
 253         &adc    ("esi","edx");
 254         &mul    (&DWP($s3,"esp"));              # h3*s3
 255         &add    ("edi","eax");
 256         &mov    ("eax",&DWP($h4,"esp"));
 257         &adc    ("esi","edx");
 258         &imul   ("eax",&DWP($s2,"esp"));        # h4*s2
 259         &add    ("edi","eax");
 260          &mov   ("eax",&DWP($h0,"esp"));
 261         &adc    ("esi",0);
 262
 263         &mul    (&DWP($r3,"esp"));              # h0*r3
 264          &mov   (&DWP($d2,"esp"),"edi");
 265         &xor    ("edi","edi");
 266         &add    ("esi","eax");
 267         &mov    ("eax","ebx");                  # h1
 268         &adc    ("edi","edx");
 269         &mul    (&DWP($r2,"esp"));              # h1*r2
 270         &add    ("esi","eax");
 271         &mov    ("eax","ecx");                  # h2
 272         &adc    ("edi","edx");
 273         &mul    (&DWP($r1,"esp"));              # h2*r1
 274         &add    ("esi","eax");
 275         &mov    ("eax",&DWP($h3,"esp"));
 276         &adc    ("edi","edx");
 277         &mul    (&DWP($r0,"esp"));              # h3*r0
 278         &add    ("esi","eax");
 279          &mov   ("ecx",&DWP($h4,"esp"));
 280         &adc    ("edi","edx");
 281
 282         &mov    ("edx","ecx");
 283         &imul   ("ecx",&DWP($s3,"esp"));        # h4*s3
 284         &add    ("esi","ecx");
 285          &mov   ("eax",&DWP($d0,"esp"));
 286         &adc    ("edi",0);
 287
 288         &imul   ("edx",&DWP($r0,"esp"));        # h4*r0
 289         &add    ("edx","edi");
 290
 291         &mov    ("ebx",&DWP($d1,"esp"));
 292         &mov    ("ecx",&DWP($d2,"esp"));
 293
 294         &mov    ("edi","edx");                  # last reduction step
 295         &shr    ("edx",2);
 296         &and    ("edi",3);
 297         &lea    ("edx",&DWP(0,"edx","edx",4));  # *5
 298         &add    ("eax","edx");
 299         &adc    ("ebx",0);
 300         &adc    ("ecx",0);
 301         &adc    ("esi",0);
 302         &adc    ("edi",0);
 303
 304         &cmp    ("ebp",&wparam(2));             # done yet?
 305         &jne    (&label("loop"));
 306
 307         &mov    ("edx",&wparam(0));             # ctx
 308         &stack_pop(16);
 309         &mov    (&DWP(4*0,"edx"),"eax");        # store hash value
 310         &mov    (&DWP(4*1,"edx"),"ebx");
 311         &mov    (&DWP(4*2,"edx"),"ecx");
 312         &mov    (&DWP(4*3,"edx"),"esi");
 313         &mov    (&DWP(4*4,"edx"),"edi");
 314 &set_label("nodata");
 315 &function_end("poly1305_blocks");
 316
 317 &function_begin("poly1305_emit");
 318         &mov    ("ebp",&wparam(0));             # context
 319 &set_label("enter_emit");
 320         &mov    ("edi",&wparam(1));             # output
 321         &mov    ("eax",&DWP(4*0,"ebp"));        # load hash value
 322         &mov    ("ebx",&DWP(4*1,"ebp"));
 323         &mov    ("ecx",&DWP(4*2,"ebp"));
 324         &mov    ("edx",&DWP(4*3,"ebp"));
 325         &mov    ("esi",&DWP(4*4,"ebp"));
 326
 327         &add    ("eax",5);                      # compare to modulus
 328         &adc    ("ebx",0);
 329         &adc    ("ecx",0);
 330         &adc    ("edx",0);
 331         &adc    ("esi",0);
 332         &shr    ("esi",2);                      # did it carry/borrow?
 333         &neg    ("esi");                        # do we choose hash-modulus?
 334
 335         &and    ("eax","esi");
 336         &and    ("ebx","esi");
 337         &and    ("ecx","esi");
 338         &and    ("edx","esi");
 339         &mov    (&DWP(4*0,"edi"),"eax");
 340         &mov    (&DWP(4*1,"edi"),"ebx");
 341         &mov    (&DWP(4*2,"edi"),"ecx");
 342         &mov    (&DWP(4*3,"edi"),"edx");
 343
 344         &not    ("esi");                        # or original hash value?
 345         &mov    ("eax",&DWP(4*0,"ebp"));
 346         &mov    ("ebx",&DWP(4*1,"ebp"));
 347         &mov    ("ecx",&DWP(4*2,"ebp"));
 348         &mov    ("edx",&DWP(4*3,"ebp"));
 349         &mov    ("ebp",&wparam(2));
 350         &and    ("eax","esi");
 351         &and    ("ebx","esi");
 352         &and    ("ecx","esi");
 353         &and    ("edx","esi");
 354         &or     ("eax",&DWP(4*0,"edi"));
 355         &or     ("ebx",&DWP(4*1,"edi"));
 356         &or     ("ecx",&DWP(4*2,"edi"));
 357         &or     ("edx",&DWP(4*3,"edi"));
 358
 359         &add    ("eax",&DWP(4*0,"ebp"));        # accumulate key
 360         &adc    ("ebx",&DWP(4*1,"ebp"));
 361         &adc    ("ecx",&DWP(4*2,"ebp"));
 362         &adc    ("edx",&DWP(4*3,"ebp"));
 363
 364         &mov    (&DWP(4*0,"edi"),"eax");
 365         &mov    (&DWP(4*1,"edi"),"ebx");
 366         &mov    (&DWP(4*2,"edi"),"ecx");
 367         &mov    (&DWP(4*3,"edi"),"edx");
 368 &function_end("poly1305_emit");
 369
 370 if ($sse2) {
 371 ########################################################################
 372 # Layout of opaque area is following.
 373 #
 374 #       unsigned __int32 h[5];          # current hash value base 2^26
 375 #       unsigned __int32 is_base2_26;
 376 #       unsigned __int32 r[4];          # key value base 2^32
 377 #       unsigned __int32 pad[2];
 378 #       struct { unsigned __int32 r^4, r^3, r^2, r^1; } r[9];
 379 #
 380 # where r^n are base 2^26 digits of degrees of multiplier key. There are
 381 # 5 digits, but last four are interleaved with multiples of 5, totalling
 382 # in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.
 383
 384 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("xmm$_",(0..7));
 385 my $MASK=$T2;   # borrow and keep in mind
 386
 387 &align  (32);
 388 &function_begin_B("_poly1305_init_sse2");
 389         &movdqu         ($D4,&QWP(4*6,"edi"));          # key base 2^32
 390         &lea            ("edi",&DWP(16*3,"edi"));       # size optimization
 391         &mov            ("ebp","esp");
 392         &sub            ("esp",16*(9+5));
 393         &and            ("esp",-16);
 394
 395         #&pand          ($D4,&QWP(96,"ebx"));           # magic mask
 396         &movq           ($MASK,&QWP(64,"ebx"));
 397
 398         &movdqa         ($D0,$D4);
 399         &movdqa         ($D1,$D4);
 400         &movdqa         ($D2,$D4);
 401
 402         &pand           ($D0,$MASK);                    # -> base 2^26
 403         &psrlq          ($D1,26);
 404         &psrldq         ($D2,6);
 405         &pand           ($D1,$MASK);
 406         &movdqa         ($D3,$D2);
 407         &psrlq          ($D2,4)
 408         &psrlq          ($D3,30);
 409         &pand           ($D2,$MASK);
 410         &pand           ($D3,$MASK);
 411         &psrldq         ($D4,13);
 412
 413         &lea            ("edx",&DWP(16*9,"esp"));       # size optimization
 414         &mov            ("ecx",2);
 415 &set_label("square");
 416         &movdqa         (&QWP(16*0,"esp"),$D0);
 417         &movdqa         (&QWP(16*1,"esp"),$D1);
 418         &movdqa         (&QWP(16*2,"esp"),$D2);
 419         &movdqa         (&QWP(16*3,"esp"),$D3);
 420         &movdqa         (&QWP(16*4,"esp"),$D4);
 421
 422         &movdqa         ($T1,$D1);
 423         &movdqa         ($T0,$D2);
 424         &pslld          ($T1,2);
 425         &pslld          ($T0,2);
 426         &paddd          ($T1,$D1);                      # *5
 427         &paddd          ($T0,$D2);                      # *5
 428         &movdqa         (&QWP(16*5,"esp"),$T1);
 429         &movdqa         (&QWP(16*6,"esp"),$T0);
 430         &movdqa         ($T1,$D3);
 431         &movdqa         ($T0,$D4);
 432         &pslld          ($T1,2);
 433         &pslld          ($T0,2);
 434         &paddd          ($T1,$D3);                      # *5
 435         &paddd          ($T0,$D4);                      # *5
 436         &movdqa         (&QWP(16*7,"esp"),$T1);
 437         &movdqa         (&QWP(16*8,"esp"),$T0);
 438
 439         &pshufd         ($T1,$D0,0b01000100);
 440         &movdqa         ($T0,$D1);
 441         &pshufd         ($D1,$D1,0b01000100);
 442         &pshufd         ($D2,$D2,0b01000100);
 443         &pshufd         ($D3,$D3,0b01000100);
 444         &pshufd         ($D4,$D4,0b01000100);
 445         &movdqa         (&QWP(16*0,"edx"),$T1);
 446         &movdqa         (&QWP(16*1,"edx"),$D1);
 447         &movdqa         (&QWP(16*2,"edx"),$D2);
 448         &movdqa         (&QWP(16*3,"edx"),$D3);
 449         &movdqa         (&QWP(16*4,"edx"),$D4);
 450
 451         ################################################################
 452         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
 453         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
 454         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
 455         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
 456         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
 457
 458         &pmuludq        ($D4,$D0);                      # h4*r0
 459         &pmuludq        ($D3,$D0);                      # h3*r0
 460         &pmuludq        ($D2,$D0);                      # h2*r0
 461         &pmuludq        ($D1,$D0);                      # h1*r0
 462         &pmuludq        ($D0,$T1);                      # h0*r0
 463
 464 sub pmuladd {
 465 my $load = shift;
 466 my $base = shift; $base = "esp" if (!defined($base));
 467
 468         ################################################################
 469         # As for choice to "rotate" $T0-$T2 in order to move paddq
 470         # past next multiplication. While it makes code harder to read
 471         # and doesn't have significant effect on most processors, it
 472         # makes a lot of difference on Atom, up to 30% improvement.
 473
 474         &movdqa         ($T1,$T0);
 475         &pmuludq        ($T0,&QWP(16*3,$base));         # r1*h3
 476         &movdqa         ($T2,$T1);
 477         &pmuludq        ($T1,&QWP(16*2,$base));         # r1*h2
 478         &paddq          ($D4,$T0);
 479         &movdqa         ($T0,$T2);
 480         &pmuludq        ($T2,&QWP(16*1,$base));         # r1*h1
 481         &paddq          ($D3,$T1);
 482         &$load          ($T1,5);                        # s1
 483         &pmuludq        ($T0,&QWP(16*0,$base));         # r1*h0
 484         &paddq          ($D2,$T2);
 485         &pmuludq        ($T1,&QWP(16*4,$base));         # s1*h4
 486          &$load         ($T2,2);                        # r2^n
 487         &paddq          ($D1,$T0);
 488
 489         &movdqa         ($T0,$T2);
 490         &pmuludq        ($T2,&QWP(16*2,$base));         # r2*h2
 491          &paddq         ($D0,$T1);
 492         &movdqa         ($T1,$T0);
 493         &pmuludq        ($T0,&QWP(16*1,$base));         # r2*h1
 494         &paddq          ($D4,$T2);
 495         &$load          ($T2,6);                        # s2^n
 496         &pmuludq        ($T1,&QWP(16*0,$base));         # r2*h0
 497         &paddq          ($D3,$T0);
 498         &movdqa         ($T0,$T2);
 499         &pmuludq        ($T2,&QWP(16*4,$base));         # s2*h4
 500         &paddq          ($D2,$T1);
 501         &pmuludq        ($T0,&QWP(16*3,$base));         # s2*h3
 502          &$load         ($T1,3);                        # r3^n
 503         &paddq          ($D1,$T2);
 504
 505         &movdqa         ($T2,$T1);
 506         &pmuludq        ($T1,&QWP(16*1,$base));         # r3*h1
 507          &paddq         ($D0,$T0);
 508         &$load          ($T0,7);                        # s3^n
 509         &pmuludq        ($T2,&QWP(16*0,$base));         # r3*h0
 510         &paddq          ($D4,$T1);
 511         &movdqa         ($T1,$T0);
 512         &pmuludq        ($T0,&QWP(16*4,$base));         # s3*h4
 513         &paddq          ($D3,$T2);
 514         &movdqa         ($T2,$T1);
 515         &pmuludq        ($T1,&QWP(16*3,$base));         # s3*h3
 516         &paddq          ($D2,$T0);
 517         &pmuludq        ($T2,&QWP(16*2,$base));         # s3*h2
 518          &$load         ($T0,4);                        # r4^n
 519         &paddq          ($D1,$T1);
 520
 521         &$load          ($T1,8);                        # s4^n
 522         &pmuludq        ($T0,&QWP(16*0,$base));         # r4*h0
 523          &paddq         ($D0,$T2);
 524         &movdqa         ($T2,$T1);
 525         &pmuludq        ($T1,&QWP(16*4,$base));         # s4*h4
 526         &paddq          ($D4,$T0);
 527         &movdqa         ($T0,$T2);
 528         &pmuludq        ($T2,&QWP(16*1,$base));         # s4*h1
 529         &paddq          ($D3,$T1);
 530         &movdqa         ($T1,$T0);
 531         &pmuludq        ($T0,&QWP(16*2,$base));         # s4*h2
 532         &paddq          ($D0,$T2);
 533         &pmuludq        ($T1,&QWP(16*3,$base));         # s4*h3
 534          &movdqa        ($MASK,&QWP(64,"ebx"));
 535         &paddq          ($D1,$T0);
 536         &paddq          ($D2,$T1);
 537 }
 538         &pmuladd        (sub {  my ($reg,$i)=@_;
 539                                 &movdqa ($reg,&QWP(16*$i,"esp"));
 540                              },"edx");
 541
 542 sub lazy_reduction {
 543 my $extra = shift;
 544
 545         ################################################################
 546         # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
 547         # and P. Schwabe
 548         #
 549         # [(*) see discussion in poly1305-armv4 module]
 550
 551          &movdqa        ($T0,$D3);
 552          &pand          ($D3,$MASK);
 553          &psrlq         ($T0,26);
 554          &$extra        ()                              if (defined($extra));
 555          &paddq         ($T0,$D4);                      # h3 -> h4
 556         &movdqa         ($T1,$D0);
 557         &pand           ($D0,$MASK);
 558         &psrlq          ($T1,26);
 559          &movdqa        ($D4,$T0);
 560         &paddq          ($T1,$D1);                      # h0 -> h1
 561          &psrlq         ($T0,26);
 562          &pand          ($D4,$MASK);
 563         &movdqa         ($D1,$T1);
 564         &psrlq          ($T1,26);
 565          &paddd         ($D0,$T0);                      # favour paddd when
 566                                                         # possible, because
 567                                                         # paddq is "broken"
 568                                                         # on Atom
 569          &psllq         ($T0,2);
 570         &paddq          ($T1,$D2);                      # h1 -> h2
 571          &paddq         ($T0,$D0);                      # h4 -> h0 (*)
 572         &pand           ($D1,$MASK);
 573         &movdqa         ($D2,$T1);
 574         &psrlq          ($T1,26);
 575         &pand           ($D2,$MASK);
 576         &paddd          ($T1,$D3);                      # h2 -> h3
 577          &movdqa        ($D0,$T0);
 578          &psrlq         ($T0,26);
 579         &movdqa         ($D3,$T1);
 580         &psrlq          ($T1,26);
 581          &pand          ($D0,$MASK);
 582          &paddd         ($D1,$T0);                      # h0 -> h1
 583         &pand           ($D3,$MASK);
 584         &paddd          ($D4,$T1);                      # h3 -> h4
 585 }
 586         &lazy_reduction ();
 587
 588         &dec            ("ecx");
 589         &jz             (&label("square_break"));
 590
 591         &punpcklqdq     ($D0,&QWP(16*0,"esp"));         # 0:r^1:0:r^2
 592         &punpcklqdq     ($D1,&QWP(16*1,"esp"));
 593         &punpcklqdq     ($D2,&QWP(16*2,"esp"));
 594         &punpcklqdq     ($D3,&QWP(16*3,"esp"));
 595         &punpcklqdq     ($D4,&QWP(16*4,"esp"));
 596         &jmp            (&label("square"));
 597
 598 &set_label("square_break");
 599         &psllq          ($D0,32);                       # -> r^3:0:r^4:0
 600         &psllq          ($D1,32);
 601         &psllq          ($D2,32);
 602         &psllq          ($D3,32);
 603         &psllq          ($D4,32);
 604         &por            ($D0,&QWP(16*0,"esp"));         # r^3:r^1:r^4:r^2
 605         &por            ($D1,&QWP(16*1,"esp"));
 606         &por            ($D2,&QWP(16*2,"esp"));
 607         &por            ($D3,&QWP(16*3,"esp"));
 608         &por            ($D4,&QWP(16*4,"esp"));
 609
 610         &pshufd         ($D0,$D0,0b10001101);           # -> r^1:r^2:r^3:r^4
 611         &pshufd         ($D1,$D1,0b10001101);
 612         &pshufd         ($D2,$D2,0b10001101);
 613         &pshufd         ($D3,$D3,0b10001101);
 614         &pshufd         ($D4,$D4,0b10001101);
 615
 616         &movdqu         (&QWP(16*0,"edi"),$D0);         # save the table
 617         &movdqu         (&QWP(16*1,"edi"),$D1);
 618         &movdqu         (&QWP(16*2,"edi"),$D2);
 619         &movdqu         (&QWP(16*3,"edi"),$D3);
 620         &movdqu         (&QWP(16*4,"edi"),$D4);
 621
 622         &movdqa         ($T1,$D1);
 623         &movdqa         ($T0,$D2);
 624         &pslld          ($T1,2);
 625         &pslld          ($T0,2);
 626         &paddd          ($T1,$D1);                      # *5
 627         &paddd          ($T0,$D2);                      # *5
 628         &movdqu         (&QWP(16*5,"edi"),$T1);
 629         &movdqu         (&QWP(16*6,"edi"),$T0);
 630         &movdqa         ($T1,$D3);
 631         &movdqa         ($T0,$D4);
 632         &pslld          ($T1,2);
 633         &pslld          ($T0,2);
 634         &paddd          ($T1,$D3);                      # *5
 635         &paddd          ($T0,$D4);                      # *5
 636         &movdqu         (&QWP(16*7,"edi"),$T1);
 637         &movdqu         (&QWP(16*8,"edi"),$T0);
 638
 639         &mov            ("esp","ebp");
 640         &lea            ("edi",&DWP(-16*3,"edi"));      # size de-optimization
 641         &ret            ();
 642 &function_end_B("_poly1305_init_sse2");
 643
 644 &align  (32);
 645 &function_begin("_poly1305_blocks_sse2");
 646         &mov    ("edi",&wparam(0));                     # ctx
 647         &mov    ("esi",&wparam(1));                     # inp
 648         &mov    ("ecx",&wparam(2));                     # len
 649
 650         &mov    ("eax",&DWP(4*5,"edi"));                # is_base2_26
 651         &and    ("ecx",-16);
 652         &jz     (&label("nodata"));
 653         &cmp    ("ecx",64);
 654         &jae    (&label("enter_sse2"));
 655         &test   ("eax","eax");                          # is_base2_26?
 656         &jz     (&label("enter_blocks"));
 657
 658 &set_label("enter_sse2",16);
 659         &call   (&label("pic_point"));
 660 &set_label("pic_point");
 661         &blindpop("ebx");
 662         &lea    ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
 663
 664         &test   ("eax","eax");                          # is_base2_26?
 665         &jnz    (&label("base2_26"));
 666
 667         &call   ("_poly1305_init_sse2");
 668
 669         ################################################# base 2^32 -> base 2^26
 670         &mov    ("eax",&DWP(0,"edi"));
 671         &mov    ("ecx",&DWP(3,"edi"));
 672         &mov    ("edx",&DWP(6,"edi"));
 673         &mov    ("esi",&DWP(9,"edi"));
 674         &mov    ("ebp",&DWP(13,"edi"));
 675         &mov    (&DWP(4*5,"edi"),1);                    # is_base2_26
 676
 677         &shr    ("ecx",2);
 678         &and    ("eax",0x3ffffff);
 679         &shr    ("edx",4);
 680         &and    ("ecx",0x3ffffff);
 681         &shr    ("esi",6);
 682         &and    ("edx",0x3ffffff);
 683
 684         &movd   ($D0,"eax");
 685         &movd   ($D1,"ecx");
 686         &movd   ($D2,"edx");
 687         &movd   ($D3,"esi");
 688         &movd   ($D4,"ebp");
 689
 690         &mov    ("esi",&wparam(1));                     # [reload] inp
 691         &mov    ("ecx",&wparam(2));                     # [reload] len
 692         &jmp    (&label("base2_32"));
 693
 694 &set_label("base2_26",16);
 695         &movd   ($D0,&DWP(4*0,"edi"));                  # load hash value
 696         &movd   ($D1,&DWP(4*1,"edi"));
 697         &movd   ($D2,&DWP(4*2,"edi"));
 698         &movd   ($D3,&DWP(4*3,"edi"));
 699         &movd   ($D4,&DWP(4*4,"edi"));
 700         &movdqa ($MASK,&QWP(64,"ebx"));
 701
 702 &set_label("base2_32");
 703         &mov    ("eax",&wparam(3));                     # padbit
 704         &mov    ("ebp","esp");
 705
 706         &sub    ("esp",16*(5+5+5+9+9));
 707         &and    ("esp",-16);
 708
 709         &lea    ("edi",&DWP(16*3,"edi"));               # size optimization
 710         &shl    ("eax",24);                             # padbit
 711
 712         &test   ("ecx",31);
 713         &jz     (&label("even"));
 714
 715         ################################################################
 716         # process single block, with SSE2, because it's still faster
 717         # even though half of result is discarded
 718
 719         &movdqu         ($T1,&QWP(0,"esi"));            # input
 720         &lea            ("esi",&DWP(16,"esi"));
 721
 722         &movdqa         ($T0,$T1);                      # -> base 2^26 ...
 723         &pand           ($T1,$MASK);
 724         &paddd          ($D0,$T1);                      # ... and accumuate
 725
 726         &movdqa         ($T1,$T0);
 727         &psrlq          ($T0,26);
 728         &psrldq         ($T1,6);
 729         &pand           ($T0,$MASK);
 730         &paddd          ($D1,$T0);
 731
 732         &movdqa         ($T0,$T1);
 733         &psrlq          ($T1,4);
 734         &pand           ($T1,$MASK);
 735         &paddd          ($D2,$T1);
 736
 737         &movdqa         ($T1,$T0);
 738         &psrlq          ($T0,30);
 739         &pand           ($T0,$MASK);
 740         &psrldq         ($T1,7);
 741         &paddd          ($D3,$T0);
 742
 743         &movd           ($T0,"eax");                    # padbit
 744         &paddd          ($D4,$T1);
 745          &movd          ($T1,&DWP(16*0+12,"edi"));      # r0
 746         &paddd          ($D4,$T0);
 747
 748         &movdqa         (&QWP(16*0,"esp"),$D0);
 749         &movdqa         (&QWP(16*1,"esp"),$D1);
 750         &movdqa         (&QWP(16*2,"esp"),$D2);
 751         &movdqa         (&QWP(16*3,"esp"),$D3);
 752         &movdqa         (&QWP(16*4,"esp"),$D4);
 753
 754         ################################################################
 755         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
 756         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
 757         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
 758         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
 759         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
 760
 761         &pmuludq        ($D0,$T1);                      # h4*r0
 762         &pmuludq        ($D1,$T1);                      # h3*r0
 763         &pmuludq        ($D2,$T1);                      # h2*r0
 764          &movd          ($T0,&DWP(16*1+12,"edi"));      # r1
 765         &pmuludq        ($D3,$T1);                      # h1*r0
 766         &pmuludq        ($D4,$T1);                      # h0*r0
 767
 768         &pmuladd        (sub {  my ($reg,$i)=@_;
 769                                 &movd ($reg,&DWP(16*$i+12,"edi"));
 770                              });
 771
 772         &lazy_reduction ();
 773
 774         &sub            ("ecx",16);
 775         &jz             (&label("done"));
 776
 777 &set_label("even");
 778         &lea            ("edx",&DWP(16*(5+5+5+9),"esp"));# size optimization
 779         &lea            ("eax",&DWP(-16*2,"esi"));
 780         &sub            ("ecx",64);
 781
 782         ################################################################
 783         # expand and copy pre-calculated table to stack
 784
 785         &movdqu         ($T0,&QWP(16*0,"edi"));         # r^1:r^2:r^3:r^4
 786         &pshufd         ($T1,$T0,0b01000100);           # duplicate r^3:r^4
 787         &cmovb          ("esi","eax");
 788         &pshufd         ($T0,$T0,0b11101110);           # duplicate r^1:r^2
 789         &movdqa         (&QWP(16*0,"edx"),$T1);
 790         &lea            ("eax",&DWP(16*10,"esp"));
 791         &movdqu         ($T1,&QWP(16*1,"edi"));
 792         &movdqa         (&QWP(16*(0-9),"edx"),$T0);
 793         &pshufd         ($T0,$T1,0b01000100);
 794         &pshufd         ($T1,$T1,0b11101110);
 795         &movdqa         (&QWP(16*1,"edx"),$T0);
 796         &movdqu         ($T0,&QWP(16*2,"edi"));
 797         &movdqa         (&QWP(16*(1-9),"edx"),$T1);
 798         &pshufd         ($T1,$T0,0b01000100);
 799         &pshufd         ($T0,$T0,0b11101110);
 800         &movdqa         (&QWP(16*2,"edx"),$T1);
 801         &movdqu         ($T1,&QWP(16*3,"edi"));
 802         &movdqa         (&QWP(16*(2-9),"edx"),$T0);
 803         &pshufd         ($T0,$T1,0b01000100);
 804         &pshufd         ($T1,$T1,0b11101110);
 805         &movdqa         (&QWP(16*3,"edx"),$T0);
 806         &movdqu         ($T0,&QWP(16*4,"edi"));
 807         &movdqa         (&QWP(16*(3-9),"edx"),$T1);
 808         &pshufd         ($T1,$T0,0b01000100);
 809         &pshufd         ($T0,$T0,0b11101110);
 810         &movdqa         (&QWP(16*4,"edx"),$T1);
 811         &movdqu         ($T1,&QWP(16*5,"edi"));
 812         &movdqa         (&QWP(16*(4-9),"edx"),$T0);
 813         &pshufd         ($T0,$T1,0b01000100);
 814         &pshufd         ($T1,$T1,0b11101110);
 815         &movdqa         (&QWP(16*5,"edx"),$T0);
 816         &movdqu         ($T0,&QWP(16*6,"edi"));
 817         &movdqa         (&QWP(16*(5-9),"edx"),$T1);
 818         &pshufd         ($T1,$T0,0b01000100);
 819         &pshufd         ($T0,$T0,0b11101110);
 820         &movdqa         (&QWP(16*6,"edx"),$T1);
 821         &movdqu         ($T1,&QWP(16*7,"edi"));
 822         &movdqa         (&QWP(16*(6-9),"edx"),$T0);
 823         &pshufd         ($T0,$T1,0b01000100);
 824         &pshufd         ($T1,$T1,0b11101110);
 825         &movdqa         (&QWP(16*7,"edx"),$T0);
 826         &movdqu         ($T0,&QWP(16*8,"edi"));
 827         &movdqa         (&QWP(16*(7-9),"edx"),$T1);
 828         &pshufd         ($T1,$T0,0b01000100);
 829         &pshufd         ($T0,$T0,0b11101110);
 830         &movdqa         (&QWP(16*8,"edx"),$T1);
 831         &movdqa         (&QWP(16*(8-9),"edx"),$T0);
 832
 833 sub load_input {
 834 my ($inpbase,$offbase)=@_;
 835
 836         &movdqu         ($T0,&QWP($inpbase+0,"esi"));   # load input
 837         &movdqu         ($T1,&QWP($inpbase+16,"esi"));
 838         &lea            ("esi",&DWP(16*2,"esi"));
 839
 840         &movdqa         (&QWP($offbase+16*2,"esp"),$D2);
 841         &movdqa         (&QWP($offbase+16*3,"esp"),$D3);
 842         &movdqa         (&QWP($offbase+16*4,"esp"),$D4);
 843
 844         &movdqa         ($D2,$T0);                      # splat input
 845         &movdqa         ($D3,$T1);
 846         &psrldq         ($D2,6);
 847         &psrldq         ($D3,6);
 848         &movdqa         ($D4,$T0);
 849         &punpcklqdq     ($D2,$D3);                      # 2:3
 850         &punpckhqdq     ($D4,$T1);                      # 4
 851         &punpcklqdq     ($T0,$T1);                      # 0:1
 852
 853         &movdqa         ($D3,$D2);
 854         &psrlq          ($D2,4);
 855         &psrlq          ($D3,30);
 856         &movdqa         ($T1,$T0);
 857         &psrlq          ($D4,40);                       # 4
 858         &psrlq          ($T1,26);
 859         &pand           ($T0,$MASK);                    # 0
 860         &pand           ($T1,$MASK);                    # 1
 861         &pand           ($D2,$MASK);                    # 2
 862         &pand           ($D3,$MASK);                    # 3
 863         &por            ($D4,&QWP(0,"ebx"));            # padbit, yes, always
 864
 865         &movdqa         (&QWP($offbase+16*0,"esp"),$D0) if ($offbase);
 866         &movdqa         (&QWP($offbase+16*1,"esp"),$D1) if ($offbase);
 867 }
 868         &load_input     (16*2,16*5);
 869
 870         &jbe            (&label("skip_loop"));
 871         &jmp            (&label("loop"));
 872
 873 &set_label("loop",32);
 874         ################################################################
 875         # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
 876         # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
 877         #   \___________________/
 878         # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
 879         # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
 880         #   \___________________/ \____________________/
 881         ################################################################
 882
 883         &movdqa         ($T2,&QWP(16*(0-9),"edx"));     # r0^2
 884         &movdqa         (&QWP(16*1,"eax"),$T1);
 885         &movdqa         (&QWP(16*2,"eax"),$D2);
 886         &movdqa         (&QWP(16*3,"eax"),$D3);
 887         &movdqa         (&QWP(16*4,"eax"),$D4);
 888
 889         ################################################################
 890         # d4 = h4*r0 + h0*r4   + h1*r3   + h2*r2   + h3*r1
 891         # d3 = h3*r0 + h0*r3   + h1*r2   + h2*r1   + h4*5*r4
 892         # d2 = h2*r0 + h0*r2   + h1*r1   + h3*5*r4 + h4*5*r3
 893         # d1 = h1*r0 + h0*r1   + h2*5*r4 + h3*5*r3 + h4*5*r2
 894         # d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
 895
 896         &movdqa         ($D1,$T0);
 897         &pmuludq        ($T0,$T2);                      # h0*r0
 898         &movdqa         ($D0,$T1);
 899         &pmuludq        ($T1,$T2);                      # h1*r0
 900         &pmuludq        ($D2,$T2);                      # h2*r0
 901         &pmuludq        ($D3,$T2);                      # h3*r0
 902         &pmuludq        ($D4,$T2);                      # h4*r0
 903
 904 sub pmuladd_alt {
 905 my $addr = shift;
 906
 907         &pmuludq        ($D0,&$addr(8));                # h1*s4
 908         &movdqa         ($T2,$D1);
 909         &pmuludq        ($D1,&$addr(1));                # h0*r1
 910         &paddq          ($D0,$T0);
 911         &movdqa         ($T0,$T2);
 912         &pmuludq        ($T2,&$addr(2));                # h0*r2
 913         &paddq          ($D1,$T1);
 914         &movdqa         ($T1,$T0);
 915         &pmuludq        ($T0,&$addr(3));                # h0*r3
 916         &paddq          ($D2,$T2);
 917          &movdqa        ($T2,&QWP(16*1,"eax"));         # pull h1
 918         &pmuludq        ($T1,&$addr(4));                # h0*r4
 919         &paddq          ($D3,$T0);
 920
 921         &movdqa         ($T0,$T2);
 922         &pmuludq        ($T2,&$addr(1));                # h1*r1
 923          &paddq         ($D4,$T1);
 924         &movdqa         ($T1,$T0);
 925         &pmuludq        ($T0,&$addr(2));                # h1*r2
 926         &paddq          ($D2,$T2);
 927         &movdqa         ($T2,&QWP(16*2,"eax"));         # pull h2
 928         &pmuludq        ($T1,&$addr(3));                # h1*r3
 929         &paddq          ($D3,$T0);
 930         &movdqa         ($T0,$T2);
 931         &pmuludq        ($T2,&$addr(7));                # h2*s3
 932         &paddq          ($D4,$T1);
 933         &movdqa         ($T1,$T0);
 934         &pmuludq        ($T0,&$addr(8));                # h2*s4
 935         &paddq          ($D0,$T2);
 936
 937         &movdqa         ($T2,$T1);
 938         &pmuludq        ($T1,&$addr(1));                # h2*r1
 939          &paddq         ($D1,$T0);
 940         &movdqa         ($T0,&QWP(16*3,"eax"));         # pull h3
 941         &pmuludq        ($T2,&$addr(2));                # h2*r2
 942         &paddq          ($D3,$T1);
 943         &movdqa         ($T1,$T0);
 944         &pmuludq        ($T0,&$addr(6));                # h3*s2
 945         &paddq          ($D4,$T2);
 946         &movdqa         ($T2,$T1);
 947         &pmuludq        ($T1,&$addr(7));                # h3*s3
 948         &paddq          ($D0,$T0);
 949         &movdqa         ($T0,$T2);
 950         &pmuludq        ($T2,&$addr(8));                # h3*s4
 951         &paddq          ($D1,$T1);
 952
 953         &movdqa         ($T1,&QWP(16*4,"eax"));         # pull h4
 954         &pmuludq        ($T0,&$addr(1));                # h3*r1
 955          &paddq         ($D2,$T2);
 956         &movdqa         ($T2,$T1);
 957         &pmuludq        ($T1,&$addr(8));                # h4*s4
 958         &paddq          ($D4,$T0);
 959         &movdqa         ($T0,$T2);
 960         &pmuludq        ($T2,&$addr(5));                # h4*s1
 961         &paddq          ($D3,$T1);
 962         &movdqa         ($T1,$T0);
 963         &pmuludq        ($T0,&$addr(6));                # h4*s2
 964         &paddq          ($D0,$T2);
 965          &movdqa        ($MASK,&QWP(64,"ebx"));
 966         &pmuludq        ($T1,&$addr(7));                # h4*s3
 967         &paddq          ($D1,$T0);
 968         &paddq          ($D2,$T1);
 969 }
 970         &pmuladd_alt    (sub {  my $i=shift; &QWP(16*($i-9),"edx");     });
 971
 972         &load_input     (-16*2,0);
 973         &lea            ("eax",&DWP(-16*2,"esi"));
 974         &sub            ("ecx",64);
 975
 976         &paddd          ($T0,&QWP(16*(5+0),"esp"));     # add hash value
 977         &paddd          ($T1,&QWP(16*(5+1),"esp"));
 978         &paddd          ($D2,&QWP(16*(5+2),"esp"));
 979         &paddd          ($D3,&QWP(16*(5+3),"esp"));
 980         &paddd          ($D4,&QWP(16*(5+4),"esp"));
 981
 982         &cmovb          ("esi","eax");
 983         &lea            ("eax",&DWP(16*10,"esp"));
 984
 985         &movdqa         ($T2,&QWP(16*0,"edx"));         # r0^4
 986         &movdqa         (&QWP(16*1,"esp"),$D1);
 987         &movdqa         (&QWP(16*1,"eax"),$T1);
 988         &movdqa         (&QWP(16*2,"eax"),$D2);
 989         &movdqa         (&QWP(16*3,"eax"),$D3);
 990         &movdqa         (&QWP(16*4,"eax"),$D4);
 991
 992         ################################################################
 993         # d4 += h4*r0 + h0*r4   + h1*r3   + h2*r2   + h3*r1
 994         # d3 += h3*r0 + h0*r3   + h1*r2   + h2*r1   + h4*5*r4
 995         # d2 += h2*r0 + h0*r2   + h1*r1   + h3*5*r4 + h4*5*r3
 996         # d1 += h1*r0 + h0*r1   + h2*5*r4 + h3*5*r3 + h4*5*r2
 997         # d0 += h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
 998
 999         &movdqa         ($D1,$T0);
1000         &pmuludq        ($T0,$T2);                      # h0*r0
1001         &paddq          ($T0,$D0);
1002         &movdqa         ($D0,$T1);
1003         &pmuludq        ($T1,$T2);                      # h1*r0
1004         &pmuludq        ($D2,$T2);                      # h2*r0
1005         &pmuludq        ($D3,$T2);                      # h3*r0
1006         &pmuludq        ($D4,$T2);                      # h4*r0
1007
1008         &paddq          ($T1,&QWP(16*1,"esp"));
1009         &paddq          ($D2,&QWP(16*2,"esp"));
1010         &paddq          ($D3,&QWP(16*3,"esp"));
1011         &paddq          ($D4,&QWP(16*4,"esp"));
1012
1013         &pmuladd_alt    (sub {  my $i=shift; &QWP(16*$i,"edx"); });
1014
1015         &lazy_reduction ();
1016
1017         &load_input     (16*2,16*5);
1018
1019         &ja             (&label("loop"));
1020
1021 &set_label("skip_loop");
1022         ################################################################
1023         # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
1024
1025          &pshufd        ($T2,&QWP(16*(0-9),"edx"),0x10);# r0^n
1026         &add            ("ecx",32);
1027         &jnz            (&label("long_tail"));
1028
1029         &paddd          ($T0,$D0);                      # add hash value
1030         &paddd          ($T1,$D1);
1031         &paddd          ($D2,&QWP(16*7,"esp"));
1032         &paddd          ($D3,&QWP(16*8,"esp"));
1033         &paddd          ($D4,&QWP(16*9,"esp"));
1034
1035 &set_label("long_tail");
1036
1037         &movdqa         (&QWP(16*0,"eax"),$T0);
1038         &movdqa         (&QWP(16*1,"eax"),$T1);
1039         &movdqa         (&QWP(16*2,"eax"),$D2);
1040         &movdqa         (&QWP(16*3,"eax"),$D3);
1041         &movdqa         (&QWP(16*4,"eax"),$D4);
1042
1043         ################################################################
1044         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
1045         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
1046         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
1047         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
1048         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1049
1050         &pmuludq        ($T0,$T2);                      # h0*r0
1051         &pmuludq        ($T1,$T2);                      # h1*r0
1052         &pmuludq        ($D2,$T2);                      # h2*r0
1053         &movdqa         ($D0,$T0);
1054          &pshufd        ($T0,&QWP(16*(1-9),"edx"),0x10);# r1^n
1055         &pmuludq        ($D3,$T2);                      # h3*r0
1056         &movdqa         ($D1,$T1);
1057         &pmuludq        ($D4,$T2);                      # h4*r0
1058
1059         &pmuladd        (sub {  my ($reg,$i)=@_;
1060                                 &pshufd ($reg,&QWP(16*($i-9),"edx"),0x10);
1061                              },"eax");
1062
1063         &jz             (&label("short_tail"));
1064
1065         &load_input     (-16*2,0);
1066
1067          &pshufd        ($T2,&QWP(16*0,"edx"),0x10);    # r0^n
1068         &paddd          ($T0,&QWP(16*5,"esp"));         # add hash value
1069         &paddd          ($T1,&QWP(16*6,"esp"));
1070         &paddd          ($D2,&QWP(16*7,"esp"));
1071         &paddd          ($D3,&QWP(16*8,"esp"));
1072         &paddd          ($D4,&QWP(16*9,"esp"));
1073
1074         ################################################################
1075         # multiply inp[0:1] by r^4:r^3 and accumulate
1076
1077         &movdqa         (&QWP(16*0,"esp"),$T0);
1078         &pmuludq        ($T0,$T2);                      # h0*r0
1079         &movdqa         (&QWP(16*1,"esp"),$T1);
1080         &pmuludq        ($T1,$T2);                      # h1*r0
1081         &paddq          ($D0,$T0);
1082         &movdqa         ($T0,$D2);
1083         &pmuludq        ($D2,$T2);                      # h2*r0
1084         &paddq          ($D1,$T1);
1085         &movdqa         ($T1,$D3);
1086         &pmuludq        ($D3,$T2);                      # h3*r0
1087         &paddq          ($D2,&QWP(16*2,"esp"));
1088         &movdqa         (&QWP(16*2,"esp"),$T0);
1089          &pshufd        ($T0,&QWP(16*1,"edx"),0x10);    # r1^n
1090         &paddq          ($D3,&QWP(16*3,"esp"));
1091         &movdqa         (&QWP(16*3,"esp"),$T1);
1092         &movdqa         ($T1,$D4);
1093         &pmuludq        ($D4,$T2);                      # h4*r0
1094         &paddq          ($D4,&QWP(16*4,"esp"));
1095         &movdqa         (&QWP(16*4,"esp"),$T1);
1096
1097         &pmuladd        (sub {  my ($reg,$i)=@_;
1098                                 &pshufd ($reg,&QWP(16*$i,"edx"),0x10);
1099                              });
1100
1101 &set_label("short_tail");
1102
1103         ################################################################
1104         # horizontal addition
1105
1106         &pshufd         ($T1,$D4,0b01001110);
1107         &pshufd         ($T0,$D3,0b01001110);
1108         &paddq          ($D4,$T1);
1109         &paddq          ($D3,$T0);
1110         &pshufd         ($T1,$D0,0b01001110);
1111         &pshufd         ($T0,$D1,0b01001110);
1112         &paddq          ($D0,$T1);
1113         &paddq          ($D1,$T0);
1114         &pshufd         ($T1,$D2,0b01001110);
1115         #&paddq         ($D2,$T1);
1116
1117         &lazy_reduction (sub { &paddq ($D2,$T1) });
1118
1119 &set_label("done");
1120         &movd           (&DWP(-16*3+4*0,"edi"),$D0);    # store hash value
1121         &movd           (&DWP(-16*3+4*1,"edi"),$D1);
1122         &movd           (&DWP(-16*3+4*2,"edi"),$D2);
1123         &movd           (&DWP(-16*3+4*3,"edi"),$D3);
1124         &movd           (&DWP(-16*3+4*4,"edi"),$D4);
1125         &mov    ("esp","ebp");
1126 &set_label("nodata");
1127 &function_end("_poly1305_blocks_sse2");
1128
1129 &align  (32);
1130 &function_begin("_poly1305_emit_sse2");
1131         &mov    ("ebp",&wparam(0));             # context
1132
1133         &cmp    (&DWP(4*5,"ebp"),0);            # is_base2_26?
1134         &je     (&label("enter_emit"));
1135
1136         &mov    ("eax",&DWP(4*0,"ebp"));        # load hash value
1137         &mov    ("edi",&DWP(4*1,"ebp"));
1138         &mov    ("ecx",&DWP(4*2,"ebp"));
1139         &mov    ("edx",&DWP(4*3,"ebp"));
1140         &mov    ("esi",&DWP(4*4,"ebp"));
1141
1142         &mov    ("ebx","edi");                  # base 2^26 -> base 2^32
1143         &shl    ("edi",26);
1144         &shr    ("ebx",6);
1145         &add    ("eax","edi");
1146         &mov    ("edi","ecx");
1147         &adc    ("ebx",0);
1148
1149         &shl    ("edi",20);
1150         &shr    ("ecx",12);
1151         &add    ("ebx","edi");
1152         &mov    ("edi","edx");
1153         &adc    ("ecx",0);
1154
1155         &shl    ("edi",14);
1156         &shr    ("edx",18);
1157         &add    ("ecx","edi");
1158         &mov    ("edi","esi");
1159         &adc    ("edx",0);
1160
1161         &shl    ("edi",8);
1162         &shr    ("esi",24);
1163         &add    ("edx","edi");
1164         &adc    ("esi",0);                      # can be partially reduced
1165
1166         &mov    ("edi","esi");                  # final reduction
1167         &and    ("esi",3);
1168         &shr    ("edi",2);
1169         &lea    ("ebp",&DWP(0,"edi","edi",4));  # *5
1170          &mov   ("edi",&wparam(1));             # output
1171         &add    ("eax","ebp");
1172          &mov   ("ebp",&wparam(2));             # key
1173         &adc    ("ebx",0);
1174         &adc    ("ecx",0);
1175         &adc    ("edx",0);
1176         &adc    ("esi",0);
1177
1178         &movd   ($D0,"eax");                    # offload original hash value
1179         &add    ("eax",5);                      # compare to modulus
1180         &movd   ($D1,"ebx");
1181         &adc    ("ebx",0);
1182         &movd   ($D2,"ecx");
1183         &adc    ("ecx",0);
1184         &movd   ($D3,"edx");
1185         &adc    ("edx",0);
1186         &adc    ("esi",0);
1187         &shr    ("esi",2);                      # did it carry/borrow?
1188
1189         &neg    ("esi");                        # do we choose (hash-modulus) ...
1190         &and    ("eax","esi");
1191         &and    ("ebx","esi");
1192         &and    ("ecx","esi");
1193         &and    ("edx","esi");
1194         &mov    (&DWP(4*0,"edi"),"eax");
1195         &movd   ("eax",$D0);
1196         &mov    (&DWP(4*1,"edi"),"ebx");
1197         &movd   ("ebx",$D1);
1198         &mov    (&DWP(4*2,"edi"),"ecx");
1199         &movd   ("ecx",$D2);
1200         &mov    (&DWP(4*3,"edi"),"edx");
1201         &movd   ("edx",$D3);
1202
1203         &not    ("esi");                        # ... or original hash value?
1204         &and    ("eax","esi");
1205         &and    ("ebx","esi");
1206         &or     ("eax",&DWP(4*0,"edi"));
1207         &and    ("ecx","esi");
1208         &or     ("ebx",&DWP(4*1,"edi"));
1209         &and    ("edx","esi");
1210         &or     ("ecx",&DWP(4*2,"edi"));
1211         &or     ("edx",&DWP(4*3,"edi"));
1212
1213         &add    ("eax",&DWP(4*0,"ebp"));        # accumulate key
1214         &adc    ("ebx",&DWP(4*1,"ebp"));
1215         &mov    (&DWP(4*0,"edi"),"eax");
1216         &adc    ("ecx",&DWP(4*2,"ebp"));
1217         &mov    (&DWP(4*1,"edi"),"ebx");
1218         &adc    ("edx",&DWP(4*3,"ebp"));
1219         &mov    (&DWP(4*2,"edi"),"ecx");
1220         &mov    (&DWP(4*3,"edi"),"edx");
1221 &function_end("_poly1305_emit_sse2");
1222
1223 if ($avx>1) {
1224 ########################################################################
1225 # Note that poly1305_init_avx2 operates on %xmm, I could have used
1226 # poly1305_init_sse2...
1227
1228 &align  (32);
1229 &function_begin_B("_poly1305_init_avx2");
1230         &vmovdqu        ($D4,&QWP(4*6,"edi"));          # key base 2^32
1231         &lea            ("edi",&DWP(16*3,"edi"));       # size optimization
1232         &mov            ("ebp","esp");
1233         &sub            ("esp",16*(9+5));
1234         &and            ("esp",-16);
1235
1236         #&vpand         ($D4,$D4,&QWP(96,"ebx"));       # magic mask
1237         &vmovdqa        ($MASK,&QWP(64,"ebx"));
1238
1239         &vpand          ($D0,$D4,$MASK);                # -> base 2^26
1240         &vpsrlq         ($D1,$D4,26);
1241         &vpsrldq        ($D3,$D4,6);
1242         &vpand          ($D1,$D1,$MASK);
1243         &vpsrlq         ($D2,$D3,4)
1244         &vpsrlq         ($D3,$D3,30);
1245         &vpand          ($D2,$D2,$MASK);
1246         &vpand          ($D3,$D3,$MASK);
1247         &vpsrldq        ($D4,$D4,13);
1248
1249         &lea            ("edx",&DWP(16*9,"esp"));       # size optimization
1250         &mov            ("ecx",2);
1251 &set_label("square");
1252         &vmovdqa        (&QWP(16*0,"esp"),$D0);
1253         &vmovdqa        (&QWP(16*1,"esp"),$D1);
1254         &vmovdqa        (&QWP(16*2,"esp"),$D2);
1255         &vmovdqa        (&QWP(16*3,"esp"),$D3);
1256         &vmovdqa        (&QWP(16*4,"esp"),$D4);
1257
1258         &vpslld         ($T1,$D1,2);
1259         &vpslld         ($T0,$D2,2);
1260         &vpaddd         ($T1,$T1,$D1);                  # *5
1261         &vpaddd         ($T0,$T0,$D2);                  # *5
1262         &vmovdqa        (&QWP(16*5,"esp"),$T1);
1263         &vmovdqa        (&QWP(16*6,"esp"),$T0);
1264         &vpslld         ($T1,$D3,2);
1265         &vpslld         ($T0,$D4,2);
1266         &vpaddd         ($T1,$T1,$D3);                  # *5
1267         &vpaddd         ($T0,$T0,$D4);                  # *5
1268         &vmovdqa        (&QWP(16*7,"esp"),$T1);
1269         &vmovdqa        (&QWP(16*8,"esp"),$T0);
1270
1271         &vpshufd        ($T0,$D0,0b01000100);
1272         &vmovdqa        ($T1,$D1);
1273         &vpshufd        ($D1,$D1,0b01000100);
1274         &vpshufd        ($D2,$D2,0b01000100);
1275         &vpshufd        ($D3,$D3,0b01000100);
1276         &vpshufd        ($D4,$D4,0b01000100);
1277         &vmovdqa        (&QWP(16*0,"edx"),$T0);
1278         &vmovdqa        (&QWP(16*1,"edx"),$D1);
1279         &vmovdqa        (&QWP(16*2,"edx"),$D2);
1280         &vmovdqa        (&QWP(16*3,"edx"),$D3);
1281         &vmovdqa        (&QWP(16*4,"edx"),$D4);
1282
1283         ################################################################
1284         # d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
1285         # d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
1286         # d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
1287         # d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
1288         # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1289
1290         &vpmuludq       ($D4,$D4,$D0);                  # h4*r0
1291         &vpmuludq       ($D3,$D3,$D0);                  # h3*r0
1292         &vpmuludq       ($D2,$D2,$D0);                  # h2*r0
1293         &vpmuludq       ($D1,$D1,$D0);                  # h1*r0
1294         &vpmuludq       ($D0,$T0,$D0);                  # h0*r0
1295
1296         &vpmuludq       ($T0,$T1,&QWP(16*3,"edx"));     # r1*h3
1297         &vpaddq         ($D4,$D4,$T0);
1298         &vpmuludq       ($T2,$T1,&QWP(16*2,"edx"));     # r1*h2
1299         &vpaddq         ($D3,$D3,$T2);
1300         &vpmuludq       ($T0,$T1,&QWP(16*1,"edx"));     # r1*h1
1301         &vpaddq         ($D2,$D2,$T0);
1302         &vmovdqa        ($T2,&QWP(16*5,"esp"));         # s1
1303         &vpmuludq       ($T1,$T1,&QWP(16*0,"edx"));     # r1*h0
1304         &vpaddq         ($D1,$D1,$T1);
1305          &vmovdqa       ($T0,&QWP(16*2,"esp"));         # r2
1306         &vpmuludq       ($T2,$T2,&QWP(16*4,"edx"));     # s1*h4
1307         &vpaddq         ($D0,$D0,$T2);
1308
1309         &vpmuludq       ($T1,$T0,&QWP(16*2,"edx"));     # r2*h2
1310         &vpaddq         ($D4,$D4,$T1);
1311         &vpmuludq       ($T2,$T0,&QWP(16*1,"edx"));     # r2*h1
1312         &vpaddq         ($D3,$D3,$T2);
1313         &vmovdqa        ($T1,&QWP(16*6,"esp"));         # s2
1314         &vpmuludq       ($T0,$T0,&QWP(16*0,"edx"));     # r2*h0
1315         &vpaddq         ($D2,$D2,$T0);
1316         &vpmuludq       ($T2,$T1,&QWP(16*4,"edx"));     # s2*h4
1317         &vpaddq         ($D1,$D1,$T2);
1318          &vmovdqa       ($T0,&QWP(16*3,"esp"));         # r3
1319         &vpmuludq       ($T1,$T1,&QWP(16*3,"edx"));     # s2*h3
1320         &vpaddq         ($D0,$D0,$T1);
1321
1322         &vpmuludq       ($T2,$T0,&QWP(16*1,"edx"));     # r3*h1
1323         &vpaddq         ($D4,$D4,$T2);
1324         &vmovdqa        ($T1,&QWP(16*7,"esp"));         # s3
1325         &vpmuludq       ($T0,$T0,&QWP(16*0,"edx"));     # r3*h0
1326         &vpaddq         ($D3,$D3,$T0);
1327         &vpmuludq       ($T2,$T1,&QWP(16*4,"edx"));     # s3*h4
1328         &vpaddq         ($D2,$D2,$T2);
1329         &vpmuludq       ($T0,$T1,&QWP(16*3,"edx"));     # s3*h3
1330         &vpaddq         ($D1,$D1,$T0);
1331          &vmovdqa       ($T2,&QWP(16*4,"esp"));         # r4
1332         &vpmuludq       ($T1,$T1,&QWP(16*2,"edx"));     # s3*h2
1333         &vpaddq         ($D0,$D0,$T1);
1334
1335         &vmovdqa        ($T0,&QWP(16*8,"esp"));         # s4
1336         &vpmuludq       ($T2,$T2,&QWP(16*0,"edx"));     # r4*h0
1337         &vpaddq         ($D4,$D4,$T2);
1338         &vpmuludq       ($T1,$T0,&QWP(16*4,"edx"));     # s4*h4
1339         &vpaddq         ($D3,$D3,$T1);
1340         &vpmuludq       ($T2,$T0,&QWP(16*1,"edx"));     # s4*h1
1341         &vpaddq         ($D0,$D0,$T2);
1342         &vpmuludq       ($T1,$T0,&QWP(16*2,"edx"));     # s4*h2
1343         &vpaddq         ($D1,$D1,$T1);
1344          &vmovdqa       ($MASK,&QWP(64,"ebx"));
1345         &vpmuludq       ($T0,$T0,&QWP(16*3,"edx"));     # s4*h3
1346         &vpaddq         ($D2,$D2,$T0);
1347
1348         ################################################################
1349         # lazy reduction
1350          &vpsrlq        ($T0,$D3,26);
1351          &vpand         ($D3,$D3,$MASK);
1352         &vpsrlq         ($T1,$D0,26);
1353         &vpand          ($D0,$D0,$MASK);
1354          &vpaddq        ($D4,$D4,$T0);                  # h3 -> h4
1355         &vpaddq         ($D1,$D1,$T1);                  # h0 -> h1
1356          &vpsrlq        ($T0,$D4,26);
1357          &vpand         ($D4,$D4,$MASK);
1358         &vpsrlq         ($T1,$D1,26);
1359         &vpand          ($D1,$D1,$MASK);
1360         &vpaddq         ($D2,$D2,$T1);                  # h1 -> h2
1361          &vpaddd        ($D0,$D0,$T0);
1362          &vpsllq        ($T0,$T0,2);
1363         &vpsrlq         ($T1,$D2,26);
1364         &vpand          ($D2,$D2,$MASK);
1365          &vpaddd        ($D0,$D0,$T0);                  # h4 -> h0
1366         &vpaddd         ($D3,$D3,$T1);                  # h2 -> h3
1367         &vpsrlq         ($T1,$D3,26);
1368          &vpsrlq        ($T0,$D0,26);
1369          &vpand         ($D0,$D0,$MASK);
1370         &vpand          ($D3,$D3,$MASK);
1371          &vpaddd        ($D1,$D1,$T0);                  # h0 -> h1
1372         &vpaddd         ($D4,$D4,$T1);                  # h3 -> h4
1373
1374         &dec            ("ecx");
1375         &jz             (&label("square_break"));
1376
1377         &vpunpcklqdq    ($D0,$D0,&QWP(16*0,"esp"));     # 0:r^1:0:r^2
1378         &vpunpcklqdq    ($D1,$D1,&QWP(16*1,"esp"));
1379         &vpunpcklqdq    ($D2,$D2,&QWP(16*2,"esp"));
1380         &vpunpcklqdq    ($D3,$D3,&QWP(16*3,"esp"));
1381         &vpunpcklqdq    ($D4,$D4,&QWP(16*4,"esp"));
1382         &jmp            (&label("square"));
1383
1384 &set_label("square_break");
1385         &vpsllq         ($D0,$D0,32);                   # -> r^3:0:r^4:0
1386         &vpsllq         ($D1,$D1,32);
1387         &vpsllq         ($D2,$D2,32);
1388         &vpsllq         ($D3,$D3,32);
1389         &vpsllq         ($D4,$D4,32);
1390         &vpor           ($D0,$D0,&QWP(16*0,"esp"));     # r^3:r^1:r^4:r^2
1391         &vpor           ($D1,$D1,&QWP(16*1,"esp"));
1392         &vpor           ($D2,$D2,&QWP(16*2,"esp"));
1393         &vpor           ($D3,$D3,&QWP(16*3,"esp"));
1394         &vpor           ($D4,$D4,&QWP(16*4,"esp"));
1395
1396         &vpshufd        ($D0,$D0,0b10001101);           # -> r^1:r^2:r^3:r^4
1397         &vpshufd        ($D1,$D1,0b10001101);
1398         &vpshufd        ($D2,$D2,0b10001101);
1399         &vpshufd        ($D3,$D3,0b10001101);
1400         &vpshufd        ($D4,$D4,0b10001101);
1401
1402         &vmovdqu        (&QWP(16*0,"edi"),$D0);         # save the table
1403         &vmovdqu        (&QWP(16*1,"edi"),$D1);
1404         &vmovdqu        (&QWP(16*2,"edi"),$D2);
1405         &vmovdqu        (&QWP(16*3,"edi"),$D3);
1406         &vmovdqu        (&QWP(16*4,"edi"),$D4);
1407
1408         &vpslld         ($T1,$D1,2);
1409         &vpslld         ($T0,$D2,2);
1410         &vpaddd         ($T1,$T1,$D1);                  # *5
1411         &vpaddd         ($T0,$T0,$D2);                  # *5
1412         &vmovdqu        (&QWP(16*5,"edi"),$T1);
1413         &vmovdqu        (&QWP(16*6,"edi"),$T0);
1414         &vpslld         ($T1,$D3,2);
1415         &vpslld         ($T0,$D4,2);
1416         &vpaddd         ($T1,$T1,$D3);                  # *5
1417         &vpaddd         ($T0,$T0,$D4);                  # *5
1418         &vmovdqu        (&QWP(16*7,"edi"),$T1);
1419         &vmovdqu        (&QWP(16*8,"edi"),$T0);
1420
1421         &mov            ("esp","ebp");
1422         &lea            ("edi",&DWP(-16*3,"edi"));      # size de-optimization
1423         &ret            ();
1424 &function_end_B("_poly1305_init_avx2");
1425
1426 ########################################################################
1427 # now it's time to switch to %ymm
1428
1429 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("ymm$_",(0..7));
1430 my $MASK=$T2;
1431
1432 sub X { my $reg=shift; $reg=~s/^ymm/xmm/; $reg; }
1433
1434 &align  (32);
1435 &function_begin("_poly1305_blocks_avx2");
1436         &mov    ("edi",&wparam(0));                     # ctx
1437         &mov    ("esi",&wparam(1));                     # inp
1438         &mov    ("ecx",&wparam(2));                     # len
1439
1440         &mov    ("eax",&DWP(4*5,"edi"));                # is_base2_26
1441         &and    ("ecx",-16);
1442         &jz     (&label("nodata"));
1443         &cmp    ("ecx",64);
1444         &jae    (&label("enter_avx2"));
1445         &test   ("eax","eax");                          # is_base2_26?
1446         &jz     (&label("enter_blocks"));
1447
1448 &set_label("enter_avx2");
1449         &vzeroupper     ();
1450
1451         &call   (&label("pic_point"));
1452 &set_label("pic_point");
1453         &blindpop("ebx");
1454         &lea    ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
1455
1456         &test   ("eax","eax");                          # is_base2_26?
1457         &jnz    (&label("base2_26"));
1458
1459         &call   ("_poly1305_init_avx2");
1460
1461         ################################################# base 2^32 -> base 2^26
1462         &mov    ("eax",&DWP(0,"edi"));
1463         &mov    ("ecx",&DWP(3,"edi"));
1464         &mov    ("edx",&DWP(6,"edi"));
1465         &mov    ("esi",&DWP(9,"edi"));
1466         &mov    ("ebp",&DWP(13,"edi"));
1467
1468         &shr    ("ecx",2);
1469         &and    ("eax",0x3ffffff);
1470         &shr    ("edx",4);
1471         &and    ("ecx",0x3ffffff);
1472         &shr    ("esi",6);
1473         &and    ("edx",0x3ffffff);
1474
1475         &mov    (&DWP(4*0,"edi"),"eax");
1476         &mov    (&DWP(4*1,"edi"),"ecx");
1477         &mov    (&DWP(4*2,"edi"),"edx");
1478         &mov    (&DWP(4*3,"edi"),"esi");
1479         &mov    (&DWP(4*4,"edi"),"ebp");
1480         &mov    (&DWP(4*5,"edi"),1);                    # is_base2_26
1481
1482         &mov    ("esi",&wparam(1));                     # [reload] inp
1483         &mov    ("ecx",&wparam(2));                     # [reload] len
1484
1485 &set_label("base2_26");
1486         &mov    ("eax",&wparam(3));                     # padbit
1487         &mov    ("ebp","esp");
1488
1489         &sub    ("esp",32*(5+9));
1490         &and    ("esp",-512);                           # ensure that frame
1491                                                         # doesn't cross page
1492                                                         # boundary, which is
1493                                                         # essential for
1494                                                         # misaligned 32-byte
1495                                                         # loads
1496
1497         ################################################################
1498         # expand and copy pre-calculated table to stack
1499
1500         &vmovdqu        (&X($D0),&QWP(16*(3+0),"edi"));
1501         &lea            ("edx",&DWP(32*5+128,"esp"));   # +128 size optimization
1502         &vmovdqu        (&X($D1),&QWP(16*(3+1),"edi"));
1503         &vmovdqu        (&X($D2),&QWP(16*(3+2),"edi"));
1504         &vmovdqu        (&X($D3),&QWP(16*(3+3),"edi"));
1505         &vmovdqu        (&X($D4),&QWP(16*(3+4),"edi"));
1506         &lea            ("edi",&DWP(16*3,"edi"));       # size optimization
1507         &vpermq         ($D0,$D0,0b01000000);           # 00001234 -> 12343434
1508         &vpermq         ($D1,$D1,0b01000000);
1509         &vpermq         ($D2,$D2,0b01000000);
1510         &vpermq         ($D3,$D3,0b01000000);
1511         &vpermq         ($D4,$D4,0b01000000);
1512         &vpshufd        ($D0,$D0,0b11001000);           # 12343434 -> 14243444
1513         &vpshufd        ($D1,$D1,0b11001000);
1514         &vpshufd        ($D2,$D2,0b11001000);
1515         &vpshufd        ($D3,$D3,0b11001000);
1516         &vpshufd        ($D4,$D4,0b11001000);
1517         &vmovdqa        (&QWP(32*0-128,"edx"),$D0);
1518         &vmovdqu        (&X($D0),&QWP(16*5,"edi"));
1519         &vmovdqa        (&QWP(32*1-128,"edx"),$D1);
1520         &vmovdqu        (&X($D1),&QWP(16*6,"edi"));
1521         &vmovdqa        (&QWP(32*2-128,"edx"),$D2);
1522         &vmovdqu        (&X($D2),&QWP(16*7,"edi"));
1523         &vmovdqa        (&QWP(32*3-128,"edx"),$D3);
1524         &vmovdqu        (&X($D3),&QWP(16*8,"edi"));
1525         &vmovdqa        (&QWP(32*4-128,"edx"),$D4);
1526         &vpermq         ($D0,$D0,0b01000000);
1527         &vpermq         ($D1,$D1,0b01000000);
1528         &vpermq         ($D2,$D2,0b01000000);
1529         &vpermq         ($D3,$D3,0b01000000);
1530         &vpshufd        ($D0,$D0,0b11001000);
1531         &vpshufd        ($D1,$D1,0b11001000);
1532         &vpshufd        ($D2,$D2,0b11001000);
1533         &vpshufd        ($D3,$D3,0b11001000);
1534         &vmovdqa        (&QWP(32*5-128,"edx"),$D0);
1535         &vmovd          (&X($D0),&DWP(-16*3+4*0,"edi"));# load hash value
1536         &vmovdqa        (&QWP(32*6-128,"edx"),$D1);
1537         &vmovd          (&X($D1),&DWP(-16*3+4*1,"edi"));
1538         &vmovdqa        (&QWP(32*7-128,"edx"),$D2);
1539         &vmovd          (&X($D2),&DWP(-16*3+4*2,"edi"));
1540         &vmovdqa        (&QWP(32*8-128,"edx"),$D3);
1541         &vmovd          (&X($D3),&DWP(-16*3+4*3,"edi"));
1542         &vmovd          (&X($D4),&DWP(-16*3+4*4,"edi"));
1543         &vmovdqa        ($MASK,&QWP(64,"ebx"));
1544         &neg            ("eax");                        # padbit
1545
1546         &test           ("ecx",63);
1547         &jz             (&label("even"));
1548
1549         &mov            ("edx","ecx");
1550         &and            ("ecx",-64);
1551         &and            ("edx",63);
1552
1553         &vmovdqu        (&X($T0),&QWP(16*0,"esi"));
1554         &cmp            ("edx",32);
1555         &jb             (&label("one"));
1556
1557         &vmovdqu        (&X($T1),&QWP(16*1,"esi"));
1558         &je             (&label("two"));
1559
1560         &vinserti128    ($T0,$T0,&QWP(16*2,"esi"),1);
1561         &lea            ("esi",&DWP(16*3,"esi"));
1562         &lea            ("ebx",&DWP(8,"ebx"));          # three padbits
1563         &lea            ("edx",&DWP(32*5+128+8,"esp")); # --:r^1:r^2:r^3 (*)
1564         &jmp            (&label("tail"));
1565
1566 &set_label("two");
1567         &lea            ("esi",&DWP(16*2,"esi"));
1568         &lea            ("ebx",&DWP(16,"ebx"));         # two padbits
1569         &lea            ("edx",&DWP(32*5+128+16,"esp"));# --:--:r^1:r^2 (*)
1570         &jmp            (&label("tail"));
1571
1572 &set_label("one");
1573         &lea            ("esi",&DWP(16*1,"esi"));
1574         &vpxor          ($T1,$T1,$T1);
1575         &lea            ("ebx",&DWP(32,"ebx","eax",8)); # one or no padbits
1576         &lea            ("edx",&DWP(32*5+128+24,"esp"));# --:--:--:r^1 (*)
1577         &jmp            (&label("tail"));
1578
1579 # (*)   spots marked with '--' are data from next table entry, but they
1580 #       are multiplied by 0 and therefore rendered insignificant
1581
1582 &set_label("even",32);
1583         &vmovdqu        (&X($T0),&QWP(16*0,"esi"));     # load input
1584         &vmovdqu        (&X($T1),&QWP(16*1,"esi"));
1585         &vinserti128    ($T0,$T0,&QWP(16*2,"esi"),1);
1586         &vinserti128    ($T1,$T1,&QWP(16*3,"esi"),1);
1587         &lea            ("esi",&DWP(16*4,"esi"));
1588         &sub            ("ecx",64);
1589         &jz             (&label("tail"));
1590
1591 &set_label("loop");
1592         ################################################################
1593         # ((inp[0]*r^4+r[4])*r^4+r[8])*r^4
1594         # ((inp[1]*r^4+r[5])*r^4+r[9])*r^3
1595         # ((inp[2]*r^4+r[6])*r^4+r[10])*r^2
1596         # ((inp[3]*r^4+r[7])*r^4+r[11])*r^1
1597         #   \________/ \_______/
1598         ################################################################
1599
1600 sub vsplat_input {
1601         &vmovdqa        (&QWP(32*2,"esp"),$D2);
1602         &vpsrldq        ($D2,$T0,6);                    # splat input
1603         &vmovdqa        (&QWP(32*0,"esp"),$D0);
1604         &vpsrldq        ($D0,$T1,6);
1605         &vmovdqa        (&QWP(32*1,"esp"),$D1);
1606         &vpunpckhqdq    ($D1,$T0,$T1);                  # 4
1607         &vpunpcklqdq    ($T0,$T0,$T1);                  # 0:1
1608         &vpunpcklqdq    ($D2,$D2,$D0);                  # 2:3
1609
1610         &vpsrlq         ($D0,$D2,30);
1611         &vpsrlq         ($D2,$D2,4);
1612         &vpsrlq         ($T1,$T0,26);
1613         &vpsrlq         ($D1,$D1,40);                   # 4
1614         &vpand          ($D2,$D2,$MASK);                # 2
1615         &vpand          ($T0,$T0,$MASK);                # 0
1616         &vpand          ($T1,$T1,$MASK);                # 1
1617         &vpand          ($D0,$D0,$MASK);                # 3 (*)
1618         &vpor           ($D1,$D1,&QWP(0,"ebx"));        # padbit, yes, always
1619
1620         # (*)   note that output is counterintuitive, inp[3:4] is
1621         #       returned in $D1-2, while $D3-4 are preserved;
1622 }
1623         &vsplat_input   ();
1624
1625 sub vpmuladd {
1626 my $addr = shift;
1627
1628         &vpaddq         ($D2,$D2,&QWP(32*2,"esp"));     # add hash value
1629         &vpaddq         ($T0,$T0,&QWP(32*0,"esp"));
1630         &vpaddq         ($T1,$T1,&QWP(32*1,"esp"));
1631         &vpaddq         ($D0,$D0,$D3);
1632         &vpaddq         ($D1,$D1,$D4);
1633
1634         ################################################################
1635         # d3 = h2*r1   + h0*r3 + h1*r2   + h3*r0   + h4*5*r4
1636         # d4 = h2*r2   + h0*r4 + h1*r3   + h3*r1   + h4*r0
1637         # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1
1638         # d1 = h2*5*r4 + h0*r1 + h1*r0   + h3*5*r3 + h4*5*r2
1639         # d2 = h2*r0   + h0*r2 + h1*r1   + h3*5*r4 + h4*5*r3
1640
1641         &vpmuludq       ($D3,$D2,&$addr(1));            # d3 = h2*r1
1642          &vmovdqa       (QWP(32*1,"esp"),$T1);
1643         &vpmuludq       ($D4,$D2,&$addr(2));            # d4 = h2*r2
1644          &vmovdqa       (QWP(32*3,"esp"),$D0);
1645         &vpmuludq       ($D0,$D2,&$addr(7));            # d0 = h2*s3
1646          &vmovdqa       (QWP(32*4,"esp"),$D1);
1647         &vpmuludq       ($D1,$D2,&$addr(8));            # d1 = h2*s4
1648         &vpmuludq       ($D2,$D2,&$addr(0));            # d2 = h2*r0
1649
1650         &vpmuludq       ($T2,$T0,&$addr(3));            # h0*r3
1651         &vpaddq         ($D3,$D3,$T2);                  # d3 += h0*r3
1652         &vpmuludq       ($T1,$T0,&$addr(4));            # h0*r4
1653         &vpaddq         ($D4,$D4,$T1);                  # d4 + h0*r4
1654         &vpmuludq       ($T2,$T0,&$addr(0));            # h0*r0
1655         &vpaddq         ($D0,$D0,$T2);                  # d0 + h0*r0
1656          &vmovdqa       ($T2,&QWP(32*1,"esp"));         # h1
1657         &vpmuludq       ($T1,$T0,&$addr(1));            # h0*r1
1658         &vpaddq         ($D1,$D1,$T1);                  # d1 += h0*r1
1659         &vpmuludq       ($T0,$T0,&$addr(2));            # h0*r2
1660         &vpaddq         ($D2,$D2,$T0);                  # d2 += h0*r2
1661
1662         &vpmuludq       ($T1,$T2,&$addr(2));            # h1*r2
1663         &vpaddq         ($D3,$D3,$T1);                  # d3 += h1*r2
1664         &vpmuludq       ($T0,$T2,&$addr(3));            # h1*r3
1665         &vpaddq         ($D4,$D4,$T0);                  # d4 += h1*r3
1666         &vpmuludq       ($T1,$T2,&$addr(8));            # h1*s4
1667         &vpaddq         ($D0,$D0,$T1);                  # d0 += h1*s4
1668          &vmovdqa       ($T1,&QWP(32*3,"esp"));         # h3
1669         &vpmuludq       ($T0,$T2,&$addr(0));            # h1*r0
1670         &vpaddq         ($D1,$D1,$T0);                  # d1 += h1*r0
1671         &vpmuludq       ($T2,$T2,&$addr(1));            # h1*r1
1672         &vpaddq         ($D2,$D2,$T2);                  # d2 += h1*r1
1673
1674         &vpmuludq       ($T0,$T1,&$addr(0));            # h3*r0
1675         &vpaddq         ($D3,$D3,$T0);                  # d3 += h3*r0
1676         &vpmuludq       ($T2,$T1,&$addr(1));            # h3*r1
1677         &vpaddq         ($D4,$D4,$T2);                  # d4 += h3*r1
1678         &vpmuludq       ($T0,$T1,&$addr(6));            # h3*s2
1679         &vpaddq         ($D0,$D0,$T0);                  # d0 += h3*s2
1680          &vmovdqa       ($T0,&QWP(32*4,"esp"));         # h4
1681         &vpmuludq       ($T2,$T1,&$addr(7));            # h3*s3
1682         &vpaddq         ($D1,$D1,$T2);                  # d1+= h3*s3
1683         &vpmuludq       ($T1,$T1,&$addr(8));            # h3*s4
1684         &vpaddq         ($D2,$D2,$T1);                  # d2 += h3*s4
1685
1686         &vpmuludq       ($T2,$T0,&$addr(8));            # h4*s4
1687         &vpaddq         ($D3,$D3,$T2);                  # d3 += h4*s4
1688         &vpmuludq       ($T1,$T0,&$addr(5));            # h4*s1
1689         &vpaddq         ($D0,$D0,$T1);                  # d0 += h4*s1
1690         &vpmuludq       ($T2,$T0,&$addr(0));            # h4*r0
1691         &vpaddq         ($D4,$D4,$T2);                  # d4 += h4*r0
1692          &vmovdqa       ($MASK,&QWP(64,"ebx"));
1693         &vpmuludq       ($T1,$T0,&$addr(6));            # h4*s2
1694         &vpaddq         ($D1,$D1,$T1);                  # d1 += h4*s2
1695         &vpmuludq       ($T0,$T0,&$addr(7));            # h4*s3
1696         &vpaddq         ($D2,$D2,$T0);                  # d2 += h4*s3
1697 }
1698         &vpmuladd       (sub {  my $i=shift; &QWP(32*$i-128,"edx");     });
1699
1700 sub vlazy_reduction {
1701         ################################################################
1702         # lazy reduction
1703
1704          &vpsrlq        ($T0,$D3,26);
1705          &vpand         ($D3,$D3,$MASK);
1706         &vpsrlq         ($T1,$D0,26);
1707         &vpand          ($D0,$D0,$MASK);
1708          &vpaddq        ($D4,$D4,$T0);                  # h3 -> h4
1709         &vpaddq         ($D1,$D1,$T1);                  # h0 -> h1
1710          &vpsrlq        ($T0,$D4,26);
1711          &vpand         ($D4,$D4,$MASK);
1712         &vpsrlq         ($T1,$D1,26);
1713         &vpand          ($D1,$D1,$MASK);
1714         &vpaddq         ($D2,$D2,$T1);                  # h1 -> h2
1715          &vpaddq        ($D0,$D0,$T0);
1716          &vpsllq        ($T0,$T0,2);
1717         &vpsrlq         ($T1,$D2,26);
1718         &vpand          ($D2,$D2,$MASK);
1719          &vpaddq        ($D0,$D0,$T0);                  # h4 -> h0
1720         &vpaddq         ($D3,$D3,$T1);                  # h2 -> h3
1721         &vpsrlq         ($T1,$D3,26);
1722          &vpsrlq        ($T0,$D0,26);
1723          &vpand         ($D0,$D0,$MASK);
1724         &vpand          ($D3,$D3,$MASK);
1725          &vpaddq        ($D1,$D1,$T0);                  # h0 -> h1
1726         &vpaddq         ($D4,$D4,$T1);                  # h3 -> h4
1727 }
1728         &vlazy_reduction();
1729
1730         &vmovdqu        (&X($T0),&QWP(16*0,"esi"));     # load input
1731         &vmovdqu        (&X($T1),&QWP(16*1,"esi"));
1732         &vinserti128    ($T0,$T0,&QWP(16*2,"esi"),1);
1733         &vinserti128    ($T1,$T1,&QWP(16*3,"esi"),1);
1734         &lea            ("esi",&DWP(16*4,"esi"));
1735         &sub            ("ecx",64);
1736         &jnz            (&label("loop"));
1737
1738 &set_label("tail");
1739         &vsplat_input   ();
1740         &and            ("ebx",-64);                    # restore pointer
1741
1742         &vpmuladd       (sub {  my $i=shift; &QWP(4+32*$i-128,"edx");   });
1743
1744         ################################################################
1745         # horizontal addition
1746
1747         &vpsrldq        ($T0,$D4,8);
1748         &vpsrldq        ($T1,$D3,8);
1749         &vpaddq         ($D4,$D4,$T0);
1750         &vpsrldq        ($T0,$D0,8);
1751         &vpaddq         ($D3,$D3,$T1);
1752         &vpsrldq        ($T1,$D1,8);
1753         &vpaddq         ($D0,$D0,$T0);
1754         &vpsrldq        ($T0,$D2,8);
1755         &vpaddq         ($D1,$D1,$T1);
1756         &vpermq         ($T1,$D4,2);                    # keep folding
1757         &vpaddq         ($D2,$D2,$T0);
1758         &vpermq         ($T0,$D3,2);
1759         &vpaddq         ($D4,$D4,$T1);
1760         &vpermq         ($T1,$D0,2);
1761         &vpaddq         ($D3,$D3,$T0);
1762         &vpermq         ($T0,$D1,2);
1763         &vpaddq         ($D0,$D0,$T1);
1764         &vpermq         ($T1,$D2,2);
1765         &vpaddq         ($D1,$D1,$T0);
1766         &vpaddq         ($D2,$D2,$T1);
1767
1768         &vlazy_reduction();
1769
1770         &cmp            ("ecx",0);
1771         &je             (&label("done"));
1772
1773         ################################################################
1774         # clear all but single word
1775
1776         &vpshufd        (&X($D0),&X($D0),0b11111100);
1777         &lea            ("edx",&DWP(32*5+128,"esp"));   # restore pointer
1778         &vpshufd        (&X($D1),&X($D1),0b11111100);
1779         &vpshufd        (&X($D2),&X($D2),0b11111100);
1780         &vpshufd        (&X($D3),&X($D3),0b11111100);
1781         &vpshufd        (&X($D4),&X($D4),0b11111100);
1782         &jmp            (&label("even"));
1783
1784 &set_label("done",16);
1785         &vmovd          (&DWP(-16*3+4*0,"edi"),&X($D0));# store hash value
1786         &vmovd          (&DWP(-16*3+4*1,"edi"),&X($D1));
1787         &vmovd          (&DWP(-16*3+4*2,"edi"),&X($D2));
1788         &vmovd          (&DWP(-16*3+4*3,"edi"),&X($D3));
1789         &vmovd          (&DWP(-16*3+4*4,"edi"),&X($D4));
1790         &vzeroupper     ();
1791         &mov    ("esp","ebp");
1792 &set_label("nodata");
1793 &function_end("_poly1305_blocks_avx2");
1794 }
1795 &set_label("const_sse2",64);
1796         &data_word(1<<24,0,     1<<24,0,        1<<24,0,        1<<24,0);
1797         &data_word(0,0,         0,0,            0,0,            0,0);
1798         &data_word(0x03ffffff,0,0x03ffffff,0,   0x03ffffff,0,   0x03ffffff,0);
1799         &data_word(0x0fffffff,0x0ffffffc,0x0ffffffc,0x0ffffffc);
1800 }
1801 &asciz  ("Poly1305 for x86, CRYPTOGAMS by <appro\@openssl.org>");
1802 &align  (4);
1803
1804 &asm_finish();
1805
1806 close STDOUT;