crypto/chacha/asm/chacha-x86.pl

   1 #!/usr/bin/env perl
   2 #
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9 #
  10 # January 2015
  11 #
  12 # ChaCha20 for x86.
  13 #
  14 # Performance in cycles per byte out of large buffer.
  15 #
  16 #               1xIALU/gcc      4xSSSE3
  17 # Pentium       17.5/+80%
  18 # PIII          14.2/+60%
  19 # P4            18.6/+84%
  20 # Core2         9.56/+89%       4.83
  21 # Westmere      9.50/+45%       3.35
  22 # Sandy Bridge  10.5/+47%       3.20
  23 # Haswell       8.15/+50%       2.83
  24 # Silvermont    17.4/+36%       8.35
  25 # Sledgehammer  10.2/+54%
  26 # Bulldozer     13.4/+50%       4.38(*)
  27 #
  28 # (*)   Bulldozer actually executes 4xXOP code path that delivers 3.55;
  29
  30 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  31 push(@INC,"${dir}","${dir}../../perlasm");
  32 require "x86asm.pl";
  33
  34 $output=pop;
  35 open STDOUT,">$output";
  36
  37 &asm_init($ARGV[0],"chacha-x86.pl",$ARGV[$#ARGV] eq "386");
  38
  39 $xmm=$ymm=0;
  40 for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); }
  41
  42 $ymm=1 if ($xmm &&
  43                 `$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  44                         =~ /GNU assembler version ([2-9]\.[0-9]+)/ &&
  45                 $1>=2.19);      # first version supporting AVX
  46
  47 $ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32n" &&
  48                 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/ &&
  49                 $1>=2.03);      # first version supporting AVX
  50
  51 $ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32" &&
  52                 `ml 2>&1` =~ /Version ([0-9]+)\./ &&
  53                 $1>=10);        # first version supporting AVX
  54
  55 $ymm=1 if ($xmm && !$ymm &&
  56                 `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/ &&
  57                 $2>=3.0);       # first version supporting AVX
  58
  59 $a="eax";
  60 ($b,$b_)=("ebx","ebp");
  61 ($c,$c_)=("ecx","esi");
  62 ($d,$d_)=("edx","edi");
  63
  64 sub QUARTERROUND {
  65 my ($ai,$bi,$ci,$di,$i)=@_;
  66 my ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+1)&3),($ai,$bi,$ci,$di)); # next
  67 my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
  68
  69         #       a   b   c   d
  70         #
  71         #       0   4   8  12 < even round
  72         #       1   5   9  13
  73         #       2   6  10  14
  74         #       3   7  11  15
  75         #       0   5  10  15 < odd round
  76         #       1   6  11  12
  77         #       2   7   8  13
  78         #       3   4   9  14
  79
  80         if ($i==0) {
  81             my $j=4;
  82             ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-$j--)&3),($ap,$bp,$cp,$dp));
  83         } elsif ($i==3) {
  84             my $j=0;
  85             ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+$j++)&3),($an,$bn,$cn,$dn));
  86         } elsif ($i==4) {
  87             my $j=4;
  88             ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_+$j--)&3),($ap,$bp,$cp,$dp));
  89         } elsif ($i==7) {
  90             my $j=0;
  91             ($an,$bn,$cn,$dn)=map(($_&~3)+(($_-$j++)&3),($an,$bn,$cn,$dn));
  92         }
  93
  94         #&add   ($a,$b);                        # see elsewhere
  95         &xor    ($d,$a);
  96          &mov   (&DWP(4*$cp,"esp"),$c_)         if ($ai>0 && $ai<3);
  97         &rol    ($d,16);
  98          &mov   (&DWP(4*$bp,"esp"),$b_)         if ($i!=0);
  99         &add    ($c,$d);
 100          &mov   ($c_,&DWP(4*$cn,"esp"))         if ($ai>0 && $ai<3);
 101         &xor    ($b,$c);
 102          &mov   ($d_,&DWP(4*$dn,"esp"))         if ($di!=$dn);
 103         &rol    ($b,12);
 104          &mov   ($b_,&DWP(4*$bn,"esp"))         if ($i<7);
 105          &mov   ($b_,&DWP(128,"esp"))           if ($i==7);     # loop counter
 106         &add    ($a,$b);
 107         &xor    ($d,$a);
 108         &mov    (&DWP(4*$ai,"esp"),$a);
 109         &rol    ($d,8);
 110         &mov    ($a,&DWP(4*$an,"esp"));
 111         &add    ($c,$d);
 112         &mov    (&DWP(4*$di,"esp"),$d)          if ($di!=$dn);
 113         &mov    ($d_,$d)                        if ($di==$dn);
 114         &xor    ($b,$c);
 115          &add   ($a,$b_)                        if ($i<7);      # elsewhere
 116         &rol    ($b,7);
 117
 118         ($b,$b_)=($b_,$b);
 119         ($c,$c_)=($c_,$c);
 120         ($d,$d_)=($d_,$d);
 121 }
 122
 123 &static_label("ssse3_shortcut");
 124 &static_label("xop_shortcut");
 125 &static_label("ssse3_data");
 126 &static_label("pic_point");
 127
 128 &function_begin("ChaCha20_ctr32");
 129         &xor    ("eax","eax");
 130         &cmp    ("eax",&wparam(2));             # len==0?
 131         &je     (&label("no_data"));
 132 if ($xmm) {
 133         &call   (&label("pic_point"));
 134 &set_label("pic_point");
 135         &blindpop("eax");
 136         &picmeup("ebp","OPENSSL_ia32cap_P","eax",&label("pic_point"));
 137         &test   (&DWP(0,"ebp"),1<<24);          # test FXSR bit
 138         &jz     (&label("x86"));
 139         &test   (&DWP(4,"ebp"),1<<9);           # test SSSE3 bit
 140         &jz     (&label("x86"));
 141         &jmp    (&label("ssse3_shortcut"));
 142 &set_label("x86");
 143 }
 144         &mov    ("esi",&wparam(3));             # key
 145         &mov    ("edi",&wparam(4));             # counter and nonce
 146
 147         &stack_push(33);
 148
 149         &mov    ("eax",&DWP(4*0,"esi"));        # copy key
 150         &mov    ("ebx",&DWP(4*1,"esi"));
 151         &mov    ("ecx",&DWP(4*2,"esi"));
 152         &mov    ("edx",&DWP(4*3,"esi"));
 153         &mov    (&DWP(64+4*4,"esp"),"eax");
 154         &mov    (&DWP(64+4*5,"esp"),"ebx");
 155         &mov    (&DWP(64+4*6,"esp"),"ecx");
 156         &mov    (&DWP(64+4*7,"esp"),"edx");
 157         &mov    ("eax",&DWP(4*4,"esi"));
 158         &mov    ("ebx",&DWP(4*5,"esi"));
 159         &mov    ("ecx",&DWP(4*6,"esi"));
 160         &mov    ("edx",&DWP(4*7,"esi"));
 161         &mov    (&DWP(64+4*8,"esp"),"eax");
 162         &mov    (&DWP(64+4*9,"esp"),"ebx");
 163         &mov    (&DWP(64+4*10,"esp"),"ecx");
 164         &mov    (&DWP(64+4*11,"esp"),"edx");
 165         &mov    ("eax",&DWP(4*0,"edi"));        # copy counter and nonce
 166         &mov    ("ebx",&DWP(4*1,"edi"));
 167         &mov    ("ecx",&DWP(4*2,"edi"));
 168         &mov    ("edx",&DWP(4*3,"edi"));
 169         &sub    ("eax",1);
 170         &mov    (&DWP(64+4*12,"esp"),"eax");
 171         &mov    (&DWP(64+4*13,"esp"),"ebx");
 172         &mov    (&DWP(64+4*14,"esp"),"ecx");
 173         &mov    (&DWP(64+4*15,"esp"),"edx");
 174         &jmp    (&label("entry"));
 175
 176 &set_label("outer_loop",16);
 177         &mov    (&wparam(1),$b);                # save input
 178         &mov    (&wparam(0),$a);                # save output
 179         &mov    (&wparam(2),$c);                # save len
 180 &set_label("entry");
 181         &mov    ($a,0x61707865);
 182         &mov    (&DWP(4*1,"esp"),0x3320646e);
 183         &mov    (&DWP(4*2,"esp"),0x79622d32);
 184         &mov    (&DWP(4*3,"esp"),0x6b206574);
 185
 186         &mov    ($b, &DWP(64+4*5,"esp"));       # copy key material
 187         &mov    ($b_,&DWP(64+4*6,"esp"));
 188         &mov    ($c, &DWP(64+4*10,"esp"));
 189         &mov    ($c_,&DWP(64+4*11,"esp"));
 190         &mov    ($d, &DWP(64+4*13,"esp"));
 191         &mov    ($d_,&DWP(64+4*14,"esp"));
 192         &mov    (&DWP(4*5,"esp"),$b);
 193         &mov    (&DWP(4*6,"esp"),$b_);
 194         &mov    (&DWP(4*10,"esp"),$c);
 195         &mov    (&DWP(4*11,"esp"),$c_);
 196         &mov    (&DWP(4*13,"esp"),$d);
 197         &mov    (&DWP(4*14,"esp"),$d_);
 198
 199         &mov    ($b, &DWP(64+4*7,"esp"));
 200         &mov    ($d_,&DWP(64+4*15,"esp"));
 201         &mov    ($d, &DWP(64+4*12,"esp"));
 202         &mov    ($b_,&DWP(64+4*4,"esp"));
 203         &mov    ($c, &DWP(64+4*8,"esp"));
 204         &mov    ($c_,&DWP(64+4*9,"esp"));
 205         &add    ($d,1);                         # counter value
 206         &mov    (&DWP(4*7,"esp"),$b);
 207         &mov    (&DWP(4*15,"esp"),$d_);
 208         &mov    (&DWP(64+4*12,"esp"),$d);       # save counter value
 209
 210         &mov    ($b,10);                        # loop counter
 211         &jmp    (&label("loop"));
 212
 213 &set_label("loop",16);
 214         &add    ($a,$b_);                       # elsewhere
 215         &mov    (&DWP(128,"esp"),$b);           # save loop counter
 216         &mov    ($b,$b_);
 217         &QUARTERROUND(0, 4, 8, 12, 0);
 218         &QUARTERROUND(1, 5, 9, 13, 1);
 219         &QUARTERROUND(2, 6,10, 14, 2);
 220         &QUARTERROUND(3, 7,11, 15, 3);
 221         &QUARTERROUND(0, 5,10, 15, 4);
 222         &QUARTERROUND(1, 6,11, 12, 5);
 223         &QUARTERROUND(2, 7, 8, 13, 6);
 224         &QUARTERROUND(3, 4, 9, 14, 7);
 225         &dec    ($b);
 226         &jnz    (&label("loop"));
 227
 228         &mov    ($b,&wparam(2));                # load len
 229
 230         &add    ($a,0x61707865);                # accumulate key material
 231         &add    ($b_,&DWP(64+4*4,"esp"));
 232         &add    ($c, &DWP(64+4*8,"esp"));
 233         &add    ($c_,&DWP(64+4*9,"esp"));
 234
 235         &cmp    ($b,64);
 236         &jb     (&label("tail"));
 237
 238         &mov    ($b,&wparam(1));                # load input pointer
 239         &add    ($d, &DWP(64+4*12,"esp"));
 240         &add    ($d_,&DWP(64+4*14,"esp"));
 241
 242         &xor    ($a, &DWP(4*0,$b));             # xor with input
 243         &xor    ($b_,&DWP(4*4,$b));
 244         &mov    (&DWP(4*0,"esp"),$a);
 245         &mov    ($a,&wparam(0));                # load output pointer
 246         &xor    ($c, &DWP(4*8,$b));
 247         &xor    ($c_,&DWP(4*9,$b));
 248         &xor    ($d, &DWP(4*12,$b));
 249         &xor    ($d_,&DWP(4*14,$b));
 250         &mov    (&DWP(4*4,$a),$b_);             # write output
 251         &mov    (&DWP(4*8,$a),$c);
 252         &mov    (&DWP(4*9,$a),$c_);
 253         &mov    (&DWP(4*12,$a),$d);
 254         &mov    (&DWP(4*14,$a),$d_);
 255
 256         &mov    ($b_,&DWP(4*1,"esp"));
 257         &mov    ($c, &DWP(4*2,"esp"));
 258         &mov    ($c_,&DWP(4*3,"esp"));
 259         &mov    ($d, &DWP(4*5,"esp"));
 260         &mov    ($d_,&DWP(4*6,"esp"));
 261         &add    ($b_,0x3320646e);               # accumulate key material
 262         &add    ($c, 0x79622d32);
 263         &add    ($c_,0x6b206574);
 264         &add    ($d, &DWP(64+4*5,"esp"));
 265         &add    ($d_,&DWP(64+4*6,"esp"));
 266         &xor    ($b_,&DWP(4*1,$b));
 267         &xor    ($c, &DWP(4*2,$b));
 268         &xor    ($c_,&DWP(4*3,$b));
 269         &xor    ($d, &DWP(4*5,$b));
 270         &xor    ($d_,&DWP(4*6,$b));
 271         &mov    (&DWP(4*1,$a),$b_);
 272         &mov    (&DWP(4*2,$a),$c);
 273         &mov    (&DWP(4*3,$a),$c_);
 274         &mov    (&DWP(4*5,$a),$d);
 275         &mov    (&DWP(4*6,$a),$d_);
 276
 277         &mov    ($b_,&DWP(4*7,"esp"));
 278         &mov    ($c, &DWP(4*10,"esp"));
 279         &mov    ($c_,&DWP(4*11,"esp"));
 280         &mov    ($d, &DWP(4*13,"esp"));
 281         &mov    ($d_,&DWP(4*15,"esp"));
 282         &add    ($b_,&DWP(64+4*7,"esp"));
 283         &add    ($c, &DWP(64+4*10,"esp"));
 284         &add    ($c_,&DWP(64+4*11,"esp"));
 285         &add    ($d, &DWP(64+4*13,"esp"));
 286         &add    ($d_,&DWP(64+4*15,"esp"));
 287         &xor    ($b_,&DWP(4*7,$b));
 288         &xor    ($c, &DWP(4*10,$b));
 289         &xor    ($c_,&DWP(4*11,$b));
 290         &xor    ($d, &DWP(4*13,$b));
 291         &xor    ($d_,&DWP(4*15,$b));
 292         &lea    ($b,&DWP(4*16,$b));
 293         &mov    (&DWP(4*7,$a),$b_);
 294         &mov    ($b_,&DWP(4*0,"esp"));
 295         &mov    (&DWP(4*10,$a),$c);
 296         &mov    ($c,&wparam(2));                # len
 297         &mov    (&DWP(4*11,$a),$c_);
 298         &mov    (&DWP(4*13,$a),$d);
 299         &mov    (&DWP(4*15,$a),$d_);
 300         &mov    (&DWP(4*0,$a),$b_);
 301         &lea    ($a,&DWP(4*16,$a));
 302         &sub    ($c,64);
 303         &jnz    (&label("outer_loop"));
 304
 305         &jmp    (&label("done"));
 306
 307 &set_label("tail");
 308         &add    ($d, &DWP(64+4*12,"esp"));
 309         &add    ($d_,&DWP(64+4*14,"esp"));
 310         &mov    (&DWP(4*0,"esp"),$a);
 311         &mov    (&DWP(4*4,"esp"),$b_);
 312         &mov    (&DWP(4*8,"esp"),$c);
 313         &mov    (&DWP(4*9,"esp"),$c_);
 314         &mov    (&DWP(4*12,"esp"),$d);
 315         &mov    (&DWP(4*14,"esp"),$d_);
 316
 317         &mov    ($b_,&DWP(4*1,"esp"));
 318         &mov    ($c, &DWP(4*2,"esp"));
 319         &mov    ($c_,&DWP(4*3,"esp"));
 320         &mov    ($d, &DWP(4*5,"esp"));
 321         &mov    ($d_,&DWP(4*6,"esp"));
 322         &add    ($b_,0x3320646e);               # accumulate key material
 323         &add    ($c, 0x79622d32);
 324         &add    ($c_,0x6b206574);
 325         &add    ($d, &DWP(64+4*5,"esp"));
 326         &add    ($d_,&DWP(64+4*6,"esp"));
 327         &mov    (&DWP(4*1,"esp"),$b_);
 328         &mov    (&DWP(4*2,"esp"),$c);
 329         &mov    (&DWP(4*3,"esp"),$c_);
 330         &mov    (&DWP(4*5,"esp"),$d);
 331         &mov    (&DWP(4*6,"esp"),$d_);
 332
 333         &mov    ($b_,&DWP(4*7,"esp"));
 334         &mov    ($c, &DWP(4*10,"esp"));
 335         &mov    ($c_,&DWP(4*11,"esp"));
 336         &mov    ($d, &DWP(4*13,"esp"));
 337         &mov    ($d_,&DWP(4*15,"esp"));
 338         &add    ($b_,&DWP(64+4*7,"esp"));
 339         &add    ($c, &DWP(64+4*10,"esp"));
 340         &add    ($c_,&DWP(64+4*11,"esp"));
 341         &add    ($d, &DWP(64+4*13,"esp"));
 342         &add    ($d_,&DWP(64+4*15,"esp"));
 343         &mov    (&DWP(4*7,"esp"),$b_);
 344         &mov    ($b_,&wparam(1));               # load input
 345         &mov    (&DWP(4*10,"esp"),$c);
 346         &mov    ($c,&wparam(0));                # load output
 347         &mov    (&DWP(4*11,"esp"),$c_);
 348         &xor    ($c_,$c_);
 349         &mov    (&DWP(4*13,"esp"),$d);
 350         &mov    (&DWP(4*15,"esp"),$d_);
 351
 352         &xor    ("eax","eax");
 353         &xor    ("edx","edx");
 354 &set_label("tail_loop");
 355         &movb   ("al",&BP(0,$c_,$b_));
 356         &movb   ("dl",&BP(0,"esp",$c_));
 357         &lea    ($c_,&DWP(1,$c_));
 358         &xor    ("al","dl");
 359         &mov    (&BP(-1,$c,$c_),"al");
 360         &dec    ($b);
 361         &jnz    (&label("tail_loop"));
 362
 363 &set_label("done");
 364         &stack_pop(33);
 365 &set_label("no_data");
 366 &function_end("ChaCha20_ctr32");
 367
 368 if ($xmm) {
 369 my ($xa,$xa_,$xb,$xb_,$xc,$xc_,$xd,$xd_)=map("xmm$_",(0..7));
 370 my ($out,$inp,$len)=("edi","esi","ecx");
 371
 372 sub QUARTERROUND_SSSE3 {
 373 my ($ai,$bi,$ci,$di,$i)=@_;
 374 my ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+1)&3),($ai,$bi,$ci,$di)); # next
 375 my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
 376
 377         #       a   b   c   d
 378         #
 379         #       0   4   8  12 < even round
 380         #       1   5   9  13
 381         #       2   6  10  14
 382         #       3   7  11  15
 383         #       0   5  10  15 < odd round
 384         #       1   6  11  12
 385         #       2   7   8  13
 386         #       3   4   9  14
 387
 388         if ($i==0) {
 389             my $j=4;
 390             ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-$j--)&3),($ap,$bp,$cp,$dp));
 391         } elsif ($i==3) {
 392             my $j=0;
 393             ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+$j++)&3),($an,$bn,$cn,$dn));
 394         } elsif ($i==4) {
 395             my $j=4;
 396             ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_+$j--)&3),($ap,$bp,$cp,$dp));
 397         } elsif ($i==7) {
 398             my $j=0;
 399             ($an,$bn,$cn,$dn)=map(($_&~3)+(($_-$j++)&3),($an,$bn,$cn,$dn));
 400         }
 401
 402         #&paddd ($xa,$xb);                      # see elsewhere
 403         #&pxor  ($xd,$xa);                      # see elsewhere
 404          &movdqa(&QWP(16*$cp-128,"ebx"),$xc_)   if ($ai>0 && $ai<3);
 405         &pshufb ($xd,&QWP(0,"eax"));            # rot16
 406          &movdqa(&QWP(16*$bp-128,"ebx"),$xb_)   if ($i!=0);
 407         &paddd  ($xc,$xd);
 408          &movdqa($xc_,&QWP(16*$cn-128,"ebx"))   if ($ai>0 && $ai<3);
 409         &pxor   ($xb,$xc);
 410          &movdqa($xb_,&QWP(16*$bn-128,"ebx"))   if ($i<7);
 411         &movdqa ($xa_,$xb);                     # borrow as temporary
 412         &pslld  ($xb,12);
 413         &psrld  ($xa_,20);
 414         &por    ($xb,$xa_);
 415          &movdqa($xa_,&QWP(16*$an-128,"ebx"));
 416         &paddd  ($xa,$xb);
 417          &movdqa($xd_,&QWP(16*$dn-128,"ebx"))   if ($di!=$dn);
 418         &pxor   ($xd,$xa);
 419         &movdqa (&QWP(16*$ai-128,"ebx"),$xa);
 420         &pshufb ($xd,&QWP(16,"eax"));           # rot8
 421         &paddd  ($xc,$xd);
 422         &movdqa (&QWP(16*$di-128,"ebx"),$xd)    if ($di!=$dn);
 423         &movdqa ($xd_,$xd)                      if ($di==$dn);
 424         &pxor   ($xb,$xc);
 425          &paddd ($xa_,$xb_)                     if ($i<7);      # elsewhere
 426         &movdqa ($xa,$xb);                      # borrow as temporary
 427         &pslld  ($xb,7);
 428         &psrld  ($xa,25);
 429          &pxor  ($xd_,$xa_)                     if ($i<7);      # elsewhere
 430         &por    ($xb,$xa);
 431
 432         ($xa,$xa_)=($xa_,$xa);
 433         ($xb,$xb_)=($xb_,$xb);
 434         ($xc,$xc_)=($xc_,$xc);
 435         ($xd,$xd_)=($xd_,$xd);
 436 }
 437
 438 &function_begin("ChaCha20_ssse3");
 439 &set_label("ssse3_shortcut");
 440         &test           (&DWP(4,"ebp"),1<<11);          # test XOP bit
 441         &jnz            (&label("xop_shortcut"));
 442
 443         &mov            ($out,&wparam(0));
 444         &mov            ($inp,&wparam(1));
 445         &mov            ($len,&wparam(2));
 446         &mov            ("edx",&wparam(3));             # key
 447         &mov            ("ebx",&wparam(4));             # counter and nonce
 448
 449         &mov            ("ebp","esp");
 450         &stack_push     (131);
 451         &and            ("esp",-64);
 452         &mov            (&DWP(512,"esp"),"ebp");
 453
 454         &lea            ("eax",&DWP(&label("ssse3_data")."-".
 455                                     &label("pic_point"),"eax"));
 456         &movdqu         ("xmm3",&QWP(0,"ebx"));         # counter and nonce
 457
 458         &cmp            ($len,64*4);
 459         &jb             (&label("1x"));
 460
 461         &mov            (&DWP(512+4,"esp"),"edx");      # offload pointers
 462         &mov            (&DWP(512+8,"esp"),"ebx");
 463         &sub            ($len,64*4);                    # bias len
 464         &lea            ("ebp",&DWP(256+128,"esp"));    # size optimization
 465
 466         &movdqu         ("xmm7",&QWP(0,"edx"));         # key
 467         &pshufd         ("xmm0","xmm3",0x00);
 468         &pshufd         ("xmm1","xmm3",0x55);
 469         &pshufd         ("xmm2","xmm3",0xaa);
 470         &pshufd         ("xmm3","xmm3",0xff);
 471          &paddd         ("xmm0",&QWP(16*3,"eax"));      # fix counters
 472         &pshufd         ("xmm4","xmm7",0x00);
 473         &pshufd         ("xmm5","xmm7",0x55);
 474          &psubd         ("xmm0",&QWP(16*4,"eax"));
 475         &pshufd         ("xmm6","xmm7",0xaa);
 476         &pshufd         ("xmm7","xmm7",0xff);
 477         &movdqa         (&QWP(16*12-128,"ebp"),"xmm0");
 478         &movdqa         (&QWP(16*13-128,"ebp"),"xmm1");
 479         &movdqa         (&QWP(16*14-128,"ebp"),"xmm2");
 480         &movdqa         (&QWP(16*15-128,"ebp"),"xmm3");
 481          &movdqu        ("xmm3",&QWP(16,"edx"));        # key
 482         &movdqa         (&QWP(16*4-128,"ebp"),"xmm4");
 483         &movdqa         (&QWP(16*5-128,"ebp"),"xmm5");
 484         &movdqa         (&QWP(16*6-128,"ebp"),"xmm6");
 485         &movdqa         (&QWP(16*7-128,"ebp"),"xmm7");
 486          &movdqa        ("xmm7",&QWP(16*2,"eax"));      # sigma
 487          &lea           ("ebx",&DWP(128,"esp"));        # size optimization
 488
 489         &pshufd         ("xmm0","xmm3",0x00);
 490         &pshufd         ("xmm1","xmm3",0x55);
 491         &pshufd         ("xmm2","xmm3",0xaa);
 492         &pshufd         ("xmm3","xmm3",0xff);
 493         &pshufd         ("xmm4","xmm7",0x00);
 494         &pshufd         ("xmm5","xmm7",0x55);
 495         &pshufd         ("xmm6","xmm7",0xaa);
 496         &pshufd         ("xmm7","xmm7",0xff);
 497         &movdqa         (&QWP(16*8-128,"ebp"),"xmm0");
 498         &movdqa         (&QWP(16*9-128,"ebp"),"xmm1");
 499         &movdqa         (&QWP(16*10-128,"ebp"),"xmm2");
 500         &movdqa         (&QWP(16*11-128,"ebp"),"xmm3");
 501         &movdqa         (&QWP(16*0-128,"ebp"),"xmm4");
 502         &movdqa         (&QWP(16*1-128,"ebp"),"xmm5");
 503         &movdqa         (&QWP(16*2-128,"ebp"),"xmm6");
 504         &movdqa         (&QWP(16*3-128,"ebp"),"xmm7");
 505
 506         &lea            ($inp,&DWP(128,$inp));          # size optimization
 507         &lea            ($out,&DWP(128,$out));          # size optimization
 508         &jmp            (&label("outer_loop"));
 509
 510 &set_label("outer_loop",16);
 511         #&movdqa        ("xmm0",&QWP(16*0-128,"ebp"));  # copy key material
 512         &movdqa         ("xmm1",&QWP(16*1-128,"ebp"));
 513         &movdqa         ("xmm2",&QWP(16*2-128,"ebp"));
 514         &movdqa         ("xmm3",&QWP(16*3-128,"ebp"));
 515         #&movdqa        ("xmm4",&QWP(16*4-128,"ebp"));
 516         &movdqa         ("xmm5",&QWP(16*5-128,"ebp"));
 517         &movdqa         ("xmm6",&QWP(16*6-128,"ebp"));
 518         &movdqa         ("xmm7",&QWP(16*7-128,"ebp"));
 519         #&movdqa        (&QWP(16*0-128,"ebx"),"xmm0");
 520         &movdqa         (&QWP(16*1-128,"ebx"),"xmm1");
 521         &movdqa         (&QWP(16*2-128,"ebx"),"xmm2");
 522         &movdqa         (&QWP(16*3-128,"ebx"),"xmm3");
 523         #&movdqa        (&QWP(16*4-128,"ebx"),"xmm4");
 524         &movdqa         (&QWP(16*5-128,"ebx"),"xmm5");
 525         &movdqa         (&QWP(16*6-128,"ebx"),"xmm6");
 526         &movdqa         (&QWP(16*7-128,"ebx"),"xmm7");
 527         #&movdqa        ("xmm0",&QWP(16*8-128,"ebp"));
 528         #&movdqa        ("xmm1",&QWP(16*9-128,"ebp"));
 529         &movdqa         ("xmm2",&QWP(16*10-128,"ebp"));
 530         &movdqa         ("xmm3",&QWP(16*11-128,"ebp"));
 531         &movdqa         ("xmm4",&QWP(16*12-128,"ebp"));
 532         &movdqa         ("xmm5",&QWP(16*13-128,"ebp"));
 533         &movdqa         ("xmm6",&QWP(16*14-128,"ebp"));
 534         &movdqa         ("xmm7",&QWP(16*15-128,"ebp"));
 535         &paddd          ("xmm4",&QWP(16*4,"eax"));      # counter value
 536         #&movdqa        (&QWP(16*8-128,"ebx"),"xmm0");
 537         #&movdqa        (&QWP(16*9-128,"ebx"),"xmm1");
 538         &movdqa         (&QWP(16*10-128,"ebx"),"xmm2");
 539         &movdqa         (&QWP(16*11-128,"ebx"),"xmm3");
 540         &movdqa         (&QWP(16*12-128,"ebx"),"xmm4");
 541         &movdqa         (&QWP(16*13-128,"ebx"),"xmm5");
 542         &movdqa         (&QWP(16*14-128,"ebx"),"xmm6");
 543         &movdqa         (&QWP(16*15-128,"ebx"),"xmm7");
 544         &movdqa         (&QWP(16*12-128,"ebp"),"xmm4"); # save counter value
 545
 546         &movdqa         ($xa, &QWP(16*0-128,"ebp"));
 547         &movdqa         ($xd, "xmm4");
 548         &movdqa         ($xb_,&QWP(16*4-128,"ebp"));
 549         &movdqa         ($xc, &QWP(16*8-128,"ebp"));
 550         &movdqa         ($xc_,&QWP(16*9-128,"ebp"));
 551
 552         &mov            ("edx",10);                     # loop counter
 553         &nop            ();
 554
 555 &set_label("loop",16);
 556         &paddd          ($xa,$xb_);                     # elsewhere
 557         &movdqa         ($xb,$xb_);
 558         &pxor           ($xd,$xa);                      # elsewhere
 559         &QUARTERROUND_SSSE3(0, 4, 8, 12, 0);
 560         &QUARTERROUND_SSSE3(1, 5, 9, 13, 1);
 561         &QUARTERROUND_SSSE3(2, 6,10, 14, 2);
 562         &QUARTERROUND_SSSE3(3, 7,11, 15, 3);
 563         &QUARTERROUND_SSSE3(0, 5,10, 15, 4);
 564         &QUARTERROUND_SSSE3(1, 6,11, 12, 5);
 565         &QUARTERROUND_SSSE3(2, 7, 8, 13, 6);
 566         &QUARTERROUND_SSSE3(3, 4, 9, 14, 7);
 567         &dec            ("edx");
 568         &jnz            (&label("loop"));
 569
 570         &movdqa         (&QWP(16*4-128,"ebx"),$xb_);
 571         &movdqa         (&QWP(16*8-128,"ebx"),$xc);
 572         &movdqa         (&QWP(16*9-128,"ebx"),$xc_);
 573         &movdqa         (&QWP(16*12-128,"ebx"),$xd);
 574         &movdqa         (&QWP(16*14-128,"ebx"),$xd_);
 575
 576     my ($xa0,$xa1,$xa2,$xa3,$xt0,$xt1,$xt2,$xt3)=map("xmm$_",(0..7));
 577
 578         #&movdqa        ($xa0,&QWP(16*0-128,"ebx"));    # it's there
 579         &movdqa         ($xa1,&QWP(16*1-128,"ebx"));
 580         &movdqa         ($xa2,&QWP(16*2-128,"ebx"));
 581         &movdqa         ($xa3,&QWP(16*3-128,"ebx"));
 582
 583     for($i=0;$i<256;$i+=64) {
 584         &paddd          ($xa0,&QWP($i+16*0-128,"ebp")); # accumulate key material
 585         &paddd          ($xa1,&QWP($i+16*1-128,"ebp"));
 586         &paddd          ($xa2,&QWP($i+16*2-128,"ebp"));
 587         &paddd          ($xa3,&QWP($i+16*3-128,"ebp"));
 588
 589         &movdqa         ($xt2,$xa0);            # "de-interlace" data
 590         &punpckldq      ($xa0,$xa1);
 591         &movdqa         ($xt3,$xa2);
 592         &punpckldq      ($xa2,$xa3);
 593         &punpckhdq      ($xt2,$xa1);
 594         &punpckhdq      ($xt3,$xa3);
 595         &movdqa         ($xa1,$xa0);
 596         &punpcklqdq     ($xa0,$xa2);            # "a0"
 597         &movdqa         ($xa3,$xt2);
 598         &punpcklqdq     ($xt2,$xt3);            # "a2"
 599         &punpckhqdq     ($xa1,$xa2);            # "a1"
 600         &punpckhqdq     ($xa3,$xt3);            # "a3"
 601
 602         #($xa2,$xt2)=($xt2,$xa2);
 603
 604         &movdqu         ($xt0,&QWP(64*0-128,$inp));     # load input
 605         &movdqu         ($xt1,&QWP(64*1-128,$inp));
 606         &movdqu         ($xa2,&QWP(64*2-128,$inp));
 607         &movdqu         ($xt3,&QWP(64*3-128,$inp));
 608         &lea            ($inp,&QWP($i<192?16:(64*4-16*3),$inp));
 609         &pxor           ($xt0,$xa0);
 610         &movdqa         ($xa0,&QWP($i+16*4-128,"ebx"))  if ($i<192);
 611         &pxor           ($xt1,$xa1);
 612         &movdqa         ($xa1,&QWP($i+16*5-128,"ebx"))  if ($i<192);
 613         &pxor           ($xt2,$xa2);
 614         &movdqa         ($xa2,&QWP($i+16*6-128,"ebx"))  if ($i<192);
 615         &pxor           ($xt3,$xa3);
 616         &movdqa         ($xa3,&QWP($i+16*7-128,"ebx"))  if ($i<192);
 617         &movdqu         (&QWP(64*0-128,$out),$xt0);     # store output
 618         &movdqu         (&QWP(64*1-128,$out),$xt1);
 619         &movdqu         (&QWP(64*2-128,$out),$xt2);
 620         &movdqu         (&QWP(64*3-128,$out),$xt3);
 621         &lea            ($out,&QWP($i<192?16:(64*4-16*3),$out));
 622     }
 623         &sub            ($len,64*4);
 624         &jnc            (&label("outer_loop"));
 625
 626         &add            ($len,64*4);
 627         &jz             (&label("done"));
 628
 629         &mov            ("ebx",&DWP(512+8,"esp"));      # restore pointers
 630         &lea            ($inp,&DWP(-128,$inp));
 631         &mov            ("edx",&DWP(512+4,"esp"));
 632         &lea            ($out,&DWP(-128,$out));
 633
 634         &movd           ("xmm2",&DWP(16*12-128,"ebp")); # counter value
 635         &movdqu         ("xmm3",&QWP(0,"ebx"));
 636         &paddd          ("xmm2",&QWP(16*6,"eax"));      # +four
 637         &pand           ("xmm3",&QWP(16*7,"eax"));
 638         &por            ("xmm3","xmm2");                # counter value
 639 {
 640 my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("xmm$_",(0..7));
 641
 642 sub SSSE3ROUND {        # critical path is 20 "SIMD ticks" per round
 643         &paddd          ($a,$b);
 644         &pxor           ($d,$a);
 645         &pshufb         ($d,$rot16);
 646
 647         &paddd          ($c,$d);
 648         &pxor           ($b,$c);
 649         &movdqa         ($t,$b);
 650         &psrld          ($b,20);
 651         &pslld          ($t,12);
 652         &por            ($b,$t);
 653
 654         &paddd          ($a,$b);
 655         &pxor           ($d,$a);
 656         &pshufb         ($d,$rot24);
 657
 658         &paddd          ($c,$d);
 659         &pxor           ($b,$c);
 660         &movdqa         ($t,$b);
 661         &psrld          ($b,25);
 662         &pslld          ($t,7);
 663         &por            ($b,$t);
 664 }
 665
 666 &set_label("1x");
 667         &movdqa         ($a,&QWP(16*2,"eax"));          # sigma
 668         &movdqu         ($b,&QWP(0,"edx"));
 669         &movdqu         ($c,&QWP(16,"edx"));
 670         #&movdqu        ($d,&QWP(0,"ebx"));             # already loaded
 671         &movdqa         ($rot16,&QWP(0,"eax"));
 672         &movdqa         ($rot24,&QWP(16,"eax"));
 673         &mov            (&DWP(16*3,"esp"),"ebp");
 674
 675         &movdqa         (&QWP(16*0,"esp"),$a);
 676         &movdqa         (&QWP(16*1,"esp"),$b);
 677         &movdqa         (&QWP(16*2,"esp"),$c);
 678         &movdqa         (&QWP(16*3,"esp"),$d);
 679         &mov            ("edx",10);
 680         &jmp            (&label("loop1x"));
 681
 682 &set_label("outer1x",16);
 683         &movdqa         ($d,&QWP(16*5,"eax"));          # one
 684         &movdqa         ($a,&QWP(16*0,"esp"));
 685         &movdqa         ($b,&QWP(16*1,"esp"));
 686         &movdqa         ($c,&QWP(16*2,"esp"));
 687         &paddd          ($d,&QWP(16*3,"esp"));
 688         &mov            ("edx",10);
 689         &movdqa         (&QWP(16*3,"esp"),$d);
 690         &jmp            (&label("loop1x"));
 691
 692 &set_label("loop1x",16);
 693         &SSSE3ROUND();
 694         &pshufd ($c,$c,0b01001110);
 695         &pshufd ($b,$b,0b00111001);
 696         &pshufd ($d,$d,0b10010011);
 697         &nop    ();
 698
 699         &SSSE3ROUND();
 700         &pshufd ($c,$c,0b01001110);
 701         &pshufd ($b,$b,0b10010011);
 702         &pshufd ($d,$d,0b00111001);
 703
 704         &dec            ("edx");
 705         &jnz            (&label("loop1x"));
 706
 707         &paddd          ($a,&QWP(16*0,"esp"));
 708         &paddd          ($b,&QWP(16*1,"esp"));
 709         &paddd          ($c,&QWP(16*2,"esp"));
 710         &paddd          ($d,&QWP(16*3,"esp"));
 711
 712         &cmp            ($len,64);
 713         &jb             (&label("tail"));
 714
 715         &movdqu         ($t,&QWP(16*0,$inp));
 716         &movdqu         ($t1,&QWP(16*1,$inp));
 717         &pxor           ($a,$t);                # xor with input
 718         &movdqu         ($t,&QWP(16*2,$inp));
 719         &pxor           ($b,$t1);
 720         &movdqu         ($t1,&QWP(16*3,$inp));
 721         &pxor           ($c,$t);
 722         &pxor           ($d,$t1);
 723         &lea            ($inp,&DWP(16*4,$inp)); # inp+=64
 724
 725         &movdqu         (&QWP(16*0,$out),$a);   # write output
 726         &movdqu         (&QWP(16*1,$out),$b);
 727         &movdqu         (&QWP(16*2,$out),$c);
 728         &movdqu         (&QWP(16*3,$out),$d);
 729         &lea            ($out,&DWP(16*4,$out)); # inp+=64
 730
 731         &sub            ($len,64);
 732         &jnz            (&label("outer1x"));
 733
 734         &jmp            (&label("done"));
 735
 736 &set_label("tail");
 737         &movdqa         (&QWP(16*0,"esp"),$a);
 738         &movdqa         (&QWP(16*1,"esp"),$b);
 739         &movdqa         (&QWP(16*2,"esp"),$c);
 740         &movdqa         (&QWP(16*3,"esp"),$d);
 741
 742         &xor            ("eax","eax");
 743         &xor            ("edx","edx");
 744         &xor            ("ebp","ebp");
 745
 746 &set_label("tail_loop");
 747         &movb           ("al",&BP(0,"esp","ebp"));
 748         &movb           ("dl",&BP(0,$inp,"ebp"));
 749         &lea            ("ebp",&DWP(1,"ebp"));
 750         &xor            ("al","dl");
 751         &movb           (&BP(-1,$out,"ebp"),"al");
 752         &dec            ($len);
 753         &jnz            (&label("tail_loop"));
 754 }
 755 &set_label("done");
 756         &mov            ("esp",&DWP(512,"esp"));
 757 &function_end("ChaCha20_ssse3");
 758
 759 &align  (64);
 760 &set_label("ssse3_data");
 761 &data_byte(0x2,0x3,0x0,0x1, 0x6,0x7,0x4,0x5, 0xa,0xb,0x8,0x9, 0xe,0xf,0xc,0xd);
 762 &data_byte(0x3,0x0,0x1,0x2, 0x7,0x4,0x5,0x6, 0xb,0x8,0x9,0xa, 0xf,0xc,0xd,0xe);
 763 &data_word(0x61707865,0x3320646e,0x79622d32,0x6b206574);
 764 &data_word(0,1,2,3);
 765 &data_word(4,4,4,4);
 766 &data_word(1,0,0,0);
 767 &data_word(4,0,0,0);
 768 &data_word(0,-1,-1,-1);
 769 &align  (64);
 770 }
 771 &asciz  ("ChaCha20 for x86, CRYPTOGAMS by <appro\@openssl.org>");
 772
 773 if ($xmm) {
 774 my ($xa,$xa_,$xb,$xb_,$xc,$xc_,$xd,$xd_)=map("xmm$_",(0..7));
 775 my ($out,$inp,$len)=("edi","esi","ecx");
 776
 777 sub QUARTERROUND_XOP {
 778 my ($ai,$bi,$ci,$di,$i)=@_;
 779 my ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+1)&3),($ai,$bi,$ci,$di)); # next
 780 my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
 781
 782         #       a   b   c   d
 783         #
 784         #       0   4   8  12 < even round
 785         #       1   5   9  13
 786         #       2   6  10  14
 787         #       3   7  11  15
 788         #       0   5  10  15 < odd round
 789         #       1   6  11  12
 790         #       2   7   8  13
 791         #       3   4   9  14
 792
 793         if ($i==0) {
 794             my $j=4;
 795             ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-$j--)&3),($ap,$bp,$cp,$dp));
 796         } elsif ($i==3) {
 797             my $j=0;
 798             ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+$j++)&3),($an,$bn,$cn,$dn));
 799         } elsif ($i==4) {
 800             my $j=4;
 801             ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_+$j--)&3),($ap,$bp,$cp,$dp));
 802         } elsif ($i==7) {
 803             my $j=0;
 804             ($an,$bn,$cn,$dn)=map(($_&~3)+(($_-$j++)&3),($an,$bn,$cn,$dn));
 805         }
 806
 807         #&vpaddd        ($xa,$xa,$xb);                  # see elsewhere
 808         #&vpxor         ($xd,$xd,$xa);                  # see elsewhere
 809          &vmovdqa       (&QWP(16*$cp-128,"ebx"),$xc_)   if ($ai>0 && $ai<3);
 810         &vprotd         ($xd,$xd,16);
 811          &vmovdqa       (&QWP(16*$bp-128,"ebx"),$xb_)   if ($i!=0);
 812         &vpaddd         ($xc,$xc,$xd);
 813          &vmovdqa       ($xc_,&QWP(16*$cn-128,"ebx"))   if ($ai>0 && $ai<3);
 814         &vpxor          ($xb,$i!=0?$xb:$xb_,$xc);
 815          &vmovdqa       ($xa_,&QWP(16*$an-128,"ebx"));
 816         &vprotd         ($xb,$xb,12);
 817          &vmovdqa       ($xb_,&QWP(16*$bn-128,"ebx"))   if ($i<7);
 818         &vpaddd         ($xa,$xa,$xb);
 819          &vmovdqa       ($xd_,&QWP(16*$dn-128,"ebx"))   if ($di!=$dn);
 820         &vpxor          ($xd,$xd,$xa);
 821          &vpaddd        ($xa_,$xa_,$xb_)                if ($i<7);      # elsewhere
 822         &vprotd         ($xd,$xd,8);
 823         &vmovdqa        (&QWP(16*$ai-128,"ebx"),$xa);
 824         &vpaddd         ($xc,$xc,$xd);
 825         &vmovdqa        (&QWP(16*$di-128,"ebx"),$xd)    if ($di!=$dn);
 826         &vpxor          ($xb,$xb,$xc);
 827          &vpxor         ($xd_,$di==$dn?$xd:$xd_,$xa_)   if ($i<7);      # elsewhere
 828         &vprotd         ($xb,$xb,7);
 829
 830         ($xa,$xa_)=($xa_,$xa);
 831         ($xb,$xb_)=($xb_,$xb);
 832         ($xc,$xc_)=($xc_,$xc);
 833         ($xd,$xd_)=($xd_,$xd);
 834 }
 835
 836 &function_begin("ChaCha20_xop");
 837 &set_label("xop_shortcut");
 838         &mov            ($out,&wparam(0));
 839         &mov            ($inp,&wparam(1));
 840         &mov            ($len,&wparam(2));
 841         &mov            ("edx",&wparam(3));             # key
 842         &mov            ("ebx",&wparam(4));             # counter and nonce
 843         &vzeroupper     ();
 844
 845         &mov            ("ebp","esp");
 846         &stack_push     (131);
 847         &and            ("esp",-64);
 848         &mov            (&DWP(512,"esp"),"ebp");
 849
 850         &lea            ("eax",&DWP(&label("ssse3_data")."-".
 851                                     &label("pic_point"),"eax"));
 852         &vmovdqu        ("xmm3",&QWP(0,"ebx"));         # counter and nonce
 853
 854         &cmp            ($len,64*4);
 855         &jb             (&label("1x"));
 856
 857         &mov            (&DWP(512+4,"esp"),"edx");      # offload pointers
 858         &mov            (&DWP(512+8,"esp"),"ebx");
 859         &sub            ($len,64*4);                    # bias len
 860         &lea            ("ebp",&DWP(256+128,"esp"));    # size optimization
 861
 862         &vmovdqu        ("xmm7",&QWP(0,"edx"));         # key
 863         &vpshufd        ("xmm0","xmm3",0x00);
 864         &vpshufd        ("xmm1","xmm3",0x55);
 865         &vpshufd        ("xmm2","xmm3",0xaa);
 866         &vpshufd        ("xmm3","xmm3",0xff);
 867          &vpaddd        ("xmm0","xmm0",&QWP(16*3,"eax"));       # fix counters
 868         &vpshufd        ("xmm4","xmm7",0x00);
 869         &vpshufd        ("xmm5","xmm7",0x55);
 870          &vpsubd        ("xmm0","xmm0",&QWP(16*4,"eax"));
 871         &vpshufd        ("xmm6","xmm7",0xaa);
 872         &vpshufd        ("xmm7","xmm7",0xff);
 873         &vmovdqa        (&QWP(16*12-128,"ebp"),"xmm0");
 874         &vmovdqa        (&QWP(16*13-128,"ebp"),"xmm1");
 875         &vmovdqa        (&QWP(16*14-128,"ebp"),"xmm2");
 876         &vmovdqa        (&QWP(16*15-128,"ebp"),"xmm3");
 877          &vmovdqu       ("xmm3",&QWP(16,"edx"));        # key
 878         &vmovdqa        (&QWP(16*4-128,"ebp"),"xmm4");
 879         &vmovdqa        (&QWP(16*5-128,"ebp"),"xmm5");
 880         &vmovdqa        (&QWP(16*6-128,"ebp"),"xmm6");
 881         &vmovdqa        (&QWP(16*7-128,"ebp"),"xmm7");
 882          &vmovdqa       ("xmm7",&QWP(16*2,"eax"));      # sigma
 883          &lea           ("ebx",&DWP(128,"esp"));        # size optimization
 884
 885         &vpshufd        ("xmm0","xmm3",0x00);
 886         &vpshufd        ("xmm1","xmm3",0x55);
 887         &vpshufd        ("xmm2","xmm3",0xaa);
 888         &vpshufd        ("xmm3","xmm3",0xff);
 889         &vpshufd        ("xmm4","xmm7",0x00);
 890         &vpshufd        ("xmm5","xmm7",0x55);
 891         &vpshufd        ("xmm6","xmm7",0xaa);
 892         &vpshufd        ("xmm7","xmm7",0xff);
 893         &vmovdqa        (&QWP(16*8-128,"ebp"),"xmm0");
 894         &vmovdqa        (&QWP(16*9-128,"ebp"),"xmm1");
 895         &vmovdqa        (&QWP(16*10-128,"ebp"),"xmm2");
 896         &vmovdqa        (&QWP(16*11-128,"ebp"),"xmm3");
 897         &vmovdqa        (&QWP(16*0-128,"ebp"),"xmm4");
 898         &vmovdqa        (&QWP(16*1-128,"ebp"),"xmm5");
 899         &vmovdqa        (&QWP(16*2-128,"ebp"),"xmm6");
 900         &vmovdqa        (&QWP(16*3-128,"ebp"),"xmm7");
 901
 902         &lea            ($inp,&DWP(128,$inp));          # size optimization
 903         &lea            ($out,&DWP(128,$out));          # size optimization
 904         &jmp            (&label("outer_loop"));
 905
 906 &set_label("outer_loop",32);
 907         #&vmovdqa       ("xmm0",&QWP(16*0-128,"ebp"));  # copy key material
 908         &vmovdqa        ("xmm1",&QWP(16*1-128,"ebp"));
 909         &vmovdqa        ("xmm2",&QWP(16*2-128,"ebp"));
 910         &vmovdqa        ("xmm3",&QWP(16*3-128,"ebp"));
 911         #&vmovdqa       ("xmm4",&QWP(16*4-128,"ebp"));
 912         &vmovdqa        ("xmm5",&QWP(16*5-128,"ebp"));
 913         &vmovdqa        ("xmm6",&QWP(16*6-128,"ebp"));
 914         &vmovdqa        ("xmm7",&QWP(16*7-128,"ebp"));
 915         #&vmovdqa       (&QWP(16*0-128,"ebx"),"xmm0");
 916         &vmovdqa        (&QWP(16*1-128,"ebx"),"xmm1");
 917         &vmovdqa        (&QWP(16*2-128,"ebx"),"xmm2");
 918         &vmovdqa        (&QWP(16*3-128,"ebx"),"xmm3");
 919         #&vmovdqa       (&QWP(16*4-128,"ebx"),"xmm4");
 920         &vmovdqa        (&QWP(16*5-128,"ebx"),"xmm5");
 921         &vmovdqa        (&QWP(16*6-128,"ebx"),"xmm6");
 922         &vmovdqa        (&QWP(16*7-128,"ebx"),"xmm7");
 923         #&vmovdqa       ("xmm0",&QWP(16*8-128,"ebp"));
 924         #&vmovdqa       ("xmm1",&QWP(16*9-128,"ebp"));
 925         &vmovdqa        ("xmm2",&QWP(16*10-128,"ebp"));
 926         &vmovdqa        ("xmm3",&QWP(16*11-128,"ebp"));
 927         &vmovdqa        ("xmm4",&QWP(16*12-128,"ebp"));
 928         &vmovdqa        ("xmm5",&QWP(16*13-128,"ebp"));
 929         &vmovdqa        ("xmm6",&QWP(16*14-128,"ebp"));
 930         &vmovdqa        ("xmm7",&QWP(16*15-128,"ebp"));
 931         &vpaddd         ("xmm4","xmm4",&QWP(16*4,"eax"));       # counter value
 932         #&vmovdqa       (&QWP(16*8-128,"ebx"),"xmm0");
 933         #&vmovdqa       (&QWP(16*9-128,"ebx"),"xmm1");
 934         &vmovdqa        (&QWP(16*10-128,"ebx"),"xmm2");
 935         &vmovdqa        (&QWP(16*11-128,"ebx"),"xmm3");
 936         &vmovdqa        (&QWP(16*12-128,"ebx"),"xmm4");
 937         &vmovdqa        (&QWP(16*13-128,"ebx"),"xmm5");
 938         &vmovdqa        (&QWP(16*14-128,"ebx"),"xmm6");
 939         &vmovdqa        (&QWP(16*15-128,"ebx"),"xmm7");
 940         &vmovdqa        (&QWP(16*12-128,"ebp"),"xmm4"); # save counter value
 941
 942         &vmovdqa        ($xa, &QWP(16*0-128,"ebp"));
 943         &vmovdqa        ($xd, "xmm4");
 944         &vmovdqa        ($xb_,&QWP(16*4-128,"ebp"));
 945         &vmovdqa        ($xc, &QWP(16*8-128,"ebp"));
 946         &vmovdqa        ($xc_,&QWP(16*9-128,"ebp"));
 947
 948         &mov            ("edx",10);                     # loop counter
 949         &nop            ();
 950
 951 &set_label("loop",32);
 952         &vpaddd         ($xa,$xa,$xb_);                 # elsewhere
 953         &vpxor          ($xd,$xd,$xa);                  # elsewhere
 954         &QUARTERROUND_XOP(0, 4, 8, 12, 0);
 955         &QUARTERROUND_XOP(1, 5, 9, 13, 1);
 956         &QUARTERROUND_XOP(2, 6,10, 14, 2);
 957         &QUARTERROUND_XOP(3, 7,11, 15, 3);
 958         &QUARTERROUND_XOP(0, 5,10, 15, 4);
 959         &QUARTERROUND_XOP(1, 6,11, 12, 5);
 960         &QUARTERROUND_XOP(2, 7, 8, 13, 6);
 961         &QUARTERROUND_XOP(3, 4, 9, 14, 7);
 962         &dec            ("edx");
 963         &jnz            (&label("loop"));
 964
 965         &vmovdqa        (&QWP(16*4-128,"ebx"),$xb_);
 966         &vmovdqa        (&QWP(16*8-128,"ebx"),$xc);
 967         &vmovdqa        (&QWP(16*9-128,"ebx"),$xc_);
 968         &vmovdqa        (&QWP(16*12-128,"ebx"),$xd);
 969         &vmovdqa        (&QWP(16*14-128,"ebx"),$xd_);
 970
 971     my ($xa0,$xa1,$xa2,$xa3,$xt0,$xt1,$xt2,$xt3)=map("xmm$_",(0..7));
 972
 973         #&vmovdqa       ($xa0,&QWP(16*0-128,"ebx"));    # it's there
 974         &vmovdqa        ($xa1,&QWP(16*1-128,"ebx"));
 975         &vmovdqa        ($xa2,&QWP(16*2-128,"ebx"));
 976         &vmovdqa        ($xa3,&QWP(16*3-128,"ebx"));
 977
 978     for($i=0;$i<256;$i+=64) {
 979         &vpaddd         ($xa0,$xa0,&QWP($i+16*0-128,"ebp"));    # accumulate key material
 980         &vpaddd         ($xa1,$xa1,&QWP($i+16*1-128,"ebp"));
 981         &vpaddd         ($xa2,$xa2,&QWP($i+16*2-128,"ebp"));
 982         &vpaddd         ($xa3,$xa3,&QWP($i+16*3-128,"ebp"));
 983
 984         &vpunpckldq     ($xt2,$xa0,$xa1);       # "de-interlace" data
 985         &vpunpckldq     ($xt3,$xa2,$xa3);
 986         &vpunpckhdq     ($xa0,$xa0,$xa1);
 987         &vpunpckhdq     ($xa2,$xa2,$xa3);
 988         &vpunpcklqdq    ($xa1,$xt2,$xt3);       # "a0"
 989         &vpunpckhqdq    ($xt2,$xt2,$xt3);       # "a1"
 990         &vpunpcklqdq    ($xt3,$xa0,$xa2);       # "a2"
 991         &vpunpckhqdq    ($xa3,$xa0,$xa2);       # "a3"
 992
 993         &vpxor          ($xt0,$xa1,&QWP(64*0-128,$inp));
 994         &vpxor          ($xt1,$xt2,&QWP(64*1-128,$inp));
 995         &vpxor          ($xt2,$xt3,&QWP(64*2-128,$inp));
 996         &vpxor          ($xt3,$xa3,&QWP(64*3-128,$inp));
 997         &lea            ($inp,&QWP($i<192?16:(64*4-16*3),$inp));
 998         &vmovdqa        ($xa0,&QWP($i+16*4-128,"ebx"))  if ($i<192);
 999         &vmovdqa        ($xa1,&QWP($i+16*5-128,"ebx"))  if ($i<192);
1000         &vmovdqa        ($xa2,&QWP($i+16*6-128,"ebx"))  if ($i<192);
1001         &vmovdqa        ($xa3,&QWP($i+16*7-128,"ebx"))  if ($i<192);
1002         &vmovdqu        (&QWP(64*0-128,$out),$xt0);     # store output
1003         &vmovdqu        (&QWP(64*1-128,$out),$xt1);
1004         &vmovdqu        (&QWP(64*2-128,$out),$xt2);
1005         &vmovdqu        (&QWP(64*3-128,$out),$xt3);
1006         &lea            ($out,&QWP($i<192?16:(64*4-16*3),$out));
1007     }
1008         &sub            ($len,64*4);
1009         &jnc            (&label("outer_loop"));
1010
1011         &add            ($len,64*4);
1012         &jz             (&label("done"));
1013
1014         &mov            ("ebx",&DWP(512+8,"esp"));      # restore pointers
1015         &lea            ($inp,&DWP(-128,$inp));
1016         &mov            ("edx",&DWP(512+4,"esp"));
1017         &lea            ($out,&DWP(-128,$out));
1018
1019         &vmovd          ("xmm2",&DWP(16*12-128,"ebp")); # counter value
1020         &vmovdqu        ("xmm3",&QWP(0,"ebx"));
1021         &vpaddd         ("xmm2","xmm2",&QWP(16*6,"eax"));# +four
1022         &vpand          ("xmm3","xmm3",&QWP(16*7,"eax"));
1023         &vpor           ("xmm3","xmm3","xmm2");         # counter value
1024 {
1025 my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("xmm$_",(0..7));
1026
1027 sub XOPROUND {
1028         &vpaddd         ($a,$a,$b);
1029         &vpxor          ($d,$d,$a);
1030         &vprotd         ($d,$d,16);
1031
1032         &vpaddd         ($c,$c,$d);
1033         &vpxor          ($b,$b,$c);
1034         &vprotd         ($b,$b,12);
1035
1036         &vpaddd         ($a,$a,$b);
1037         &vpxor          ($d,$d,$a);
1038         &vprotd         ($d,$d,8);
1039
1040         &vpaddd         ($c,$c,$d);
1041         &vpxor          ($b,$b,$c);
1042         &vprotd         ($b,$b,7);
1043 }
1044
1045 &set_label("1x");
1046         &vmovdqa        ($a,&QWP(16*2,"eax"));          # sigma
1047         &vmovdqu        ($b,&QWP(0,"edx"));
1048         &vmovdqu        ($c,&QWP(16,"edx"));
1049         #&vmovdqu       ($d,&QWP(0,"ebx"));             # already loaded
1050         &vmovdqa        ($rot16,&QWP(0,"eax"));
1051         &vmovdqa        ($rot24,&QWP(16,"eax"));
1052         &mov            (&DWP(16*3,"esp"),"ebp");
1053
1054         &vmovdqa        (&QWP(16*0,"esp"),$a);
1055         &vmovdqa        (&QWP(16*1,"esp"),$b);
1056         &vmovdqa        (&QWP(16*2,"esp"),$c);
1057         &vmovdqa        (&QWP(16*3,"esp"),$d);
1058         &mov            ("edx",10);
1059         &jmp            (&label("loop1x"));
1060
1061 &set_label("outer1x",16);
1062         &vmovdqa        ($d,&QWP(16*5,"eax"));          # one
1063         &vmovdqa        ($a,&QWP(16*0,"esp"));
1064         &vmovdqa        ($b,&QWP(16*1,"esp"));
1065         &vmovdqa        ($c,&QWP(16*2,"esp"));
1066         &vpaddd         ($d,$d,&QWP(16*3,"esp"));
1067         &mov            ("edx",10);
1068         &vmovdqa        (&QWP(16*3,"esp"),$d);
1069         &jmp            (&label("loop1x"));
1070
1071 &set_label("loop1x",16);
1072         &XOPROUND();
1073         &vpshufd        ($c,$c,0b01001110);
1074         &vpshufd        ($b,$b,0b00111001);
1075         &vpshufd        ($d,$d,0b10010011);
1076
1077         &XOPROUND();
1078         &vpshufd        ($c,$c,0b01001110);
1079         &vpshufd        ($b,$b,0b10010011);
1080         &vpshufd        ($d,$d,0b00111001);
1081
1082         &dec            ("edx");
1083         &jnz            (&label("loop1x"));
1084
1085         &vpaddd         ($a,$a,&QWP(16*0,"esp"));
1086         &vpaddd         ($b,$b,&QWP(16*1,"esp"));
1087         &vpaddd         ($c,$c,&QWP(16*2,"esp"));
1088         &vpaddd         ($d,$d,&QWP(16*3,"esp"));
1089
1090         &cmp            ($len,64);
1091         &jb             (&label("tail"));
1092
1093         &vpxor          ($a,$a,&QWP(16*0,$inp));        # xor with input
1094         &vpxor          ($b,$b,&QWP(16*1,$inp));
1095         &vpxor          ($c,$c,&QWP(16*2,$inp));
1096         &vpxor          ($d,$d,&QWP(16*3,$inp));
1097         &lea            ($inp,&DWP(16*4,$inp));         # inp+=64
1098
1099         &vmovdqu        (&QWP(16*0,$out),$a);           # write output
1100         &vmovdqu        (&QWP(16*1,$out),$b);
1101         &vmovdqu        (&QWP(16*2,$out),$c);
1102         &vmovdqu        (&QWP(16*3,$out),$d);
1103         &lea            ($out,&DWP(16*4,$out));         # inp+=64
1104
1105         &sub            ($len,64);
1106         &jnz            (&label("outer1x"));
1107
1108         &jmp            (&label("done"));
1109
1110 &set_label("tail");
1111         &vmovdqa        (&QWP(16*0,"esp"),$a);
1112         &vmovdqa        (&QWP(16*1,"esp"),$b);
1113         &vmovdqa        (&QWP(16*2,"esp"),$c);
1114         &vmovdqa        (&QWP(16*3,"esp"),$d);
1115
1116         &xor            ("eax","eax");
1117         &xor            ("edx","edx");
1118         &xor            ("ebp","ebp");
1119
1120 &set_label("tail_loop");
1121         &movb           ("al",&BP(0,"esp","ebp"));
1122         &movb           ("dl",&BP(0,$inp,"ebp"));
1123         &lea            ("ebp",&DWP(1,"ebp"));
1124         &xor            ("al","dl");
1125         &movb           (&BP(-1,$out,"ebp"),"al");
1126         &dec            ($len);
1127         &jnz            (&label("tail_loop"));
1128 }
1129 &set_label("done");
1130         &vzeroupper     ();
1131         &mov            ("esp",&DWP(512,"esp"));
1132 &function_end("ChaCha20_xop");
1133 }
1134
1135 &asm_finish();
1136
1137 close STDOUT;