crypto/aes/asm/aesni-x86.pl

   1 #!/usr/bin/env perl
   2
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9 #
  10 # This module implements support for Intel AES-NI extension. In
  11 # OpenSSL context it's used with Intel engine, but can also be used as
  12 # drop-in replacement for crypto/aes/asm/aes-586.pl [see below for
  13 # details].
  14
  15 $PREFIX="aesni";        # if $PREFIX is set to "AES", the script
  16                         # generates drop-in replacement for
  17                         # crypto/aes/asm/aes-586.pl:-)
  18
  19 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  20 push(@INC,"${dir}","${dir}../../perlasm");
  21 require "x86asm.pl";
  22
  23 &asm_init($ARGV[0],$0);
  24
  25 $movekey = eval($RREFIX eq "aseni" ? "*movaps" : "*movups");
  26
  27 $len="eax";
  28 $rounds="ecx";
  29 $key="edx";
  30 $inp="esi";
  31 $out="edi";
  32 $rounds_="ebx"; # backup copy for $rounds
  33 $key_="ebp";    # backup copy for $key
  34
  35 $inout0="xmm0";
  36 $inout1="xmm1";
  37 $inout2="xmm2";
  38 $rndkey0="xmm3";
  39 $rndkey1="xmm4";
  40 $ivec="xmm5";
  41 $in0="xmm6";
  42 $in1="xmm7";    $inout3="xmm7";
  43
  44 # Inline version of internal aesni_[en|de]crypt1
  45 sub aesni_inline_generate1
  46 { my $p=shift;
  47
  48     &$movekey           ($rndkey0,&QWP(0,$key));
  49     &$movekey           ($rndkey1,&QWP(16,$key));
  50     &lea                ($key,&DWP(32,$key));
  51     &pxor               ($inout0,$rndkey0);
  52     &set_label("${p}1_loop");
  53         eval"&aes${p}   ($inout0,$rndkey1)";
  54         &dec            ($rounds);
  55         &$movekey       ($rndkey1,&QWP(0,$key));
  56         &lea            ($key,&DWP(16,$key));
  57     &jnz                (&label("${p}1_loop"));
  58     eval"&aes${p}last   ($inout0,$rndkey1)";
  59 }
  60
  61 sub aesni_generate1     # fully unrolled loop
  62 { my $p=shift;
  63
  64     &function_begin_B("_aesni_${p}rypt1");
  65         &$movekey       ($rndkey0,&QWP(0,$key));
  66         &$movekey       ($rndkey1,&QWP(0x10,$key));
  67         &cmp            ($rounds,11);
  68         &pxor           ($inout0,$rndkey0);
  69         &$movekey       ($rndkey0,&QWP(0x20,$key));
  70         &lea            ($key,&DWP(0x30,$key));
  71         &jb             (&label("${p}128"));
  72         &lea            ($key,&DWP(0x20,$key));
  73         &je             (&label("${p}192"));
  74         &lea            ($key,&DWP(0x20,$key));
  75         eval"&aes${p}   ($inout0,$rndkey1)";
  76         &$movekey       ($rndkey1,&QWP(-0x40,$key));
  77         eval"&aes${p}   ($inout0,$rndkey0)";
  78         &$movekey       ($rndkey0,&QWP(-0x30,$key));
  79     &set_label("${p}192");
  80         eval"&aes${p}   ($inout0,$rndkey1)";
  81         &$movekey       ($rndkey1,&QWP(-0x20,$key));
  82         eval"&aes${p}   ($inout0,$rndkey0)";
  83         &$movekey       ($rndkey0,&QWP(-0x10,$key));
  84     &set_label("${p}128");
  85         eval"&aes${p}   ($inout0,$rndkey1)";
  86         &$movekey       ($rndkey1,&QWP(0,$key));
  87         eval"&aes${p}   ($inout0,$rndkey0)";
  88         &$movekey       ($rndkey0,&QWP(0x10,$key));
  89         eval"&aes${p}   ($inout0,$rndkey1)";
  90         &$movekey       ($rndkey1,&QWP(0x20,$key));
  91         eval"&aes${p}   ($inout0,$rndkey0)";
  92         &$movekey       ($rndkey0,&QWP(0x30,$key));
  93         eval"&aes${p}   ($inout0,$rndkey1)";
  94         &$movekey       ($rndkey1,&QWP(0x40,$key));
  95         eval"&aes${p}   ($inout0,$rndkey0)";
  96         &$movekey       ($rndkey0,&QWP(0x50,$key));
  97         eval"&aes${p}   ($inout0,$rndkey1)";
  98         &$movekey       ($rndkey1,&QWP(0x60,$key));
  99         eval"&aes${p}   ($inout0,$rndkey0)";
 100         &$movekey       ($rndkey0,&QWP(0x70,$key));
 101         eval"&aes${p}   ($inout0,$rndkey1)";
 102     eval"&aes${p}last   ($inout0,$rndkey0)";
 103     &ret();
 104     &function_end_B("_aesni_${p}rypt1");
 105 }
 106
 107 # void $PREFIX_encrypt (const void *inp,void *out,const AES_KEY *key);
 108 # &aesni_generate1("dec");
 109 &function_begin_B("${PREFIX}_encrypt");
 110         &mov    ("eax",&wparam(0));
 111         &mov    ($key,&wparam(2));
 112         &movups ($inout0,&QWP(0,"eax"));
 113         &mov    ($rounds,&DWP(240,$key));
 114         &mov    ("eax",&wparam(1));
 115         &aesni_inline_generate1("enc"); # &call ("_aesni_encrypt1");
 116         &movups (&QWP(0,"eax"),$inout0);
 117         &ret    ();
 118 &function_end_B("${PREFIX}_encrypt");
 119
 120 # void $PREFIX_decrypt (const void *inp,void *out,const AES_KEY *key);
 121 # &aesni_generate1("dec");
 122 &function_begin_B("${PREFIX}_decrypt");
 123         &mov    ("eax",&wparam(0));
 124         &mov    ($key,&wparam(2));
 125         &movups ($inout0,&QWP(0,"eax"));
 126         &mov    ($rounds,&DWP(240,$key));
 127         &mov    ("eax",&wparam(1));
 128         &aesni_inline_generate1("dec"); # &call ("_aesni_decrypt1");
 129         &movups (&QWP(0,"eax"),$inout0);
 130         &ret    ();
 131 &function_end_B("${PREFIX}_decrypt");
 132 \f
 133 # _aesni_[en|de]crypt[34] are private interfaces, N denotes interleave
 134 # factor. Why 3x subroutine is used in loops? Even though aes[enc|dec]
 135 # latency is 6, it turned out that it can be scheduled only every
 136 # *second* cycle. Thus 3x interleave is the one providing optimal
 137 # utilization, i.e. when subroutine's throughput is virtually same as
 138 # of non-interleaved subroutine [for number of input blocks up to 3].
 139 # This is why it makes no sense to implement 2x subroutine. As soon
 140 # as/if Intel improves throughput by making it possible to schedule
 141 # the instructions in question *every* cycles I would have to
 142 # implement 6x interleave and use it in loop...
 143 sub aesni_generate3
 144 { my $p=shift;
 145
 146     &function_begin_B("_aesni_${p}rypt3");
 147         &$movekey       ($rndkey0,&QWP(0,$key));
 148         &shr            ($rounds,1);
 149         &$movekey       ($rndkey1,&QWP(16,$key));
 150         &lea            ($key,&DWP(32,$key));
 151         &pxor           ($inout0,$rndkey0);
 152         &pxor           ($inout1,$rndkey0);
 153         &pxor           ($inout2,$rndkey0);
 154         &jmp            (&label("${p}3_loop"));
 155     &set_label("${p}3_loop",16);
 156         eval"&aes${p}   ($inout0,$rndkey1)";
 157         &$movekey       ($rndkey0,&QWP(0,$key));
 158         eval"&aes${p}   ($inout1,$rndkey1)";
 159         &dec            ($rounds);
 160         eval"&aes${p}   ($inout2,$rndkey1)";
 161         &$movekey       ($rndkey1,&QWP(16,$key));
 162         eval"&aes${p}   ($inout0,$rndkey0)";
 163         &lea            ($key,&DWP(32,$key));
 164         eval"&aes${p}   ($inout1,$rndkey0)";
 165         eval"&aes${p}   ($inout2,$rndkey0)";
 166         &jnz            (&label("${p}3_loop"));
 167     eval"&aes${p}       ($inout0,$rndkey1)";
 168     &$movekey           ($rndkey0,&QWP(0,$key));
 169     eval"&aes${p}       ($inout1,$rndkey1)";
 170     eval"&aes${p}       ($inout2,$rndkey1)";
 171     eval"&aes${p}last   ($inout0,$rndkey0)";
 172     eval"&aes${p}last   ($inout1,$rndkey0)";
 173     eval"&aes${p}last   ($inout2,$rndkey0)";
 174     &ret();
 175     &function_end_B("_aesni_${p}rypt3");
 176 }
 177
 178 # 4x interleave is implemented to improve small block performance,
 179 # most notably [and naturally] 4 block by ~30%. One can argue that one
 180 # should have implemented 5x as well, but improvement  would be <20%,
 181 # so it's not worth it...
 182 sub aesni_generate4
 183 { my $p=shift;
 184
 185     &function_begin_B("_aesni_${p}rypt4");
 186         &$movekey       ($rndkey0,&QWP(0,$key));
 187         &$movekey       ($rndkey1,&QWP(16,$key));
 188         &shr            ($rounds,1);
 189         &lea            ($key,&DWP(32,$key));
 190         &pxor           ($inout0,$rndkey0);
 191         &pxor           ($inout1,$rndkey0);
 192         &pxor           ($inout2,$rndkey0);
 193         &pxor           ($inout3,$rndkey0);
 194         &jmp            (&label("${p}3_loop"));
 195     &set_label("${p}3_loop",16);
 196         eval"&aes${p}   ($inout0,$rndkey1)";
 197         &$movekey       ($rndkey0,&QWP(0,$key));
 198         eval"&aes${p}   ($inout1,$rndkey1)";
 199         &dec            ($rounds);
 200         eval"&aes${p}   ($inout2,$rndkey1)";
 201         eval"&aes${p}   ($inout3,$rndkey1)";
 202         &$movekey       ($rndkey1,&QWP(16,$key));
 203         eval"&aes${p}   ($inout0,$rndkey0)";
 204         &lea            ($key,&DWP(32,$key));
 205         eval"&aes${p}   ($inout1,$rndkey0)";
 206         eval"&aes${p}   ($inout2,$rndkey0)";
 207         eval"&aes${p}   ($inout3,$rndkey0)";
 208         &jnz            (&label("${p}3_loop"));
 209     eval"&aes${p}       ($inout0,$rndkey1)";
 210     &$movekey           ($rndkey0,&QWP(0,$key));
 211     eval"&aes${p}       ($inout1,$rndkey1)";
 212     eval"&aes${p}       ($inout2,$rndkey1)";
 213     eval"&aes${p}       ($inout3,$rndkey1)";
 214     eval"&aes${p}last   ($inout0,$rndkey0)";
 215     eval"&aes${p}last   ($inout1,$rndkey0)";
 216     eval"&aes${p}last   ($inout2,$rndkey0)";
 217     eval"&aes${p}last   ($inout3,$rndkey0)";
 218     &ret();
 219     &function_end_B("_aesni_${p}rypt4");
 220 }
 221 &aesni_generate3("enc") if ($PREFIX eq "aesni");
 222 &aesni_generate3("dec");
 223 &aesni_generate4("enc") if ($PREFIX eq "aesni");
 224 &aesni_generate4("dec");
 225
 226 if ($PREFIX eq "aesni") {
 227 # void aesni_ecb_encrypt (const void *in, void *out,
 228 #                         size_t length, const AES_KEY *key,
 229 #                         int enc);
 230 &function_begin("aesni_ecb_encrypt");
 231         &mov    ($inp,&wparam(0));
 232         &mov    ($out,&wparam(1));
 233         &mov    ($len,&wparam(2));
 234         &mov    ($key,&wparam(3));
 235         &mov    ($rounds,&wparam(4));
 236         &cmp    ($len,16);
 237         &jb     (&label("ecb_ret"));
 238         &and    ($len,-16);
 239         &test   ($rounds,$rounds)
 240         &mov    ($rounds,&DWP(240,$key));
 241         &mov    ($key_,$key);           # backup $key
 242         &mov    ($rounds_,$rounds);     # backup $rounds
 243         &jz     (&label("ecb_decrypt"));
 244
 245         &sub    ($len,0x40);
 246         &jbe    (&label("ecb_enc_tail"));
 247         &jmp    (&label("ecb_enc_loop3"));
 248
 249 &set_label("ecb_enc_loop3",16);
 250         &movups ($inout0,&QWP(0,$inp));
 251         &movups ($inout1,&QWP(0x10,$inp));
 252         &movups ($inout2,&QWP(0x20,$inp));
 253         &call   ("_aesni_encrypt3");
 254         &sub    ($len,0x30);
 255         &lea    ($inp,&DWP(0x30,$inp));
 256         &lea    ($out,&DWP(0x30,$out));
 257         &movups (&QWP(-0x30,$out),$inout0);
 258         &mov    ($key,$key_);           # restore $key
 259         &movups (&QWP(-0x20,$out),$inout1);
 260         &mov    ($rounds,$rounds_);     # restore $rounds
 261         &movups (&QWP(-0x10,$out),$inout2);
 262         &ja     (&label("ecb_enc_loop3"));
 263
 264 &set_label("ecb_enc_tail");
 265         &add    ($len,0x40);
 266         &jz     (&label("ecb_ret"));
 267
 268         &cmp    ($len,0x10);
 269         &movups ($inout0,&QWP(0,$inp));
 270         &je     (&label("ecb_enc_one"));
 271         &cmp    ($len,0x20);
 272         &movups ($inout1,&QWP(0x10,$inp));
 273         &je     (&label("ecb_enc_two"));
 274         &cmp    ($len,0x30);
 275         &movups ($inout2,&QWP(0x20,$inp));
 276         &je     (&label("ecb_enc_three"));
 277         &movups ($inout3,&QWP(0x30,$inp));
 278         &call   ("_aesni_encrypt4");
 279         &movups (&QWP(0,$out),$inout0);
 280         &movups (&QWP(0x10,$out),$inout1);
 281         &movups (&QWP(0x20,$out),$inout2);
 282         &movups (&QWP(0x30,$out),$inout3);
 283         jmp     (&label("ecb_ret"));
 284
 285 &set_label("ecb_enc_one",16);
 286         &aesni_inline_generate1("enc"); # &call ("_aesni_encrypt1");
 287         &movups (&QWP(0,$out),$inout0);
 288         &jmp    (&label("ecb_ret"));
 289
 290 &set_label("ecb_enc_two",16);
 291         &call   ("_aesni_encrypt3");
 292         &movups (&QWP(0,$out),$inout0);
 293         &movups (&QWP(0x10,$out),$inout1);
 294         &jmp    (&label("ecb_ret"));
 295
 296 &set_label("ecb_enc_three",16);
 297         &call   ("_aesni_encrypt3");
 298         &movups (&QWP(0,$out),$inout0);
 299         &movups (&QWP(0x10,$out),$inout1);
 300         &movups (&QWP(0x20,$out),$inout2);
 301         &jmp    (&label("ecb_ret"));
 302
 303 &set_label("ecb_decrypt",16);
 304         &sub    ($len,0x40);
 305         &jbe    (&label("ecb_dec_tail"));
 306         &jmp    (&label("ecb_dec_loop3"));
 307
 308 &set_label("ecb_dec_loop3",16);
 309         &movups ($inout0,&QWP(0,$inp));
 310         &movups ($inout1,&QWP(0x10,$inp));
 311         &movups ($inout2,&QWP(0x20,$inp));
 312         &call   ("_aesni_decrypt3");
 313         &sub    ($len,0x30);
 314         &lea    ($inp,&DWP(0x30,$inp));
 315         &lea    ($out,&DWP(0x30,$out));
 316         &movups (&QWP(-0x30,$out),$inout0);
 317         &mov    ($key,$key_);           # restore $key
 318         &movups (&QWP(-0x20,$out),$inout1);
 319         &mov    ($rounds,$rounds_);     # restore $rounds
 320         &movups (&QWP(-0x10,$out),$inout2);
 321         &ja     (&label("ecb_dec_loop3"));
 322
 323 &set_label("ecb_dec_tail");
 324         &add    ($len,0x40);
 325         &jz     (&label("ecb_ret"));
 326
 327         &cmp    ($len,0x10);
 328         &movups ($inout0,&QWP(0,$inp));
 329         &je     (&label("ecb_dec_one"));
 330         &cmp    ($len,0x20);
 331         &movups ($inout1,&QWP(0x10,$inp));
 332         &je     (&label("ecb_dec_two"));
 333         &cmp    ($len,0x30);
 334         &movups ($inout2,&QWP(0x20,$inp));
 335         &je     (&label("ecb_dec_three"));
 336         &movups ($inout3,&QWP(0x30,$inp));
 337         &call   ("_aesni_decrypt4");
 338         &movups (&QWP(0,$out),$inout0);
 339         &movups (&QWP(0x10,$out),$inout1);
 340         &movups (&QWP(0x20,$out),$inout2);
 341         &movups (&QWP(0x30,$out),$inout3);
 342         &jmp    (&label("ecb_ret"));
 343
 344 &set_label("ecb_dec_one",16);
 345         &aesni_inline_generate1("dec"); # &call ("_aesni_decrypt3");
 346         &movups (&QWP(0,$out),$inout0);
 347         &jmp    (&label("ecb_ret"));
 348
 349 &set_label("ecb_dec_two",16);
 350         &call   ("_aesni_decrypt3");
 351         &movups (&QWP(0,$out),$inout0);
 352         &movups (&QWP(0x10,$out),$inout1);
 353         &jmp    (&label("ecb_ret"));
 354
 355 &set_label("ecb_dec_three",16);
 356         &call   ("_aesni_decrypt3");
 357         &movups (&QWP(0,$out),$inout0);
 358         &movups (&QWP(0x10,$out),$inout1);
 359         &movups (&QWP(0x20,$out),$inout2);
 360
 361 &set_label("ecb_ret");
 362 &function_end("aesni_ecb_encrypt");
 363 }
 364
 365 # void $PREFIX_cbc_encrypt (const void *inp, void *out,
 366 #                           size_t length, const AES_KEY *key,
 367 #                           unsigned char *ivp,const int enc);
 368 &function_begin("${PREFIX}_cbc_encrypt");
 369         &mov    ($inp,&wparam(0));
 370         &mov    ($out,&wparam(1));
 371         &mov    ($len,&wparam(2));
 372         &mov    ($key,&wparam(3));
 373         &test   ($len,$len);
 374         &mov    ($key_,&wparam(4));
 375         &jz     (&label("cbc_ret"));
 376
 377         &cmp    (&wparam(5),0);
 378         &movups ($ivec,&QWP(0,$key_));  # load IV
 379         &mov    ($rounds,&DWP(240,$key));
 380         &mov    ($key_,$key);           # backup $key
 381         &mov    ($rounds_,$rounds);     # backup $rounds
 382         &je     (&label("cbc_decrypt"));
 383
 384         &movaps ($inout0,$ivec);
 385         &cmp    ($len,16);
 386         &jb     (&label("cbc_enc_tail"));
 387         &sub    ($len,16);
 388         &jmp    (&label("cbc_enc_loop"));
 389
 390 &set_label("cbc_enc_loop",16);
 391         &movups ($ivec,&QWP(0,$inp));
 392         &lea    ($inp,&DWP(16,$inp));
 393         &pxor   ($inout0,$ivec);
 394         &aesni_inline_generate1("enc"); # &call ("_aesni_encrypt3");
 395         &sub    ($len,16);
 396         &lea    ($out,&DWP(16,$out));
 397         &mov    ($rounds,$rounds_);     # restore $rounds
 398         &mov    ($key,$key_);           # restore $key
 399         &movups (&QWP(-16,$out),$inout0);
 400         &jnc    (&label("cbc_enc_loop"));
 401         &add    ($len,16);
 402         &jnz    (&label("cbc_enc_tail"));
 403         &movaps ($ivec,$inout0);
 404         &jmp    (&label("cbc_ret"));
 405
 406 &set_label("cbc_enc_tail");
 407         &mov    ("ecx",$len);           # zaps $rounds
 408         &data_word(0xA4F3F689);         # rep movsb
 409         &mov    ("ecx",16);             # zero tail
 410         &sub    ("ecx",$len);
 411         &xor    ("eax","eax");          # zaps $len
 412         &data_word(0xAAF3F689);         # rep stosb
 413         &lea    ($out,&DWP(-16,$out));  # rewind $out by 1 block
 414         &mov    ($rounds,$rounds_);     # restore $rounds
 415         &mov    ($inp,$out);            # $inp and $out are the same
 416         &mov    ($key,$key_);           # restore $key
 417         &jmp    (&label("cbc_enc_loop"));
 418
 419 &set_label("cbc_decrypt",16);
 420         &sub    ($len,0x40);
 421         &jbe    (&label("cbc_dec_tail"));
 422         &jmp    (&label("cbc_dec_loop3"));
 423
 424 &set_label("cbc_dec_loop3",16);
 425         &movups ($inout0,&QWP(0,$inp));
 426         &movups ($inout1,&QWP(0x10,$inp));
 427         &movups ($inout2,&QWP(0x20,$inp));
 428         &movaps ($in0,$inout0);
 429         &movaps ($in1,$inout1);
 430         &call   ("_aesni_decrypt3");
 431         &sub    ($len,0x30);
 432         &lea    ($inp,&DWP(0x30,$inp));
 433         &lea    ($out,&DWP(0x30,$out));
 434         &pxor   ($inout0,$ivec);
 435         &pxor   ($inout1,$in0);
 436         &movups ($ivec,&QWP(-0x10,$inp));
 437         &pxor   ($inout2,$in1);
 438         &movups (&QWP(-0x30,$out),$inout0);
 439         &mov    ($rounds,$rounds_)      # restore $rounds
 440         &movups (&QWP(-0x20,$out),$inout1);
 441         &mov    ($key,$key_);           # restore $key
 442         &movups (&QWP(-0x10,$out),$inout2);
 443         &ja     (&label("cbc_dec_loop3"));
 444
 445 &set_label("cbc_dec_tail");
 446         &add    ($len,0x40);
 447         &jz     (&label("cbc_ret"));
 448
 449         &movups ($inout0,&QWP(0,$inp));
 450         &cmp    ($len,0x10);
 451         &movaps ($in0,$inout0);
 452         &jbe    (&label("cbc_dec_one"));
 453         &movups ($inout1,&QWP(0x10,$inp));
 454         &cmp    ($len,0x20);
 455         &movaps ($in1,$inout1);
 456         &jbe    (&label("cbc_dec_two"));
 457         &movups ($inout2,&QWP(0x20,$inp));
 458         &cmp    ($len,0x30);
 459         &jbe    (&label("cbc_dec_three"));
 460         &movups ($inout3,&QWP(0x30,$inp));
 461         &call   ("_aesni_decrypt4");
 462         &movups ($rndkey0,&QWP(0x10,$inp));
 463         &movups ($rndkey1,&QWP(0x20,$inp));
 464         &pxor   ($inout0,$ivec);
 465         &pxor   ($inout1,$in0);
 466         &movups ($ivec,&QWP(0x30,$inp));
 467         &movups (&QWP(0,$out),$inout0);
 468         &pxor   ($inout2,$rndkey0);
 469         &pxor   ($inout3,$rndkey1);
 470         &movups (&QWP(0x10,$out),$inout1);
 471         &movups (&QWP(0x20,$out),$inout2);
 472         &movaps ($inout0,$inout3);
 473         &lea    ($out,&DWP(0x30,$out));
 474         &jmp    (&label("cbc_dec_tail_collected"));
 475
 476 &set_label("cbc_dec_one");
 477         &aesni_inline_generate1("dec"); # &call ("_aesni_decrypt3");
 478         &pxor   ($inout0,$ivec);
 479         &movaps ($ivec,$in0);
 480         &jmp    (&label("cbc_dec_tail_collected"));
 481
 482 &set_label("cbc_dec_two");
 483         &call   ("_aesni_decrypt3");
 484         &pxor   ($inout0,$ivec);
 485         &pxor   ($inout1,$in0);
 486         &movups (&QWP(0,$out),$inout0);
 487         &movaps ($inout0,$inout1);
 488         &movaps ($ivec,$in1);
 489         &lea    ($out,&DWP(0x10,$out));
 490         &jmp    (&label("cbc_dec_tail_collected"));
 491
 492 &set_label("cbc_dec_three");
 493         &call   ("_aesni_decrypt3");
 494         &pxor   ($inout0,$ivec);
 495         &pxor   ($inout1,$in0);
 496         &pxor   ($inout2,$in1);
 497         &movups (&QWP(0,$out),$inout0);
 498         &movups (&QWP(0x10,$out),$inout1);
 499         &movaps ($inout0,$inout2);
 500         &movups ($ivec,&QWP(0x20,$inp));
 501         &lea    ($out,&DWP(0x20,$out));
 502
 503 &set_label("cbc_dec_tail_collected");
 504         &and    ($len,15);
 505         &jnz    (&label("cbc_dec_tail_partial"));
 506         &movups (&QWP(0,$out),$inout0);
 507         &jmp    (&label("cbc_ret"));
 508
 509 &set_label("cbc_dec_tail_partial");
 510         &mov    ($key_,"esp");
 511         &sub    ("esp",16);
 512         &and    ("esp",-16);
 513         &movaps (&QWP(0,"esp"),$inout0);
 514         &mov    ($inp,"esp");
 515         &mov    ("ecx",$len);
 516         &data_word(0xA4F3F689);         # rep movsb
 517         &mov    ("esp",$key_);
 518
 519 &set_label("cbc_ret");
 520         &mov    ($key_,&wparam(4));
 521         &movups (&QWP(0,$key_),$ivec);  # output IV
 522 &function_end("${PREFIX}_cbc_encrypt");
 523
 524 # Mechanical port from aesni-x86_64.pl.
 525 #
 526 # _aesni_set_encrypt_key is private interface,
 527 # input:
 528 #       "eax"   const unsigned char *userKey
 529 #       $rounds int bits
 530 #       $key    AES_KEY *key
 531 # output:
 532 #       "eax"   return code
 533 #       $round  rounds
 534
 535 &function_begin_B("_aesni_set_encrypt_key");
 536         &test   ("eax","eax");
 537         &jz     (&label("bad_pointer"));
 538         &test   ($key,$key);
 539         &jz     (&label("bad_pointer"));
 540
 541         &movups ("xmm0",&QWP(0,"eax")); # pull first 128 bits of *userKey
 542         &pxor   ("xmm4","xmm4");        # low dword of xmm4 is assumed 0
 543         &lea    ($key,&DWP(16,$key));
 544         &cmp    ($rounds,256);
 545         &je     (&label("14rounds"));
 546         &cmp    ($rounds,192);
 547         &je     (&label("12rounds"));
 548         &cmp    ($rounds,128);
 549         &jne    (&label("bad_keybits"));
 550
 551 &set_label("10rounds",16);
 552         &mov            ($rounds,9);
 553         &$movekey       (&QWP(-16,$key),"xmm0");        # round 0
 554         &aeskeygenassist("xmm1","xmm0",0x01);           # round 1
 555         &call           (&label("key_128_cold"));
 556         &aeskeygenassist("xmm1","xmm0",0x2);            # round 2
 557         &call           (&label("key_128"));
 558         &aeskeygenassist("xmm1","xmm0",0x04);           # round 3
 559         &call           (&label("key_128"));
 560         &aeskeygenassist("xmm1","xmm0",0x08);           # round 4
 561         &call           (&label("key_128"));
 562         &aeskeygenassist("xmm1","xmm0",0x10);           # round 5
 563         &call           (&label("key_128"));
 564         &aeskeygenassist("xmm1","xmm0",0x20);           # round 6
 565         &call           (&label("key_128"));
 566         &aeskeygenassist("xmm1","xmm0",0x40);           # round 7
 567         &call           (&label("key_128"));
 568         &aeskeygenassist("xmm1","xmm0",0x80);           # round 8
 569         &call           (&label("key_128"));
 570         &aeskeygenassist("xmm1","xmm0",0x1b);           # round 9
 571         &call           (&label("key_128"));
 572         &aeskeygenassist("xmm1","xmm0",0x36);           # round 10
 573         &call           (&label("key_128"));
 574         &$movekey       (&QWP(0,$key),"xmm0");
 575         &mov            (&DWP(80,$key),$rounds);
 576         &xor            ("eax","eax");
 577         &ret();
 578
 579 &set_label("key_128",16);
 580         &$movekey       (&QWP(0,$key),"xmm0");
 581         &lea            ($key,&DWP(16,$key));
 582 &set_label("key_128_cold");
 583         &shufps         ("xmm4","xmm0",0b00010000);
 584         &pxor           ("xmm0","xmm4");
 585         &shufps         ("xmm4","xmm0",0b10001100,);
 586         &pxor           ("xmm0","xmm4");
 587         &pshufd         ("xmm1","xmm1",0b11111111);     # critical path
 588         &pxor           ("xmm0","xmm1");
 589         &ret();
 590
 591 &set_label("12rounds",16);
 592         &movq           ("xmm2",&QWP(16,"eax"));        # remaining 1/3 of *userKey
 593         &mov            ($rounds,11);
 594         &$movekey       (&QWP(-16,$key),"xmm0")         # round 0
 595         &aeskeygenassist("xmm1","xmm2",0x01);           # round 1,2
 596         &call           (&label("key_192a_cold"));
 597         &aeskeygenassist("xmm1","xmm2",0x02);           # round 2,3
 598         &call           (&label("key_192b"));
 599         &aeskeygenassist("xmm1","xmm2",0x04);           # round 4,5
 600         &call           (&label("key_192a"));
 601         &aeskeygenassist("xmm1","xmm2",0x08);           # round 5,6
 602         &call           (&label("key_192b"));
 603         &aeskeygenassist("xmm1","xmm2",0x10);           # round 7,8
 604         &call           (&label("key_192a"));
 605         &aeskeygenassist("xmm1","xmm2",0x20);           # round 8,9
 606         &call           (&label("key_192b"));
 607         &aeskeygenassist("xmm1","xmm2",0x40);           # round 10,11
 608         &call           (&label("key_192a"));
 609         &aeskeygenassist("xmm1","xmm2",0x80);           # round 11,12
 610         &call           (&label("key_192b"));
 611         &$movekey       (&QWP(0,$key),"xmm0");
 612         &mov            (&DWP(48,$key),$rounds);
 613         &xor            ("eax","eax");
 614         &ret();
 615
 616 &set_label("key_192a",16);
 617         &$movekey       (&QWP(0,$key),"xmm0");
 618         &lea            ($key,&DWP(16,$key));
 619 &set_label("key_192a_cold",16);
 620         &movaps         ("xmm5","xmm2");
 621 &set_label("key_192b_warm");
 622         &shufps         ("xmm4","xmm0",0b00010000);
 623         &movaps         ("xmm3","xmm2");
 624         &pxor           ("xmm0","xmm4");
 625         &shufps         ("xmm4","xmm0",0b10001100);
 626         &pslldq         ("xmm3",4);
 627         &pxor           ("xmm0","xmm4");
 628         &pshufd         ("xmm1","xmm1",0b01010101);     # critical path
 629         &pxor           ("xmm2","xmm3");
 630         &pxor           ("xmm0","xmm1");
 631         &pshufd         ("xmm3","xmm0",0b11111111);
 632         &pxor           ("xmm2","xmm3");
 633         &ret();
 634
 635 &set_label("key_192b",16);
 636         &movaps         ("xmm3","xmm0");
 637         &shufps         ("xmm5","xmm0",0b01000100);
 638         &$movekey       (&QWP(0,$key),"xmm5");
 639         &shufps         ("xmm3","xmm2",0b01001110);
 640         &$movekey       (&QWP(16,$key),"xmm3");
 641         &lea            ($key,&DWP(32,$key));
 642         &jmp            (&label("key_192b_warm"));
 643
 644 &set_label("14rounds",16);
 645         &movups         ("xmm2",&QWP(16,"eax"));        # remaining half of *userKey
 646         &mov            ($rounds,13);
 647         &lea            ($key,&DWP(16,$key));
 648         &$movekey       (&QWP(-32,$key),"xmm0");        # round 0
 649         &$movekey       (&QWP(-16,$key),"xmm2");        # round 1
 650         &aeskeygenassist("xmm1","xmm2",0x01);           # round 2
 651         &call           (&label("key_256a_cold"));
 652         &aeskeygenassist("xmm1","xmm0",0x01);           # round 3
 653         &call           (&label("key_256b"));
 654         &aeskeygenassist("xmm1","xmm2",0x02);           # round 4
 655         &call           (&label("key_256a"));
 656         &aeskeygenassist("xmm1","xmm0",0x02);           # round 5
 657         &call           (&label("key_256b"));
 658         &aeskeygenassist("xmm1","xmm2",0x04);           # round 6
 659         &call           (&label("key_256a"));
 660         &aeskeygenassist("xmm1","xmm0",0x04);           # round 7
 661         &call           (&label("key_256b"));
 662         &aeskeygenassist("xmm1","xmm2",0x08);           # round 8
 663         &call           (&label("key_256a"));
 664         &aeskeygenassist("xmm1","xmm0",0x08);           # round 9
 665         &call           (&label("key_256b"));
 666         &aeskeygenassist("xmm1","xmm2",0x10);           # round 10
 667         &call           (&label("key_256a"));
 668         &aeskeygenassist("xmm1","xmm0",0x10);           # round 11
 669         &call           (&label("key_256b"));
 670         &aeskeygenassist("xmm1","xmm2",0x20);           # round 12
 671         &call           (&label("key_256a"));
 672         &aeskeygenassist("xmm1","xmm0",0x20);           # round 13
 673         &call           (&label("key_256b"));
 674         &aeskeygenassist("xmm1","xmm2",0x40);           # round 14
 675         &call           (&label("key_256a"));
 676         &$movekey       (&QWP(0,$key),"xmm0");
 677         &mov            (&DWP(16,$key),$rounds);
 678         &xor            ("eax","eax");
 679         &ret();
 680
 681 &set_label("key_256a",16);
 682         &$movekey       (&QWP(0,$key),"xmm2");
 683         &lea            ($key,&DWP(16,$key));
 684 &set_label("key_256a_cold");
 685         &shufps         ("xmm4","xmm0",0b00010000);
 686         &pxor           ("xmm0","xmm4");
 687         &shufps         ("xmm4","xmm0",0b10001100);
 688         &pxor           ("xmm0","xmm4");
 689         &pshufd         ("xmm1","xmm1",0b11111111);     # critical path
 690         &pxor           ("xmm0","xmm1");
 691         &ret();
 692
 693 &set_label("key_256b",16);
 694         &$movekey       (&QWP(0,$key),"xmm0");
 695         &lea            ($key,&DWP(16,$key));
 696
 697         &shufps         ("xmm4","xmm2",0b00010000);
 698         &pxor           ("xmm2","xmm4");
 699         &shufps         ("xmm4","xmm2",0b10001100);
 700         &pxor           ("xmm2","xmm4");
 701         &pshufd         ("xmm1","xmm1",0b10101010);     # critical path
 702         &pxor           ("xmm2","xmm1");
 703         &ret();
 704
 705 &set_label("bad_pointer",4);
 706         &mov    ("eax",-1);
 707         &ret    ();
 708 &set_label("bad_keybits",4);
 709         &mov    ("eax",-2);
 710         &ret    ();
 711 &function_end_B("_aesni_set_encrypt_key");
 712
 713 # int $PREFIX_set_encrypt_key (const unsigned char *userKey, int bits,
 714 #                              AES_KEY *key)
 715 &function_begin_B("${PREFIX}_set_encrypt_key");
 716         &mov    ("eax",&wparam(0));
 717         &mov    ($rounds,&wparam(1));
 718         &mov    ($key,&wparam(2));
 719         &call   ("_aesni_set_encrypt_key");
 720         &ret    ();
 721 &function_end_B("${PREFIX}_set_encrypt_key");
 722
 723 # int $PREFIX_set_decrypt_key (const unsigned char *userKey, int bits,
 724 #                              AES_KEY *key)
 725 &function_begin_B("${PREFIX}_set_decrypt_key");
 726         &mov    ("eax",&wparam(0));
 727         &mov    ($rounds,&wparam(1));
 728         &mov    ($key,&wparam(2));
 729         &call   ("_aesni_set_encrypt_key");
 730         &mov    ($key,&wparam(2));
 731         &shl    ($rounds,4)     # rounds-1 after _aesni_set_encrypt_key
 732         &test   ("eax","eax");
 733         &jnz    (&label("dec_key_ret"));
 734         &lea    ("eax",&DWP(16,$key,$rounds));  # end of key schedule
 735
 736         &$movekey       ("xmm0",&QWP(0,$key));  # just swap
 737         &$movekey       ("xmm1",&QWP(0,"eax"));
 738         &$movekey       (&QWP(0,"eax"),"xmm0");
 739         &$movekey       (&QWP(0,$key),"xmm1");
 740         &lea            ($key,&DWP(16,$key));
 741         &lea            ("eax",&DWP(-16,"eax"));
 742
 743 &set_label("dec_key_inverse");
 744         &$movekey       ("xmm0",&QWP(0,$key));  # swap and inverse
 745         &$movekey       ("xmm1",&QWP(0,"eax"));
 746         &aesimc         ("xmm0","xmm0");
 747         &aesimc         ("xmm1","xmm1");
 748         &lea            ($key,&DWP(16,$key));
 749         &lea            ("eax",&DWP(-16,"eax"));
 750         &cmp            ("eax",$key);
 751         &$movekey       (&QWP(16,"eax"),"xmm0");
 752         &$movekey       (&QWP(-16,$key),"xmm1");
 753         &ja             (&label("dec_key_inverse"));
 754
 755         &$movekey       ("xmm0",&QWP(0,$key));  # inverse middle
 756         &aesimc         ("xmm0","xmm0");
 757         &$movekey       (&QWP(0,$key),"xmm0");
 758
 759         &xor            ("eax","eax");          # return success
 760 &set_label("dec_key_ret");
 761         &ret    ();
 762 &function_end_B("${PREFIX}_set_decrypt_key");
 763 &asciz("AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>");
 764
 765 &asm_finish();