crypto/aes/asm/aesni-mb-x86_64.pl

   1 #!/usr/bin/env perl
   2
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9
  10 # Multi-buffer AES-NI procedures process several independent buffers
  11 # in parallel by interleaving independent instructions.
  12 #
  13 # Cycles per byte for interleave factor 4:
  14 #
  15 #                       asymptotic      measured
  16 #                       ---------------------------
  17 # Westmere              5.00/4=1.25     5.13/4=1.28
  18 # Atom                  15.0/4=3.75     ?15.7/4=3.93
  19 # Sandy Bridge          5.06/4=1.27     5.18/4=1.29
  20 # Ivy Bridge            5.06/4=1.27     5.14/4=1.29
  21 # Haswell               4.44/4=1.11     4.44/4=1.11
  22 # Bulldozer             5.75/4=1.44     5.76/4=1.44
  23 #
  24 # Cycles per byte for interleave factor 8 (not implemented for
  25 # pre-AVX processors, where higher interleave factor incidentally
  26 # doesn't result in improvement):
  27 #
  28 #                       asymptotic      measured
  29 #                       ---------------------------
  30 # Sandy Bridge          5.06/8=0.64     7.10/8=0.89(*)
  31 # Ivy Bridge            5.06/8=0.64     7.14/8=0.89(*)
  32 # Haswell               5.00/8=0.63     5.00/8=0.63
  33 # Bulldozer             5.75/8=0.72     5.77/8=0.72
  34 #
  35 # (*)   Sandy/Ivy Bridge are known to handle high interleave factors
  36 #       suboptimally;
  37
  38 $flavour = shift;
  39 $output  = shift;
  40 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
  41
  42 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  43
  44 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  45 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  46 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
  47 die "can't locate x86_64-xlate.pl";
  48
  49 $avx=0;
  50
  51 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  52                 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  53         $avx = ($1>=2.19) + ($1>=2.22);
  54 }
  55
  56 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
  57            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
  58         $avx = ($1>=2.09) + ($1>=2.10);
  59 }
  60
  61 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
  62            `ml64 2>&1` =~ /Version ([0-9]+)\./) {
  63         $avx = ($1>=10) + ($1>=11);
  64 }
  65
  66 if (!$avx && `$ENV{CC} -v` =~ /LLVM ([3-9]\.[0-9]+)/) {
  67         $avx = ($1>=3.0) + ($1>=3.1);
  68 }
  69
  70 open OUT,"| \"$^X\" $xlate $flavour $output";
  71 *STDOUT=*OUT;
  72
  73 # void aesni_multi_cbc_encrypt (
  74 #     struct {  void *inp,*out; int blocks; double iv[2]; } inp[8];
  75 #     const AES_KEY *key,
  76 #     int num);         /* 1 or 2 */
  77 #
  78 $inp="%rdi";    # 1st arg
  79 $key="%rsi";    # 2nd arg
  80 $num="%edx";
  81
  82 @inptr=map("%r$_",(8..11));
  83 @outptr=map("%r$_",(12..15));
  84
  85 ($rndkey0,$rndkey1)=("%xmm0","%xmm1");
  86 @out=map("%xmm$_",(2..5));
  87 @inp=map("%xmm$_",(6..9));
  88 ($counters,$mask,$zero)=map("%xmm$_",(10..12));
  89
  90 ($rounds,$one,$sink,$offset)=("%eax","%ecx","%rbp","%rbx");
  91
  92 $code.=<<___;
  93 .text
  94
  95 .extern OPENSSL_ia32cap_P
  96
  97 .globl  aesni_multi_cbc_encrypt
  98 .type   aesni_multi_cbc_encrypt,\@function,3
  99 .align  32
 100 aesni_multi_cbc_encrypt:
 101 ___
 102 $code.=<<___ if ($avx);
 103         cmp     \$2,$num
 104         jb      .Lenc_non_avx
 105         mov     OPENSSL_ia32cap_P+4(%rip),%ecx
 106         test    \$`1<<28`,%ecx                  # AVX bit
 107         jnz     _avx_cbc_enc_shortcut
 108         jmp     .Lenc_non_avx
 109 .align  16
 110 .Lenc_non_avx:
 111 ___
 112 $code.=<<___;
 113         mov     %rsp,%rax
 114         push    %rbx
 115         push    %rbp
 116         push    %r12
 117         push    %r13
 118         push    %r14
 119         push    %r15
 120 ___
 121 $code.=<<___ if ($win64);
 122         lea     -0xa8(%rsp),%rsp
 123         movaps  %xmm6,(%rsp)
 124         movaps  %xmm7,0x10(%rsp)
 125         movaps  %xmm8,0x20(%rsp)
 126         movaps  %xmm9,0x30(%rsp)
 127         movaps  %xmm10,0x40(%rsp)
 128         movaps  %xmm11,0x50(%rsp)
 129         movaps  %xmm12,0x60(%rsp)
 130         movaps  %xmm13,-0x68(%rax)      # not used, saved to share se_handler
 131         movaps  %xmm14,-0x58(%rax)
 132         movaps  %xmm15,-0x48(%rax)
 133 ___
 134 $code.=<<___;
 135         # stack layout
 136         #
 137         # +0    output sink
 138         # +16   input sink [original %rsp and $num]
 139         # +32   counters
 140
 141         sub     \$48,%rsp
 142         and     \$-64,%rsp
 143         mov     %rax,16(%rsp)                   # original %rsp
 144
 145 .Lenc4x_body:
 146         movdqu  ($key),$zero                    # 0-round key
 147         lea     0x78($key),$key                 # size optimization
 148         lea     40*2($inp),$inp
 149
 150 .Lenc4x_loop_grande:
 151         mov     $num,24(%rsp)                   # original $num
 152         xor     $num,$num
 153 ___
 154 for($i=0;$i<4;$i++) {
 155     $code.=<<___;
 156         mov     `40*$i+16-40*2`($inp),$one      # borrow $one for number of blocks
 157         mov     `40*$i+0-40*2`($inp),@inptr[$i]
 158         cmp     $num,$one
 159         mov     `40*$i+8-40*2`($inp),@outptr[$i]
 160         cmovg   $one,$num                       # find maximum
 161         test    $one,$one
 162         movdqu  `40*$i+24-40*2`($inp),@out[$i]  # load IV
 163         mov     $one,`32+4*$i`(%rsp)            # initialize counters
 164         cmovle  %rsp,@inptr[$i]                 # cancel input
 165 ___
 166 }
 167 $code.=<<___;
 168         test    $num,$num
 169         jz      .Lenc4x_done
 170
 171         movups  0x10-0x78($key),$rndkey1
 172          pxor   $zero,@out[0]
 173         movups  0x20-0x78($key),$rndkey0
 174          pxor   $zero,@out[1]
 175         mov     0xf0-0x78($key),$rounds
 176          pxor   $zero,@out[2]
 177         movdqu  (@inptr[0]),@inp[0]             # load inputs
 178          pxor   $zero,@out[3]
 179         movdqu  (@inptr[1]),@inp[1]
 180          pxor   @inp[0],@out[0]
 181         movdqu  (@inptr[2]),@inp[2]
 182          pxor   @inp[1],@out[1]
 183         movdqu  (@inptr[3]),@inp[3]
 184          pxor   @inp[2],@out[2]
 185          pxor   @inp[3],@out[3]
 186         movdqa  32(%rsp),$counters              # load counters
 187         xor     $offset,$offset
 188         jmp     .Loop_enc4x
 189
 190 .align  32
 191 .Loop_enc4x:
 192         add     \$16,$offset
 193         lea     16(%rsp),$sink                  # sink pointer
 194         mov     \$1,$one                        # constant of 1
 195         sub     $offset,$sink
 196
 197         aesenc          $rndkey1,@out[0]
 198         prefetcht0      31(@inptr[0],$offset)   # prefetch input
 199         prefetcht0      31(@inptr[1],$offset)
 200         aesenc          $rndkey1,@out[1]
 201         prefetcht0      31(@inptr[2],$offset)
 202         prefetcht0      31(@inptr[2],$offset)
 203         aesenc          $rndkey1,@out[2]
 204         aesenc          $rndkey1,@out[3]
 205         movups          0x30-0x78($key),$rndkey1
 206 ___
 207 for($i=0;$i<4;$i++) {
 208 my $rndkey = ($i&1) ? $rndkey1 : $rndkey0;
 209 $code.=<<___;
 210          cmp            `32+4*$i`(%rsp),$one
 211         aesenc          $rndkey,@out[0]
 212         aesenc          $rndkey,@out[1]
 213         aesenc          $rndkey,@out[2]
 214          cmovge         $sink,@inptr[$i]        # cancel input
 215          cmovg          $sink,@outptr[$i]       # sink output
 216         aesenc          $rndkey,@out[3]
 217         movups          `0x40+16*$i-0x78`($key),$rndkey
 218 ___
 219 }
 220 $code.=<<___;
 221          movdqa         $counters,$mask
 222         aesenc          $rndkey0,@out[0]
 223         prefetcht0      15(@outptr[0],$offset)  # prefetch output
 224         prefetcht0      15(@outptr[1],$offset)
 225         aesenc          $rndkey0,@out[1]
 226         prefetcht0      15(@outptr[2],$offset)
 227         prefetcht0      15(@outptr[3],$offset)
 228         aesenc          $rndkey0,@out[2]
 229         aesenc          $rndkey0,@out[3]
 230         movups          0x80-0x78($key),$rndkey0
 231          pxor           $zero,$zero
 232
 233         aesenc          $rndkey1,@out[0]
 234          pcmpgtd        $zero,$mask
 235          movdqu         -0x78($key),$zero       # reload 0-round key
 236         aesenc          $rndkey1,@out[1]
 237          paddd          $mask,$counters         # decrement counters
 238          movdqa         $counters,32(%rsp)      # update counters
 239         aesenc          $rndkey1,@out[2]
 240         aesenc          $rndkey1,@out[3]
 241         movups          0x90-0x78($key),$rndkey1
 242
 243         cmp     \$11,$rounds
 244
 245         aesenc          $rndkey0,@out[0]
 246         aesenc          $rndkey0,@out[1]
 247         aesenc          $rndkey0,@out[2]
 248         aesenc          $rndkey0,@out[3]
 249         movups          0xa0-0x78($key),$rndkey0
 250
 251         jb      .Lenc4x_tail
 252
 253         aesenc          $rndkey1,@out[0]
 254         aesenc          $rndkey1,@out[1]
 255         aesenc          $rndkey1,@out[2]
 256         aesenc          $rndkey1,@out[3]
 257         movups          0xb0-0x78($key),$rndkey1
 258
 259         aesenc          $rndkey0,@out[0]
 260         aesenc          $rndkey0,@out[1]
 261         aesenc          $rndkey0,@out[2]
 262         aesenc          $rndkey0,@out[3]
 263         movups          0xc0-0x78($key),$rndkey0
 264
 265         je      .Lenc4x_tail
 266
 267         aesenc          $rndkey1,@out[0]
 268         aesenc          $rndkey1,@out[1]
 269         aesenc          $rndkey1,@out[2]
 270         aesenc          $rndkey1,@out[3]
 271         movups          0xd0-0x78($key),$rndkey1
 272
 273         aesenc          $rndkey0,@out[0]
 274         aesenc          $rndkey0,@out[1]
 275         aesenc          $rndkey0,@out[2]
 276         aesenc          $rndkey0,@out[3]
 277         movups          0xe0-0x78($key),$rndkey0
 278         jmp     .Lenc4x_tail
 279
 280 .align  32
 281 .Lenc4x_tail:
 282         aesenc          $rndkey1,@out[0]
 283         aesenc          $rndkey1,@out[1]
 284         aesenc          $rndkey1,@out[2]
 285         aesenc          $rndkey1,@out[3]
 286          movdqu         (@inptr[0],$offset),@inp[0]
 287         movdqu          0x10-0x78($key),$rndkey1
 288
 289         aesenclast      $rndkey0,@out[0]
 290          movdqu         (@inptr[1],$offset),@inp[1]
 291          pxor           $zero,@inp[0]
 292         aesenclast      $rndkey0,@out[1]
 293          movdqu         (@inptr[2],$offset),@inp[2]
 294          pxor           $zero,@inp[1]
 295         aesenclast      $rndkey0,@out[2]
 296          movdqu         (@inptr[3],$offset),@inp[3]
 297          pxor           $zero,@inp[2]
 298         aesenclast      $rndkey0,@out[3]
 299         movdqu          0x20-0x78($key),$rndkey0
 300          pxor           $zero,@inp[3]
 301
 302         movups          @out[0],-16(@outptr[0],$offset)
 303          pxor           @inp[0],@out[0]
 304         movups          @out[1],-16(@outptr[1],$offset)
 305          pxor           @inp[1],@out[1]
 306         movups          @out[2],-16(@outptr[2],$offset)
 307          pxor           @inp[2],@out[2]
 308         movups          @out[3],-16(@outptr[3],$offset)
 309          pxor           @inp[3],@out[3]
 310
 311         dec     $num
 312         jnz     .Loop_enc4x
 313
 314         mov     16(%rsp),%rax                   # original %rsp
 315         mov     24(%rsp),$num
 316
 317         #pxor   @inp[0],@out[0]
 318         #pxor   @inp[1],@out[1]
 319         #movdqu @out[0],`40*0+24-40*2`($inp)    # output iv FIX ME!
 320         #pxor   @inp[2],@out[2]
 321         #movdqu @out[1],`40*1+24-40*2`($inp)
 322         #pxor   @inp[3],@out[3]
 323         #movdqu @out[2],`40*2+24-40*2`($inp)    # won't fix, let caller
 324         #movdqu @out[3],`40*3+24-40*2`($inp)    # figure this out...
 325
 326         lea     `40*4`($inp),$inp
 327         dec     $num
 328         jnz     .Lenc4x_loop_grande
 329
 330 .Lenc4x_done:
 331 ___
 332 $code.=<<___ if ($win64);
 333         movaps  -0xd8(%rax),%xmm6
 334         movaps  -0xc8(%rax),%xmm7
 335         movaps  -0xb8(%rax),%xmm8
 336         movaps  -0xa8(%rax),%xmm9
 337         movaps  -0x98(%rax),%xmm10
 338         movaps  -0x88(%rax),%xmm11
 339         movaps  -0x78(%rax),%xmm12
 340         #movaps -0x68(%rax),%xmm13
 341         #movaps -0x58(%rax),%xmm14
 342         #movaps -0x48(%rax),%xmm15
 343 ___
 344 $code.=<<___;
 345         mov     -48(%rax),%r15
 346         mov     -40(%rax),%r14
 347         mov     -32(%rax),%r13
 348         mov     -24(%rax),%r12
 349         mov     -16(%rax),%rbp
 350         mov     -8(%rax),%rbx
 351         lea     (%rax),%rsp
 352 .Lenc4x_epilogue:
 353         ret
 354 .size   aesni_multi_cbc_encrypt,.-aesni_multi_cbc_encrypt
 355
 356 .globl  aesni_multi_cbc_decrypt
 357 .type   aesni_multi_cbc_decrypt,\@function,3
 358 .align  32
 359 aesni_multi_cbc_decrypt:
 360 ___
 361 $code.=<<___ if ($avx);
 362         cmp     \$2,$num
 363         jb      .Ldec_non_avx
 364         mov     OPENSSL_ia32cap_P+4(%rip),%ecx
 365         test    \$`1<<28`,%ecx                  # AVX bit
 366         jnz     _avx_cbc_dec_shortcut
 367         jmp     .Ldec_non_avx
 368 .align  16
 369 .Ldec_non_avx:
 370 ___
 371 $code.=<<___;
 372         mov     %rsp,%rax
 373         push    %rbx
 374         push    %rbp
 375         push    %r12
 376         push    %r13
 377         push    %r14
 378         push    %r15
 379 ___
 380 $code.=<<___ if ($win64);
 381         lea     -0xa8(%rsp),%rsp
 382         movaps  %xmm6,(%rsp)
 383         movaps  %xmm7,0x10(%rsp)
 384         movaps  %xmm8,0x20(%rsp)
 385         movaps  %xmm9,0x30(%rsp)
 386         movaps  %xmm10,0x40(%rsp)
 387         movaps  %xmm11,0x50(%rsp)
 388         movaps  %xmm12,0x60(%rsp)
 389         movaps  %xmm13,-0x68(%rax)      # not used, saved to share se_handler
 390         movaps  %xmm14,-0x58(%rax)
 391         movaps  %xmm15,-0x48(%rax)
 392 ___
 393 $code.=<<___;
 394         # stack layout
 395         #
 396         # +0    output sink
 397         # +16   input sink [original %rsp and $num]
 398         # +32   counters
 399
 400         sub     \$48,%rsp
 401         and     \$-64,%rsp
 402         mov     %rax,16(%rsp)                   # original %rsp
 403
 404 .Ldec4x_body:
 405         movdqu  ($key),$zero                    # 0-round key
 406         lea     0x78($key),$key                 # size optimization
 407         lea     40*2($inp),$inp
 408
 409 .Ldec4x_loop_grande:
 410         mov     $num,24(%rsp)                   # original $num
 411         xor     $num,$num
 412 ___
 413 for($i=0;$i<4;$i++) {
 414     $code.=<<___;
 415         mov     `40*$i+16-40*2`($inp),$one      # borrow $one for number of blocks
 416         mov     `40*$i+0-40*2`($inp),@inptr[$i]
 417         cmp     $num,$one
 418         mov     `40*$i+8-40*2`($inp),@outptr[$i]
 419         cmovg   $one,$num                       # find maximum
 420         test    $one,$one
 421         movdqu  `40*$i+24-40*2`($inp),@inp[$i]  # load IV
 422         mov     $one,`32+4*$i`(%rsp)            # initialize counters
 423         cmovle  %rsp,@inptr[$i]                 # cancel input
 424 ___
 425 }
 426 $code.=<<___;
 427         test    $num,$num
 428         jz      .Ldec4x_done
 429
 430         movups  0x10-0x78($key),$rndkey1
 431         movups  0x20-0x78($key),$rndkey0
 432         mov     0xf0-0x78($key),$rounds
 433         movdqu  (@inptr[0]),@out[0]             # load inputs
 434         movdqu  (@inptr[1]),@out[1]
 435          pxor   $zero,@out[0]
 436         movdqu  (@inptr[2]),@out[2]
 437          pxor   $zero,@out[1]
 438         movdqu  (@inptr[3]),@out[3]
 439          pxor   $zero,@out[2]
 440          pxor   $zero,@out[3]
 441         movdqa  32(%rsp),$counters              # load counters
 442         xor     $offset,$offset
 443         jmp     .Loop_dec4x
 444
 445 .align  32
 446 .Loop_dec4x:
 447         add     \$16,$offset
 448         lea     16(%rsp),$sink                  # sink pointer
 449         mov     \$1,$one                        # constant of 1
 450         sub     $offset,$sink
 451
 452         aesdec          $rndkey1,@out[0]
 453         prefetcht0      31(@inptr[0],$offset)   # prefetch input
 454         prefetcht0      31(@inptr[1],$offset)
 455         aesdec          $rndkey1,@out[1]
 456         prefetcht0      31(@inptr[2],$offset)
 457         prefetcht0      31(@inptr[3],$offset)
 458         aesdec          $rndkey1,@out[2]
 459         aesdec          $rndkey1,@out[3]
 460         movups          0x30-0x78($key),$rndkey1
 461 ___
 462 for($i=0;$i<4;$i++) {
 463 my $rndkey = ($i&1) ? $rndkey1 : $rndkey0;
 464 $code.=<<___;
 465          cmp            `32+4*$i`(%rsp),$one
 466         aesdec          $rndkey,@out[0]
 467         aesdec          $rndkey,@out[1]
 468         aesdec          $rndkey,@out[2]
 469          cmovge         $sink,@inptr[$i]        # cancel input
 470          cmovg          $sink,@outptr[$i]       # sink output
 471         aesdec          $rndkey,@out[3]
 472         movups          `0x40+16*$i-0x78`($key),$rndkey
 473 ___
 474 }
 475 $code.=<<___;
 476          movdqa         $counters,$mask
 477         aesdec          $rndkey0,@out[0]
 478         prefetcht0      15(@outptr[0],$offset)  # prefetch output
 479         prefetcht0      15(@outptr[1],$offset)
 480         aesdec          $rndkey0,@out[1]
 481         prefetcht0      15(@outptr[2],$offset)
 482         prefetcht0      15(@outptr[3],$offset)
 483         aesdec          $rndkey0,@out[2]
 484         aesdec          $rndkey0,@out[3]
 485         movups          0x80-0x78($key),$rndkey0
 486          pxor           $zero,$zero
 487
 488         aesdec          $rndkey1,@out[0]
 489          pcmpgtd        $zero,$mask
 490          movdqu         -0x78($key),$zero       # reload 0-round key
 491         aesdec          $rndkey1,@out[1]
 492          paddd          $mask,$counters         # decrement counters
 493          movdqa         $counters,32(%rsp)      # update counters
 494         aesdec          $rndkey1,@out[2]
 495         aesdec          $rndkey1,@out[3]
 496         movups          0x90-0x78($key),$rndkey1
 497
 498         cmp     \$11,$rounds
 499
 500         aesdec          $rndkey0,@out[0]
 501         aesdec          $rndkey0,@out[1]
 502         aesdec          $rndkey0,@out[2]
 503         aesdec          $rndkey0,@out[3]
 504         movups          0xa0-0x78($key),$rndkey0
 505
 506         jb      .Ldec4x_tail
 507
 508         aesdec          $rndkey1,@out[0]
 509         aesdec          $rndkey1,@out[1]
 510         aesdec          $rndkey1,@out[2]
 511         aesdec          $rndkey1,@out[3]
 512         movups          0xb0-0x78($key),$rndkey1
 513
 514         aesdec          $rndkey0,@out[0]
 515         aesdec          $rndkey0,@out[1]
 516         aesdec          $rndkey0,@out[2]
 517         aesdec          $rndkey0,@out[3]
 518         movups          0xc0-0x78($key),$rndkey0
 519
 520         je      .Ldec4x_tail
 521
 522         aesdec          $rndkey1,@out[0]
 523         aesdec          $rndkey1,@out[1]
 524         aesdec          $rndkey1,@out[2]
 525         aesdec          $rndkey1,@out[3]
 526         movups          0xd0-0x78($key),$rndkey1
 527
 528         aesdec          $rndkey0,@out[0]
 529         aesdec          $rndkey0,@out[1]
 530         aesdec          $rndkey0,@out[2]
 531         aesdec          $rndkey0,@out[3]
 532         movups          0xe0-0x78($key),$rndkey0
 533         jmp     .Ldec4x_tail
 534
 535 .align  32
 536 .Ldec4x_tail:
 537         aesdec          $rndkey1,@out[0]
 538         aesdec          $rndkey1,@out[1]
 539         aesdec          $rndkey1,@out[2]
 540          pxor           $rndkey0,@inp[0]
 541          pxor           $rndkey0,@inp[1]
 542         aesdec          $rndkey1,@out[3]
 543         movdqu          0x10-0x78($key),$rndkey1
 544          pxor           $rndkey0,@inp[2]
 545          pxor           $rndkey0,@inp[3]
 546         movdqu          0x20-0x78($key),$rndkey0
 547
 548         aesdeclast      @inp[0],@out[0]
 549         aesdeclast      @inp[1],@out[1]
 550          movdqu         -16(@inptr[0],$offset),@inp[0]  # load next IV
 551          movdqu         -16(@inptr[1],$offset),@inp[1]
 552         aesdeclast      @inp[2],@out[2]
 553         aesdeclast      @inp[3],@out[3]
 554          movdqu         -16(@inptr[2],$offset),@inp[2]
 555          movdqu         -16(@inptr[3],$offset),@inp[3]
 556
 557         movups          @out[0],-16(@outptr[0],$offset)
 558          movdqu         (@inptr[0],$offset),@out[0]
 559         movups          @out[1],-16(@outptr[1],$offset)
 560          movdqu         (@inptr[1],$offset),@out[1]
 561          pxor           $zero,@out[0]
 562         movups          @out[2],-16(@outptr[2],$offset)
 563          movdqu         (@inptr[2],$offset),@out[2]
 564          pxor           $zero,@out[1]
 565         movups          @out[3],-16(@outptr[3],$offset)
 566          movdqu         (@inptr[3],$offset),@out[3]
 567          pxor           $zero,@out[2]
 568          pxor           $zero,@out[3]
 569
 570         dec     $num
 571         jnz     .Loop_dec4x
 572
 573         mov     16(%rsp),%rax                   # original %rsp
 574         mov     24(%rsp),$num
 575
 576         lea     `40*4`($inp),$inp
 577         dec     $num
 578         jnz     .Ldec4x_loop_grande
 579
 580 .Ldec4x_done:
 581 ___
 582 $code.=<<___ if ($win64);
 583         movaps  -0xd8(%rax),%xmm6
 584         movaps  -0xc8(%rax),%xmm7
 585         movaps  -0xb8(%rax),%xmm8
 586         movaps  -0xa8(%rax),%xmm9
 587         movaps  -0x98(%rax),%xmm10
 588         movaps  -0x88(%rax),%xmm11
 589         movaps  -0x78(%rax),%xmm12
 590         #movaps -0x68(%rax),%xmm13
 591         #movaps -0x58(%rax),%xmm14
 592         #movaps -0x48(%rax),%xmm15
 593 ___
 594 $code.=<<___;
 595         mov     -48(%rax),%r15
 596         mov     -40(%rax),%r14
 597         mov     -32(%rax),%r13
 598         mov     -24(%rax),%r12
 599         mov     -16(%rax),%rbp
 600         mov     -8(%rax),%rbx
 601         lea     (%rax),%rsp
 602 .Ldec4x_epilogue:
 603         ret
 604 .size   aesni_multi_cbc_decrypt,.-aesni_multi_cbc_decrypt
 605 ___
 606
 607                                                 if ($avx) {{{
 608 my @ptr=map("%r$_",(8..15));
 609 my $offload=$sink;
 610
 611 my @out=map("%xmm$_",(2..9));
 612 my @inp=map("%xmm$_",(10..13));
 613 my ($counters,$zero)=("%xmm14","%xmm15");
 614
 615 $code.=<<___;
 616 .type   aesni_multi_cbc_encrypt_avx,\@function,3
 617 .align  32
 618 aesni_multi_cbc_encrypt_avx:
 619 _avx_cbc_enc_shortcut:
 620         mov     %rsp,%rax
 621         push    %rbx
 622         push    %rbp
 623         push    %r12
 624         push    %r13
 625         push    %r14
 626         push    %r15
 627 ___
 628 $code.=<<___ if ($win64);
 629         lea     -0xa8(%rsp),%rsp
 630         movaps  %xmm6,(%rsp)
 631         movaps  %xmm7,0x10(%rsp)
 632         movaps  %xmm8,0x20(%rsp)
 633         movaps  %xmm9,0x30(%rsp)
 634         movaps  %xmm10,0x40(%rsp)
 635         movaps  %xmm11,0x50(%rsp)
 636         movaps  %xmm12,-0x78(%rax)
 637         movaps  %xmm13,-0x68(%rax)
 638         movaps  %xmm14,-0x58(%rax)
 639         movaps  %xmm15,-0x48(%rax)
 640 ___
 641 $code.=<<___;
 642         # stack layout
 643         #
 644         # +0    output sink
 645         # +16   input sink [original %rsp and $num]
 646         # +32   counters
 647         # +64   distances between inputs and outputs
 648         # +128  off-load area for @inp[0..3]
 649
 650         sub     \$192,%rsp
 651         and     \$-128,%rsp
 652         mov     %rax,16(%rsp)                   # original %rsp
 653
 654 .Lenc8x_body:
 655         vzeroupper
 656         vmovdqu ($key),$zero                    # 0-round key
 657         lea     0x78($key),$key                 # size optimization
 658         lea     40*4($inp),$inp
 659         shr     \$1,$num
 660
 661 .Lenc8x_loop_grande:
 662         #mov    $num,24(%rsp)                   # original $num
 663         xor     $num,$num
 664 ___
 665 for($i=0;$i<8;$i++) {
 666   my $temp = $i ? $offload : $offset;
 667     $code.=<<___;
 668         mov     `40*$i+16-40*4`($inp),$one      # borrow $one for number of blocks
 669         mov     `40*$i+0-40*4`($inp),@ptr[$i]   # input pointer
 670         cmp     $num,$one
 671         mov     `40*$i+8-40*4`($inp),$temp      # output pointer
 672         cmovg   $one,$num                       # find maximum
 673         test    $one,$one
 674         vmovdqu `40*$i+24-40*4`($inp),@out[$i]  # load IV
 675         mov     $one,`32+4*$i`(%rsp)            # initialize counters
 676         cmovle  %rsp,@ptr[$i]                   # cancel input
 677         sub     @ptr[$i],$temp                  # distance between input and output
 678         mov     $temp,`64+8*$i`(%rsp)           # initialize distances
 679 ___
 680 }
 681 $code.=<<___;
 682         test    $num,$num
 683         jz      .Lenc8x_done
 684
 685         vmovups 0x10-0x78($key),$rndkey1
 686         vmovups 0x20-0x78($key),$rndkey0
 687         mov     0xf0-0x78($key),$rounds
 688
 689         vpxor   (@ptr[0]),$zero,@inp[0]         # load inputs and xor with 0-round
 690          lea    128(%rsp),$offload              # offload area
 691         vpxor   (@ptr[1]),$zero,@inp[1]
 692         vpxor   (@ptr[2]),$zero,@inp[2]
 693         vpxor   (@ptr[3]),$zero,@inp[3]
 694          vpxor  @inp[0],@out[0],@out[0]
 695         vpxor   (@ptr[4]),$zero,@inp[0]
 696          vpxor  @inp[1],@out[1],@out[1]
 697         vpxor   (@ptr[5]),$zero,@inp[1]
 698          vpxor  @inp[2],@out[2],@out[2]
 699         vpxor   (@ptr[6]),$zero,@inp[2]
 700          vpxor  @inp[3],@out[3],@out[3]
 701         vpxor   (@ptr[7]),$zero,@inp[3]
 702          vpxor  @inp[0],@out[4],@out[4]
 703         mov     \$1,$one                        # constant of 1
 704          vpxor  @inp[1],@out[5],@out[5]
 705          vpxor  @inp[2],@out[6],@out[6]
 706          vpxor  @inp[3],@out[7],@out[7]
 707         jmp     .Loop_enc8x
 708
 709 .align  32
 710 .Loop_enc8x:
 711 ___
 712 for($i=0;$i<8;$i++) {
 713 my $rndkey=($i&1)?$rndkey0:$rndkey1;
 714 $code.=<<___;
 715         vaesenc         $rndkey,@out[0],@out[0]
 716          cmp            32+4*$i(%rsp),$one
 717 ___
 718 $code.=<<___ if ($i);
 719          mov            64+8*$i(%rsp),$offset
 720 ___
 721 $code.=<<___;
 722         vaesenc         $rndkey,@out[1],@out[1]
 723         prefetcht0      31(@ptr[$i])                    # prefetch input
 724         vaesenc         $rndkey,@out[2],@out[2]
 725 ___
 726 $code.=<<___ if ($i>1);
 727         prefetcht0      15(@ptr[$i-2])                  # prefetch output
 728 ___
 729 $code.=<<___;
 730         vaesenc         $rndkey,@out[3],@out[3]
 731          lea            (@ptr[$i],$offset),$offset
 732          cmovge         %rsp,@ptr[$i]                   # cancel input
 733         vaesenc         $rndkey,@out[4],@out[4]
 734          cmovg          %rsp,$offset                    # sink output
 735         vaesenc         $rndkey,@out[5],@out[5]
 736          sub            @ptr[$i],$offset
 737         vaesenc         $rndkey,@out[6],@out[6]
 738          vpxor          16(@ptr[$i]),$zero,@inp[$i%4]   # load input and xor with 0-round
 739          mov            $offset,64+8*$i(%rsp)
 740         vaesenc         $rndkey,@out[7],@out[7]
 741         vmovups         `16*(3+$i)-0x78`($key),$rndkey
 742          lea            16(@ptr[$i],$offset),@ptr[$i]   # switch to output
 743 ___
 744 $code.=<<___ if ($i<4)
 745          vmovdqu        @inp[$i%4],`16*$i`($offload)    # off-load
 746 ___
 747 }
 748 $code.=<<___;
 749          vmovdqu        32(%rsp),$counters
 750         prefetcht0      15(@ptr[$i-2])                  # prefetch output
 751         prefetcht0      15(@ptr[$i-1])
 752         cmp     \$11,$rounds
 753         jb      .Lenc8x_tail
 754
 755         vaesenc         $rndkey1,@out[0],@out[0]
 756         vaesenc         $rndkey1,@out[1],@out[1]
 757         vaesenc         $rndkey1,@out[2],@out[2]
 758         vaesenc         $rndkey1,@out[3],@out[3]
 759         vaesenc         $rndkey1,@out[4],@out[4]
 760         vaesenc         $rndkey1,@out[5],@out[5]
 761         vaesenc         $rndkey1,@out[6],@out[6]
 762         vaesenc         $rndkey1,@out[7],@out[7]
 763         vmovups         0xb0-0x78($key),$rndkey1
 764
 765         vaesenc         $rndkey0,@out[0],@out[0]
 766         vaesenc         $rndkey0,@out[1],@out[1]
 767         vaesenc         $rndkey0,@out[2],@out[2]
 768         vaesenc         $rndkey0,@out[3],@out[3]
 769         vaesenc         $rndkey0,@out[4],@out[4]
 770         vaesenc         $rndkey0,@out[5],@out[5]
 771         vaesenc         $rndkey0,@out[6],@out[6]
 772         vaesenc         $rndkey0,@out[7],@out[7]
 773         vmovups         0xc0-0x78($key),$rndkey0
 774         je      .Lenc8x_tail
 775
 776         vaesenc         $rndkey1,@out[0],@out[0]
 777         vaesenc         $rndkey1,@out[1],@out[1]
 778         vaesenc         $rndkey1,@out[2],@out[2]
 779         vaesenc         $rndkey1,@out[3],@out[3]
 780         vaesenc         $rndkey1,@out[4],@out[4]
 781         vaesenc         $rndkey1,@out[5],@out[5]
 782         vaesenc         $rndkey1,@out[6],@out[6]
 783         vaesenc         $rndkey1,@out[7],@out[7]
 784         vmovups         0xd0-0x78($key),$rndkey1
 785
 786         vaesenc         $rndkey0,@out[0],@out[0]
 787         vaesenc         $rndkey0,@out[1],@out[1]
 788         vaesenc         $rndkey0,@out[2],@out[2]
 789         vaesenc         $rndkey0,@out[3],@out[3]
 790         vaesenc         $rndkey0,@out[4],@out[4]
 791         vaesenc         $rndkey0,@out[5],@out[5]
 792         vaesenc         $rndkey0,@out[6],@out[6]
 793         vaesenc         $rndkey0,@out[7],@out[7]
 794         vmovups         0xe0-0x78($key),$rndkey0
 795
 796 .Lenc8x_tail:
 797         vaesenc         $rndkey1,@out[0],@out[0]
 798          vpxor          $zero,$zero,$zero
 799         vaesenc         $rndkey1,@out[1],@out[1]
 800         vaesenc         $rndkey1,@out[2],@out[2]
 801          vpcmpgtd       $zero,$counters,$zero
 802         vaesenc         $rndkey1,@out[3],@out[3]
 803         vaesenc         $rndkey1,@out[4],@out[4]
 804          vpaddd         $counters,$zero,$zero           # decrement counters
 805          vmovdqu        48(%rsp),$counters
 806         vaesenc         $rndkey1,@out[5],@out[5]
 807          mov            64(%rsp),$offset                # pre-load 1st offset
 808         vaesenc         $rndkey1,@out[6],@out[6]
 809         vaesenc         $rndkey1,@out[7],@out[7]
 810         vmovups         0x10-0x78($key),$rndkey1
 811
 812         vaesenclast     $rndkey0,@out[0],@out[0]
 813          vmovdqa        $zero,32(%rsp)                  # update counters
 814          vpxor          $zero,$zero,$zero
 815         vaesenclast     $rndkey0,@out[1],@out[1]
 816         vaesenclast     $rndkey0,@out[2],@out[2]
 817          vpcmpgtd       $zero,$counters,$zero
 818         vaesenclast     $rndkey0,@out[3],@out[3]
 819         vaesenclast     $rndkey0,@out[4],@out[4]
 820          vpaddd         $zero,$counters,$counters       # decrement counters
 821          vmovdqu        -0x78($key),$zero               # 0-round
 822         vaesenclast     $rndkey0,@out[5],@out[5]
 823         vaesenclast     $rndkey0,@out[6],@out[6]
 824          vmovdqa        $counters,48(%rsp)              # update counters
 825         vaesenclast     $rndkey0,@out[7],@out[7]
 826         vmovups         0x20-0x78($key),$rndkey0
 827
 828         vmovups         @out[0],-16(@ptr[0])            # write output
 829          sub            $offset,@ptr[0]                 # switch to input
 830          vpxor          0x00($offload),@out[0],@out[0]
 831         vmovups         @out[1],-16(@ptr[1])
 832          sub            `64+1*8`(%rsp),@ptr[1]
 833          vpxor          0x10($offload),@out[1],@out[1]
 834         vmovups         @out[2],-16(@ptr[2])
 835          sub            `64+2*8`(%rsp),@ptr[2]
 836          vpxor          0x20($offload),@out[2],@out[2]
 837         vmovups         @out[3],-16(@ptr[3])
 838          sub            `64+3*8`(%rsp),@ptr[3]
 839          vpxor          0x30($offload),@out[3],@out[3]
 840         vmovups         @out[4],-16(@ptr[4])
 841          sub            `64+4*8`(%rsp),@ptr[4]
 842          vpxor          @inp[0],@out[4],@out[4]
 843         vmovups         @out[5],-16(@ptr[5])
 844          sub            `64+5*8`(%rsp),@ptr[5]
 845          vpxor          @inp[1],@out[5],@out[5]
 846         vmovups         @out[6],-16(@ptr[6])
 847          sub            `64+6*8`(%rsp),@ptr[6]
 848          vpxor          @inp[2],@out[6],@out[6]
 849         vmovups         @out[7],-16(@ptr[7])
 850          sub            `64+7*8`(%rsp),@ptr[7]
 851          vpxor          @inp[3],@out[7],@out[7]
 852
 853         dec     $num
 854         jnz     .Loop_enc8x
 855
 856         mov     16(%rsp),%rax                   # original %rsp
 857         #mov    24(%rsp),$num
 858         #lea    `40*8`($inp),$inp
 859         #dec    $num
 860         #jnz    .Lenc8x_loop_grande
 861
 862 .Lenc8x_done:
 863         vzeroupper
 864 ___
 865 $code.=<<___ if ($win64);
 866         movaps  -0xd8(%rax),%xmm6
 867         movaps  -0xc8(%rax),%xmm7
 868         movaps  -0xb8(%rax),%xmm8
 869         movaps  -0xa8(%rax),%xmm9
 870         movaps  -0x98(%rax),%xmm10
 871         movaps  -0x88(%rax),%xmm11
 872         movaps  -0x78(%rax),%xmm12
 873         movaps  -0x68(%rax),%xmm13
 874         movaps  -0x58(%rax),%xmm14
 875         movaps  -0x48(%rax),%xmm15
 876 ___
 877 $code.=<<___;
 878         mov     -48(%rax),%r15
 879         mov     -40(%rax),%r14
 880         mov     -32(%rax),%r13
 881         mov     -24(%rax),%r12
 882         mov     -16(%rax),%rbp
 883         mov     -8(%rax),%rbx
 884         lea     (%rax),%rsp
 885 .Lenc8x_epilogue:
 886         ret
 887 .size   aesni_multi_cbc_encrypt_avx,.-aesni_multi_cbc_encrypt_avx
 888
 889 .type   aesni_multi_cbc_decrypt_avx,\@function,3
 890 .align  32
 891 aesni_multi_cbc_decrypt_avx:
 892 _avx_cbc_dec_shortcut:
 893         mov     %rsp,%rax
 894         push    %rbx
 895         push    %rbp
 896         push    %r12
 897         push    %r13
 898         push    %r14
 899         push    %r15
 900 ___
 901 $code.=<<___ if ($win64);
 902         lea     -0xa8(%rsp),%rsp
 903         movaps  %xmm6,(%rsp)
 904         movaps  %xmm7,0x10(%rsp)
 905         movaps  %xmm8,0x20(%rsp)
 906         movaps  %xmm9,0x30(%rsp)
 907         movaps  %xmm10,0x40(%rsp)
 908         movaps  %xmm11,0x50(%rsp)
 909         movaps  %xmm12,-0x78(%rax)
 910         movaps  %xmm13,-0x68(%rax)
 911         movaps  %xmm14,-0x58(%rax)
 912         movaps  %xmm15,-0x48(%rax)
 913 ___
 914 $code.=<<___;
 915         # stack layout
 916         #
 917         # +0    output sink
 918         # +16   input sink [original %rsp and $num]
 919         # +32   counters
 920         # +64   distances between inputs and outputs
 921         # +128  off-load area for @inp[0..3]
 922         # +192  IV/input offload
 923
 924         sub     \$256,%rsp
 925         and     \$-256,%rsp
 926         sub     \$192,%rsp
 927         mov     %rax,16(%rsp)                   # original %rsp
 928
 929 .Ldec8x_body:
 930         vzeroupper
 931         vmovdqu ($key),$zero                    # 0-round key
 932         lea     0x78($key),$key                 # size optimization
 933         lea     40*4($inp),$inp
 934         shr     \$1,$num
 935
 936 .Ldec8x_loop_grande:
 937         #mov    $num,24(%rsp)                   # original $num
 938         xor     $num,$num
 939 ___
 940 for($i=0;$i<8;$i++) {
 941   my $temp = $i ? $offload : $offset;
 942     $code.=<<___;
 943         mov     `40*$i+16-40*4`($inp),$one      # borrow $one for number of blocks
 944         mov     `40*$i+0-40*4`($inp),@ptr[$i]   # input pointer
 945         cmp     $num,$one
 946         mov     `40*$i+8-40*4`($inp),$temp      # output pointer
 947         cmovg   $one,$num                       # find maximum
 948         test    $one,$one
 949         vmovdqu `40*$i+24-40*4`($inp),@out[$i]  # load IV
 950         mov     $one,`32+4*$i`(%rsp)            # initialize counters
 951         cmovle  %rsp,@ptr[$i]                   # cancel input
 952         sub     @ptr[$i],$temp                  # distance between input and output
 953         mov     $temp,`64+8*$i`(%rsp)           # initialize distances
 954         vmovdqu @out[$i],`192+16*$i`(%rsp)      # offload IV
 955 ___
 956 }
 957 $code.=<<___;
 958         test    $num,$num
 959         jz      .Ldec8x_done
 960
 961         vmovups 0x10-0x78($key),$rndkey1
 962         vmovups 0x20-0x78($key),$rndkey0
 963         mov     0xf0-0x78($key),$rounds
 964          lea    192+128(%rsp),$offload          # offload area
 965
 966         vmovdqu (@ptr[0]),@out[0]               # load inputs
 967         vmovdqu (@ptr[1]),@out[1]
 968         vmovdqu (@ptr[2]),@out[2]
 969         vmovdqu (@ptr[3]),@out[3]
 970         vmovdqu (@ptr[4]),@out[4]
 971         vmovdqu (@ptr[5]),@out[5]
 972         vmovdqu (@ptr[6]),@out[6]
 973         vmovdqu (@ptr[7]),@out[7]
 974         vmovdqu @out[0],0x00($offload)          # offload inputs
 975         vpxor   $zero,@out[0],@out[0]           # xor inputs with 0-round
 976         vmovdqu @out[1],0x10($offload)
 977         vpxor   $zero,@out[1],@out[1]
 978         vmovdqu @out[2],0x20($offload)
 979         vpxor   $zero,@out[2],@out[2]
 980         vmovdqu @out[3],0x30($offload)
 981         vpxor   $zero,@out[3],@out[3]
 982         vmovdqu @out[4],0x40($offload)
 983         vpxor   $zero,@out[4],@out[4]
 984         vmovdqu @out[5],0x50($offload)
 985         vpxor   $zero,@out[5],@out[5]
 986         vmovdqu @out[6],0x60($offload)
 987         vpxor   $zero,@out[6],@out[6]
 988         vmovdqu @out[7],0x70($offload)
 989         vpxor   $zero,@out[7],@out[7]
 990         xor     \$0x80,$offload
 991         mov     \$1,$one                        # constant of 1
 992         jmp     .Loop_dec8x
 993
 994 .align  32
 995 .Loop_dec8x:
 996 ___
 997 for($i=0;$i<8;$i++) {
 998 my $rndkey=($i&1)?$rndkey0:$rndkey1;
 999 $code.=<<___;
1000         vaesdec         $rndkey,@out[0],@out[0]
1001          cmp            32+4*$i(%rsp),$one
1002 ___
1003 $code.=<<___ if ($i);
1004          mov            64+8*$i(%rsp),$offset
1005 ___
1006 $code.=<<___;
1007         vaesdec         $rndkey,@out[1],@out[1]
1008         prefetcht0      31(@ptr[$i])                    # prefetch input
1009         vaesdec         $rndkey,@out[2],@out[2]
1010 ___
1011 $code.=<<___ if ($i>1);
1012         prefetcht0      15(@ptr[$i-2])                  # prefetch output
1013 ___
1014 $code.=<<___;
1015         vaesdec         $rndkey,@out[3],@out[3]
1016          lea            (@ptr[$i],$offset),$offset
1017          cmovge         %rsp,@ptr[$i]                   # cancel input
1018         vaesdec         $rndkey,@out[4],@out[4]
1019          cmovg          %rsp,$offset                    # sink output
1020         vaesdec         $rndkey,@out[5],@out[5]
1021          sub            @ptr[$i],$offset
1022         vaesdec         $rndkey,@out[6],@out[6]
1023          vmovdqu        16(@ptr[$i]),@inp[$i%4]         # load input
1024          mov            $offset,64+8*$i(%rsp)
1025         vaesdec         $rndkey,@out[7],@out[7]
1026         vmovups         `16*(3+$i)-0x78`($key),$rndkey
1027          lea            16(@ptr[$i],$offset),@ptr[$i]   # switch to output
1028 ___
1029 $code.=<<___ if ($i<4);
1030          vmovdqu        @inp[$i%4],`128+16*$i`(%rsp)    # off-load
1031 ___
1032 }
1033 $code.=<<___;
1034          vmovdqu        32(%rsp),$counters
1035         prefetcht0      15(@ptr[$i-2])                  # prefetch output
1036         prefetcht0      15(@ptr[$i-1])
1037         cmp     \$11,$rounds
1038         jb      .Ldec8x_tail
1039
1040         vaesdec         $rndkey1,@out[0],@out[0]
1041         vaesdec         $rndkey1,@out[1],@out[1]
1042         vaesdec         $rndkey1,@out[2],@out[2]
1043         vaesdec         $rndkey1,@out[3],@out[3]
1044         vaesdec         $rndkey1,@out[4],@out[4]
1045         vaesdec         $rndkey1,@out[5],@out[5]
1046         vaesdec         $rndkey1,@out[6],@out[6]
1047         vaesdec         $rndkey1,@out[7],@out[7]
1048         vmovups         0xb0-0x78($key),$rndkey1
1049
1050         vaesdec         $rndkey0,@out[0],@out[0]
1051         vaesdec         $rndkey0,@out[1],@out[1]
1052         vaesdec         $rndkey0,@out[2],@out[2]
1053         vaesdec         $rndkey0,@out[3],@out[3]
1054         vaesdec         $rndkey0,@out[4],@out[4]
1055         vaesdec         $rndkey0,@out[5],@out[5]
1056         vaesdec         $rndkey0,@out[6],@out[6]
1057         vaesdec         $rndkey0,@out[7],@out[7]
1058         vmovups         0xc0-0x78($key),$rndkey0
1059         je      .Ldec8x_tail
1060
1061         vaesdec         $rndkey1,@out[0],@out[0]
1062         vaesdec         $rndkey1,@out[1],@out[1]
1063         vaesdec         $rndkey1,@out[2],@out[2]
1064         vaesdec         $rndkey1,@out[3],@out[3]
1065         vaesdec         $rndkey1,@out[4],@out[4]
1066         vaesdec         $rndkey1,@out[5],@out[5]
1067         vaesdec         $rndkey1,@out[6],@out[6]
1068         vaesdec         $rndkey1,@out[7],@out[7]
1069         vmovups         0xd0-0x78($key),$rndkey1
1070
1071         vaesdec         $rndkey0,@out[0],@out[0]
1072         vaesdec         $rndkey0,@out[1],@out[1]
1073         vaesdec         $rndkey0,@out[2],@out[2]
1074         vaesdec         $rndkey0,@out[3],@out[3]
1075         vaesdec         $rndkey0,@out[4],@out[4]
1076         vaesdec         $rndkey0,@out[5],@out[5]
1077         vaesdec         $rndkey0,@out[6],@out[6]
1078         vaesdec         $rndkey0,@out[7],@out[7]
1079         vmovups         0xe0-0x78($key),$rndkey0
1080
1081 .Ldec8x_tail:
1082         vaesdec         $rndkey1,@out[0],@out[0]
1083          vpxor          $zero,$zero,$zero
1084         vaesdec         $rndkey1,@out[1],@out[1]
1085         vaesdec         $rndkey1,@out[2],@out[2]
1086          vpcmpgtd       $zero,$counters,$zero
1087         vaesdec         $rndkey1,@out[3],@out[3]
1088         vaesdec         $rndkey1,@out[4],@out[4]
1089          vpaddd         $counters,$zero,$zero           # decrement counters
1090          vmovdqu        48(%rsp),$counters
1091         vaesdec         $rndkey1,@out[5],@out[5]
1092          mov            64(%rsp),$offset                # pre-load 1st offset
1093         vaesdec         $rndkey1,@out[6],@out[6]
1094         vaesdec         $rndkey1,@out[7],@out[7]
1095         vmovups         0x10-0x78($key),$rndkey1
1096
1097         vaesdeclast     $rndkey0,@out[0],@out[0]
1098          vmovdqa        $zero,32(%rsp)                  # update counters
1099          vpxor          $zero,$zero,$zero
1100         vaesdeclast     $rndkey0,@out[1],@out[1]
1101         vpxor           0x00($offload),@out[0],@out[0]  # xor with IV
1102         vaesdeclast     $rndkey0,@out[2],@out[2]
1103         vpxor           0x10($offload),@out[1],@out[1]
1104          vpcmpgtd       $zero,$counters,$zero
1105         vaesdeclast     $rndkey0,@out[3],@out[3]
1106         vpxor           0x20($offload),@out[2],@out[2]
1107         vaesdeclast     $rndkey0,@out[4],@out[4]
1108         vpxor           0x30($offload),@out[3],@out[3]
1109          vpaddd         $zero,$counters,$counters       # decrement counters
1110          vmovdqu        -0x78($key),$zero               # 0-round
1111         vaesdeclast     $rndkey0,@out[5],@out[5]
1112         vpxor           0x40($offload),@out[4],@out[4]
1113         vaesdeclast     $rndkey0,@out[6],@out[6]
1114         vpxor           0x50($offload),@out[5],@out[5]
1115          vmovdqa        $counters,48(%rsp)              # update counters
1116         vaesdeclast     $rndkey0,@out[7],@out[7]
1117         vpxor           0x60($offload),@out[6],@out[6]
1118         vmovups         0x20-0x78($key),$rndkey0
1119
1120         vmovups         @out[0],-16(@ptr[0])            # write output
1121          sub            $offset,@ptr[0]                 # switch to input
1122          vmovdqu        128+0(%rsp),@out[0]
1123         vpxor           0x70($offload),@out[7],@out[7]
1124         vmovups         @out[1],-16(@ptr[1])
1125          sub            `64+1*8`(%rsp),@ptr[1]
1126          vmovdqu        @out[0],0x00($offload)
1127          vpxor          $zero,@out[0],@out[0]
1128          vmovdqu        128+16(%rsp),@out[1]
1129         vmovups         @out[2],-16(@ptr[2])
1130          sub            `64+2*8`(%rsp),@ptr[2]
1131          vmovdqu        @out[1],0x10($offload)
1132          vpxor          $zero,@out[1],@out[1]
1133          vmovdqu        128+32(%rsp),@out[2]
1134         vmovups         @out[3],-16(@ptr[3])
1135          sub            `64+3*8`(%rsp),@ptr[3]
1136          vmovdqu        @out[2],0x20($offload)
1137          vpxor          $zero,@out[2],@out[2]
1138          vmovdqu        128+48(%rsp),@out[3]
1139         vmovups         @out[4],-16(@ptr[4])
1140          sub            `64+4*8`(%rsp),@ptr[4]
1141          vmovdqu        @out[3],0x30($offload)
1142          vpxor          $zero,@out[3],@out[3]
1143          vmovdqu        @inp[0],0x40($offload)
1144          vpxor          @inp[0],$zero,@out[4]
1145         vmovups         @out[5],-16(@ptr[5])
1146          sub            `64+5*8`(%rsp),@ptr[5]
1147          vmovdqu        @inp[1],0x50($offload)
1148          vpxor          @inp[1],$zero,@out[5]
1149         vmovups         @out[6],-16(@ptr[6])
1150          sub            `64+6*8`(%rsp),@ptr[6]
1151          vmovdqu        @inp[2],0x60($offload)
1152          vpxor          @inp[2],$zero,@out[6]
1153         vmovups         @out[7],-16(@ptr[7])
1154          sub            `64+7*8`(%rsp),@ptr[7]
1155          vmovdqu        @inp[3],0x70($offload)
1156          vpxor          @inp[3],$zero,@out[7]
1157
1158         xor     \$128,$offload
1159         dec     $num
1160         jnz     .Loop_dec8x
1161
1162         mov     16(%rsp),%rax                   # original %rsp
1163         #mov    24(%rsp),$num
1164         #lea    `40*8`($inp),$inp
1165         #dec    $num
1166         #jnz    .Ldec8x_loop_grande
1167
1168 .Ldec8x_done:
1169         vzeroupper
1170 ___
1171 $code.=<<___ if ($win64);
1172         movaps  -0xd8(%rax),%xmm6
1173         movaps  -0xc8(%rax),%xmm7
1174         movaps  -0xb8(%rax),%xmm8
1175         movaps  -0xa8(%rax),%xmm9
1176         movaps  -0x98(%rax),%xmm10
1177         movaps  -0x88(%rax),%xmm11
1178         movaps  -0x78(%rax),%xmm12
1179         movaps  -0x68(%rax),%xmm13
1180         movaps  -0x58(%rax),%xmm14
1181         movaps  -0x48(%rax),%xmm15
1182 ___
1183 $code.=<<___;
1184         mov     -48(%rax),%r15
1185         mov     -40(%rax),%r14
1186         mov     -32(%rax),%r13
1187         mov     -24(%rax),%r12
1188         mov     -16(%rax),%rbp
1189         mov     -8(%rax),%rbx
1190         lea     (%rax),%rsp
1191 .Ldec8x_epilogue:
1192         ret
1193 .size   aesni_multi_cbc_decrypt_avx,.-aesni_multi_cbc_decrypt_avx
1194 ___
1195                                                 }}}
1196
1197 if ($win64) {
1198 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1199 #               CONTEXT *context,DISPATCHER_CONTEXT *disp)
1200 $rec="%rcx";
1201 $frame="%rdx";
1202 $context="%r8";
1203 $disp="%r9";
1204
1205 $code.=<<___;
1206 .extern __imp_RtlVirtualUnwind
1207 .type   se_handler,\@abi-omnipotent
1208 .align  16
1209 se_handler:
1210         push    %rsi
1211         push    %rdi
1212         push    %rbx
1213         push    %rbp
1214         push    %r12
1215         push    %r13
1216         push    %r14
1217         push    %r15
1218         pushfq
1219         sub     \$64,%rsp
1220
1221         mov     120($context),%rax      # pull context->Rax
1222         mov     248($context),%rbx      # pull context->Rip
1223
1224         mov     8($disp),%rsi           # disp->ImageBase
1225         mov     56($disp),%r11          # disp->HandlerData
1226
1227         mov     0(%r11),%r10d           # HandlerData[0]
1228         lea     (%rsi,%r10),%r10        # prologue label
1229         cmp     %r10,%rbx               # context->Rip<.Lprologue
1230         jb      .Lin_prologue
1231
1232         mov     152($context),%rax      # pull context->Rsp
1233
1234         mov     4(%r11),%r10d           # HandlerData[1]
1235         lea     (%rsi,%r10),%r10        # epilogue label
1236         cmp     %r10,%rbx               # context->Rip>=.Lepilogue
1237         jae     .Lin_prologue
1238
1239         mov     16(%rax),%rax           # pull saved stack pointer
1240
1241         mov     -8(%rax),%rbx
1242         mov     -16(%rax),%rbp
1243         mov     -24(%rax),%r12
1244         mov     -32(%rax),%r13
1245         mov     -40(%rax),%r14
1246         mov     -48(%rax),%r15
1247         mov     %rbx,144($context)      # restore context->Rbx
1248         mov     %rbp,160($context)      # restore context->Rbp
1249         mov     %r12,216($context)      # restore cotnext->R12
1250         mov     %r13,224($context)      # restore cotnext->R13
1251         mov     %r14,232($context)      # restore cotnext->R14
1252         mov     %r15,240($context)      # restore cotnext->R15
1253
1254         lea     -56-10*16(%rax),%rsi
1255         lea     512($context),%rdi      # &context.Xmm6
1256         mov     \$20,%ecx
1257         .long   0xa548f3fc              # cld; rep movsq
1258
1259 .Lin_prologue:
1260         mov     8(%rax),%rdi
1261         mov     16(%rax),%rsi
1262         mov     %rax,152($context)      # restore context->Rsp
1263         mov     %rsi,168($context)      # restore context->Rsi
1264         mov     %rdi,176($context)      # restore context->Rdi
1265
1266         mov     40($disp),%rdi          # disp->ContextRecord
1267         mov     $context,%rsi           # context
1268         mov     \$154,%ecx              # sizeof(CONTEXT)
1269         .long   0xa548f3fc              # cld; rep movsq
1270
1271         mov     $disp,%rsi
1272         xor     %rcx,%rcx               # arg1, UNW_FLAG_NHANDLER
1273         mov     8(%rsi),%rdx            # arg2, disp->ImageBase
1274         mov     0(%rsi),%r8             # arg3, disp->ControlPc
1275         mov     16(%rsi),%r9            # arg4, disp->FunctionEntry
1276         mov     40(%rsi),%r10           # disp->ContextRecord
1277         lea     56(%rsi),%r11           # &disp->HandlerData
1278         lea     24(%rsi),%r12           # &disp->EstablisherFrame
1279         mov     %r10,32(%rsp)           # arg5
1280         mov     %r11,40(%rsp)           # arg6
1281         mov     %r12,48(%rsp)           # arg7
1282         mov     %rcx,56(%rsp)           # arg8, (NULL)
1283         call    *__imp_RtlVirtualUnwind(%rip)
1284
1285         mov     \$1,%eax                # ExceptionContinueSearch
1286         add     \$64,%rsp
1287         popfq
1288         pop     %r15
1289         pop     %r14
1290         pop     %r13
1291         pop     %r12
1292         pop     %rbp
1293         pop     %rbx
1294         pop     %rdi
1295         pop     %rsi
1296         ret
1297 .size   se_handler,.-se_handler
1298
1299 .section        .pdata
1300 .align  4
1301         .rva    .LSEH_begin_aesni_multi_cbc_encrypt
1302         .rva    .LSEH_end_aesni_multi_cbc_encrypt
1303         .rva    .LSEH_info_aesni_multi_cbc_encrypt
1304         .rva    .LSEH_begin_aesni_multi_cbc_decrypt
1305         .rva    .LSEH_end_aesni_multi_cbc_decrypt
1306         .rva    .LSEH_info_aesni_multi_cbc_decrypt
1307 ___
1308 $code.=<<___ if ($avx);
1309         .rva    .LSEH_begin_aesni_multi_cbc_encrypt_avx
1310         .rva    .LSEH_end_aesni_multi_cbc_encrypt_avx
1311         .rva    .LSEH_info_aesni_multi_cbc_encrypt_avx
1312         .rva    .LSEH_begin_aesni_multi_cbc_decrypt_avx
1313         .rva    .LSEH_end_aesni_multi_cbc_decrypt_avx
1314         .rva    .LSEH_info_aesni_multi_cbc_decrypt_avx
1315 ___
1316 $code.=<<___;
1317 .section        .xdata
1318 .align  8
1319 .LSEH_info_aesni_multi_cbc_encrypt:
1320         .byte   9,0,0,0
1321         .rva    se_handler
1322         .rva    .Lenc4x_body,.Lenc4x_epilogue           # HandlerData[]
1323 .LSEH_info_aesni_multi_cbc_decrypt:
1324         .byte   9,0,0,0
1325         .rva    se_handler
1326         .rva    .Ldec4x_body,.Ldec4x_epilogue           # HandlerData[]
1327 ___
1328 $code.=<<___ if ($avx);
1329 .LSEH_info_aesni_multi_cbc_encrypt_avx:
1330         .byte   9,0,0,0
1331         .rva    se_handler
1332         .rva    .Lenc8x_body,.Lenc8x_epilogue           # HandlerData[]
1333 .LSEH_info_aesni_multi_cbc_decrypt_avx:
1334         .byte   9,0,0,0
1335         .rva    se_handler
1336         .rva    .Ldec8x_body,.Ldec8x_epilogue           # HandlerData[]
1337 ___
1338 }
1339 ####################################################################
1340
1341 sub rex {
1342   local *opcode=shift;
1343   my ($dst,$src)=@_;
1344   my $rex=0;
1345
1346     $rex|=0x04                  if($dst>=8);
1347     $rex|=0x01                  if($src>=8);
1348     push @opcode,$rex|0x40      if($rex);
1349 }
1350
1351 sub aesni {
1352   my $line=shift;
1353   my @opcode=(0x66);
1354
1355     if ($line=~/(aeskeygenassist)\s+\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1356         rex(\@opcode,$4,$3);
1357         push @opcode,0x0f,0x3a,0xdf;
1358         push @opcode,0xc0|($3&7)|(($4&7)<<3);   # ModR/M
1359         my $c=$2;
1360         push @opcode,$c=~/^0/?oct($c):$c;
1361         return ".byte\t".join(',',@opcode);
1362     }
1363     elsif ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1364         my %opcodelet = (
1365                 "aesimc" => 0xdb,
1366                 "aesenc" => 0xdc,       "aesenclast" => 0xdd,
1367                 "aesdec" => 0xde,       "aesdeclast" => 0xdf
1368         );
1369         return undef if (!defined($opcodelet{$1}));
1370         rex(\@opcode,$3,$2);
1371         push @opcode,0x0f,0x38,$opcodelet{$1};
1372         push @opcode,0xc0|($2&7)|(($3&7)<<3);   # ModR/M
1373         return ".byte\t".join(',',@opcode);
1374     }
1375     elsif ($line=~/(aes[a-z]+)\s+([0x1-9a-fA-F]*)\(%rsp\),\s*%xmm([0-9]+)/) {
1376         my %opcodelet = (
1377                 "aesenc" => 0xdc,       "aesenclast" => 0xdd,
1378                 "aesdec" => 0xde,       "aesdeclast" => 0xdf
1379         );
1380         return undef if (!defined($opcodelet{$1}));
1381         my $off = $2;
1382         push @opcode,0x44 if ($3>=8);
1383         push @opcode,0x0f,0x38,$opcodelet{$1};
1384         push @opcode,0x44|(($3&7)<<3),0x24;     # ModR/M
1385         push @opcode,($off=~/^0/?oct($off):$off)&0xff;
1386         return ".byte\t".join(',',@opcode);
1387     }
1388     return $line;
1389 }
1390
1391 $code =~ s/\`([^\`]*)\`/eval($1)/gem;
1392 $code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem;
1393
1394 print $code;
1395 close STDOUT;