crypto/aes/asm/aesni-mb-x86_64.pl

   1 #!/usr/bin/env perl
   2
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9
  10 # Multi-buffer AES-NI procedures process several independent buffers
  11 # in parallel by interleaving independent instructions.
  12 #
  13 # Cycles per byte for interleave factor 4:
  14 #
  15 #                       asymptotic      measured
  16 #                       ---------------------------
  17 # Westmere              5.00/4=1.25     5.13/4=1.28
  18 # Atom                  15.0/4=3.75     ?15.7/4=3.93
  19 # Sandy Bridge          5.06/4=1.27     5.18/4=1.29
  20 # Ivy Bridge            5.06/4=1.27     5.14/4=1.29
  21 # Haswell               4.44/4=1.11     4.44/4=1.11
  22 # Bulldozer             5.75/4=1.44     5.76/4=1.44
  23 #
  24 # Cycles per byte for interleave factor 8 (not implemented for
  25 # pre-AVX processors, where higher interleave factor incidentally
  26 # doesn't result in improvement):
  27 #
  28 #                       asymptotic      measured
  29 #                       ---------------------------
  30 # Sandy Bridge          5.06/8=0.64     7.10/8=0.89(*)
  31 # Ivy Bridge            5.06/8=0.64     7.14/8=0.89(*)
  32 # Haswell               5.00/8=0.63     5.00/8=0.63
  33 # Bulldozer             5.75/8=0.72     5.77/8=0.72
  34 #
  35 # (*)   Sandy/Ivy Bridge are known to handle high interleave factors
  36 #       suboptimally;
  37
  38 $flavour = shift;
  39 $output  = shift;
  40 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
  41
  42 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  43
  44 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  45 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  46 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
  47 die "can't locate x86_64-xlate.pl";
  48
  49 $avx=0;
  50
  51 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  52                 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  53         $avx = ($1>=2.19) + ($1>=2.22);
  54 }
  55
  56 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
  57            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
  58         $avx = ($1>=2.09) + ($1>=2.10);
  59 }
  60
  61 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
  62            `ml64 2>&1` =~ /Version ([0-9]+)\./) {
  63         $avx = ($1>=10) + ($1>=11);
  64 }
  65
  66 open OUT,"| \"$^X\" $xlate $flavour $output";
  67 *STDOUT=*OUT;
  68
  69 # void aesni_multi_cbc_encrypt (
  70 #     struct {  void *inp,*out; int blocks; double iv[2]; } inp[8];
  71 #     const AES_KEY *key,
  72 #     int num);         /* 1 or 2 */
  73 #
  74 $inp="%rdi";    # 1st arg
  75 $key="%rsi";    # 2nd arg
  76 $num="%edx";
  77
  78 @inptr=map("%r$_",(8..11));
  79 @outptr=map("%r$_",(12..15));
  80
  81 ($rndkey0,$rndkey1)=("%xmm0","%xmm1");
  82 @out=map("%xmm$_",(2..5));
  83 @inp=map("%xmm$_",(6..9));
  84 ($counters,$mask,$zero)=map("%xmm$_",(10..12));
  85
  86 ($rounds,$one,$sink,$offset)=("%eax","%ecx","%rbp","%rbx");
  87
  88 $code.=<<___;
  89 .text
  90
  91 .extern OPENSSL_ia32cap_P
  92
  93 .globl  aesni_multi_cbc_encrypt
  94 .type   aesni_multi_cbc_encrypt,\@function,3
  95 .align  32
  96 aesni_multi_cbc_encrypt:
  97 ___
  98 $code.=<<___ if ($avx);
  99         cmp     \$2,$num
 100         jb      .Lenc_non_avx
 101         mov     OPENSSL_ia32cap_P+4(%rip),%ecx
 102         test    \$`1<<28`,%ecx                  # AVX bit
 103         jnz     _avx_cbc_enc_shortcut
 104         jmp     .Lenc_non_avx
 105 .align  16
 106 .Lenc_non_avx:
 107 ___
 108 $code.=<<___;
 109         mov     %rsp,%rax
 110         push    %rbx
 111         push    %rbp
 112         push    %r12
 113         push    %r13
 114         push    %r14
 115         push    %r15
 116 ___
 117 $code.=<<___ if ($win64);
 118         lea     -0xa8(%rsp),%rsp
 119         movaps  %xmm6,(%rsp)
 120         movaps  %xmm7,0x10(%rsp)
 121         movaps  %xmm8,0x20(%rsp)
 122         movaps  %xmm9,0x30(%rsp)
 123         movaps  %xmm10,0x40(%rsp)
 124         movaps  %xmm11,0x50(%rsp)
 125         movaps  %xmm12,0x60(%rsp)
 126         movaps  %xmm13,-0x68(%rax)      # not used, saved to share se_handler
 127         movaps  %xmm14,-0x58(%rax)
 128         movaps  %xmm15,-0x48(%rax)
 129 ___
 130 $code.=<<___;
 131         # stack layout
 132         #
 133         # +0    output sink
 134         # +16   input sink [original %rsp and $num]
 135         # +32   counters
 136
 137         sub     \$48,%rsp
 138         and     \$-64,%rsp
 139         mov     %rax,16(%rsp)                   # original %rsp
 140
 141 .Lenc4x_body:
 142         movdqu  ($key),$zero                    # 0-round key
 143         lea     0x78($key),$key                 # size optimization
 144         lea     40*2($inp),$inp
 145
 146 .Lenc4x_loop_grande:
 147         mov     $num,24(%rsp)                   # original $num
 148         xor     $num,$num
 149 ___
 150 for($i=0;$i<4;$i++) {
 151     $code.=<<___;
 152         mov     `40*$i+16-40*2`($inp),$one      # borrow $one for number of blocks
 153         mov     `40*$i+0-40*2`($inp),@inptr[$i]
 154         cmp     $num,$one
 155         mov     `40*$i+8-40*2`($inp),@outptr[$i]
 156         cmovg   $one,$num                       # find maximum
 157         test    $one,$one
 158         movdqu  `40*$i+24-40*2`($inp),@out[$i]  # load IV
 159         mov     $one,`32+4*$i`(%rsp)            # initialize counters
 160         cmovle  %rsp,@inptr[$i]                 # cancel input
 161 ___
 162 }
 163 $code.=<<___;
 164         test    $num,$num
 165         jz      .Lenc4x_done
 166
 167         movups  0x10-0x78($key),$rndkey1
 168          pxor   $zero,@out[0]
 169         movups  0x20-0x78($key),$rndkey0
 170          pxor   $zero,@out[1]
 171         mov     0xf0-0x78($key),$rounds
 172          pxor   $zero,@out[2]
 173         movdqu  (@inptr[0]),@inp[0]             # load inputs
 174          pxor   $zero,@out[3]
 175         movdqu  (@inptr[1]),@inp[1]
 176          pxor   @inp[0],@out[0]
 177         movdqu  (@inptr[2]),@inp[2]
 178          pxor   @inp[1],@out[1]
 179         movdqu  (@inptr[3]),@inp[3]
 180          pxor   @inp[2],@out[2]
 181          pxor   @inp[3],@out[3]
 182         movdqa  32(%rsp),$counters              # load counters
 183         xor     $offset,$offset
 184         jmp     .Loop_enc4x
 185
 186 .align  32
 187 .Loop_enc4x:
 188         add     \$16,$offset
 189         lea     16(%rsp),$sink                  # sink pointer
 190         mov     \$1,$one                        # constant of 1
 191         sub     $offset,$sink
 192
 193         aesenc          $rndkey1,@out[0]
 194         prefetcht0      31(@inptr[0],$offset)   # prefetch input
 195         prefetcht0      31(@inptr[1],$offset)
 196         aesenc          $rndkey1,@out[1]
 197         prefetcht0      31(@inptr[2],$offset)
 198         prefetcht0      31(@inptr[2],$offset)
 199         aesenc          $rndkey1,@out[2]
 200         aesenc          $rndkey1,@out[3]
 201         movups          0x30-0x78($key),$rndkey1
 202 ___
 203 for($i=0;$i<4;$i++) {
 204 my $rndkey = ($i&1) ? $rndkey1 : $rndkey0;
 205 $code.=<<___;
 206          cmp            `32+4*$i`(%rsp),$one
 207         aesenc          $rndkey,@out[0]
 208         aesenc          $rndkey,@out[1]
 209         aesenc          $rndkey,@out[2]
 210          cmovge         $sink,@inptr[$i]        # cancel input
 211          cmovg          $sink,@outptr[$i]       # sink output
 212         aesenc          $rndkey,@out[3]
 213         movups          `0x40+16*$i-0x78`($key),$rndkey
 214 ___
 215 }
 216 $code.=<<___;
 217          movdqa         $counters,$mask
 218         aesenc          $rndkey0,@out[0]
 219         prefetcht0      15(@outptr[0],$offset)  # prefetch output
 220         prefetcht0      15(@outptr[1],$offset)
 221         aesenc          $rndkey0,@out[1]
 222         prefetcht0      15(@outptr[2],$offset)
 223         prefetcht0      15(@outptr[3],$offset)
 224         aesenc          $rndkey0,@out[2]
 225         aesenc          $rndkey0,@out[3]
 226         movups          0x80-0x78($key),$rndkey0
 227          pxor           $zero,$zero
 228
 229         aesenc          $rndkey1,@out[0]
 230          pcmpgtd        $zero,$mask
 231          movdqu         -0x78($key),$zero       # reload 0-round key
 232         aesenc          $rndkey1,@out[1]
 233          paddd          $mask,$counters         # decrement counters
 234          movdqa         $counters,32(%rsp)      # update counters
 235         aesenc          $rndkey1,@out[2]
 236         aesenc          $rndkey1,@out[3]
 237         movups          0x90-0x78($key),$rndkey1
 238
 239         cmp     \$11,$rounds
 240
 241         aesenc          $rndkey0,@out[0]
 242         aesenc          $rndkey0,@out[1]
 243         aesenc          $rndkey0,@out[2]
 244         aesenc          $rndkey0,@out[3]
 245         movups          0xa0-0x78($key),$rndkey0
 246
 247         jb      .Lenc4x_tail
 248
 249         aesenc          $rndkey1,@out[0]
 250         aesenc          $rndkey1,@out[1]
 251         aesenc          $rndkey1,@out[2]
 252         aesenc          $rndkey1,@out[3]
 253         movups          0xb0-0x78($key),$rndkey1
 254
 255         aesenc          $rndkey0,@out[0]
 256         aesenc          $rndkey0,@out[1]
 257         aesenc          $rndkey0,@out[2]
 258         aesenc          $rndkey0,@out[3]
 259         movups          0xc0-0x78($key),$rndkey0
 260
 261         je      .Lenc4x_tail
 262
 263         aesenc          $rndkey1,@out[0]
 264         aesenc          $rndkey1,@out[1]
 265         aesenc          $rndkey1,@out[2]
 266         aesenc          $rndkey1,@out[3]
 267         movups          0xd0-0x78($key),$rndkey1
 268
 269         aesenc          $rndkey0,@out[0]
 270         aesenc          $rndkey0,@out[1]
 271         aesenc          $rndkey0,@out[2]
 272         aesenc          $rndkey0,@out[3]
 273         movups          0xe0-0x78($key),$rndkey0
 274         jmp     .Lenc4x_tail
 275
 276 .align  32
 277 .Lenc4x_tail:
 278         aesenc          $rndkey1,@out[0]
 279         aesenc          $rndkey1,@out[1]
 280         aesenc          $rndkey1,@out[2]
 281         aesenc          $rndkey1,@out[3]
 282          movdqu         (@inptr[0],$offset),@inp[0]
 283         movdqu          0x10-0x78($key),$rndkey1
 284
 285         aesenclast      $rndkey0,@out[0]
 286          movdqu         (@inptr[1],$offset),@inp[1]
 287          pxor           $zero,@inp[0]
 288         aesenclast      $rndkey0,@out[1]
 289          movdqu         (@inptr[2],$offset),@inp[2]
 290          pxor           $zero,@inp[1]
 291         aesenclast      $rndkey0,@out[2]
 292          movdqu         (@inptr[3],$offset),@inp[3]
 293          pxor           $zero,@inp[2]
 294         aesenclast      $rndkey0,@out[3]
 295         movdqu          0x20-0x78($key),$rndkey0
 296          pxor           $zero,@inp[3]
 297
 298         movups          @out[0],-16(@outptr[0],$offset)
 299          pxor           @inp[0],@out[0]
 300         movups          @out[1],-16(@outptr[1],$offset)
 301          pxor           @inp[1],@out[1]
 302         movups          @out[2],-16(@outptr[2],$offset)
 303          pxor           @inp[2],@out[2]
 304         movups          @out[3],-16(@outptr[3],$offset)
 305          pxor           @inp[3],@out[3]
 306
 307         dec     $num
 308         jnz     .Loop_enc4x
 309
 310         mov     16(%rsp),%rax                   # original %rsp
 311         mov     24(%rsp),$num
 312
 313         #pxor   @inp[0],@out[0]
 314         #pxor   @inp[1],@out[1]
 315         #movdqu @out[0],`40*0+24-40*2`($inp)    # output iv FIX ME!
 316         #pxor   @inp[2],@out[2]
 317         #movdqu @out[1],`40*1+24-40*2`($inp)
 318         #pxor   @inp[3],@out[3]
 319         #movdqu @out[2],`40*2+24-40*2`($inp)    # won't fix, let caller
 320         #movdqu @out[3],`40*3+24-40*2`($inp)    # figure this out...
 321
 322         lea     `40*4`($inp),$inp
 323         dec     $num
 324         jnz     .Lenc4x_loop_grande
 325
 326 .Lenc4x_done:
 327 ___
 328 $code.=<<___ if ($win64);
 329         movaps  -0xd8(%rax),%xmm6
 330         movaps  -0xc8(%rax),%xmm7
 331         movaps  -0xb8(%rax),%xmm8
 332         movaps  -0xa8(%rax),%xmm9
 333         movaps  -0x98(%rax),%xmm10
 334         movaps  -0x88(%rax),%xmm11
 335         movaps  -0x78(%rax),%xmm12
 336         #movaps -0x68(%rax),%xmm13
 337         #movaps -0x58(%rax),%xmm14
 338         #movaps -0x48(%rax),%xmm15
 339 ___
 340 $code.=<<___;
 341         mov     -48(%rax),%r15
 342         mov     -40(%rax),%r14
 343         mov     -32(%rax),%r13
 344         mov     -24(%rax),%r12
 345         mov     -16(%rax),%rbp
 346         mov     -8(%rax),%rbx
 347         lea     (%rax),%rsp
 348 .Lenc4x_epilogue:
 349         ret
 350 .size   aesni_multi_cbc_encrypt,.-aesni_multi_cbc_encrypt
 351
 352 .globl  aesni_multi_cbc_decrypt
 353 .type   aesni_multi_cbc_decrypt,\@function,3
 354 .align  32
 355 aesni_multi_cbc_decrypt:
 356 ___
 357 $code.=<<___ if ($avx);
 358         cmp     \$2,$num
 359         jb      .Ldec_non_avx
 360         mov     OPENSSL_ia32cap_P+4(%rip),%ecx
 361         test    \$`1<<28`,%ecx                  # AVX bit
 362         jnz     _avx_cbc_dec_shortcut
 363         jmp     .Ldec_non_avx
 364 .align  16
 365 .Ldec_non_avx:
 366 ___
 367 $code.=<<___;
 368         mov     %rsp,%rax
 369         push    %rbx
 370         push    %rbp
 371         push    %r12
 372         push    %r13
 373         push    %r14
 374         push    %r15
 375 ___
 376 $code.=<<___ if ($win64);
 377         lea     -0xa8(%rsp),%rsp
 378         movaps  %xmm6,(%rsp)
 379         movaps  %xmm7,0x10(%rsp)
 380         movaps  %xmm8,0x20(%rsp)
 381         movaps  %xmm9,0x30(%rsp)
 382         movaps  %xmm10,0x40(%rsp)
 383         movaps  %xmm11,0x50(%rsp)
 384         movaps  %xmm12,0x60(%rsp)
 385         movaps  %xmm13,-0x68(%rax)      # not used, saved to share se_handler
 386         movaps  %xmm14,-0x58(%rax)
 387         movaps  %xmm15,-0x48(%rax)
 388 ___
 389 $code.=<<___;
 390         # stack layout
 391         #
 392         # +0    output sink
 393         # +16   input sink [original %rsp and $num]
 394         # +32   counters
 395
 396         sub     \$48,%rsp
 397         and     \$-64,%rsp
 398         mov     %rax,16(%rsp)                   # original %rsp
 399
 400 .Ldec4x_body:
 401         movdqu  ($key),$zero                    # 0-round key
 402         lea     0x78($key),$key                 # size optimization
 403         lea     40*2($inp),$inp
 404
 405 .Ldec4x_loop_grande:
 406         mov     $num,24(%rsp)                   # original $num
 407         xor     $num,$num
 408 ___
 409 for($i=0;$i<4;$i++) {
 410     $code.=<<___;
 411         mov     `40*$i+16-40*2`($inp),$one      # borrow $one for number of blocks
 412         mov     `40*$i+0-40*2`($inp),@inptr[$i]
 413         cmp     $num,$one
 414         mov     `40*$i+8-40*2`($inp),@outptr[$i]
 415         cmovg   $one,$num                       # find maximum
 416         test    $one,$one
 417         movdqu  `40*$i+24-40*2`($inp),@inp[$i]  # load IV
 418         mov     $one,`32+4*$i`(%rsp)            # initialize counters
 419         cmovle  %rsp,@inptr[$i]                 # cancel input
 420 ___
 421 }
 422 $code.=<<___;
 423         test    $num,$num
 424         jz      .Ldec4x_done
 425
 426         movups  0x10-0x78($key),$rndkey1
 427         movups  0x20-0x78($key),$rndkey0
 428         mov     0xf0-0x78($key),$rounds
 429         movdqu  (@inptr[0]),@out[0]             # load inputs
 430         movdqu  (@inptr[1]),@out[1]
 431          pxor   $zero,@out[0]
 432         movdqu  (@inptr[2]),@out[2]
 433          pxor   $zero,@out[1]
 434         movdqu  (@inptr[3]),@out[3]
 435          pxor   $zero,@out[2]
 436          pxor   $zero,@out[3]
 437         movdqa  32(%rsp),$counters              # load counters
 438         xor     $offset,$offset
 439         jmp     .Loop_dec4x
 440
 441 .align  32
 442 .Loop_dec4x:
 443         add     \$16,$offset
 444         lea     16(%rsp),$sink                  # sink pointer
 445         mov     \$1,$one                        # constant of 1
 446         sub     $offset,$sink
 447
 448         aesdec          $rndkey1,@out[0]
 449         prefetcht0      31(@inptr[0],$offset)   # prefetch input
 450         prefetcht0      31(@inptr[1],$offset)
 451         aesdec          $rndkey1,@out[1]
 452         prefetcht0      31(@inptr[2],$offset)
 453         prefetcht0      31(@inptr[3],$offset)
 454         aesdec          $rndkey1,@out[2]
 455         aesdec          $rndkey1,@out[3]
 456         movups          0x30-0x78($key),$rndkey1
 457 ___
 458 for($i=0;$i<4;$i++) {
 459 my $rndkey = ($i&1) ? $rndkey1 : $rndkey0;
 460 $code.=<<___;
 461          cmp            `32+4*$i`(%rsp),$one
 462         aesdec          $rndkey,@out[0]
 463         aesdec          $rndkey,@out[1]
 464         aesdec          $rndkey,@out[2]
 465          cmovge         $sink,@inptr[$i]        # cancel input
 466          cmovg          $sink,@outptr[$i]       # sink output
 467         aesdec          $rndkey,@out[3]
 468         movups          `0x40+16*$i-0x78`($key),$rndkey
 469 ___
 470 }
 471 $code.=<<___;
 472          movdqa         $counters,$mask
 473         aesdec          $rndkey0,@out[0]
 474         prefetcht0      15(@outptr[0],$offset)  # prefetch output
 475         prefetcht0      15(@outptr[1],$offset)
 476         aesdec          $rndkey0,@out[1]
 477         prefetcht0      15(@outptr[2],$offset)
 478         prefetcht0      15(@outptr[3],$offset)
 479         aesdec          $rndkey0,@out[2]
 480         aesdec          $rndkey0,@out[3]
 481         movups          0x80-0x78($key),$rndkey0
 482          pxor           $zero,$zero
 483
 484         aesdec          $rndkey1,@out[0]
 485          pcmpgtd        $zero,$mask
 486          movdqu         -0x78($key),$zero       # reload 0-round key
 487         aesdec          $rndkey1,@out[1]
 488          paddd          $mask,$counters         # decrement counters
 489          movdqa         $counters,32(%rsp)      # update counters
 490         aesdec          $rndkey1,@out[2]
 491         aesdec          $rndkey1,@out[3]
 492         movups          0x90-0x78($key),$rndkey1
 493
 494         cmp     \$11,$rounds
 495
 496         aesdec          $rndkey0,@out[0]
 497         aesdec          $rndkey0,@out[1]
 498         aesdec          $rndkey0,@out[2]
 499         aesdec          $rndkey0,@out[3]
 500         movups          0xa0-0x78($key),$rndkey0
 501
 502         jb      .Ldec4x_tail
 503
 504         aesdec          $rndkey1,@out[0]
 505         aesdec          $rndkey1,@out[1]
 506         aesdec          $rndkey1,@out[2]
 507         aesdec          $rndkey1,@out[3]
 508         movups          0xb0-0x78($key),$rndkey1
 509
 510         aesdec          $rndkey0,@out[0]
 511         aesdec          $rndkey0,@out[1]
 512         aesdec          $rndkey0,@out[2]
 513         aesdec          $rndkey0,@out[3]
 514         movups          0xc0-0x78($key),$rndkey0
 515
 516         je      .Ldec4x_tail
 517
 518         aesdec          $rndkey1,@out[0]
 519         aesdec          $rndkey1,@out[1]
 520         aesdec          $rndkey1,@out[2]
 521         aesdec          $rndkey1,@out[3]
 522         movups          0xd0-0x78($key),$rndkey1
 523
 524         aesdec          $rndkey0,@out[0]
 525         aesdec          $rndkey0,@out[1]
 526         aesdec          $rndkey0,@out[2]
 527         aesdec          $rndkey0,@out[3]
 528         movups          0xe0-0x78($key),$rndkey0
 529         jmp     .Ldec4x_tail
 530
 531 .align  32
 532 .Ldec4x_tail:
 533         aesdec          $rndkey1,@out[0]
 534         aesdec          $rndkey1,@out[1]
 535         aesdec          $rndkey1,@out[2]
 536          pxor           $rndkey0,@inp[0]
 537          pxor           $rndkey0,@inp[1]
 538         aesdec          $rndkey1,@out[3]
 539         movdqu          0x10-0x78($key),$rndkey1
 540          pxor           $rndkey0,@inp[2]
 541          pxor           $rndkey0,@inp[3]
 542         movdqu          0x20-0x78($key),$rndkey0
 543
 544         aesdeclast      @inp[0],@out[0]
 545         aesdeclast      @inp[1],@out[1]
 546          movdqu         -16(@inptr[0],$offset),@inp[0]  # load next IV
 547          movdqu         -16(@inptr[1],$offset),@inp[1]
 548         aesdeclast      @inp[2],@out[2]
 549         aesdeclast      @inp[3],@out[3]
 550          movdqu         -16(@inptr[2],$offset),@inp[2]
 551          movdqu         -16(@inptr[3],$offset),@inp[3]
 552
 553         movups          @out[0],-16(@outptr[0],$offset)
 554          movdqu         (@inptr[0],$offset),@out[0]
 555         movups          @out[1],-16(@outptr[1],$offset)
 556          movdqu         (@inptr[1],$offset),@out[1]
 557          pxor           $zero,@out[0]
 558         movups          @out[2],-16(@outptr[2],$offset)
 559          movdqu         (@inptr[2],$offset),@out[2]
 560          pxor           $zero,@out[1]
 561         movups          @out[3],-16(@outptr[3],$offset)
 562          movdqu         (@inptr[3],$offset),@out[3]
 563          pxor           $zero,@out[2]
 564          pxor           $zero,@out[3]
 565
 566         dec     $num
 567         jnz     .Loop_dec4x
 568
 569         mov     16(%rsp),%rax                   # original %rsp
 570         mov     24(%rsp),$num
 571
 572         lea     `40*4`($inp),$inp
 573         dec     $num
 574         jnz     .Ldec4x_loop_grande
 575
 576 .Ldec4x_done:
 577 ___
 578 $code.=<<___ if ($win64);
 579         movaps  -0xd8(%rax),%xmm6
 580         movaps  -0xc8(%rax),%xmm7
 581         movaps  -0xb8(%rax),%xmm8
 582         movaps  -0xa8(%rax),%xmm9
 583         movaps  -0x98(%rax),%xmm10
 584         movaps  -0x88(%rax),%xmm11
 585         movaps  -0x78(%rax),%xmm12
 586         #movaps -0x68(%rax),%xmm13
 587         #movaps -0x58(%rax),%xmm14
 588         #movaps -0x48(%rax),%xmm15
 589 ___
 590 $code.=<<___;
 591         mov     -48(%rax),%r15
 592         mov     -40(%rax),%r14
 593         mov     -32(%rax),%r13
 594         mov     -24(%rax),%r12
 595         mov     -16(%rax),%rbp
 596         mov     -8(%rax),%rbx
 597         lea     (%rax),%rsp
 598 .Ldec4x_epilogue:
 599         ret
 600 .size   aesni_multi_cbc_decrypt,.-aesni_multi_cbc_decrypt
 601 ___
 602
 603                                                 if ($avx) {{{
 604 my @ptr=map("%r$_",(8..15));
 605 my $offload=$sink;
 606
 607 my @out=map("%xmm$_",(2..9));
 608 my @inp=map("%xmm$_",(10..13));
 609 my ($counters,$zero)=("%xmm14","%xmm15");
 610
 611 $code.=<<___;
 612 .type   aesni_multi_cbc_encrypt_avx,\@function,3
 613 .align  32
 614 aesni_multi_cbc_encrypt_avx:
 615 _avx_cbc_enc_shortcut:
 616         mov     %rsp,%rax
 617         push    %rbx
 618         push    %rbp
 619         push    %r12
 620         push    %r13
 621         push    %r14
 622         push    %r15
 623 ___
 624 $code.=<<___ if ($win64);
 625         lea     -0xa8(%rsp),%rsp
 626         movaps  %xmm6,(%rsp)
 627         movaps  %xmm7,0x10(%rsp)
 628         movaps  %xmm8,0x20(%rsp)
 629         movaps  %xmm9,0x30(%rsp)
 630         movaps  %xmm10,0x40(%rsp)
 631         movaps  %xmm11,0x50(%rsp)
 632         movaps  %xmm12,-0x78(%rax)
 633         movaps  %xmm13,-0x68(%rax)
 634         movaps  %xmm14,-0x58(%rax)
 635         movaps  %xmm15,-0x48(%rax)
 636 ___
 637 $code.=<<___;
 638         # stack layout
 639         #
 640         # +0    output sink
 641         # +16   input sink [original %rsp and $num]
 642         # +32   counters
 643         # +64   distances between inputs and outputs
 644         # +128  off-load area for @inp[0..3]
 645
 646         sub     \$192,%rsp
 647         and     \$-128,%rsp
 648         mov     %rax,16(%rsp)                   # original %rsp
 649
 650 .Lenc8x_body:
 651         vzeroupper
 652         vmovdqu ($key),$zero                    # 0-round key
 653         lea     0x78($key),$key                 # size optimization
 654         lea     40*4($inp),$inp
 655         shr     \$1,$num
 656
 657 .Lenc8x_loop_grande:
 658         #mov    $num,24(%rsp)                   # original $num
 659         xor     $num,$num
 660 ___
 661 for($i=0;$i<8;$i++) {
 662   my $temp = $i ? $offload : $offset;
 663     $code.=<<___;
 664         mov     `40*$i+16-40*4`($inp),$one      # borrow $one for number of blocks
 665         mov     `40*$i+0-40*4`($inp),@ptr[$i]   # input pointer
 666         cmp     $num,$one
 667         mov     `40*$i+8-40*4`($inp),$temp      # output pointer
 668         cmovg   $one,$num                       # find maximum
 669         test    $one,$one
 670         vmovdqu `40*$i+24-40*4`($inp),@out[$i]  # load IV
 671         mov     $one,`32+4*$i`(%rsp)            # initialize counters
 672         cmovle  %rsp,@ptr[$i]                   # cancel input
 673         sub     @ptr[$i],$temp                  # distance between input and output
 674         mov     $temp,`64+8*$i`(%rsp)           # initialize distances
 675 ___
 676 }
 677 $code.=<<___;
 678         test    $num,$num
 679         jz      .Lenc8x_done
 680
 681         vmovups 0x10-0x78($key),$rndkey1
 682         vmovups 0x20-0x78($key),$rndkey0
 683         mov     0xf0-0x78($key),$rounds
 684
 685         vpxor   (@ptr[0]),$zero,@inp[0]         # load inputs and xor with 0-round
 686          lea    128(%rsp),$offload              # offload area
 687         vpxor   (@ptr[1]),$zero,@inp[1]
 688         vpxor   (@ptr[2]),$zero,@inp[2]
 689         vpxor   (@ptr[3]),$zero,@inp[3]
 690          vpxor  @inp[0],@out[0],@out[0]
 691         vpxor   (@ptr[4]),$zero,@inp[0]
 692          vpxor  @inp[1],@out[1],@out[1]
 693         vpxor   (@ptr[5]),$zero,@inp[1]
 694          vpxor  @inp[2],@out[2],@out[2]
 695         vpxor   (@ptr[6]),$zero,@inp[2]
 696          vpxor  @inp[3],@out[3],@out[3]
 697         vpxor   (@ptr[7]),$zero,@inp[3]
 698          vpxor  @inp[0],@out[4],@out[4]
 699         mov     \$1,$one                        # constant of 1
 700          vpxor  @inp[1],@out[5],@out[5]
 701          vpxor  @inp[2],@out[6],@out[6]
 702          vpxor  @inp[3],@out[7],@out[7]
 703         jmp     .Loop_enc8x
 704
 705 .align  32
 706 .Loop_enc8x:
 707 ___
 708 for($i=0;$i<8;$i++) {
 709 my $rndkey=($i&1)?$rndkey0:$rndkey1;
 710 $code.=<<___;
 711         vaesenc         $rndkey,@out[0],@out[0]
 712          cmp            32+4*$i(%rsp),$one
 713 ___
 714 $code.=<<___ if ($i);
 715          mov            64+8*$i(%rsp),$offset
 716 ___
 717 $code.=<<___;
 718         vaesenc         $rndkey,@out[1],@out[1]
 719         prefetcht0      31(@ptr[$i])                    # prefetch input
 720         vaesenc         $rndkey,@out[2],@out[2]
 721 ___
 722 $code.=<<___ if ($i>1);
 723         prefetcht0      15(@ptr[$i-2])                  # prefetch output
 724 ___
 725 $code.=<<___;
 726         vaesenc         $rndkey,@out[3],@out[3]
 727          lea            (@ptr[$i],$offset),$offset
 728          cmovge         %rsp,@ptr[$i]                   # cancel input
 729         vaesenc         $rndkey,@out[4],@out[4]
 730          cmovg          %rsp,$offset                    # sink output
 731         vaesenc         $rndkey,@out[5],@out[5]
 732          sub            @ptr[$i],$offset
 733         vaesenc         $rndkey,@out[6],@out[6]
 734          vpxor          16(@ptr[$i]),$zero,@inp[$i%4]   # load input and xor with 0-round
 735          mov            $offset,64+8*$i(%rsp)
 736         vaesenc         $rndkey,@out[7],@out[7]
 737         vmovups         `16*(3+$i)-0x78`($key),$rndkey
 738          lea            16(@ptr[$i],$offset),@ptr[$i]   # switch to output
 739 ___
 740 $code.=<<___ if ($i<4)
 741          vmovdqu        @inp[$i%4],`16*$i`($offload)    # off-load
 742 ___
 743 }
 744 $code.=<<___;
 745          vmovdqu        32(%rsp),$counters
 746         prefetcht0      15(@ptr[$i-2])                  # prefetch output
 747         prefetcht0      15(@ptr[$i-1])
 748         cmp     \$11,$rounds
 749         jb      .Lenc8x_tail
 750
 751         vaesenc         $rndkey1,@out[0],@out[0]
 752         vaesenc         $rndkey1,@out[1],@out[1]
 753         vaesenc         $rndkey1,@out[2],@out[2]
 754         vaesenc         $rndkey1,@out[3],@out[3]
 755         vaesenc         $rndkey1,@out[4],@out[4]
 756         vaesenc         $rndkey1,@out[5],@out[5]
 757         vaesenc         $rndkey1,@out[6],@out[6]
 758         vaesenc         $rndkey1,@out[7],@out[7]
 759         vmovups         0xb0-0x78($key),$rndkey1
 760
 761         vaesenc         $rndkey0,@out[0],@out[0]
 762         vaesenc         $rndkey0,@out[1],@out[1]
 763         vaesenc         $rndkey0,@out[2],@out[2]
 764         vaesenc         $rndkey0,@out[3],@out[3]
 765         vaesenc         $rndkey0,@out[4],@out[4]
 766         vaesenc         $rndkey0,@out[5],@out[5]
 767         vaesenc         $rndkey0,@out[6],@out[6]
 768         vaesenc         $rndkey0,@out[7],@out[7]
 769         vmovups         0xc0-0x78($key),$rndkey0
 770         je      .Lenc8x_tail
 771
 772         vaesenc         $rndkey1,@out[0],@out[0]
 773         vaesenc         $rndkey1,@out[1],@out[1]
 774         vaesenc         $rndkey1,@out[2],@out[2]
 775         vaesenc         $rndkey1,@out[3],@out[3]
 776         vaesenc         $rndkey1,@out[4],@out[4]
 777         vaesenc         $rndkey1,@out[5],@out[5]
 778         vaesenc         $rndkey1,@out[6],@out[6]
 779         vaesenc         $rndkey1,@out[7],@out[7]
 780         vmovups         0xd0-0x78($key),$rndkey1
 781
 782         vaesenc         $rndkey0,@out[0],@out[0]
 783         vaesenc         $rndkey0,@out[1],@out[1]
 784         vaesenc         $rndkey0,@out[2],@out[2]
 785         vaesenc         $rndkey0,@out[3],@out[3]
 786         vaesenc         $rndkey0,@out[4],@out[4]
 787         vaesenc         $rndkey0,@out[5],@out[5]
 788         vaesenc         $rndkey0,@out[6],@out[6]
 789         vaesenc         $rndkey0,@out[7],@out[7]
 790         vmovups         0xe0-0x78($key),$rndkey0
 791
 792 .Lenc8x_tail:
 793         vaesenc         $rndkey1,@out[0],@out[0]
 794          vpxor          $zero,$zero,$zero
 795         vaesenc         $rndkey1,@out[1],@out[1]
 796         vaesenc         $rndkey1,@out[2],@out[2]
 797          vpcmpgtd       $zero,$counters,$zero
 798         vaesenc         $rndkey1,@out[3],@out[3]
 799         vaesenc         $rndkey1,@out[4],@out[4]
 800          vpaddd         $counters,$zero,$zero           # decrement counters
 801          vmovdqu        48(%rsp),$counters
 802         vaesenc         $rndkey1,@out[5],@out[5]
 803          mov            64(%rsp),$offset                # pre-load 1st offset
 804         vaesenc         $rndkey1,@out[6],@out[6]
 805         vaesenc         $rndkey1,@out[7],@out[7]
 806         vmovups         0x10-0x78($key),$rndkey1
 807
 808         vaesenclast     $rndkey0,@out[0],@out[0]
 809          vmovdqa        $zero,32(%rsp)                  # update counters
 810          vpxor          $zero,$zero,$zero
 811         vaesenclast     $rndkey0,@out[1],@out[1]
 812         vaesenclast     $rndkey0,@out[2],@out[2]
 813          vpcmpgtd       $zero,$counters,$zero
 814         vaesenclast     $rndkey0,@out[3],@out[3]
 815         vaesenclast     $rndkey0,@out[4],@out[4]
 816          vpaddd         $zero,$counters,$counters       # decrement counters
 817          vmovdqu        -0x78($key),$zero               # 0-round
 818         vaesenclast     $rndkey0,@out[5],@out[5]
 819         vaesenclast     $rndkey0,@out[6],@out[6]
 820          vmovdqa        $counters,48(%rsp)              # update counters
 821         vaesenclast     $rndkey0,@out[7],@out[7]
 822         vmovups         0x20-0x78($key),$rndkey0
 823
 824         vmovups         @out[0],-16(@ptr[0])            # write output
 825          sub            $offset,@ptr[0]                 # switch to input
 826          vpxor          0x00($offload),@out[0],@out[0]
 827         vmovups         @out[1],-16(@ptr[1])
 828          sub            `64+1*8`(%rsp),@ptr[1]
 829          vpxor          0x10($offload),@out[1],@out[1]
 830         vmovups         @out[2],-16(@ptr[2])
 831          sub            `64+2*8`(%rsp),@ptr[2]
 832          vpxor          0x20($offload),@out[2],@out[2]
 833         vmovups         @out[3],-16(@ptr[3])
 834          sub            `64+3*8`(%rsp),@ptr[3]
 835          vpxor          0x30($offload),@out[3],@out[3]
 836         vmovups         @out[4],-16(@ptr[4])
 837          sub            `64+4*8`(%rsp),@ptr[4]
 838          vpxor          @inp[0],@out[4],@out[4]
 839         vmovups         @out[5],-16(@ptr[5])
 840          sub            `64+5*8`(%rsp),@ptr[5]
 841          vpxor          @inp[1],@out[5],@out[5]
 842         vmovups         @out[6],-16(@ptr[6])
 843          sub            `64+6*8`(%rsp),@ptr[6]
 844          vpxor          @inp[2],@out[6],@out[6]
 845         vmovups         @out[7],-16(@ptr[7])
 846          sub            `64+7*8`(%rsp),@ptr[7]
 847          vpxor          @inp[3],@out[7],@out[7]
 848
 849         dec     $num
 850         jnz     .Loop_enc8x
 851
 852         mov     16(%rsp),%rax                   # original %rsp
 853         #mov    24(%rsp),$num
 854         #lea    `40*8`($inp),$inp
 855         #dec    $num
 856         #jnz    .Lenc8x_loop_grande
 857
 858 .Lenc8x_done:
 859         vzeroupper
 860 ___
 861 $code.=<<___ if ($win64);
 862         movaps  -0xd8(%rax),%xmm6
 863         movaps  -0xc8(%rax),%xmm7
 864         movaps  -0xb8(%rax),%xmm8
 865         movaps  -0xa8(%rax),%xmm9
 866         movaps  -0x98(%rax),%xmm10
 867         movaps  -0x88(%rax),%xmm11
 868         movaps  -0x78(%rax),%xmm12
 869         movaps  -0x68(%rax),%xmm13
 870         movaps  -0x58(%rax),%xmm14
 871         movaps  -0x48(%rax),%xmm15
 872 ___
 873 $code.=<<___;
 874         mov     -48(%rax),%r15
 875         mov     -40(%rax),%r14
 876         mov     -32(%rax),%r13
 877         mov     -24(%rax),%r12
 878         mov     -16(%rax),%rbp
 879         mov     -8(%rax),%rbx
 880         lea     (%rax),%rsp
 881 .Lenc8x_epilogue:
 882         ret
 883 .size   aesni_multi_cbc_encrypt_avx,.-aesni_multi_cbc_encrypt_avx
 884
 885 .type   aesni_multi_cbc_decrypt_avx,\@function,3
 886 .align  32
 887 aesni_multi_cbc_decrypt_avx:
 888 _avx_cbc_dec_shortcut:
 889         mov     %rsp,%rax
 890         push    %rbx
 891         push    %rbp
 892         push    %r12
 893         push    %r13
 894         push    %r14
 895         push    %r15
 896 ___
 897 $code.=<<___ if ($win64);
 898         lea     -0xa8(%rsp),%rsp
 899         movaps  %xmm6,(%rsp)
 900         movaps  %xmm7,0x10(%rsp)
 901         movaps  %xmm8,0x20(%rsp)
 902         movaps  %xmm9,0x30(%rsp)
 903         movaps  %xmm10,0x40(%rsp)
 904         movaps  %xmm11,0x50(%rsp)
 905         movaps  %xmm12,-0x78(%rax)
 906         movaps  %xmm13,-0x68(%rax)
 907         movaps  %xmm14,-0x58(%rax)
 908         movaps  %xmm15,-0x48(%rax)
 909 ___
 910 $code.=<<___;
 911         # stack layout
 912         #
 913         # +0    output sink
 914         # +16   input sink [original %rsp and $num]
 915         # +32   counters
 916         # +64   distances between inputs and outputs
 917         # +128  off-load area for @inp[0..3]
 918         # +192  IV/input offload
 919
 920         sub     \$256,%rsp
 921         and     \$-256,%rsp
 922         sub     \$192,%rsp
 923         mov     %rax,16(%rsp)                   # original %rsp
 924
 925 .Ldec8x_body:
 926         vzeroupper
 927         vmovdqu ($key),$zero                    # 0-round key
 928         lea     0x78($key),$key                 # size optimization
 929         lea     40*4($inp),$inp
 930         shr     \$1,$num
 931
 932 .Ldec8x_loop_grande:
 933         #mov    $num,24(%rsp)                   # original $num
 934         xor     $num,$num
 935 ___
 936 for($i=0;$i<8;$i++) {
 937   my $temp = $i ? $offload : $offset;
 938     $code.=<<___;
 939         mov     `40*$i+16-40*4`($inp),$one      # borrow $one for number of blocks
 940         mov     `40*$i+0-40*4`($inp),@ptr[$i]   # input pointer
 941         cmp     $num,$one
 942         mov     `40*$i+8-40*4`($inp),$temp      # output pointer
 943         cmovg   $one,$num                       # find maximum
 944         test    $one,$one
 945         vmovdqu `40*$i+24-40*4`($inp),@out[$i]  # load IV
 946         mov     $one,`32+4*$i`(%rsp)            # initialize counters
 947         cmovle  %rsp,@ptr[$i]                   # cancel input
 948         sub     @ptr[$i],$temp                  # distance between input and output
 949         mov     $temp,`64+8*$i`(%rsp)           # initialize distances
 950         vmovdqu @out[$i],`192+16*$i`(%rsp)      # offload IV
 951 ___
 952 }
 953 $code.=<<___;
 954         test    $num,$num
 955         jz      .Ldec8x_done
 956
 957         vmovups 0x10-0x78($key),$rndkey1
 958         vmovups 0x20-0x78($key),$rndkey0
 959         mov     0xf0-0x78($key),$rounds
 960          lea    192+128(%rsp),$offload          # offload area
 961
 962         vmovdqu (@ptr[0]),@out[0]               # load inputs
 963         vmovdqu (@ptr[1]),@out[1]
 964         vmovdqu (@ptr[2]),@out[2]
 965         vmovdqu (@ptr[3]),@out[3]
 966         vmovdqu (@ptr[4]),@out[4]
 967         vmovdqu (@ptr[5]),@out[5]
 968         vmovdqu (@ptr[6]),@out[6]
 969         vmovdqu (@ptr[7]),@out[7]
 970         vmovdqu @out[0],0x00($offload)          # offload inputs
 971         vpxor   $zero,@out[0],@out[0]           # xor inputs with 0-round
 972         vmovdqu @out[1],0x10($offload)
 973         vpxor   $zero,@out[1],@out[1]
 974         vmovdqu @out[2],0x20($offload)
 975         vpxor   $zero,@out[2],@out[2]
 976         vmovdqu @out[3],0x30($offload)
 977         vpxor   $zero,@out[3],@out[3]
 978         vmovdqu @out[4],0x40($offload)
 979         vpxor   $zero,@out[4],@out[4]
 980         vmovdqu @out[5],0x50($offload)
 981         vpxor   $zero,@out[5],@out[5]
 982         vmovdqu @out[6],0x60($offload)
 983         vpxor   $zero,@out[6],@out[6]
 984         vmovdqu @out[7],0x70($offload)
 985         vpxor   $zero,@out[7],@out[7]
 986         xor     \$0x80,$offload
 987         mov     \$1,$one                        # constant of 1
 988         jmp     .Loop_dec8x
 989
 990 .align  32
 991 .Loop_dec8x:
 992 ___
 993 for($i=0;$i<8;$i++) {
 994 my $rndkey=($i&1)?$rndkey0:$rndkey1;
 995 $code.=<<___;
 996         vaesdec         $rndkey,@out[0],@out[0]
 997          cmp            32+4*$i(%rsp),$one
 998 ___
 999 $code.=<<___ if ($i);
1000          mov            64+8*$i(%rsp),$offset
1001 ___
1002 $code.=<<___;
1003         vaesdec         $rndkey,@out[1],@out[1]
1004         prefetcht0      31(@ptr[$i])                    # prefetch input
1005         vaesdec         $rndkey,@out[2],@out[2]
1006 ___
1007 $code.=<<___ if ($i>1);
1008         prefetcht0      15(@ptr[$i-2])                  # prefetch output
1009 ___
1010 $code.=<<___;
1011         vaesdec         $rndkey,@out[3],@out[3]
1012          lea            (@ptr[$i],$offset),$offset
1013          cmovge         %rsp,@ptr[$i]                   # cancel input
1014         vaesdec         $rndkey,@out[4],@out[4]
1015          cmovg          %rsp,$offset                    # sink output
1016         vaesdec         $rndkey,@out[5],@out[5]
1017          sub            @ptr[$i],$offset
1018         vaesdec         $rndkey,@out[6],@out[6]
1019          vmovdqu        16(@ptr[$i]),@inp[$i%4]         # load input
1020          mov            $offset,64+8*$i(%rsp)
1021         vaesdec         $rndkey,@out[7],@out[7]
1022         vmovups         `16*(3+$i)-0x78`($key),$rndkey
1023          lea            16(@ptr[$i],$offset),@ptr[$i]   # switch to output
1024 ___
1025 $code.=<<___ if ($i<4);
1026          vmovdqu        @inp[$i%4],`128+16*$i`(%rsp)    # off-load
1027 ___
1028 }
1029 $code.=<<___;
1030          vmovdqu        32(%rsp),$counters
1031         prefetcht0      15(@ptr[$i-2])                  # prefetch output
1032         prefetcht0      15(@ptr[$i-1])
1033         cmp     \$11,$rounds
1034         jb      .Ldec8x_tail
1035
1036         vaesdec         $rndkey1,@out[0],@out[0]
1037         vaesdec         $rndkey1,@out[1],@out[1]
1038         vaesdec         $rndkey1,@out[2],@out[2]
1039         vaesdec         $rndkey1,@out[3],@out[3]
1040         vaesdec         $rndkey1,@out[4],@out[4]
1041         vaesdec         $rndkey1,@out[5],@out[5]
1042         vaesdec         $rndkey1,@out[6],@out[6]
1043         vaesdec         $rndkey1,@out[7],@out[7]
1044         vmovups         0xb0-0x78($key),$rndkey1
1045
1046         vaesdec         $rndkey0,@out[0],@out[0]
1047         vaesdec         $rndkey0,@out[1],@out[1]
1048         vaesdec         $rndkey0,@out[2],@out[2]
1049         vaesdec         $rndkey0,@out[3],@out[3]
1050         vaesdec         $rndkey0,@out[4],@out[4]
1051         vaesdec         $rndkey0,@out[5],@out[5]
1052         vaesdec         $rndkey0,@out[6],@out[6]
1053         vaesdec         $rndkey0,@out[7],@out[7]
1054         vmovups         0xc0-0x78($key),$rndkey0
1055         je      .Ldec8x_tail
1056
1057         vaesdec         $rndkey1,@out[0],@out[0]
1058         vaesdec         $rndkey1,@out[1],@out[1]
1059         vaesdec         $rndkey1,@out[2],@out[2]
1060         vaesdec         $rndkey1,@out[3],@out[3]
1061         vaesdec         $rndkey1,@out[4],@out[4]
1062         vaesdec         $rndkey1,@out[5],@out[5]
1063         vaesdec         $rndkey1,@out[6],@out[6]
1064         vaesdec         $rndkey1,@out[7],@out[7]
1065         vmovups         0xd0-0x78($key),$rndkey1
1066
1067         vaesdec         $rndkey0,@out[0],@out[0]
1068         vaesdec         $rndkey0,@out[1],@out[1]
1069         vaesdec         $rndkey0,@out[2],@out[2]
1070         vaesdec         $rndkey0,@out[3],@out[3]
1071         vaesdec         $rndkey0,@out[4],@out[4]
1072         vaesdec         $rndkey0,@out[5],@out[5]
1073         vaesdec         $rndkey0,@out[6],@out[6]
1074         vaesdec         $rndkey0,@out[7],@out[7]
1075         vmovups         0xe0-0x78($key),$rndkey0
1076
1077 .Ldec8x_tail:
1078         vaesdec         $rndkey1,@out[0],@out[0]
1079          vpxor          $zero,$zero,$zero
1080         vaesdec         $rndkey1,@out[1],@out[1]
1081         vaesdec         $rndkey1,@out[2],@out[2]
1082          vpcmpgtd       $zero,$counters,$zero
1083         vaesdec         $rndkey1,@out[3],@out[3]
1084         vaesdec         $rndkey1,@out[4],@out[4]
1085          vpaddd         $counters,$zero,$zero           # decrement counters
1086          vmovdqu        48(%rsp),$counters
1087         vaesdec         $rndkey1,@out[5],@out[5]
1088          mov            64(%rsp),$offset                # pre-load 1st offset
1089         vaesdec         $rndkey1,@out[6],@out[6]
1090         vaesdec         $rndkey1,@out[7],@out[7]
1091         vmovups         0x10-0x78($key),$rndkey1
1092
1093         vaesdeclast     $rndkey0,@out[0],@out[0]
1094          vmovdqa        $zero,32(%rsp)                  # update counters
1095          vpxor          $zero,$zero,$zero
1096         vaesdeclast     $rndkey0,@out[1],@out[1]
1097         vpxor           0x00($offload),@out[0],@out[0]  # xor with IV
1098         vaesdeclast     $rndkey0,@out[2],@out[2]
1099         vpxor           0x10($offload),@out[1],@out[1]
1100          vpcmpgtd       $zero,$counters,$zero
1101         vaesdeclast     $rndkey0,@out[3],@out[3]
1102         vpxor           0x20($offload),@out[2],@out[2]
1103         vaesdeclast     $rndkey0,@out[4],@out[4]
1104         vpxor           0x30($offload),@out[3],@out[3]
1105          vpaddd         $zero,$counters,$counters       # decrement counters
1106          vmovdqu        -0x78($key),$zero               # 0-round
1107         vaesdeclast     $rndkey0,@out[5],@out[5]
1108         vpxor           0x40($offload),@out[4],@out[4]
1109         vaesdeclast     $rndkey0,@out[6],@out[6]
1110         vpxor           0x50($offload),@out[5],@out[5]
1111          vmovdqa        $counters,48(%rsp)              # update counters
1112         vaesdeclast     $rndkey0,@out[7],@out[7]
1113         vpxor           0x60($offload),@out[6],@out[6]
1114         vmovups         0x20-0x78($key),$rndkey0
1115
1116         vmovups         @out[0],-16(@ptr[0])            # write output
1117          sub            $offset,@ptr[0]                 # switch to input
1118          vmovdqu        128+0(%rsp),@out[0]
1119         vpxor           0x70($offload),@out[7],@out[7]
1120         vmovups         @out[1],-16(@ptr[1])
1121          sub            `64+1*8`(%rsp),@ptr[1]
1122          vmovdqu        @out[0],0x00($offload)
1123          vpxor          $zero,@out[0],@out[0]
1124          vmovdqu        128+16(%rsp),@out[1]
1125         vmovups         @out[2],-16(@ptr[2])
1126          sub            `64+2*8`(%rsp),@ptr[2]
1127          vmovdqu        @out[1],0x10($offload)
1128          vpxor          $zero,@out[1],@out[1]
1129          vmovdqu        128+32(%rsp),@out[2]
1130         vmovups         @out[3],-16(@ptr[3])
1131          sub            `64+3*8`(%rsp),@ptr[3]
1132          vmovdqu        @out[2],0x20($offload)
1133          vpxor          $zero,@out[2],@out[2]
1134          vmovdqu        128+48(%rsp),@out[3]
1135         vmovups         @out[4],-16(@ptr[4])
1136          sub            `64+4*8`(%rsp),@ptr[4]
1137          vmovdqu        @out[3],0x30($offload)
1138          vpxor          $zero,@out[3],@out[3]
1139          vmovdqu        @inp[0],0x40($offload)
1140          vpxor          @inp[0],$zero,@out[4]
1141         vmovups         @out[5],-16(@ptr[5])
1142          sub            `64+5*8`(%rsp),@ptr[5]
1143          vmovdqu        @inp[1],0x50($offload)
1144          vpxor          @inp[1],$zero,@out[5]
1145         vmovups         @out[6],-16(@ptr[6])
1146          sub            `64+6*8`(%rsp),@ptr[6]
1147          vmovdqu        @inp[2],0x60($offload)
1148          vpxor          @inp[2],$zero,@out[6]
1149         vmovups         @out[7],-16(@ptr[7])
1150          sub            `64+7*8`(%rsp),@ptr[7]
1151          vmovdqu        @inp[3],0x70($offload)
1152          vpxor          @inp[3],$zero,@out[7]
1153
1154         xor     \$128,$offload
1155         dec     $num
1156         jnz     .Loop_dec8x
1157
1158         mov     16(%rsp),%rax                   # original %rsp
1159         #mov    24(%rsp),$num
1160         #lea    `40*8`($inp),$inp
1161         #dec    $num
1162         #jnz    .Ldec8x_loop_grande
1163
1164 .Ldec8x_done:
1165         vzeroupper
1166 ___
1167 $code.=<<___ if ($win64);
1168         movaps  -0xd8(%rax),%xmm6
1169         movaps  -0xc8(%rax),%xmm7
1170         movaps  -0xb8(%rax),%xmm8
1171         movaps  -0xa8(%rax),%xmm9
1172         movaps  -0x98(%rax),%xmm10
1173         movaps  -0x88(%rax),%xmm11
1174         movaps  -0x78(%rax),%xmm12
1175         movaps  -0x68(%rax),%xmm13
1176         movaps  -0x58(%rax),%xmm14
1177         movaps  -0x48(%rax),%xmm15
1178 ___
1179 $code.=<<___;
1180         mov     -48(%rax),%r15
1181         mov     -40(%rax),%r14
1182         mov     -32(%rax),%r13
1183         mov     -24(%rax),%r12
1184         mov     -16(%rax),%rbp
1185         mov     -8(%rax),%rbx
1186         lea     (%rax),%rsp
1187 .Ldec8x_epilogue:
1188         ret
1189 .size   aesni_multi_cbc_decrypt_avx,.-aesni_multi_cbc_decrypt_avx
1190 ___
1191                                                 }}}
1192
1193 if ($win64) {
1194 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1195 #               CONTEXT *context,DISPATCHER_CONTEXT *disp)
1196 $rec="%rcx";
1197 $frame="%rdx";
1198 $context="%r8";
1199 $disp="%r9";
1200
1201 $code.=<<___;
1202 .extern __imp_RtlVirtualUnwind
1203 .type   se_handler,\@abi-omnipotent
1204 .align  16
1205 se_handler:
1206         push    %rsi
1207         push    %rdi
1208         push    %rbx
1209         push    %rbp
1210         push    %r12
1211         push    %r13
1212         push    %r14
1213         push    %r15
1214         pushfq
1215         sub     \$64,%rsp
1216
1217         mov     120($context),%rax      # pull context->Rax
1218         mov     248($context),%rbx      # pull context->Rip
1219
1220         mov     8($disp),%rsi           # disp->ImageBase
1221         mov     56($disp),%r11          # disp->HandlerData
1222
1223         mov     0(%r11),%r10d           # HandlerData[0]
1224         lea     (%rsi,%r10),%r10        # prologue label
1225         cmp     %r10,%rbx               # context->Rip<.Lprologue
1226         jb      .Lin_prologue
1227
1228         mov     152($context),%rax      # pull context->Rsp
1229
1230         mov     4(%r11),%r10d           # HandlerData[1]
1231         lea     (%rsi,%r10),%r10        # epilogue label
1232         cmp     %r10,%rbx               # context->Rip>=.Lepilogue
1233         jae     .Lin_prologue
1234
1235         mov     16(%rax),%rax           # pull saved stack pointer
1236
1237         mov     -8(%rax),%rbx
1238         mov     -16(%rax),%rbp
1239         mov     -24(%rax),%r12
1240         mov     -32(%rax),%r13
1241         mov     -40(%rax),%r14
1242         mov     -48(%rax),%r15
1243         mov     %rbx,144($context)      # restore context->Rbx
1244         mov     %rbp,160($context)      # restore context->Rbp
1245         mov     %r12,216($context)      # restore cotnext->R12
1246         mov     %r13,224($context)      # restore cotnext->R13
1247         mov     %r14,232($context)      # restore cotnext->R14
1248         mov     %r15,240($context)      # restore cotnext->R15
1249
1250         lea     -56-10*16(%rax),%rsi
1251         lea     512($context),%rdi      # &context.Xmm6
1252         mov     \$20,%ecx
1253         .long   0xa548f3fc              # cld; rep movsq
1254
1255 .Lin_prologue:
1256         mov     8(%rax),%rdi
1257         mov     16(%rax),%rsi
1258         mov     %rax,152($context)      # restore context->Rsp
1259         mov     %rsi,168($context)      # restore context->Rsi
1260         mov     %rdi,176($context)      # restore context->Rdi
1261
1262         mov     40($disp),%rdi          # disp->ContextRecord
1263         mov     $context,%rsi           # context
1264         mov     \$154,%ecx              # sizeof(CONTEXT)
1265         .long   0xa548f3fc              # cld; rep movsq
1266
1267         mov     $disp,%rsi
1268         xor     %rcx,%rcx               # arg1, UNW_FLAG_NHANDLER
1269         mov     8(%rsi),%rdx            # arg2, disp->ImageBase
1270         mov     0(%rsi),%r8             # arg3, disp->ControlPc
1271         mov     16(%rsi),%r9            # arg4, disp->FunctionEntry
1272         mov     40(%rsi),%r10           # disp->ContextRecord
1273         lea     56(%rsi),%r11           # &disp->HandlerData
1274         lea     24(%rsi),%r12           # &disp->EstablisherFrame
1275         mov     %r10,32(%rsp)           # arg5
1276         mov     %r11,40(%rsp)           # arg6
1277         mov     %r12,48(%rsp)           # arg7
1278         mov     %rcx,56(%rsp)           # arg8, (NULL)
1279         call    *__imp_RtlVirtualUnwind(%rip)
1280
1281         mov     \$1,%eax                # ExceptionContinueSearch
1282         add     \$64,%rsp
1283         popfq
1284         pop     %r15
1285         pop     %r14
1286         pop     %r13
1287         pop     %r12
1288         pop     %rbp
1289         pop     %rbx
1290         pop     %rdi
1291         pop     %rsi
1292         ret
1293 .size   se_handler,.-se_handler
1294
1295 .section        .pdata
1296 .align  4
1297         .rva    .LSEH_begin_aesni_multi_cbc_encrypt
1298         .rva    .LSEH_end_aesni_multi_cbc_encrypt
1299         .rva    .LSEH_info_aesni_multi_cbc_encrypt
1300         .rva    .LSEH_begin_aesni_multi_cbc_decrypt
1301         .rva    .LSEH_end_aesni_multi_cbc_decrypt
1302         .rva    .LSEH_info_aesni_multi_cbc_decrypt
1303 ___
1304 $code.=<<___ if ($avx);
1305         .rva    .LSEH_begin_aesni_multi_cbc_encrypt_avx
1306         .rva    .LSEH_end_aesni_multi_cbc_encrypt_avx
1307         .rva    .LSEH_info_aesni_multi_cbc_encrypt_avx
1308         .rva    .LSEH_begin_aesni_multi_cbc_decrypt_avx
1309         .rva    .LSEH_end_aesni_multi_cbc_decrypt_avx
1310         .rva    .LSEH_info_aesni_multi_cbc_decrypt_avx
1311 ___
1312 $code.=<<___;
1313 .section        .xdata
1314 .align  8
1315 .LSEH_info_aesni_multi_cbc_encrypt:
1316         .byte   9,0,0,0
1317         .rva    se_handler
1318         .rva    .Lenc4x_body,.Lenc4x_epilogue           # HandlerData[]
1319 .LSEH_info_aesni_multi_cbc_decrypt:
1320         .byte   9,0,0,0
1321         .rva    se_handler
1322         .rva    .Ldec4x_body,.Ldec4x_epilogue           # HandlerData[]
1323 ___
1324 $code.=<<___ if ($avx);
1325 .LSEH_info_aesni_multi_cbc_encrypt_avx:
1326         .byte   9,0,0,0
1327         .rva    se_handler
1328         .rva    .Lenc8x_body,.Lenc8x_epilogue           # HandlerData[]
1329 .LSEH_info_aesni_multi_cbc_decrypt_avx:
1330         .byte   9,0,0,0
1331         .rva    se_handler
1332         .rva    .Ldec8x_body,.Ldec8x_epilogue           # HandlerData[]
1333 ___
1334 }
1335 ####################################################################
1336
1337 sub rex {
1338   local *opcode=shift;
1339   my ($dst,$src)=@_;
1340   my $rex=0;
1341
1342     $rex|=0x04                  if($dst>=8);
1343     $rex|=0x01                  if($src>=8);
1344     push @opcode,$rex|0x40      if($rex);
1345 }
1346
1347 sub aesni {
1348   my $line=shift;
1349   my @opcode=(0x66);
1350
1351     if ($line=~/(aeskeygenassist)\s+\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1352         rex(\@opcode,$4,$3);
1353         push @opcode,0x0f,0x3a,0xdf;
1354         push @opcode,0xc0|($3&7)|(($4&7)<<3);   # ModR/M
1355         my $c=$2;
1356         push @opcode,$c=~/^0/?oct($c):$c;
1357         return ".byte\t".join(',',@opcode);
1358     }
1359     elsif ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1360         my %opcodelet = (
1361                 "aesimc" => 0xdb,
1362                 "aesenc" => 0xdc,       "aesenclast" => 0xdd,
1363                 "aesdec" => 0xde,       "aesdeclast" => 0xdf
1364         );
1365         return undef if (!defined($opcodelet{$1}));
1366         rex(\@opcode,$3,$2);
1367         push @opcode,0x0f,0x38,$opcodelet{$1};
1368         push @opcode,0xc0|($2&7)|(($3&7)<<3);   # ModR/M
1369         return ".byte\t".join(',',@opcode);
1370     }
1371     elsif ($line=~/(aes[a-z]+)\s+([0x1-9a-fA-F]*)\(%rsp\),\s*%xmm([0-9]+)/) {
1372         my %opcodelet = (
1373                 "aesenc" => 0xdc,       "aesenclast" => 0xdd,
1374                 "aesdec" => 0xde,       "aesdeclast" => 0xdf
1375         );
1376         return undef if (!defined($opcodelet{$1}));
1377         my $off = $2;
1378         push @opcode,0x44 if ($3>=8);
1379         push @opcode,0x0f,0x38,$opcodelet{$1};
1380         push @opcode,0x44|(($3&7)<<3),0x24;     # ModR/M
1381         push @opcode,($off=~/^0/?oct($off):$off)&0xff;
1382         return ".byte\t".join(',',@opcode);
1383     }
1384     return $line;
1385 }
1386
1387 $code =~ s/\`([^\`]*)\`/eval($1)/gem;
1388 $code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem;
1389
1390 print $code;
1391 close STDOUT;