x86_64: Don't assume 8-byte pointer size
[openssl.git] / crypto / aes / asm / aesni-mb-x86_64.pl
1 #! /usr/bin/env perl
2 # Copyright 2013-2016 The OpenSSL Project Authors. All Rights Reserved.
3 #
4 # Licensed under the Apache License 2.0 (the "License").  You may not use
5 # this file except in compliance with the License.  You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
8
9
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
16
17 # Multi-buffer AES-NI procedures process several independent buffers
18 # in parallel by interleaving independent instructions.
19 #
20 # Cycles per byte for interleave factor 4:
21 #
22 #                       asymptotic      measured
23 #                       ---------------------------
24 # Westmere              5.00/4=1.25     5.13/4=1.28
25 # Atom                  15.0/4=3.75     ?15.7/4=3.93
26 # Sandy Bridge          5.06/4=1.27     5.18/4=1.29
27 # Ivy Bridge            5.06/4=1.27     5.14/4=1.29
28 # Haswell               4.44/4=1.11     4.44/4=1.11
29 # Bulldozer             5.75/4=1.44     5.76/4=1.44
30 #
31 # Cycles per byte for interleave factor 8 (not implemented for
32 # pre-AVX processors, where higher interleave factor incidentally
33 # doesn't result in improvement):
34 #
35 #                       asymptotic      measured
36 #                       ---------------------------
37 # Sandy Bridge          5.06/8=0.64     7.10/8=0.89(*)
38 # Ivy Bridge            5.06/8=0.64     7.14/8=0.89(*)
39 # Haswell               5.00/8=0.63     5.00/8=0.63
40 # Bulldozer             5.75/8=0.72     5.77/8=0.72
41 #
42 # (*)   Sandy/Ivy Bridge are known to handle high interleave factors
43 #       suboptimally;
44
45 # $output is the last argument if it looks like a file (it has an extension)
46 # $flavour is the first argument if it doesn't look like a file
47 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
48 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
49
50 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
51
52 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
53 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
54 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
55 die "can't locate x86_64-xlate.pl";
56
57 push(@INC,"${dir}","${dir}../../perlasm");
58 require "x86_64-support.pl";
59
60 $ptr_size=&pointer_size($flavour);
61
62 $avx=0;
63
64 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
65                 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
66         $avx = ($1>=2.19) + ($1>=2.22);
67 }
68
69 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
70            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
71         $avx = ($1>=2.09) + ($1>=2.10);
72 }
73
74 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
75            `ml64 2>&1` =~ /Version ([0-9]+)\./) {
76         $avx = ($1>=10) + ($1>=11);
77 }
78
79 if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) {
80         $avx = ($2>=3.0) + ($2>3.0);
81 }
82
83 open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""
84     or die "can't call $xlate: $!";
85 *STDOUT=*OUT;
86
87 # void aesni_multi_cbc_encrypt (
88 #     struct {  void *inp,*out; int blocks; double iv[2]; } inp[8];
89 #     const AES_KEY *key,
90 #     int num);         /* 1 or 2 */
91 #
92 $inp="%rdi";    # 1st arg
93 $key="%rsi";    # 2nd arg
94 $num="%edx";
95
96 $inp_elm_size=2*$ptr_size+8+16;
97
98 @inptr=map("%r$_",(8..11));
99 @outptr=map("%r$_",(12..15));
100
101 ($rndkey0,$rndkey1)=("%xmm0","%xmm1");
102 @out=map("%xmm$_",(2..5));
103 @inp=map("%xmm$_",(6..9));
104 ($counters,$mask,$zero)=map("%xmm$_",(10..12));
105
106 ($rounds,$one,$sink,$offset)=("%eax","%ecx","%rbp","%rbx");
107
108 $code.=<<___;
109 .text
110
111 .extern OPENSSL_ia32cap_P
112
113 .globl  aesni_multi_cbc_encrypt
114 .type   aesni_multi_cbc_encrypt,\@function,3
115 .align  32
116 aesni_multi_cbc_encrypt:
117 .cfi_startproc
118 ___
119 $code.=<<___ if ($avx);
120         cmp     \$2,$num
121         jb      .Lenc_non_avx
122         mov     OPENSSL_ia32cap_P+4(%rip),%ecx
123         test    \$`1<<28`,%ecx                  # AVX bit
124         jnz     _avx_cbc_enc_shortcut
125         jmp     .Lenc_non_avx
126 .align  16
127 .Lenc_non_avx:
128 ___
129 $code.=<<___;
130         mov     %rsp,%rax
131 .cfi_def_cfa_register   %rax
132         push    %rbx
133 .cfi_push       %rbx
134         push    %rbp
135 .cfi_push       %rbp
136         push    %r12
137 .cfi_push       %r12
138         push    %r13
139 .cfi_push       %r13
140         push    %r14
141 .cfi_push       %r14
142         push    %r15
143 .cfi_push       %r15
144 ___
145 $code.=<<___ if ($win64);
146         lea     -0xa8(%rsp),%rsp
147         movaps  %xmm6,(%rsp)
148         movaps  %xmm7,0x10(%rsp)
149         movaps  %xmm8,0x20(%rsp)
150         movaps  %xmm9,0x30(%rsp)
151         movaps  %xmm10,0x40(%rsp)
152         movaps  %xmm11,0x50(%rsp)
153         movaps  %xmm12,0x60(%rsp)
154         movaps  %xmm13,-0x68(%rax)      # not used, saved to share se_handler
155         movaps  %xmm14,-0x58(%rax)
156         movaps  %xmm15,-0x48(%rax)
157 ___
158 $code.=<<___;
159         # stack layout
160         #
161         # +0    output sink
162         # +16   input sink [original %rsp and $num]
163         # +32   counters
164
165         sub     \$48,%rsp
166         and     \$-64,%rsp
167         mov     %rax,16(%rsp)                   # original %rsp
168 .cfi_cfa_expression     %rsp+16,deref,+8
169
170 .Lenc4x_body:
171         movdqu  ($key),$zero                    # 0-round key
172         lea     0x78($key),$key                 # size optimization
173         lea     $inp_elm_size*2($inp),$inp
174
175 .Lenc4x_loop_grande:
176         mov     $num,24(%rsp)                   # original $num
177         xor     $num,$num
178 ___
179 for($i=0;$i<4;$i++) {
180     $inptr_reg=&pointer_register($flavour,@inptr[$i]);
181     $outptr_reg=&pointer_register($flavour,@outptr[$i]);
182     $code.=<<___;
183         # borrow $one for number of blocks
184         mov     `$inp_elm_size*$i+2*$ptr_size-$inp_elm_size*2`($inp),$one
185         mov     `$inp_elm_size*$i+0-$inp_elm_size*2`($inp),$inptr_reg
186         cmp     $num,$one
187         mov     `$inp_elm_size*$i+$ptr_size-$inp_elm_size*2`($inp),$outptr_reg
188         cmovg   $one,$num                       # find maximum
189         test    $one,$one
190         # load IV
191         movdqu  `$inp_elm_size*$i+2*$ptr_size+8-$inp_elm_size*2`($inp),@out[$i]
192         mov     $one,`32+4*$i`(%rsp)            # initialize counters
193         cmovle  %rsp,@inptr[$i]                 # cancel input
194 ___
195 }
196 $code.=<<___;
197         test    $num,$num
198         jz      .Lenc4x_done
199
200         movups  0x10-0x78($key),$rndkey1
201          pxor   $zero,@out[0]
202         movups  0x20-0x78($key),$rndkey0
203          pxor   $zero,@out[1]
204         mov     0xf0-0x78($key),$rounds
205          pxor   $zero,@out[2]
206         movdqu  (@inptr[0]),@inp[0]             # load inputs
207          pxor   $zero,@out[3]
208         movdqu  (@inptr[1]),@inp[1]
209          pxor   @inp[0],@out[0]
210         movdqu  (@inptr[2]),@inp[2]
211          pxor   @inp[1],@out[1]
212         movdqu  (@inptr[3]),@inp[3]
213          pxor   @inp[2],@out[2]
214          pxor   @inp[3],@out[3]
215         movdqa  32(%rsp),$counters              # load counters
216         xor     $offset,$offset
217         jmp     .Loop_enc4x
218
219 .align  32
220 .Loop_enc4x:
221         add     \$16,$offset
222         lea     16(%rsp),$sink                  # sink pointer
223         mov     \$1,$one                        # constant of 1
224         sub     $offset,$sink
225
226         aesenc          $rndkey1,@out[0]
227         prefetcht0      31(@inptr[0],$offset)   # prefetch input
228         prefetcht0      31(@inptr[1],$offset)
229         aesenc          $rndkey1,@out[1]
230         prefetcht0      31(@inptr[2],$offset)
231         prefetcht0      31(@inptr[2],$offset)
232         aesenc          $rndkey1,@out[2]
233         aesenc          $rndkey1,@out[3]
234         movups          0x30-0x78($key),$rndkey1
235 ___
236 for($i=0;$i<4;$i++) {
237 my $rndkey = ($i&1) ? $rndkey1 : $rndkey0;
238 $code.=<<___;
239          cmp            `32+4*$i`(%rsp),$one
240         aesenc          $rndkey,@out[0]
241         aesenc          $rndkey,@out[1]
242         aesenc          $rndkey,@out[2]
243          cmovge         $sink,@inptr[$i]        # cancel input
244          cmovg          $sink,@outptr[$i]       # sink output
245         aesenc          $rndkey,@out[3]
246         movups          `0x40+16*$i-0x78`($key),$rndkey
247 ___
248 }
249 $code.=<<___;
250          movdqa         $counters,$mask
251         aesenc          $rndkey0,@out[0]
252         prefetcht0      15(@outptr[0],$offset)  # prefetch output
253         prefetcht0      15(@outptr[1],$offset)
254         aesenc          $rndkey0,@out[1]
255         prefetcht0      15(@outptr[2],$offset)
256         prefetcht0      15(@outptr[3],$offset)
257         aesenc          $rndkey0,@out[2]
258         aesenc          $rndkey0,@out[3]
259         movups          0x80-0x78($key),$rndkey0
260          pxor           $zero,$zero
261
262         aesenc          $rndkey1,@out[0]
263          pcmpgtd        $zero,$mask
264          movdqu         -0x78($key),$zero       # reload 0-round key
265         aesenc          $rndkey1,@out[1]
266          paddd          $mask,$counters         # decrement counters
267          movdqa         $counters,32(%rsp)      # update counters
268         aesenc          $rndkey1,@out[2]
269         aesenc          $rndkey1,@out[3]
270         movups          0x90-0x78($key),$rndkey1
271
272         cmp     \$11,$rounds
273
274         aesenc          $rndkey0,@out[0]
275         aesenc          $rndkey0,@out[1]
276         aesenc          $rndkey0,@out[2]
277         aesenc          $rndkey0,@out[3]
278         movups          0xa0-0x78($key),$rndkey0
279
280         jb      .Lenc4x_tail
281
282         aesenc          $rndkey1,@out[0]
283         aesenc          $rndkey1,@out[1]
284         aesenc          $rndkey1,@out[2]
285         aesenc          $rndkey1,@out[3]
286         movups          0xb0-0x78($key),$rndkey1
287
288         aesenc          $rndkey0,@out[0]
289         aesenc          $rndkey0,@out[1]
290         aesenc          $rndkey0,@out[2]
291         aesenc          $rndkey0,@out[3]
292         movups          0xc0-0x78($key),$rndkey0
293
294         je      .Lenc4x_tail
295
296         aesenc          $rndkey1,@out[0]
297         aesenc          $rndkey1,@out[1]
298         aesenc          $rndkey1,@out[2]
299         aesenc          $rndkey1,@out[3]
300         movups          0xd0-0x78($key),$rndkey1
301
302         aesenc          $rndkey0,@out[0]
303         aesenc          $rndkey0,@out[1]
304         aesenc          $rndkey0,@out[2]
305         aesenc          $rndkey0,@out[3]
306         movups          0xe0-0x78($key),$rndkey0
307         jmp     .Lenc4x_tail
308
309 .align  32
310 .Lenc4x_tail:
311         aesenc          $rndkey1,@out[0]
312         aesenc          $rndkey1,@out[1]
313         aesenc          $rndkey1,@out[2]
314         aesenc          $rndkey1,@out[3]
315          movdqu         (@inptr[0],$offset),@inp[0]
316         movdqu          0x10-0x78($key),$rndkey1
317
318         aesenclast      $rndkey0,@out[0]
319          movdqu         (@inptr[1],$offset),@inp[1]
320          pxor           $zero,@inp[0]
321         aesenclast      $rndkey0,@out[1]
322          movdqu         (@inptr[2],$offset),@inp[2]
323          pxor           $zero,@inp[1]
324         aesenclast      $rndkey0,@out[2]
325          movdqu         (@inptr[3],$offset),@inp[3]
326          pxor           $zero,@inp[2]
327         aesenclast      $rndkey0,@out[3]
328         movdqu          0x20-0x78($key),$rndkey0
329          pxor           $zero,@inp[3]
330
331         movups          @out[0],-16(@outptr[0],$offset)
332          pxor           @inp[0],@out[0]
333         movups          @out[1],-16(@outptr[1],$offset)
334          pxor           @inp[1],@out[1]
335         movups          @out[2],-16(@outptr[2],$offset)
336          pxor           @inp[2],@out[2]
337         movups          @out[3],-16(@outptr[3],$offset)
338          pxor           @inp[3],@out[3]
339
340         dec     $num
341         jnz     .Loop_enc4x
342
343         mov     16(%rsp),%rax                   # original %rsp
344 .cfi_def_cfa    %rax,8
345         mov     24(%rsp),$num
346
347         #pxor   @inp[0],@out[0]
348         #pxor   @inp[1],@out[1]
349         # output iv FIX ME!
350         #movdqu @out[0],`$inp_elm_size*0+2*$ptr_size+8-$inp_elm_size*2`($inp)
351         #pxor   @inp[2],@out[2]
352         #movdqu @out[1],`$inp_elm_size*1+2*$ptr_size+8-$inp_elm_size*2`($inp)
353         #pxor   @inp[3],@out[3]
354         #movdqu @out[2],`$inp_elm_size*2+2*$ptr_size+8-$inp_elm_size*2`($inp)   # won't fix, let caller
355         #movdqu @out[3],`$inp_elm_size*3+2*$ptr_size+8-$inp_elm_size*2`($inp)   # figure this out...
356
357         lea     `$inp_elm_size*4`($inp),$inp
358         dec     $num
359         jnz     .Lenc4x_loop_grande
360
361 .Lenc4x_done:
362 ___
363 $code.=<<___ if ($win64);
364         movaps  -0xd8(%rax),%xmm6
365         movaps  -0xc8(%rax),%xmm7
366         movaps  -0xb8(%rax),%xmm8
367         movaps  -0xa8(%rax),%xmm9
368         movaps  -0x98(%rax),%xmm10
369         movaps  -0x88(%rax),%xmm11
370         movaps  -0x78(%rax),%xmm12
371         #movaps -0x68(%rax),%xmm13
372         #movaps -0x58(%rax),%xmm14
373         #movaps -0x48(%rax),%xmm15
374 ___
375 $code.=<<___;
376         mov     -48(%rax),%r15
377 .cfi_restore    %r15
378         mov     -40(%rax),%r14
379 .cfi_restore    %r14
380         mov     -32(%rax),%r13
381 .cfi_restore    %r13
382         mov     -24(%rax),%r12
383 .cfi_restore    %r12
384         mov     -16(%rax),%rbp
385 .cfi_restore    %rbp
386         mov     -8(%rax),%rbx
387 .cfi_restore    %rbx
388         lea     (%rax),%rsp
389 .cfi_def_cfa_register   %rsp
390 .Lenc4x_epilogue:
391         ret
392 .cfi_endproc
393 .size   aesni_multi_cbc_encrypt,.-aesni_multi_cbc_encrypt
394
395 .globl  aesni_multi_cbc_decrypt
396 .type   aesni_multi_cbc_decrypt,\@function,3
397 .align  32
398 aesni_multi_cbc_decrypt:
399 .cfi_startproc
400 ___
401 $code.=<<___ if ($avx);
402         cmp     \$2,$num
403         jb      .Ldec_non_avx
404         mov     OPENSSL_ia32cap_P+4(%rip),%ecx
405         test    \$`1<<28`,%ecx                  # AVX bit
406         jnz     _avx_cbc_dec_shortcut
407         jmp     .Ldec_non_avx
408 .align  16
409 .Ldec_non_avx:
410 ___
411 $code.=<<___;
412         mov     %rsp,%rax
413 .cfi_def_cfa_register   %rax
414         push    %rbx
415 .cfi_push       %rbx
416         push    %rbp
417 .cfi_push       %rbp
418         push    %r12
419 .cfi_push       %r12
420         push    %r13
421 .cfi_push       %r13
422         push    %r14
423 .cfi_push       %r14
424         push    %r15
425 .cfi_push       %r15
426 ___
427 $code.=<<___ if ($win64);
428         lea     -0xa8(%rsp),%rsp
429         movaps  %xmm6,(%rsp)
430         movaps  %xmm7,0x10(%rsp)
431         movaps  %xmm8,0x20(%rsp)
432         movaps  %xmm9,0x30(%rsp)
433         movaps  %xmm10,0x40(%rsp)
434         movaps  %xmm11,0x50(%rsp)
435         movaps  %xmm12,0x60(%rsp)
436         movaps  %xmm13,-0x68(%rax)      # not used, saved to share se_handler
437         movaps  %xmm14,-0x58(%rax)
438         movaps  %xmm15,-0x48(%rax)
439 ___
440 $code.=<<___;
441         # stack layout
442         #
443         # +0    output sink
444         # +16   input sink [original %rsp and $num]
445         # +32   counters
446
447         sub     \$48,%rsp
448         and     \$-64,%rsp
449         mov     %rax,16(%rsp)                   # original %rsp
450 .cfi_cfa_expression     %rsp+16,deref,+8
451
452 .Ldec4x_body:
453         movdqu  ($key),$zero                    # 0-round key
454         lea     0x78($key),$key                 # size optimization
455         lea     $inp_elm_size*2($inp),$inp
456
457 .Ldec4x_loop_grande:
458         mov     $num,24(%rsp)                   # original $num
459         xor     $num,$num
460 ___
461 for($i=0;$i<4;$i++) {
462     $inptr_reg=&pointer_register($flavour,@inptr[$i]);
463     $outptr_reg=&pointer_register($flavour,@outptr[$i]);
464     $code.=<<___;
465         # borrow $one for number of blocks
466         mov     `$inp_elm_size*$i+2*$ptr_size-$inp_elm_size*2`($inp),$one
467         mov     `$inp_elm_size*$i+0-$inp_elm_size*2`($inp),$inptr_reg
468         cmp     $num,$one
469         mov     `$inp_elm_size*$i+$ptr_size-$inp_elm_size*2`($inp),$outptr_reg
470         cmovg   $one,$num                       # find maximum
471         test    $one,$one
472         # load IV
473         movdqu  `$inp_elm_size*$i+2*$ptr_size+8-$inp_elm_size*2`($inp),@inp[$i]
474         mov     $one,`32+4*$i`(%rsp)            # initialize counters
475         cmovle  %rsp,@inptr[$i]                 # cancel input
476 ___
477 }
478 $code.=<<___;
479         test    $num,$num
480         jz      .Ldec4x_done
481
482         movups  0x10-0x78($key),$rndkey1
483         movups  0x20-0x78($key),$rndkey0
484         mov     0xf0-0x78($key),$rounds
485         movdqu  (@inptr[0]),@out[0]             # load inputs
486         movdqu  (@inptr[1]),@out[1]
487          pxor   $zero,@out[0]
488         movdqu  (@inptr[2]),@out[2]
489          pxor   $zero,@out[1]
490         movdqu  (@inptr[3]),@out[3]
491          pxor   $zero,@out[2]
492          pxor   $zero,@out[3]
493         movdqa  32(%rsp),$counters              # load counters
494         xor     $offset,$offset
495         jmp     .Loop_dec4x
496
497 .align  32
498 .Loop_dec4x:
499         add     \$16,$offset
500         lea     16(%rsp),$sink                  # sink pointer
501         mov     \$1,$one                        # constant of 1
502         sub     $offset,$sink
503
504         aesdec          $rndkey1,@out[0]
505         prefetcht0      31(@inptr[0],$offset)   # prefetch input
506         prefetcht0      31(@inptr[1],$offset)
507         aesdec          $rndkey1,@out[1]
508         prefetcht0      31(@inptr[2],$offset)
509         prefetcht0      31(@inptr[3],$offset)
510         aesdec          $rndkey1,@out[2]
511         aesdec          $rndkey1,@out[3]
512         movups          0x30-0x78($key),$rndkey1
513 ___
514 for($i=0;$i<4;$i++) {
515 my $rndkey = ($i&1) ? $rndkey1 : $rndkey0;
516 $code.=<<___;
517          cmp            `32+4*$i`(%rsp),$one
518         aesdec          $rndkey,@out[0]
519         aesdec          $rndkey,@out[1]
520         aesdec          $rndkey,@out[2]
521          cmovge         $sink,@inptr[$i]        # cancel input
522          cmovg          $sink,@outptr[$i]       # sink output
523         aesdec          $rndkey,@out[3]
524         movups          `0x40+16*$i-0x78`($key),$rndkey
525 ___
526 }
527 $code.=<<___;
528          movdqa         $counters,$mask
529         aesdec          $rndkey0,@out[0]
530         prefetcht0      15(@outptr[0],$offset)  # prefetch output
531         prefetcht0      15(@outptr[1],$offset)
532         aesdec          $rndkey0,@out[1]
533         prefetcht0      15(@outptr[2],$offset)
534         prefetcht0      15(@outptr[3],$offset)
535         aesdec          $rndkey0,@out[2]
536         aesdec          $rndkey0,@out[3]
537         movups          0x80-0x78($key),$rndkey0
538          pxor           $zero,$zero
539
540         aesdec          $rndkey1,@out[0]
541          pcmpgtd        $zero,$mask
542          movdqu         -0x78($key),$zero       # reload 0-round key
543         aesdec          $rndkey1,@out[1]
544          paddd          $mask,$counters         # decrement counters
545          movdqa         $counters,32(%rsp)      # update counters
546         aesdec          $rndkey1,@out[2]
547         aesdec          $rndkey1,@out[3]
548         movups          0x90-0x78($key),$rndkey1
549
550         cmp     \$11,$rounds
551
552         aesdec          $rndkey0,@out[0]
553         aesdec          $rndkey0,@out[1]
554         aesdec          $rndkey0,@out[2]
555         aesdec          $rndkey0,@out[3]
556         movups          0xa0-0x78($key),$rndkey0
557
558         jb      .Ldec4x_tail
559
560         aesdec          $rndkey1,@out[0]
561         aesdec          $rndkey1,@out[1]
562         aesdec          $rndkey1,@out[2]
563         aesdec          $rndkey1,@out[3]
564         movups          0xb0-0x78($key),$rndkey1
565
566         aesdec          $rndkey0,@out[0]
567         aesdec          $rndkey0,@out[1]
568         aesdec          $rndkey0,@out[2]
569         aesdec          $rndkey0,@out[3]
570         movups          0xc0-0x78($key),$rndkey0
571
572         je      .Ldec4x_tail
573
574         aesdec          $rndkey1,@out[0]
575         aesdec          $rndkey1,@out[1]
576         aesdec          $rndkey1,@out[2]
577         aesdec          $rndkey1,@out[3]
578         movups          0xd0-0x78($key),$rndkey1
579
580         aesdec          $rndkey0,@out[0]
581         aesdec          $rndkey0,@out[1]
582         aesdec          $rndkey0,@out[2]
583         aesdec          $rndkey0,@out[3]
584         movups          0xe0-0x78($key),$rndkey0
585         jmp     .Ldec4x_tail
586
587 .align  32
588 .Ldec4x_tail:
589         aesdec          $rndkey1,@out[0]
590         aesdec          $rndkey1,@out[1]
591         aesdec          $rndkey1,@out[2]
592          pxor           $rndkey0,@inp[0]
593          pxor           $rndkey0,@inp[1]
594         aesdec          $rndkey1,@out[3]
595         movdqu          0x10-0x78($key),$rndkey1
596          pxor           $rndkey0,@inp[2]
597          pxor           $rndkey0,@inp[3]
598         movdqu          0x20-0x78($key),$rndkey0
599
600         aesdeclast      @inp[0],@out[0]
601         aesdeclast      @inp[1],@out[1]
602          movdqu         -16(@inptr[0],$offset),@inp[0]  # load next IV
603          movdqu         -16(@inptr[1],$offset),@inp[1]
604         aesdeclast      @inp[2],@out[2]
605         aesdeclast      @inp[3],@out[3]
606          movdqu         -16(@inptr[2],$offset),@inp[2]
607          movdqu         -16(@inptr[3],$offset),@inp[3]
608
609         movups          @out[0],-16(@outptr[0],$offset)
610          movdqu         (@inptr[0],$offset),@out[0]
611         movups          @out[1],-16(@outptr[1],$offset)
612          movdqu         (@inptr[1],$offset),@out[1]
613          pxor           $zero,@out[0]
614         movups          @out[2],-16(@outptr[2],$offset)
615          movdqu         (@inptr[2],$offset),@out[2]
616          pxor           $zero,@out[1]
617         movups          @out[3],-16(@outptr[3],$offset)
618          movdqu         (@inptr[3],$offset),@out[3]
619          pxor           $zero,@out[2]
620          pxor           $zero,@out[3]
621
622         dec     $num
623         jnz     .Loop_dec4x
624
625         mov     16(%rsp),%rax                   # original %rsp
626 .cfi_def_cfa    %rax,8
627         mov     24(%rsp),$num
628
629         lea     `$inp_elm_size*4`($inp),$inp
630         dec     $num
631         jnz     .Ldec4x_loop_grande
632
633 .Ldec4x_done:
634 ___
635 $code.=<<___ if ($win64);
636         movaps  -0xd8(%rax),%xmm6
637         movaps  -0xc8(%rax),%xmm7
638         movaps  -0xb8(%rax),%xmm8
639         movaps  -0xa8(%rax),%xmm9
640         movaps  -0x98(%rax),%xmm10
641         movaps  -0x88(%rax),%xmm11
642         movaps  -0x78(%rax),%xmm12
643         #movaps -0x68(%rax),%xmm13
644         #movaps -0x58(%rax),%xmm14
645         #movaps -0x48(%rax),%xmm15
646 ___
647 $code.=<<___;
648         mov     -48(%rax),%r15
649 .cfi_restore    %r15
650         mov     -40(%rax),%r14
651 .cfi_restore    %r14
652         mov     -32(%rax),%r13
653 .cfi_restore    %r13
654         mov     -24(%rax),%r12
655 .cfi_restore    %r12
656         mov     -16(%rax),%rbp
657 .cfi_restore    %rbp
658         mov     -8(%rax),%rbx
659 .cfi_restore    %rbx
660         lea     (%rax),%rsp
661 .cfi_def_cfa_register   %rsp
662 .Ldec4x_epilogue:
663         ret
664 .cfi_endproc
665 .size   aesni_multi_cbc_decrypt,.-aesni_multi_cbc_decrypt
666 ___
667
668                                                 if ($avx) {{{
669 my @ptr=map("%r$_",(8..15));
670 my $offload=$sink;
671
672 my @out=map("%xmm$_",(2..9));
673 my @inp=map("%xmm$_",(10..13));
674 my ($counters,$zero)=("%xmm14","%xmm15");
675
676 $code.=<<___;
677 .type   aesni_multi_cbc_encrypt_avx,\@function,3
678 .align  32
679 aesni_multi_cbc_encrypt_avx:
680 .cfi_startproc
681 _avx_cbc_enc_shortcut:
682         mov     %rsp,%rax
683 .cfi_def_cfa_register   %rax
684         push    %rbx
685 .cfi_push       %rbx
686         push    %rbp
687 .cfi_push       %rbp
688         push    %r12
689 .cfi_push       %r12
690         push    %r13
691 .cfi_push       %r13
692         push    %r14
693 .cfi_push       %r14
694         push    %r15
695 .cfi_push       %r15
696 ___
697 $code.=<<___ if ($win64);
698         lea     -0xa8(%rsp),%rsp
699         movaps  %xmm6,(%rsp)
700         movaps  %xmm7,0x10(%rsp)
701         movaps  %xmm8,0x20(%rsp)
702         movaps  %xmm9,0x30(%rsp)
703         movaps  %xmm10,0x40(%rsp)
704         movaps  %xmm11,0x50(%rsp)
705         movaps  %xmm12,-0x78(%rax)
706         movaps  %xmm13,-0x68(%rax)
707         movaps  %xmm14,-0x58(%rax)
708         movaps  %xmm15,-0x48(%rax)
709 ___
710 $code.=<<___;
711         # stack layout
712         #
713         # +0    output sink
714         # +16   input sink [original %rsp and $num]
715         # +32   counters
716         # +64   distances between inputs and outputs
717         # +128  off-load area for @inp[0..3]
718
719         sub     \$192,%rsp
720         and     \$-128,%rsp
721         mov     %rax,16(%rsp)                   # original %rsp
722 .cfi_cfa_expression     %rsp+16,deref,+8
723
724 .Lenc8x_body:
725         vzeroupper
726         vmovdqu ($key),$zero                    # 0-round key
727         lea     0x78($key),$key                 # size optimization
728         lea     `$inp_elm_size*4`($inp),$inp
729         shr     \$1,$num
730
731 .Lenc8x_loop_grande:
732         #mov    $num,24(%rsp)                   # original $num
733         xor     $num,$num
734 ___
735 for($i=0;$i<8;$i++) {
736   my $temp = $i ? $offload : $offset;
737     $ptr_reg=&pointer_register($flavour,@ptr[$i]);
738     $temp_reg=&pointer_register($flavour,$temp);
739     $code.=<<___;
740         # borrow $one for number of blocks
741         mov     `$inp_elm_size*$i+2*$ptr_size-$inp_elm_size*4`($inp),$one
742         # input pointer
743         mov     `$inp_elm_size*$i+0-$inp_elm_size*4`($inp),$ptr_reg
744         cmp     $num,$one
745         # output pointer
746         mov     `$inp_elm_size*$i+$ptr_size-$inp_elm_size*4`($inp),$temp_reg
747         cmovg   $one,$num                       # find maximum
748         test    $one,$one
749         # load IV
750         vmovdqu `$inp_elm_size*$i+2*$ptr_size+8-$inp_elm_size*4`($inp),@out[$i]
751         mov     $one,`32+4*$i`(%rsp)            # initialize counters
752         cmovle  %rsp,@ptr[$i]                   # cancel input
753         sub     @ptr[$i],$temp                  # distance between input and output
754         mov     $temp,`64+8*$i`(%rsp)           # initialize distances
755 ___
756 }
757 $code.=<<___;
758         test    $num,$num
759         jz      .Lenc8x_done
760
761         vmovups 0x10-0x78($key),$rndkey1
762         vmovups 0x20-0x78($key),$rndkey0
763         mov     0xf0-0x78($key),$rounds
764
765         vpxor   (@ptr[0]),$zero,@inp[0]         # load inputs and xor with 0-round
766          lea    128(%rsp),$offload              # offload area
767         vpxor   (@ptr[1]),$zero,@inp[1]
768         vpxor   (@ptr[2]),$zero,@inp[2]
769         vpxor   (@ptr[3]),$zero,@inp[3]
770          vpxor  @inp[0],@out[0],@out[0]
771         vpxor   (@ptr[4]),$zero,@inp[0]
772          vpxor  @inp[1],@out[1],@out[1]
773         vpxor   (@ptr[5]),$zero,@inp[1]
774          vpxor  @inp[2],@out[2],@out[2]
775         vpxor   (@ptr[6]),$zero,@inp[2]
776          vpxor  @inp[3],@out[3],@out[3]
777         vpxor   (@ptr[7]),$zero,@inp[3]
778          vpxor  @inp[0],@out[4],@out[4]
779         mov     \$1,$one                        # constant of 1
780          vpxor  @inp[1],@out[5],@out[5]
781          vpxor  @inp[2],@out[6],@out[6]
782          vpxor  @inp[3],@out[7],@out[7]
783         jmp     .Loop_enc8x
784
785 .align  32
786 .Loop_enc8x:
787 ___
788 for($i=0;$i<8;$i++) {
789 my $rndkey=($i&1)?$rndkey0:$rndkey1;
790 $code.=<<___;
791         vaesenc         $rndkey,@out[0],@out[0]
792          cmp            32+4*$i(%rsp),$one
793 ___
794 $code.=<<___ if ($i);
795          mov            64+8*$i(%rsp),$offset
796 ___
797 $code.=<<___;
798         vaesenc         $rndkey,@out[1],@out[1]
799         prefetcht0      31(@ptr[$i])                    # prefetch input
800         vaesenc         $rndkey,@out[2],@out[2]
801 ___
802 $code.=<<___ if ($i>1);
803         prefetcht0      15(@ptr[$i-2])                  # prefetch output
804 ___
805 $code.=<<___;
806         vaesenc         $rndkey,@out[3],@out[3]
807          lea            (@ptr[$i],$offset),$offset
808          cmovge         %rsp,@ptr[$i]                   # cancel input
809         vaesenc         $rndkey,@out[4],@out[4]
810          cmovg          %rsp,$offset                    # sink output
811         vaesenc         $rndkey,@out[5],@out[5]
812          sub            @ptr[$i],$offset
813         vaesenc         $rndkey,@out[6],@out[6]
814          vpxor          16(@ptr[$i]),$zero,@inp[$i%4]   # load input and xor with 0-round
815          mov            $offset,64+8*$i(%rsp)
816         vaesenc         $rndkey,@out[7],@out[7]
817         vmovups         `16*(3+$i)-0x78`($key),$rndkey
818          lea            16(@ptr[$i],$offset),@ptr[$i]   # switch to output
819 ___
820 $code.=<<___ if ($i<4)
821          vmovdqu        @inp[$i%4],`16*$i`($offload)    # off-load
822 ___
823 }
824 $code.=<<___;
825          vmovdqu        32(%rsp),$counters
826         prefetcht0      15(@ptr[$i-2])                  # prefetch output
827         prefetcht0      15(@ptr[$i-1])
828         cmp     \$11,$rounds
829         jb      .Lenc8x_tail
830
831         vaesenc         $rndkey1,@out[0],@out[0]
832         vaesenc         $rndkey1,@out[1],@out[1]
833         vaesenc         $rndkey1,@out[2],@out[2]
834         vaesenc         $rndkey1,@out[3],@out[3]
835         vaesenc         $rndkey1,@out[4],@out[4]
836         vaesenc         $rndkey1,@out[5],@out[5]
837         vaesenc         $rndkey1,@out[6],@out[6]
838         vaesenc         $rndkey1,@out[7],@out[7]
839         vmovups         0xb0-0x78($key),$rndkey1
840
841         vaesenc         $rndkey0,@out[0],@out[0]
842         vaesenc         $rndkey0,@out[1],@out[1]
843         vaesenc         $rndkey0,@out[2],@out[2]
844         vaesenc         $rndkey0,@out[3],@out[3]
845         vaesenc         $rndkey0,@out[4],@out[4]
846         vaesenc         $rndkey0,@out[5],@out[5]
847         vaesenc         $rndkey0,@out[6],@out[6]
848         vaesenc         $rndkey0,@out[7],@out[7]
849         vmovups         0xc0-0x78($key),$rndkey0
850         je      .Lenc8x_tail
851
852         vaesenc         $rndkey1,@out[0],@out[0]
853         vaesenc         $rndkey1,@out[1],@out[1]
854         vaesenc         $rndkey1,@out[2],@out[2]
855         vaesenc         $rndkey1,@out[3],@out[3]
856         vaesenc         $rndkey1,@out[4],@out[4]
857         vaesenc         $rndkey1,@out[5],@out[5]
858         vaesenc         $rndkey1,@out[6],@out[6]
859         vaesenc         $rndkey1,@out[7],@out[7]
860         vmovups         0xd0-0x78($key),$rndkey1
861
862         vaesenc         $rndkey0,@out[0],@out[0]
863         vaesenc         $rndkey0,@out[1],@out[1]
864         vaesenc         $rndkey0,@out[2],@out[2]
865         vaesenc         $rndkey0,@out[3],@out[3]
866         vaesenc         $rndkey0,@out[4],@out[4]
867         vaesenc         $rndkey0,@out[5],@out[5]
868         vaesenc         $rndkey0,@out[6],@out[6]
869         vaesenc         $rndkey0,@out[7],@out[7]
870         vmovups         0xe0-0x78($key),$rndkey0
871
872 .Lenc8x_tail:
873         vaesenc         $rndkey1,@out[0],@out[0]
874          vpxor          $zero,$zero,$zero
875         vaesenc         $rndkey1,@out[1],@out[1]
876         vaesenc         $rndkey1,@out[2],@out[2]
877          vpcmpgtd       $zero,$counters,$zero
878         vaesenc         $rndkey1,@out[3],@out[3]
879         vaesenc         $rndkey1,@out[4],@out[4]
880          vpaddd         $counters,$zero,$zero           # decrement counters
881          vmovdqu        48(%rsp),$counters
882         vaesenc         $rndkey1,@out[5],@out[5]
883          mov            64(%rsp),$offset                # pre-load 1st offset
884         vaesenc         $rndkey1,@out[6],@out[6]
885         vaesenc         $rndkey1,@out[7],@out[7]
886         vmovups         0x10-0x78($key),$rndkey1
887
888         vaesenclast     $rndkey0,@out[0],@out[0]
889          vmovdqa        $zero,32(%rsp)                  # update counters
890          vpxor          $zero,$zero,$zero
891         vaesenclast     $rndkey0,@out[1],@out[1]
892         vaesenclast     $rndkey0,@out[2],@out[2]
893          vpcmpgtd       $zero,$counters,$zero
894         vaesenclast     $rndkey0,@out[3],@out[3]
895         vaesenclast     $rndkey0,@out[4],@out[4]
896          vpaddd         $zero,$counters,$counters       # decrement counters
897          vmovdqu        -0x78($key),$zero               # 0-round
898         vaesenclast     $rndkey0,@out[5],@out[5]
899         vaesenclast     $rndkey0,@out[6],@out[6]
900          vmovdqa        $counters,48(%rsp)              # update counters
901         vaesenclast     $rndkey0,@out[7],@out[7]
902         vmovups         0x20-0x78($key),$rndkey0
903
904         vmovups         @out[0],-16(@ptr[0])            # write output
905          sub            $offset,@ptr[0]                 # switch to input
906          vpxor          0x00($offload),@out[0],@out[0]
907         vmovups         @out[1],-16(@ptr[1])
908          sub            `64+1*8`(%rsp),@ptr[1]
909          vpxor          0x10($offload),@out[1],@out[1]
910         vmovups         @out[2],-16(@ptr[2])
911          sub            `64+2*8`(%rsp),@ptr[2]
912          vpxor          0x20($offload),@out[2],@out[2]
913         vmovups         @out[3],-16(@ptr[3])
914          sub            `64+3*8`(%rsp),@ptr[3]
915          vpxor          0x30($offload),@out[3],@out[3]
916         vmovups         @out[4],-16(@ptr[4])
917          sub            `64+4*8`(%rsp),@ptr[4]
918          vpxor          @inp[0],@out[4],@out[4]
919         vmovups         @out[5],-16(@ptr[5])
920          sub            `64+5*8`(%rsp),@ptr[5]
921          vpxor          @inp[1],@out[5],@out[5]
922         vmovups         @out[6],-16(@ptr[6])
923          sub            `64+6*8`(%rsp),@ptr[6]
924          vpxor          @inp[2],@out[6],@out[6]
925         vmovups         @out[7],-16(@ptr[7])
926          sub            `64+7*8`(%rsp),@ptr[7]
927          vpxor          @inp[3],@out[7],@out[7]
928
929         dec     $num
930         jnz     .Loop_enc8x
931
932         mov     16(%rsp),%rax                   # original %rsp
933 .cfi_def_cfa    %rax,8
934         #mov    24(%rsp),$num
935         #lea    `$inp_elm_size*8`($inp),$inp
936         #dec    $num
937         #jnz    .Lenc8x_loop_grande
938
939 .Lenc8x_done:
940         vzeroupper
941 ___
942 $code.=<<___ if ($win64);
943         movaps  -0xd8(%rax),%xmm6
944         movaps  -0xc8(%rax),%xmm7
945         movaps  -0xb8(%rax),%xmm8
946         movaps  -0xa8(%rax),%xmm9
947         movaps  -0x98(%rax),%xmm10
948         movaps  -0x88(%rax),%xmm11
949         movaps  -0x78(%rax),%xmm12
950         movaps  -0x68(%rax),%xmm13
951         movaps  -0x58(%rax),%xmm14
952         movaps  -0x48(%rax),%xmm15
953 ___
954 $code.=<<___;
955         mov     -48(%rax),%r15
956 .cfi_restore    %r15
957         mov     -40(%rax),%r14
958 .cfi_restore    %r14
959         mov     -32(%rax),%r13
960 .cfi_restore    %r13
961         mov     -24(%rax),%r12
962 .cfi_restore    %r12
963         mov     -16(%rax),%rbp
964 .cfi_restore    %rbp
965         mov     -8(%rax),%rbx
966 .cfi_restore    %rbx
967         lea     (%rax),%rsp
968 .cfi_def_cfa_register   %rsp
969 .Lenc8x_epilogue:
970         ret
971 .cfi_endproc
972 .size   aesni_multi_cbc_encrypt_avx,.-aesni_multi_cbc_encrypt_avx
973
974 .type   aesni_multi_cbc_decrypt_avx,\@function,3
975 .align  32
976 aesni_multi_cbc_decrypt_avx:
977 .cfi_startproc
978 _avx_cbc_dec_shortcut:
979         mov     %rsp,%rax
980 .cfi_def_cfa_register   %rax
981         push    %rbx
982 .cfi_push       %rbx
983         push    %rbp
984 .cfi_push       %rbp
985         push    %r12
986 .cfi_push       %r12
987         push    %r13
988 .cfi_push       %r13
989         push    %r14
990 .cfi_push       %r14
991         push    %r15
992 .cfi_push       %r15
993 ___
994 $code.=<<___ if ($win64);
995         lea     -0xa8(%rsp),%rsp
996         movaps  %xmm6,(%rsp)
997         movaps  %xmm7,0x10(%rsp)
998         movaps  %xmm8,0x20(%rsp)
999         movaps  %xmm9,0x30(%rsp)
1000         movaps  %xmm10,0x40(%rsp)
1001         movaps  %xmm11,0x50(%rsp)
1002         movaps  %xmm12,-0x78(%rax)
1003         movaps  %xmm13,-0x68(%rax)
1004         movaps  %xmm14,-0x58(%rax)
1005         movaps  %xmm15,-0x48(%rax)
1006 ___
1007 $code.=<<___;
1008         # stack layout
1009         #
1010         # +0    output sink
1011         # +16   input sink [original %rsp and $num]
1012         # +32   counters
1013         # +64   distances between inputs and outputs
1014         # +128  off-load area for @inp[0..3]
1015         # +192  IV/input offload
1016
1017         sub     \$256,%rsp
1018         and     \$-256,%rsp
1019         sub     \$192,%rsp
1020         mov     %rax,16(%rsp)                   # original %rsp
1021 .cfi_cfa_expression     %rsp+16,deref,+8
1022
1023 .Ldec8x_body:
1024         vzeroupper
1025         vmovdqu ($key),$zero                    # 0-round key
1026         lea     0x78($key),$key                 # size optimization
1027         lea     `$inp_elm_size*4`($inp),$inp
1028         shr     \$1,$num
1029
1030 .Ldec8x_loop_grande:
1031         #mov    $num,24(%rsp)                   # original $num
1032         xor     $num,$num
1033 ___
1034 for($i=0;$i<8;$i++) {
1035   my $temp = $i ? $offload : $offset;
1036     $ptr_reg=&pointer_register($flavour,@ptr[$i]);
1037     $temp_reg=&pointer_register($flavour,$temp);
1038     $code.=<<___;
1039         # borrow $one for number of blocks
1040         mov     `$inp_elm_size*$i+2*$ptr_size-$inp_elm_size*4`($inp),$one
1041         # input pointer
1042         mov     `$inp_elm_size*$i+0-$inp_elm_size*4`($inp),$ptr_reg
1043         cmp     $num,$one
1044         # output pointer
1045         mov     `$inp_elm_size*$i+$ptr_size-$inp_elm_size*4`($inp),$temp_reg
1046         cmovg   $one,$num                       # find maximum
1047         test    $one,$one
1048         # load IV
1049         vmovdqu `$inp_elm_size*$i+2*$ptr_size+8-$inp_elm_size*4`($inp),@out[$i]
1050         mov     $one,`32+4*$i`(%rsp)            # initialize counters
1051         cmovle  %rsp,@ptr[$i]                   # cancel input
1052         sub     @ptr[$i],$temp                  # distance between input and output
1053         mov     $temp,`64+8*$i`(%rsp)           # initialize distances
1054         vmovdqu @out[$i],`192+16*$i`(%rsp)      # offload IV
1055 ___
1056 }
1057 $code.=<<___;
1058         test    $num,$num
1059         jz      .Ldec8x_done
1060
1061         vmovups 0x10-0x78($key),$rndkey1
1062         vmovups 0x20-0x78($key),$rndkey0
1063         mov     0xf0-0x78($key),$rounds
1064          lea    192+128(%rsp),$offload          # offload area
1065
1066         vmovdqu (@ptr[0]),@out[0]               # load inputs
1067         vmovdqu (@ptr[1]),@out[1]
1068         vmovdqu (@ptr[2]),@out[2]
1069         vmovdqu (@ptr[3]),@out[3]
1070         vmovdqu (@ptr[4]),@out[4]
1071         vmovdqu (@ptr[5]),@out[5]
1072         vmovdqu (@ptr[6]),@out[6]
1073         vmovdqu (@ptr[7]),@out[7]
1074         vmovdqu @out[0],0x00($offload)          # offload inputs
1075         vpxor   $zero,@out[0],@out[0]           # xor inputs with 0-round
1076         vmovdqu @out[1],0x10($offload)
1077         vpxor   $zero,@out[1],@out[1]
1078         vmovdqu @out[2],0x20($offload)
1079         vpxor   $zero,@out[2],@out[2]
1080         vmovdqu @out[3],0x30($offload)
1081         vpxor   $zero,@out[3],@out[3]
1082         vmovdqu @out[4],0x40($offload)
1083         vpxor   $zero,@out[4],@out[4]
1084         vmovdqu @out[5],0x50($offload)
1085         vpxor   $zero,@out[5],@out[5]
1086         vmovdqu @out[6],0x60($offload)
1087         vpxor   $zero,@out[6],@out[6]
1088         vmovdqu @out[7],0x70($offload)
1089         vpxor   $zero,@out[7],@out[7]
1090         xor     \$0x80,$offload
1091         mov     \$1,$one                        # constant of 1
1092         jmp     .Loop_dec8x
1093
1094 .align  32
1095 .Loop_dec8x:
1096 ___
1097 for($i=0;$i<8;$i++) {
1098 my $rndkey=($i&1)?$rndkey0:$rndkey1;
1099 $code.=<<___;
1100         vaesdec         $rndkey,@out[0],@out[0]
1101          cmp            32+4*$i(%rsp),$one
1102 ___
1103 $code.=<<___ if ($i);
1104          mov            64+8*$i(%rsp),$offset
1105 ___
1106 $code.=<<___;
1107         vaesdec         $rndkey,@out[1],@out[1]
1108         prefetcht0      31(@ptr[$i])                    # prefetch input
1109         vaesdec         $rndkey,@out[2],@out[2]
1110 ___
1111 $code.=<<___ if ($i>1);
1112         prefetcht0      15(@ptr[$i-2])                  # prefetch output
1113 ___
1114 $code.=<<___;
1115         vaesdec         $rndkey,@out[3],@out[3]
1116          lea            (@ptr[$i],$offset),$offset
1117          cmovge         %rsp,@ptr[$i]                   # cancel input
1118         vaesdec         $rndkey,@out[4],@out[4]
1119          cmovg          %rsp,$offset                    # sink output
1120         vaesdec         $rndkey,@out[5],@out[5]
1121          sub            @ptr[$i],$offset
1122         vaesdec         $rndkey,@out[6],@out[6]
1123          vmovdqu        16(@ptr[$i]),@inp[$i%4]         # load input
1124          mov            $offset,64+8*$i(%rsp)
1125         vaesdec         $rndkey,@out[7],@out[7]
1126         vmovups         `16*(3+$i)-0x78`($key),$rndkey
1127          lea            16(@ptr[$i],$offset),@ptr[$i]   # switch to output
1128 ___
1129 $code.=<<___ if ($i<4);
1130          vmovdqu        @inp[$i%4],`128+16*$i`(%rsp)    # off-load
1131 ___
1132 }
1133 $code.=<<___;
1134          vmovdqu        32(%rsp),$counters
1135         prefetcht0      15(@ptr[$i-2])                  # prefetch output
1136         prefetcht0      15(@ptr[$i-1])
1137         cmp     \$11,$rounds
1138         jb      .Ldec8x_tail
1139
1140         vaesdec         $rndkey1,@out[0],@out[0]
1141         vaesdec         $rndkey1,@out[1],@out[1]
1142         vaesdec         $rndkey1,@out[2],@out[2]
1143         vaesdec         $rndkey1,@out[3],@out[3]
1144         vaesdec         $rndkey1,@out[4],@out[4]
1145         vaesdec         $rndkey1,@out[5],@out[5]
1146         vaesdec         $rndkey1,@out[6],@out[6]
1147         vaesdec         $rndkey1,@out[7],@out[7]
1148         vmovups         0xb0-0x78($key),$rndkey1
1149
1150         vaesdec         $rndkey0,@out[0],@out[0]
1151         vaesdec         $rndkey0,@out[1],@out[1]
1152         vaesdec         $rndkey0,@out[2],@out[2]
1153         vaesdec         $rndkey0,@out[3],@out[3]
1154         vaesdec         $rndkey0,@out[4],@out[4]
1155         vaesdec         $rndkey0,@out[5],@out[5]
1156         vaesdec         $rndkey0,@out[6],@out[6]
1157         vaesdec         $rndkey0,@out[7],@out[7]
1158         vmovups         0xc0-0x78($key),$rndkey0
1159         je      .Ldec8x_tail
1160
1161         vaesdec         $rndkey1,@out[0],@out[0]
1162         vaesdec         $rndkey1,@out[1],@out[1]
1163         vaesdec         $rndkey1,@out[2],@out[2]
1164         vaesdec         $rndkey1,@out[3],@out[3]
1165         vaesdec         $rndkey1,@out[4],@out[4]
1166         vaesdec         $rndkey1,@out[5],@out[5]
1167         vaesdec         $rndkey1,@out[6],@out[6]
1168         vaesdec         $rndkey1,@out[7],@out[7]
1169         vmovups         0xd0-0x78($key),$rndkey1
1170
1171         vaesdec         $rndkey0,@out[0],@out[0]
1172         vaesdec         $rndkey0,@out[1],@out[1]
1173         vaesdec         $rndkey0,@out[2],@out[2]
1174         vaesdec         $rndkey0,@out[3],@out[3]
1175         vaesdec         $rndkey0,@out[4],@out[4]
1176         vaesdec         $rndkey0,@out[5],@out[5]
1177         vaesdec         $rndkey0,@out[6],@out[6]
1178         vaesdec         $rndkey0,@out[7],@out[7]
1179         vmovups         0xe0-0x78($key),$rndkey0
1180
1181 .Ldec8x_tail:
1182         vaesdec         $rndkey1,@out[0],@out[0]
1183          vpxor          $zero,$zero,$zero
1184         vaesdec         $rndkey1,@out[1],@out[1]
1185         vaesdec         $rndkey1,@out[2],@out[2]
1186          vpcmpgtd       $zero,$counters,$zero
1187         vaesdec         $rndkey1,@out[3],@out[3]
1188         vaesdec         $rndkey1,@out[4],@out[4]
1189          vpaddd         $counters,$zero,$zero           # decrement counters
1190          vmovdqu        48(%rsp),$counters
1191         vaesdec         $rndkey1,@out[5],@out[5]
1192          mov            64(%rsp),$offset                # pre-load 1st offset
1193         vaesdec         $rndkey1,@out[6],@out[6]
1194         vaesdec         $rndkey1,@out[7],@out[7]
1195         vmovups         0x10-0x78($key),$rndkey1
1196
1197         vaesdeclast     $rndkey0,@out[0],@out[0]
1198          vmovdqa        $zero,32(%rsp)                  # update counters
1199          vpxor          $zero,$zero,$zero
1200         vaesdeclast     $rndkey0,@out[1],@out[1]
1201         vpxor           0x00($offload),@out[0],@out[0]  # xor with IV
1202         vaesdeclast     $rndkey0,@out[2],@out[2]
1203         vpxor           0x10($offload),@out[1],@out[1]
1204          vpcmpgtd       $zero,$counters,$zero
1205         vaesdeclast     $rndkey0,@out[3],@out[3]
1206         vpxor           0x20($offload),@out[2],@out[2]
1207         vaesdeclast     $rndkey0,@out[4],@out[4]
1208         vpxor           0x30($offload),@out[3],@out[3]
1209          vpaddd         $zero,$counters,$counters       # decrement counters
1210          vmovdqu        -0x78($key),$zero               # 0-round
1211         vaesdeclast     $rndkey0,@out[5],@out[5]
1212         vpxor           0x40($offload),@out[4],@out[4]
1213         vaesdeclast     $rndkey0,@out[6],@out[6]
1214         vpxor           0x50($offload),@out[5],@out[5]
1215          vmovdqa        $counters,48(%rsp)              # update counters
1216         vaesdeclast     $rndkey0,@out[7],@out[7]
1217         vpxor           0x60($offload),@out[6],@out[6]
1218         vmovups         0x20-0x78($key),$rndkey0
1219
1220         vmovups         @out[0],-16(@ptr[0])            # write output
1221          sub            $offset,@ptr[0]                 # switch to input
1222          vmovdqu        128+0(%rsp),@out[0]
1223         vpxor           0x70($offload),@out[7],@out[7]
1224         vmovups         @out[1],-16(@ptr[1])
1225          sub            `64+1*8`(%rsp),@ptr[1]
1226          vmovdqu        @out[0],0x00($offload)
1227          vpxor          $zero,@out[0],@out[0]
1228          vmovdqu        128+16(%rsp),@out[1]
1229         vmovups         @out[2],-16(@ptr[2])
1230          sub            `64+2*8`(%rsp),@ptr[2]
1231          vmovdqu        @out[1],0x10($offload)
1232          vpxor          $zero,@out[1],@out[1]
1233          vmovdqu        128+32(%rsp),@out[2]
1234         vmovups         @out[3],-16(@ptr[3])
1235          sub            `64+3*8`(%rsp),@ptr[3]
1236          vmovdqu        @out[2],0x20($offload)
1237          vpxor          $zero,@out[2],@out[2]
1238          vmovdqu        128+48(%rsp),@out[3]
1239         vmovups         @out[4],-16(@ptr[4])
1240          sub            `64+4*8`(%rsp),@ptr[4]
1241          vmovdqu        @out[3],0x30($offload)
1242          vpxor          $zero,@out[3],@out[3]
1243          vmovdqu        @inp[0],0x40($offload)
1244          vpxor          @inp[0],$zero,@out[4]
1245         vmovups         @out[5],-16(@ptr[5])
1246          sub            `64+5*8`(%rsp),@ptr[5]
1247          vmovdqu        @inp[1],0x50($offload)
1248          vpxor          @inp[1],$zero,@out[5]
1249         vmovups         @out[6],-16(@ptr[6])
1250          sub            `64+6*8`(%rsp),@ptr[6]
1251          vmovdqu        @inp[2],0x60($offload)
1252          vpxor          @inp[2],$zero,@out[6]
1253         vmovups         @out[7],-16(@ptr[7])
1254          sub            `64+7*8`(%rsp),@ptr[7]
1255          vmovdqu        @inp[3],0x70($offload)
1256          vpxor          @inp[3],$zero,@out[7]
1257
1258         xor     \$128,$offload
1259         dec     $num
1260         jnz     .Loop_dec8x
1261
1262         mov     16(%rsp),%rax                   # original %rsp
1263 .cfi_def_cfa    %rax,8
1264         #mov    24(%rsp),$num
1265         #lea    `$inp_elm_size*8`($inp),$inp
1266         #dec    $num
1267         #jnz    .Ldec8x_loop_grande
1268
1269 .Ldec8x_done:
1270         vzeroupper
1271 ___
1272 $code.=<<___ if ($win64);
1273         movaps  -0xd8(%rax),%xmm6
1274         movaps  -0xc8(%rax),%xmm7
1275         movaps  -0xb8(%rax),%xmm8
1276         movaps  -0xa8(%rax),%xmm9
1277         movaps  -0x98(%rax),%xmm10
1278         movaps  -0x88(%rax),%xmm11
1279         movaps  -0x78(%rax),%xmm12
1280         movaps  -0x68(%rax),%xmm13
1281         movaps  -0x58(%rax),%xmm14
1282         movaps  -0x48(%rax),%xmm15
1283 ___
1284 $code.=<<___;
1285         mov     -48(%rax),%r15
1286 .cfi_restore    %r15
1287         mov     -40(%rax),%r14
1288 .cfi_restore    %r14
1289         mov     -32(%rax),%r13
1290 .cfi_restore    %r13
1291         mov     -24(%rax),%r12
1292 .cfi_restore    %r12
1293         mov     -16(%rax),%rbp
1294 .cfi_restore    %rbp
1295         mov     -8(%rax),%rbx
1296 .cfi_restore    %rbx
1297         lea     (%rax),%rsp
1298 .cfi_def_cfa_register   %rsp
1299 .Ldec8x_epilogue:
1300         ret
1301 .cfi_endproc
1302 .size   aesni_multi_cbc_decrypt_avx,.-aesni_multi_cbc_decrypt_avx
1303 ___
1304                                                 }}}
1305
1306 if ($win64) {
1307 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1308 #               CONTEXT *context,DISPATCHER_CONTEXT *disp)
1309 $rec="%rcx";
1310 $frame="%rdx";
1311 $context="%r8";
1312 $disp="%r9";
1313
1314 $code.=<<___;
1315 .extern __imp_RtlVirtualUnwind
1316 .type   se_handler,\@abi-omnipotent
1317 .align  16
1318 se_handler:
1319         push    %rsi
1320         push    %rdi
1321         push    %rbx
1322         push    %rbp
1323         push    %r12
1324         push    %r13
1325         push    %r14
1326         push    %r15
1327         pushfq
1328         sub     \$64,%rsp
1329
1330         mov     120($context),%rax      # pull context->Rax
1331         mov     248($context),%rbx      # pull context->Rip
1332
1333         mov     8($disp),%rsi           # disp->ImageBase
1334         mov     56($disp),%r11          # disp->HandlerData
1335
1336         mov     0(%r11),%r10d           # HandlerData[0]
1337         lea     (%rsi,%r10),%r10        # prologue label
1338         cmp     %r10,%rbx               # context->Rip<.Lprologue
1339         jb      .Lin_prologue
1340
1341         mov     152($context),%rax      # pull context->Rsp
1342
1343         mov     4(%r11),%r10d           # HandlerData[1]
1344         lea     (%rsi,%r10),%r10        # epilogue label
1345         cmp     %r10,%rbx               # context->Rip>=.Lepilogue
1346         jae     .Lin_prologue
1347
1348         mov     16(%rax),%rax           # pull saved stack pointer
1349
1350         mov     -8(%rax),%rbx
1351         mov     -16(%rax),%rbp
1352         mov     -24(%rax),%r12
1353         mov     -32(%rax),%r13
1354         mov     -40(%rax),%r14
1355         mov     -48(%rax),%r15
1356         mov     %rbx,144($context)      # restore context->Rbx
1357         mov     %rbp,160($context)      # restore context->Rbp
1358         mov     %r12,216($context)      # restore context->R12
1359         mov     %r13,224($context)      # restore context->R13
1360         mov     %r14,232($context)      # restore context->R14
1361         mov     %r15,240($context)      # restore context->R15
1362
1363         lea     -56-10*16(%rax),%rsi
1364         lea     512($context),%rdi      # &context.Xmm6
1365         mov     \$20,%ecx
1366         .long   0xa548f3fc              # cld; rep movsq
1367
1368 .Lin_prologue:
1369         mov     8(%rax),%rdi
1370         mov     16(%rax),%rsi
1371         mov     %rax,152($context)      # restore context->Rsp
1372         mov     %rsi,168($context)      # restore context->Rsi
1373         mov     %rdi,176($context)      # restore context->Rdi
1374
1375         mov     40($disp),%rdi          # disp->ContextRecord
1376         mov     $context,%rsi           # context
1377         mov     \$154,%ecx              # sizeof(CONTEXT)
1378         .long   0xa548f3fc              # cld; rep movsq
1379
1380         mov     $disp,%rsi
1381         xor     %rcx,%rcx               # arg1, UNW_FLAG_NHANDLER
1382         mov     8(%rsi),%rdx            # arg2, disp->ImageBase
1383         mov     0(%rsi),%r8             # arg3, disp->ControlPc
1384         mov     16(%rsi),%r9            # arg4, disp->FunctionEntry
1385         mov     40(%rsi),%r10           # disp->ContextRecord
1386         lea     56(%rsi),%r11           # &disp->HandlerData
1387         lea     24(%rsi),%r12           # &disp->EstablisherFrame
1388         mov     %r10,32(%rsp)           # arg5
1389         mov     %r11,40(%rsp)           # arg6
1390         mov     %r12,48(%rsp)           # arg7
1391         mov     %rcx,56(%rsp)           # arg8, (NULL)
1392         call    *__imp_RtlVirtualUnwind(%rip)
1393
1394         mov     \$1,%eax                # ExceptionContinueSearch
1395         add     \$64,%rsp
1396         popfq
1397         pop     %r15
1398         pop     %r14
1399         pop     %r13
1400         pop     %r12
1401         pop     %rbp
1402         pop     %rbx
1403         pop     %rdi
1404         pop     %rsi
1405         ret
1406 .size   se_handler,.-se_handler
1407
1408 .section        .pdata
1409 .align  4
1410         .rva    .LSEH_begin_aesni_multi_cbc_encrypt
1411         .rva    .LSEH_end_aesni_multi_cbc_encrypt
1412         .rva    .LSEH_info_aesni_multi_cbc_encrypt
1413         .rva    .LSEH_begin_aesni_multi_cbc_decrypt
1414         .rva    .LSEH_end_aesni_multi_cbc_decrypt
1415         .rva    .LSEH_info_aesni_multi_cbc_decrypt
1416 ___
1417 $code.=<<___ if ($avx);
1418         .rva    .LSEH_begin_aesni_multi_cbc_encrypt_avx
1419         .rva    .LSEH_end_aesni_multi_cbc_encrypt_avx
1420         .rva    .LSEH_info_aesni_multi_cbc_encrypt_avx
1421         .rva    .LSEH_begin_aesni_multi_cbc_decrypt_avx
1422         .rva    .LSEH_end_aesni_multi_cbc_decrypt_avx
1423         .rva    .LSEH_info_aesni_multi_cbc_decrypt_avx
1424 ___
1425 $code.=<<___;
1426 .section        .xdata
1427 .align  8
1428 .LSEH_info_aesni_multi_cbc_encrypt:
1429         .byte   9,0,0,0
1430         .rva    se_handler
1431         .rva    .Lenc4x_body,.Lenc4x_epilogue           # HandlerData[]
1432 .LSEH_info_aesni_multi_cbc_decrypt:
1433         .byte   9,0,0,0
1434         .rva    se_handler
1435         .rva    .Ldec4x_body,.Ldec4x_epilogue           # HandlerData[]
1436 ___
1437 $code.=<<___ if ($avx);
1438 .LSEH_info_aesni_multi_cbc_encrypt_avx:
1439         .byte   9,0,0,0
1440         .rva    se_handler
1441         .rva    .Lenc8x_body,.Lenc8x_epilogue           # HandlerData[]
1442 .LSEH_info_aesni_multi_cbc_decrypt_avx:
1443         .byte   9,0,0,0
1444         .rva    se_handler
1445         .rva    .Ldec8x_body,.Ldec8x_epilogue           # HandlerData[]
1446 ___
1447 }
1448 ####################################################################
1449
1450 sub rex {
1451   local *opcode=shift;
1452   my ($dst,$src)=@_;
1453   my $rex=0;
1454
1455     $rex|=0x04                  if($dst>=8);
1456     $rex|=0x01                  if($src>=8);
1457     push @opcode,$rex|0x40      if($rex);
1458 }
1459
1460 sub aesni {
1461   my $line=shift;
1462   my @opcode=(0x66);
1463
1464     if ($line=~/(aeskeygenassist)\s+\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1465         rex(\@opcode,$4,$3);
1466         push @opcode,0x0f,0x3a,0xdf;
1467         push @opcode,0xc0|($3&7)|(($4&7)<<3);   # ModR/M
1468         my $c=$2;
1469         push @opcode,$c=~/^0/?oct($c):$c;
1470         return ".byte\t".join(',',@opcode);
1471     }
1472     elsif ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1473         my %opcodelet = (
1474                 "aesimc" => 0xdb,
1475                 "aesenc" => 0xdc,       "aesenclast" => 0xdd,
1476                 "aesdec" => 0xde,       "aesdeclast" => 0xdf
1477         );
1478         return undef if (!defined($opcodelet{$1}));
1479         rex(\@opcode,$3,$2);
1480         push @opcode,0x0f,0x38,$opcodelet{$1};
1481         push @opcode,0xc0|($2&7)|(($3&7)<<3);   # ModR/M
1482         return ".byte\t".join(',',@opcode);
1483     }
1484     elsif ($line=~/(aes[a-z]+)\s+([0x1-9a-fA-F]*)\(%rsp\),\s*%xmm([0-9]+)/) {
1485         my %opcodelet = (
1486                 "aesenc" => 0xdc,       "aesenclast" => 0xdd,
1487                 "aesdec" => 0xde,       "aesdeclast" => 0xdf
1488         );
1489         return undef if (!defined($opcodelet{$1}));
1490         my $off = $2;
1491         push @opcode,0x44 if ($3>=8);
1492         push @opcode,0x0f,0x38,$opcodelet{$1};
1493         push @opcode,0x44|(($3&7)<<3),0x24;     # ModR/M
1494         push @opcode,($off=~/^0/?oct($off):$off)&0xff;
1495         return ".byte\t".join(',',@opcode);
1496     }
1497     return $line;
1498 }
1499
1500 $code =~ s/\`([^\`]*)\`/eval($1)/gem;
1501 $code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem;
1502
1503 print $code;
1504 close STDOUT or die "error closing STDOUT: $!";