aesni-mb-x86_64.pl: add Win64 SEH.
[openssl.git] / crypto / aes / asm / aesni-mb-x86_64.pl
1 #!/usr/bin/env perl
2
3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9
10 # Multi-buffer AES-NI procedures process several independent buffers
11 # in parallel by interleaving independent instructions.
12 #
13 # Cycles per byte for interleave factor 4:
14 #
15 #                       asymptotic      measured
16 #                       ---------------------------
17 # Westmere              5.00/4=1.25     5.13/4=1.28
18 # Atom                  15.0/4=3.75     ?15.7/4=3.93
19 # Sandy Bridge          5.06/4=1.27     5.18/4=1.29
20 # Ivy Bridge            5.06/4=1.27     5.14/4=1.29
21 # Haswell               4.44/4=1.11     4.44/4=1.11
22 # Bulldozer             5.75/4=1.44     5.76/4=1.44
23 #
24 # Cycles per byte for interleave factor 8 (not implemented for
25 # pre-AVX processors, where higher interleave factor incidentally
26 # doesn't result in improvement):
27 #
28 #                       asymptotic      measured
29 #                       ---------------------------
30 # Sandy Bridge          5.06/8=0.64     7.10/8=0.89(*)
31 # Ivy Bridge            5.06/8=0.64     7.14/8=0.89(*)
32 # Haswell               5.00/8=0.63     5.00/8=0.63
33 # Bulldozer             5.75/8=0.72     5.77/8=0.72
34 #
35 # (*)   Sandy/Ivy Bridge are known to handle high interleave factors
36 #       suboptimally;
37
38 $flavour = shift;
39 $output  = shift;
40 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
41
42 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
43
44 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
45 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
46 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
47 die "can't locate x86_64-xlate.pl";
48
49 $avx=0;
50
51 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
52                 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
53         $avx = ($1>=2.19) + ($1>=2.22);
54 }
55
56 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
57            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
58         $avx = ($1>=2.09) + ($1>=2.10);
59 }
60
61 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
62            `ml64 2>&1` =~ /Version ([0-9]+)\./) {
63         $avx = ($1>=10) + ($1>=11);
64 }
65
66 open OUT,"| \"$^X\" $xlate $flavour $output";
67 *STDOUT=*OUT;
68
69 # void aesni_multi_cbc_encrypt (
70 #     struct {  void *inp,*out; int blocks; double iv[2]; } inp[8];
71 #     const AES_KEY *key,
72 #     int num);         /* 1 or 2 */
73 #
74 $inp="%rdi";    # 1st arg
75 $key="%rsi";    # 2nd arg
76 $num="%edx";
77
78 @inptr=map("%r$_",(8..11));
79 @outptr=map("%r$_",(12..15));
80
81 ($rndkey0,$rndkey1)=("%xmm0","%xmm1");
82 @out=map("%xmm$_",(2..5));
83 @inp=map("%xmm$_",(6..9));
84 ($counters,$mask,$zero)=map("%xmm$_",(10..12));
85
86 ($rounds,$one,$sink,$offset)=("%eax","%ecx","%rbp","%rbx");
87
88 $code.=<<___;
89 .text
90
91 .extern OPENSSL_ia32cap_P
92
93 .globl  aesni_multi_cbc_encrypt
94 .type   aesni_multi_cbc_encrypt,\@function,3
95 .align  32
96 aesni_multi_cbc_encrypt:
97 ___
98 $code.=<<___ if ($avx);
99         cmp     \$2,$num
100         jb      .Lenc_non_avx
101         mov     OPENSSL_ia32cap_P+4(%rip),%ecx
102         test    \$`1<<28`,%ecx                  # AVX bit
103         jnz     _avx_cbc_enc_shortcut
104         jmp     .Lenc_non_avx
105 .align  16
106 .Lenc_non_avx:
107 ___
108 $code.=<<___;
109         mov     %rsp,%rax
110         push    %rbx
111         push    %rbp
112         push    %r12
113         push    %r13
114         push    %r14
115         push    %r15
116 ___
117 $code.=<<___ if ($win64);
118         lea     -0xa8(%rsp),%rsp
119         movaps  %xmm6,(%rsp)
120         movaps  %xmm7,0x10(%rsp)
121         movaps  %xmm8,0x20(%rsp)
122         movaps  %xmm9,0x30(%rsp)
123         movaps  %xmm10,0x40(%rsp)
124         movaps  %xmm11,0x50(%rsp)
125         movaps  %xmm12,0x60(%rsp)
126         movaps  %xmm13,-0x68(%rax)      # not used, saved to share se_handler 
127         movaps  %xmm14,-0x58(%rax)
128         movaps  %xmm15,-0x48(%rax)
129 ___
130 $code.=<<___;
131         # stack layout
132         #
133         # +0    output sink
134         # +16   input sink [original %rsp and $num]
135         # +32   counters
136
137         sub     \$48,%rsp
138         and     \$-64,%rsp
139         mov     %rax,16(%rsp)                   # original %rsp
140
141 .Lenc4x_body:
142         movdqu  ($key),$zero                    # 0-round key
143         lea     0x78($key),$key                 # size optimization
144         lea     40*2($inp),$inp
145
146 .Lenc4x_loop_grande:
147         mov     $num,24(%rsp)                   # original $num
148         xor     $num,$num
149 ___
150 for($i=0;$i<4;$i++) {
151     $code.=<<___;
152         mov     `40*$i+16-40*2`($inp),$one      # borrow $one for number of blocks
153         mov     `40*$i+0-40*2`($inp),@inptr[$i]
154         cmp     $num,$one
155         mov     `40*$i+8-40*2`($inp),@outptr[$i]
156         cmovg   $one,$num                       # find maximum
157         test    $one,$one
158         movdqu  `40*$i+24-40*2`($inp),@out[$i]  # load IV
159         mov     $one,`32+4*$i`(%rsp)            # initialize counters
160         cmovle  %rsp,@inptr[$i]                 # cancel input
161 ___
162 }
163 $code.=<<___;
164         test    $num,$num
165         jz      .Lenc4x_done
166
167         movups  0x10-0x78($key),$rndkey1
168          pxor   $zero,@out[0]
169         movups  0x20-0x78($key),$rndkey0
170          pxor   $zero,@out[1]
171         mov     0xf0-0x78($key),$rounds
172          pxor   $zero,@out[2]
173         movdqu  (@inptr[0]),@inp[0]             # load inputs
174          pxor   $zero,@out[3]
175         movdqu  (@inptr[1]),@inp[1]
176          pxor   @inp[0],@out[0]
177         movdqu  (@inptr[2]),@inp[2]
178          pxor   @inp[1],@out[1]
179         movdqu  (@inptr[3]),@inp[3]
180          pxor   @inp[2],@out[2]
181          pxor   @inp[3],@out[3]
182         movdqa  32(%rsp),$counters              # load counters
183         xor     $offset,$offset
184         jmp     .Loop_enc4x
185
186 .align  32
187 .Loop_enc4x:
188         add     \$16,$offset
189         lea     16(%rsp),$sink                  # sink pointer
190         mov     \$1,$one                        # constant of 1
191         sub     $offset,$sink
192
193         aesenc          $rndkey1,@out[0]
194         prefetcht0      31(@inptr[0],$offset)   # prefetch input
195         prefetcht0      31(@inptr[1],$offset)
196         aesenc          $rndkey1,@out[1]
197         prefetcht0      31(@inptr[2],$offset)
198         prefetcht0      31(@inptr[2],$offset)
199         aesenc          $rndkey1,@out[2]
200         aesenc          $rndkey1,@out[3]
201         movups          0x30-0x78($key),$rndkey1
202 ___
203 for($i=0;$i<4;$i++) {
204 my $rndkey = ($i&1) ? $rndkey1 : $rndkey0;
205 $code.=<<___;
206          cmp            `32+4*$i`(%rsp),$one
207         aesenc          $rndkey,@out[0]
208         aesenc          $rndkey,@out[1]
209         aesenc          $rndkey,@out[2]
210          cmovge         $sink,@inptr[$i]        # cancel input
211          cmovg          $sink,@outptr[$i]       # sink output
212         aesenc          $rndkey,@out[3]
213         movups          `0x40+16*$i-0x78`($key),$rndkey
214 ___
215 }
216 $code.=<<___;
217          movdqa         $counters,$mask
218         aesenc          $rndkey0,@out[0]
219         prefetcht0      15(@outptr[0],$offset)  # prefetch output
220         prefetcht0      15(@outptr[1],$offset)
221         aesenc          $rndkey0,@out[1]
222         prefetcht0      15(@outptr[2],$offset)
223         prefetcht0      15(@outptr[3],$offset)
224         aesenc          $rndkey0,@out[2]
225         aesenc          $rndkey0,@out[3]
226         movups          0x80-0x78($key),$rndkey0
227          pxor           $zero,$zero
228
229         aesenc          $rndkey1,@out[0]
230          pcmpgtd        $zero,$mask
231          movdqu         -0x78($key),$zero       # reload 0-round key
232         aesenc          $rndkey1,@out[1]
233          paddd          $mask,$counters         # decrement counters
234          movdqa         $counters,32(%rsp)      # update counters
235         aesenc          $rndkey1,@out[2]
236         aesenc          $rndkey1,@out[3]
237         movups          0x90-0x78($key),$rndkey1
238
239         cmp     \$11,$rounds
240
241         aesenc          $rndkey0,@out[0]
242         aesenc          $rndkey0,@out[1]
243         aesenc          $rndkey0,@out[2]
244         aesenc          $rndkey0,@out[3]
245         movups          0xa0-0x78($key),$rndkey0
246
247         jb      .Lenc4x_tail
248
249         aesenc          $rndkey1,@out[0]
250         aesenc          $rndkey1,@out[1]
251         aesenc          $rndkey1,@out[2]
252         aesenc          $rndkey1,@out[3]
253         movups          0xb0-0x78($key),$rndkey1
254
255         aesenc          $rndkey0,@out[0]
256         aesenc          $rndkey0,@out[1]
257         aesenc          $rndkey0,@out[2]
258         aesenc          $rndkey0,@out[3]
259         movups          0xc0-0x78($key),$rndkey0
260
261         je      .Lenc4x_tail
262
263         aesenc          $rndkey1,@out[0]
264         aesenc          $rndkey1,@out[1]
265         aesenc          $rndkey1,@out[2]
266         aesenc          $rndkey1,@out[3]
267         movups          0xd0-0x78($key),$rndkey1
268
269         aesenc          $rndkey0,@out[0]
270         aesenc          $rndkey0,@out[1]
271         aesenc          $rndkey0,@out[2]
272         aesenc          $rndkey0,@out[3]
273         movups          0xe0-0x78($key),$rndkey0
274         jmp     .Lenc4x_tail
275
276 .align  32
277 .Lenc4x_tail:
278         aesenc          $rndkey1,@out[0]
279         aesenc          $rndkey1,@out[1]
280         aesenc          $rndkey1,@out[2]
281         aesenc          $rndkey1,@out[3]
282          movdqu         (@inptr[0],$offset),@inp[0]
283         movdqu          0x10-0x78($key),$rndkey1
284
285         aesenclast      $rndkey0,@out[0]
286          movdqu         (@inptr[1],$offset),@inp[1]
287          pxor           $zero,@inp[0]
288         aesenclast      $rndkey0,@out[1]
289          movdqu         (@inptr[2],$offset),@inp[2]
290          pxor           $zero,@inp[1]
291         aesenclast      $rndkey0,@out[2]
292          movdqu         (@inptr[3],$offset),@inp[3]
293          pxor           $zero,@inp[2]
294         aesenclast      $rndkey0,@out[3]
295         movdqu          0x20-0x78($key),$rndkey0
296          pxor           $zero,@inp[3]
297
298         movups          @out[0],-16(@outptr[0],$offset)
299          pxor           @inp[0],@out[0]
300         movups          @out[1],-16(@outptr[1],$offset) 
301          pxor           @inp[1],@out[1]
302         movups          @out[2],-16(@outptr[2],$offset) 
303          pxor           @inp[2],@out[2]
304         movups          @out[3],-16(@outptr[3],$offset)
305          pxor           @inp[3],@out[3]
306
307         dec     $num
308         jnz     .Loop_enc4x
309
310         mov     16(%rsp),%rax                   # original %rsp
311         mov     24(%rsp),$num
312
313         #pxor   @inp[0],@out[0]
314         #pxor   @inp[1],@out[1]
315         #movdqu @out[0],`40*0+24-40*2`($inp)    # output iv FIX ME!
316         #pxor   @inp[2],@out[2]
317         #movdqu @out[1],`40*1+24-40*2`($inp)
318         #pxor   @inp[3],@out[3]
319         #movdqu @out[2],`40*2+24-40*2`($inp)    # won't fix, let caller
320         #movdqu @out[3],`40*3+24-40*2`($inp)    # figure this out...
321
322         lea     `40*4`($inp),$inp
323         dec     $num
324         jnz     .Lenc4x_loop_grande
325
326 .Lenc4x_done:
327 ___
328 $code.=<<___ if ($win64);
329         movaps  -0xd8(%rax),%xmm6
330         movaps  -0xc8(%rax),%xmm7
331         movaps  -0xb8(%rax),%xmm8
332         movaps  -0xa8(%rax),%xmm9
333         movaps  -0x98(%rax),%xmm10
334         movaps  -0x88(%rax),%xmm11
335         movaps  -0x78(%rax),%xmm12
336         #movaps -0x68(%rax),%xmm13
337         #movaps -0x58(%rax),%xmm14
338         #movaps -0x48(%rax),%xmm15
339 ___
340 $code.=<<___;
341         mov     -48(%rax),%r15
342         mov     -40(%rax),%r14
343         mov     -32(%rax),%r13
344         mov     -24(%rax),%r12
345         mov     -16(%rax),%rbp
346         mov     -8(%rax),%rbx
347         lea     (%rax),%rsp
348 .Lenc4x_epilogue:
349         ret
350 .size   aesni_multi_cbc_encrypt,.-aesni_multi_cbc_encrypt
351
352 .globl  aesni_multi_cbc_decrypt
353 .type   aesni_multi_cbc_decrypt,\@function,3
354 .align  32
355 aesni_multi_cbc_decrypt:
356 ___
357 $code.=<<___ if ($avx);
358         cmp     \$2,$num
359         jb      .Ldec_non_avx
360         mov     OPENSSL_ia32cap_P+4(%rip),%ecx
361         test    \$`1<<28`,%ecx                  # AVX bit
362         jnz     _avx_cbc_dec_shortcut
363         jmp     .Ldec_non_avx
364 .align  16
365 .Ldec_non_avx:
366 ___
367 $code.=<<___;
368         mov     %rsp,%rax
369         push    %rbx
370         push    %rbp
371         push    %r12
372         push    %r13
373         push    %r14
374         push    %r15
375 ___
376 $code.=<<___ if ($win64);
377         lea     -0xa8(%rsp),%rsp
378         movaps  %xmm6,(%rsp)
379         movaps  %xmm7,0x10(%rsp)
380         movaps  %xmm8,0x20(%rsp)
381         movaps  %xmm9,0x30(%rsp)
382         movaps  %xmm10,0x40(%rsp)
383         movaps  %xmm11,0x50(%rsp)
384         movaps  %xmm12,0x60(%rsp)
385         movaps  %xmm13,-0x68(%rax)      # not used, saved to share se_handler 
386         movaps  %xmm14,-0x58(%rax)
387         movaps  %xmm15,-0x48(%rax)
388 ___
389 $code.=<<___;
390         # stack layout
391         #
392         # +0    output sink
393         # +16   input sink [original %rsp and $num]
394         # +32   counters
395
396         sub     \$48,%rsp
397         and     \$-64,%rsp
398         mov     %rax,16(%rsp)                   # original %rsp
399
400 .Ldec4x_body:
401         movdqu  ($key),$zero                    # 0-round key
402         lea     0x78($key),$key                 # size optimization
403         lea     40*2($inp),$inp
404
405 .Ldec4x_loop_grande:
406         mov     $num,24(%rsp)                   # original $num
407         xor     $num,$num
408 ___
409 for($i=0;$i<4;$i++) {
410     $code.=<<___;
411         mov     `40*$i+16-40*2`($inp),$one      # borrow $one for number of blocks
412         mov     `40*$i+0-40*2`($inp),@inptr[$i]
413         cmp     $num,$one
414         mov     `40*$i+8-40*2`($inp),@outptr[$i]
415         cmovg   $one,$num                       # find maximum
416         test    $one,$one
417         movdqu  `40*$i+24-40*2`($inp),@inp[$i]  # load IV
418         mov     $one,`32+4*$i`(%rsp)            # initialize counters
419         cmovle  %rsp,@inptr[$i]                 # cancel input
420 ___
421 }
422 $code.=<<___;
423         test    $num,$num
424         jz      .Ldec4x_done
425
426         movups  0x10-0x78($key),$rndkey1
427         movups  0x20-0x78($key),$rndkey0
428         mov     0xf0-0x78($key),$rounds
429         movdqu  (@inptr[0]),@out[0]             # load inputs
430         movdqu  (@inptr[1]),@out[1]
431          pxor   $zero,@out[0]
432         movdqu  (@inptr[2]),@out[2]
433          pxor   $zero,@out[1]
434         movdqu  (@inptr[3]),@out[3]
435          pxor   $zero,@out[2]
436          pxor   $zero,@out[3]
437         movdqa  32(%rsp),$counters              # load counters
438         xor     $offset,$offset
439         jmp     .Loop_dec4x
440
441 .align  32
442 .Loop_dec4x:
443         add     \$16,$offset
444         lea     16(%rsp),$sink                  # sink pointer
445         mov     \$1,$one                        # constant of 1
446         sub     $offset,$sink
447
448         aesdec          $rndkey1,@out[0]
449         prefetcht0      31(@inptr[0],$offset)   # prefetch input
450         prefetcht0      31(@inptr[1],$offset)
451         aesdec          $rndkey1,@out[1]
452         prefetcht0      31(@inptr[2],$offset)
453         prefetcht0      31(@inptr[3],$offset)
454         aesdec          $rndkey1,@out[2]
455         aesdec          $rndkey1,@out[3]
456         movups          0x30-0x78($key),$rndkey1
457 ___
458 for($i=0;$i<4;$i++) {
459 my $rndkey = ($i&1) ? $rndkey1 : $rndkey0;
460 $code.=<<___;
461          cmp            `32+4*$i`(%rsp),$one
462         aesdec          $rndkey,@out[0]
463         aesdec          $rndkey,@out[1]
464         aesdec          $rndkey,@out[2]
465          cmovge         $sink,@inptr[$i]        # cancel input
466          cmovg          $sink,@outptr[$i]       # sink output
467         aesdec          $rndkey,@out[3]
468         movups          `0x40+16*$i-0x78`($key),$rndkey
469 ___
470 }
471 $code.=<<___;
472          movdqa         $counters,$mask
473         aesdec          $rndkey0,@out[0]
474         prefetcht0      15(@outptr[0],$offset)  # prefetch output
475         prefetcht0      15(@outptr[1],$offset)
476         aesdec          $rndkey0,@out[1]
477         prefetcht0      15(@outptr[2],$offset)
478         prefetcht0      15(@outptr[3],$offset)
479         aesdec          $rndkey0,@out[2]
480         aesdec          $rndkey0,@out[3]
481         movups          0x80-0x78($key),$rndkey0
482          pxor           $zero,$zero
483
484         aesdec          $rndkey1,@out[0]
485          pcmpgtd        $zero,$mask
486          movdqu         -0x78($key),$zero       # reload 0-round key
487         aesdec          $rndkey1,@out[1]
488          paddd          $mask,$counters         # decrement counters
489          movdqa         $counters,32(%rsp)      # update counters
490         aesdec          $rndkey1,@out[2]
491         aesdec          $rndkey1,@out[3]
492         movups          0x90-0x78($key),$rndkey1
493
494         cmp     \$11,$rounds
495
496         aesdec          $rndkey0,@out[0]
497         aesdec          $rndkey0,@out[1]
498         aesdec          $rndkey0,@out[2]
499         aesdec          $rndkey0,@out[3]
500         movups          0xa0-0x78($key),$rndkey0
501
502         jb      .Ldec4x_tail
503
504         aesdec          $rndkey1,@out[0]
505         aesdec          $rndkey1,@out[1]
506         aesdec          $rndkey1,@out[2]
507         aesdec          $rndkey1,@out[3]
508         movups          0xb0-0x78($key),$rndkey1
509
510         aesdec          $rndkey0,@out[0]
511         aesdec          $rndkey0,@out[1]
512         aesdec          $rndkey0,@out[2]
513         aesdec          $rndkey0,@out[3]
514         movups          0xc0-0x78($key),$rndkey0
515
516         je      .Ldec4x_tail
517
518         aesdec          $rndkey1,@out[0]
519         aesdec          $rndkey1,@out[1]
520         aesdec          $rndkey1,@out[2]
521         aesdec          $rndkey1,@out[3]
522         movups          0xd0-0x78($key),$rndkey1
523
524         aesdec          $rndkey0,@out[0]
525         aesdec          $rndkey0,@out[1]
526         aesdec          $rndkey0,@out[2]
527         aesdec          $rndkey0,@out[3]
528         movups          0xe0-0x78($key),$rndkey0
529         jmp     .Ldec4x_tail
530
531 .align  32
532 .Ldec4x_tail:
533         aesdec          $rndkey1,@out[0]
534         aesdec          $rndkey1,@out[1]
535         aesdec          $rndkey1,@out[2]
536          pxor           $rndkey0,@inp[0]
537          pxor           $rndkey0,@inp[1]
538         aesdec          $rndkey1,@out[3]
539         movdqu          0x10-0x78($key),$rndkey1
540          pxor           $rndkey0,@inp[2]
541          pxor           $rndkey0,@inp[3]
542         movdqu          0x20-0x78($key),$rndkey0
543
544         aesdeclast      @inp[0],@out[0]
545         aesdeclast      @inp[1],@out[1]
546          movdqu         -16(@inptr[0],$offset),@inp[0]  # load next IV
547          movdqu         -16(@inptr[1],$offset),@inp[1]
548         aesdeclast      @inp[2],@out[2]
549         aesdeclast      @inp[3],@out[3]
550          movdqu         -16(@inptr[2],$offset),@inp[2]
551          movdqu         -16(@inptr[3],$offset),@inp[3]
552
553         movups          @out[0],-16(@outptr[0],$offset)
554          movdqu         (@inptr[0],$offset),@out[0]
555         movups          @out[1],-16(@outptr[1],$offset) 
556          movdqu         (@inptr[1],$offset),@out[1]
557          pxor           $zero,@out[0]
558         movups          @out[2],-16(@outptr[2],$offset) 
559          movdqu         (@inptr[2],$offset),@out[2]
560          pxor           $zero,@out[1]
561         movups          @out[3],-16(@outptr[3],$offset)
562          movdqu         (@inptr[3],$offset),@out[3]
563          pxor           $zero,@out[2]
564          pxor           $zero,@out[3]
565
566         dec     $num
567         jnz     .Loop_dec4x
568
569         mov     16(%rsp),%rax                   # original %rsp
570         mov     24(%rsp),$num
571
572         lea     `40*4`($inp),$inp
573         dec     $num
574         jnz     .Ldec4x_loop_grande
575
576 .Ldec4x_done:
577 ___
578 $code.=<<___ if ($win64);
579         movaps  -0xd8(%rax),%xmm6
580         movaps  -0xc8(%rax),%xmm7
581         movaps  -0xb8(%rax),%xmm8
582         movaps  -0xa8(%rax),%xmm9
583         movaps  -0x98(%rax),%xmm10
584         movaps  -0x88(%rax),%xmm11
585         movaps  -0x78(%rax),%xmm12
586         #movaps -0x68(%rax),%xmm13
587         #movaps -0x58(%rax),%xmm14
588         #movaps -0x48(%rax),%xmm15
589 ___
590 $code.=<<___;
591         mov     -48(%rax),%r15
592         mov     -40(%rax),%r14
593         mov     -32(%rax),%r13
594         mov     -24(%rax),%r12
595         mov     -16(%rax),%rbp
596         mov     -8(%rax),%rbx
597         lea     (%rax),%rsp
598 .Ldec4x_epilogue:
599         ret
600 .size   aesni_multi_cbc_decrypt,.-aesni_multi_cbc_decrypt
601 ___
602
603                                                 if ($avx) {{{
604 my @ptr=map("%r$_",(8..15));
605 my $offload=$sink;
606
607 my @out=map("%xmm$_",(2..9));
608 my @inp=map("%xmm$_",(10..13));
609 my ($counters,$zero)=("%xmm14","%xmm15");
610
611 $code.=<<___;
612 .type   aesni_multi_cbc_encrypt_avx,\@function,3
613 .align  32
614 aesni_multi_cbc_encrypt_avx:
615 _avx_cbc_enc_shortcut:
616         mov     %rsp,%rax
617         push    %rbx
618         push    %rbp
619         push    %r12
620         push    %r13
621         push    %r14
622         push    %r15
623 ___
624 $code.=<<___ if ($win64);
625         lea     -0xa8(%rsp),%rsp
626         movaps  %xmm6,(%rsp)
627         movaps  %xmm7,0x10(%rsp)
628         movaps  %xmm8,0x20(%rsp)
629         movaps  %xmm9,0x30(%rsp)
630         movaps  %xmm10,0x40(%rsp)
631         movaps  %xmm11,0x50(%rsp)
632         movaps  %xmm12,-0x78(%rax)
633         movaps  %xmm13,-0x68(%rax)
634         movaps  %xmm14,-0x58(%rax)
635         movaps  %xmm15,-0x48(%rax)
636 ___
637 $code.=<<___;
638         # stack layout
639         #
640         # +0    output sink
641         # +16   input sink [original %rsp and $num]
642         # +32   counters
643         # +64   distances between inputs and outputs
644         # +128  off-load area for @inp[0..3]
645
646         sub     \$192,%rsp
647         and     \$-128,%rsp
648         mov     %rax,16(%rsp)                   # original %rsp
649
650 .Lenc8x_body:
651         vzeroupper
652         vmovdqu ($key),$zero                    # 0-round key
653         lea     0x78($key),$key                 # size optimization
654         lea     40*4($inp),$inp
655         shr     \$1,$num
656
657 .Lenc8x_loop_grande:
658         #mov    $num,24(%rsp)                   # original $num
659         xor     $num,$num
660 ___
661 for($i=0;$i<8;$i++) {
662   my $temp = $i ? $offload : $offset;
663     $code.=<<___;
664         mov     `40*$i+16-40*4`($inp),$one      # borrow $one for number of blocks
665         mov     `40*$i+0-40*4`($inp),@ptr[$i]   # input pointer
666         cmp     $num,$one
667         mov     `40*$i+8-40*4`($inp),$temp      # output pointer
668         cmovg   $one,$num                       # find maximum
669         test    $one,$one
670         vmovdqu `40*$i+24-40*4`($inp),@out[$i]  # load IV
671         mov     $one,`32+4*$i`(%rsp)            # initialize counters
672         cmovle  %rsp,@ptr[$i]                   # cancel input
673         sub     @ptr[$i],$temp                  # distance between input and output
674         mov     $temp,`64+8*$i`(%rsp)           # initialize distances
675 ___
676 }
677 $code.=<<___;
678         test    $num,$num
679         jz      .Lenc8x_done
680
681         vmovups 0x10-0x78($key),$rndkey1
682         vmovups 0x20-0x78($key),$rndkey0
683         mov     0xf0-0x78($key),$rounds
684
685         vpxor   (@ptr[0]),$zero,@inp[0]         # load inputs and xor with 0-round
686          lea    128(%rsp),$offload              # offload area
687         vpxor   (@ptr[1]),$zero,@inp[1]
688         vpxor   (@ptr[2]),$zero,@inp[2]
689         vpxor   (@ptr[3]),$zero,@inp[3]
690          vpxor  @inp[0],@out[0],@out[0]
691         vpxor   (@ptr[4]),$zero,@inp[0]
692          vpxor  @inp[1],@out[1],@out[1]
693         vpxor   (@ptr[5]),$zero,@inp[1]
694          vpxor  @inp[2],@out[2],@out[2]
695         vpxor   (@ptr[6]),$zero,@inp[2]
696          vpxor  @inp[3],@out[3],@out[3]
697         vpxor   (@ptr[7]),$zero,@inp[3]
698          vpxor  @inp[0],@out[4],@out[4]
699         mov     \$1,$one                        # constant of 1
700          vpxor  @inp[1],@out[5],@out[5]
701          vpxor  @inp[2],@out[6],@out[6]
702          vpxor  @inp[3],@out[7],@out[7]
703         jmp     .Loop_enc8x
704
705 .align  32
706 .Loop_enc8x:
707 ___
708 for($i=0;$i<8;$i++) {
709 my $rndkey=($i&1)?$rndkey0:$rndkey1;
710 $code.=<<___;
711         vaesenc         $rndkey,@out[0],@out[0]
712          cmp            32+4*$i(%rsp),$one
713 ___
714 $code.=<<___ if ($i);
715          mov            64+8*$i(%rsp),$offset
716 ___
717 $code.=<<___;
718         vaesenc         $rndkey,@out[1],@out[1]
719         prefetcht0      31(@ptr[$i])                    # prefetch input
720         vaesenc         $rndkey,@out[2],@out[2]
721 ___
722 $code.=<<___ if ($i>1);
723         prefetcht0      15(@ptr[$i-2])                  # prefetch output
724 ___
725 $code.=<<___;
726         vaesenc         $rndkey,@out[3],@out[3]
727          lea            (@ptr[$i],$offset),$offset
728          cmovge         %rsp,@ptr[$i]                   # cancel input
729         vaesenc         $rndkey,@out[4],@out[4]
730          cmovg          %rsp,$offset                    # sink output
731         vaesenc         $rndkey,@out[5],@out[5]
732          sub            @ptr[$i],$offset
733         vaesenc         $rndkey,@out[6],@out[6]
734          vpxor          16(@ptr[$i]),$zero,@inp[$i%4]   # load input and xor with 0-round
735          mov            $offset,64+8*$i(%rsp)
736         vaesenc         $rndkey,@out[7],@out[7]
737         vmovups         `16*(3+$i)-0x78`($key),$rndkey
738          lea            16(@ptr[$i],$offset),@ptr[$i]   # switch to output
739 ___
740 $code.=<<___ if ($i<4)
741          vmovdqu        @inp[$i%4],`16*$i`($offload)    # off-load
742 ___
743 }
744 $code.=<<___;
745          vmovdqu        32(%rsp),$counters
746         prefetcht0      15(@ptr[$i-2])                  # prefetch output
747         prefetcht0      15(@ptr[$i-1])
748         cmp     \$11,$rounds
749         jb      .Lenc8x_tail
750
751         vaesenc         $rndkey1,@out[0],@out[0]
752         vaesenc         $rndkey1,@out[1],@out[1]
753         vaesenc         $rndkey1,@out[2],@out[2]
754         vaesenc         $rndkey1,@out[3],@out[3]
755         vaesenc         $rndkey1,@out[4],@out[4]
756         vaesenc         $rndkey1,@out[5],@out[5]
757         vaesenc         $rndkey1,@out[6],@out[6]
758         vaesenc         $rndkey1,@out[7],@out[7]
759         vmovups         0xb0-0x78($key),$rndkey1
760
761         vaesenc         $rndkey0,@out[0],@out[0]
762         vaesenc         $rndkey0,@out[1],@out[1]
763         vaesenc         $rndkey0,@out[2],@out[2]
764         vaesenc         $rndkey0,@out[3],@out[3]
765         vaesenc         $rndkey0,@out[4],@out[4]
766         vaesenc         $rndkey0,@out[5],@out[5]
767         vaesenc         $rndkey0,@out[6],@out[6]
768         vaesenc         $rndkey0,@out[7],@out[7]
769         vmovups         0xc0-0x78($key),$rndkey0
770         je      .Lenc8x_tail
771
772         vaesenc         $rndkey1,@out[0],@out[0]
773         vaesenc         $rndkey1,@out[1],@out[1]
774         vaesenc         $rndkey1,@out[2],@out[2]
775         vaesenc         $rndkey1,@out[3],@out[3]
776         vaesenc         $rndkey1,@out[4],@out[4]
777         vaesenc         $rndkey1,@out[5],@out[5]
778         vaesenc         $rndkey1,@out[6],@out[6]
779         vaesenc         $rndkey1,@out[7],@out[7]
780         vmovups         0xd0-0x78($key),$rndkey1
781
782         vaesenc         $rndkey0,@out[0],@out[0]
783         vaesenc         $rndkey0,@out[1],@out[1]
784         vaesenc         $rndkey0,@out[2],@out[2]
785         vaesenc         $rndkey0,@out[3],@out[3]
786         vaesenc         $rndkey0,@out[4],@out[4]
787         vaesenc         $rndkey0,@out[5],@out[5]
788         vaesenc         $rndkey0,@out[6],@out[6]
789         vaesenc         $rndkey0,@out[7],@out[7]
790         vmovups         0xe0-0x78($key),$rndkey0
791
792 .Lenc8x_tail:
793         vaesenc         $rndkey1,@out[0],@out[0]
794          vpxor          $zero,$zero,$zero
795         vaesenc         $rndkey1,@out[1],@out[1]
796         vaesenc         $rndkey1,@out[2],@out[2]
797          vpcmpgtd       $zero,$counters,$zero
798         vaesenc         $rndkey1,@out[3],@out[3]
799         vaesenc         $rndkey1,@out[4],@out[4]
800          vpaddd         $counters,$zero,$zero           # decrement counters
801          vmovdqu        48(%rsp),$counters
802         vaesenc         $rndkey1,@out[5],@out[5]
803          mov            64(%rsp),$offset                # pre-load 1st offset
804         vaesenc         $rndkey1,@out[6],@out[6]
805         vaesenc         $rndkey1,@out[7],@out[7]
806         vmovups         0x10-0x78($key),$rndkey1
807
808         vaesenclast     $rndkey0,@out[0],@out[0]
809          vmovdqa        $zero,32(%rsp)                  # update counters
810          vpxor          $zero,$zero,$zero
811         vaesenclast     $rndkey0,@out[1],@out[1]
812         vaesenclast     $rndkey0,@out[2],@out[2]
813          vpcmpgtd       $zero,$counters,$zero
814         vaesenclast     $rndkey0,@out[3],@out[3]
815         vaesenclast     $rndkey0,@out[4],@out[4]
816          vpaddd         $zero,$counters,$counters       # decrement counters
817          vmovdqu        -0x78($key),$zero               # 0-round
818         vaesenclast     $rndkey0,@out[5],@out[5]
819         vaesenclast     $rndkey0,@out[6],@out[6]
820          vmovdqa        $counters,48(%rsp)              # update counters
821         vaesenclast     $rndkey0,@out[7],@out[7]
822         vmovups         0x20-0x78($key),$rndkey0
823
824         vmovups         @out[0],-16(@ptr[0])            # write output
825          sub            $offset,@ptr[0]                 # switch to input
826          vpxor          0x00($offload),@out[0],@out[0]
827         vmovups         @out[1],-16(@ptr[1])    
828          sub            `64+1*8`(%rsp),@ptr[1]
829          vpxor          0x10($offload),@out[1],@out[1]
830         vmovups         @out[2],-16(@ptr[2])    
831          sub            `64+2*8`(%rsp),@ptr[2]
832          vpxor          0x20($offload),@out[2],@out[2]
833         vmovups         @out[3],-16(@ptr[3])
834          sub            `64+3*8`(%rsp),@ptr[3]
835          vpxor          0x30($offload),@out[3],@out[3]
836         vmovups         @out[4],-16(@ptr[4])
837          sub            `64+4*8`(%rsp),@ptr[4]
838          vpxor          @inp[0],@out[4],@out[4]
839         vmovups         @out[5],-16(@ptr[5])    
840          sub            `64+5*8`(%rsp),@ptr[5]
841          vpxor          @inp[1],@out[5],@out[5]
842         vmovups         @out[6],-16(@ptr[6])    
843          sub            `64+6*8`(%rsp),@ptr[6]
844          vpxor          @inp[2],@out[6],@out[6]
845         vmovups         @out[7],-16(@ptr[7])
846          sub            `64+7*8`(%rsp),@ptr[7]
847          vpxor          @inp[3],@out[7],@out[7]
848
849         dec     $num
850         jnz     .Loop_enc8x
851
852         mov     16(%rsp),%rax                   # original %rsp
853         #mov    24(%rsp),$num
854         #lea    `40*8`($inp),$inp
855         #dec    $num
856         #jnz    .Lenc8x_loop_grande
857
858 .Lenc8x_done:
859         vzeroupper
860 ___
861 $code.=<<___ if ($win64);
862         movaps  -0xd8(%rax),%xmm6
863         movaps  -0xc8(%rax),%xmm7
864         movaps  -0xb8(%rax),%xmm8
865         movaps  -0xa8(%rax),%xmm9
866         movaps  -0x98(%rax),%xmm10
867         movaps  -0x88(%rax),%xmm11
868         movaps  -0x78(%rax),%xmm12
869         movaps  -0x68(%rax),%xmm13
870         movaps  -0x58(%rax),%xmm14
871         movaps  -0x48(%rax),%xmm15
872 ___
873 $code.=<<___;
874         mov     -48(%rax),%r15
875         mov     -40(%rax),%r14
876         mov     -32(%rax),%r13
877         mov     -24(%rax),%r12
878         mov     -16(%rax),%rbp
879         mov     -8(%rax),%rbx
880         lea     (%rax),%rsp
881 .Lenc8x_epilogue:
882         ret
883 .size   aesni_multi_cbc_encrypt_avx,.-aesni_multi_cbc_encrypt_avx
884
885 .type   aesni_multi_cbc_decrypt_avx,\@function,3
886 .align  32
887 aesni_multi_cbc_decrypt_avx:
888 _avx_cbc_dec_shortcut:
889         mov     %rsp,%rax
890         push    %rbx
891         push    %rbp
892         push    %r12
893         push    %r13
894         push    %r14
895         push    %r15
896 ___
897 $code.=<<___ if ($win64);
898         lea     -0xa8(%rsp),%rsp
899         movaps  %xmm6,(%rsp)
900         movaps  %xmm7,0x10(%rsp)
901         movaps  %xmm8,0x20(%rsp)
902         movaps  %xmm9,0x30(%rsp)
903         movaps  %xmm10,0x40(%rsp)
904         movaps  %xmm11,0x50(%rsp)
905         movaps  %xmm12,-0x78(%rax)
906         movaps  %xmm13,-0x68(%rax)
907         movaps  %xmm14,-0x58(%rax)
908         movaps  %xmm15,-0x48(%rax)
909 ___
910 $code.=<<___;
911         # stack layout
912         #
913         # +0    output sink
914         # +16   input sink [original %rsp and $num]
915         # +32   counters
916         # +64   distances between inputs and outputs
917         # +128  off-load area for @inp[0..3]
918         # +192  IV/input offload
919
920         sub     \$256,%rsp
921         and     \$-256,%rsp
922         sub     \$192,%rsp
923         mov     %rax,16(%rsp)                   # original %rsp
924
925 .Ldec8x_body:
926         vzeroupper
927         vmovdqu ($key),$zero                    # 0-round key
928         lea     0x78($key),$key                 # size optimization
929         lea     40*4($inp),$inp
930         shr     \$1,$num
931
932 .Ldec8x_loop_grande:
933         #mov    $num,24(%rsp)                   # original $num
934         xor     $num,$num
935 ___
936 for($i=0;$i<8;$i++) {
937   my $temp = $i ? $offload : $offset;
938     $code.=<<___;
939         mov     `40*$i+16-40*4`($inp),$one      # borrow $one for number of blocks
940         mov     `40*$i+0-40*4`($inp),@ptr[$i]   # input pointer
941         cmp     $num,$one
942         mov     `40*$i+8-40*4`($inp),$temp      # output pointer
943         cmovg   $one,$num                       # find maximum
944         test    $one,$one
945         vmovdqu `40*$i+24-40*4`($inp),@out[$i]  # load IV
946         mov     $one,`32+4*$i`(%rsp)            # initialize counters
947         cmovle  %rsp,@ptr[$i]                   # cancel input
948         sub     @ptr[$i],$temp                  # distance between input and output
949         mov     $temp,`64+8*$i`(%rsp)           # initialize distances
950         vmovdqu @out[$i],`192+16*$i`(%rsp)      # offload IV
951 ___
952 }
953 $code.=<<___;
954         test    $num,$num
955         jz      .Ldec8x_done
956
957         vmovups 0x10-0x78($key),$rndkey1
958         vmovups 0x20-0x78($key),$rndkey0
959         mov     0xf0-0x78($key),$rounds
960          lea    192+128(%rsp),$offload          # offload area
961
962         vmovdqu (@ptr[0]),@out[0]               # load inputs
963         vmovdqu (@ptr[1]),@out[1]
964         vmovdqu (@ptr[2]),@out[2]
965         vmovdqu (@ptr[3]),@out[3]
966         vmovdqu (@ptr[4]),@out[4]
967         vmovdqu (@ptr[5]),@out[5]
968         vmovdqu (@ptr[6]),@out[6]
969         vmovdqu (@ptr[7]),@out[7]
970         vmovdqu @out[0],0x00($offload)          # offload inputs
971         vpxor   $zero,@out[0],@out[0]           # xor inputs with 0-round
972         vmovdqu @out[1],0x10($offload)
973         vpxor   $zero,@out[1],@out[1]
974         vmovdqu @out[2],0x20($offload)
975         vpxor   $zero,@out[2],@out[2]
976         vmovdqu @out[3],0x30($offload)
977         vpxor   $zero,@out[3],@out[3]
978         vmovdqu @out[4],0x40($offload)
979         vpxor   $zero,@out[4],@out[4]
980         vmovdqu @out[5],0x50($offload)
981         vpxor   $zero,@out[5],@out[5]
982         vmovdqu @out[6],0x60($offload)
983         vpxor   $zero,@out[6],@out[6]
984         vmovdqu @out[7],0x70($offload)
985         vpxor   $zero,@out[7],@out[7]
986         xor     \$0x80,$offload
987         mov     \$1,$one                        # constant of 1
988         jmp     .Loop_dec8x
989
990 .align  32
991 .Loop_dec8x:
992 ___
993 for($i=0;$i<8;$i++) {
994 my $rndkey=($i&1)?$rndkey0:$rndkey1;
995 $code.=<<___;
996         vaesdec         $rndkey,@out[0],@out[0]
997          cmp            32+4*$i(%rsp),$one
998 ___
999 $code.=<<___ if ($i);
1000          mov            64+8*$i(%rsp),$offset
1001 ___
1002 $code.=<<___;
1003         vaesdec         $rndkey,@out[1],@out[1]
1004         prefetcht0      31(@ptr[$i])                    # prefetch input
1005         vaesdec         $rndkey,@out[2],@out[2]
1006 ___
1007 $code.=<<___ if ($i>1);
1008         prefetcht0      15(@ptr[$i-2])                  # prefetch output
1009 ___
1010 $code.=<<___;
1011         vaesdec         $rndkey,@out[3],@out[3]
1012          lea            (@ptr[$i],$offset),$offset
1013          cmovge         %rsp,@ptr[$i]                   # cancel input
1014         vaesdec         $rndkey,@out[4],@out[4]
1015          cmovg          %rsp,$offset                    # sink output
1016         vaesdec         $rndkey,@out[5],@out[5]
1017          sub            @ptr[$i],$offset
1018         vaesdec         $rndkey,@out[6],@out[6]
1019          vmovdqu        16(@ptr[$i]),@inp[$i%4]         # load input
1020          mov            $offset,64+8*$i(%rsp)
1021         vaesdec         $rndkey,@out[7],@out[7]
1022         vmovups         `16*(3+$i)-0x78`($key),$rndkey
1023          lea            16(@ptr[$i],$offset),@ptr[$i]   # switch to output
1024 ___
1025 $code.=<<___ if ($i<4);
1026          vmovdqu        @inp[$i%4],`128+16*$i`(%rsp)    # off-load
1027 ___
1028 }
1029 $code.=<<___;
1030          vmovdqu        32(%rsp),$counters
1031         prefetcht0      15(@ptr[$i-2])                  # prefetch output
1032         prefetcht0      15(@ptr[$i-1])
1033         cmp     \$11,$rounds
1034         jb      .Ldec8x_tail
1035
1036         vaesdec         $rndkey1,@out[0],@out[0]
1037         vaesdec         $rndkey1,@out[1],@out[1]
1038         vaesdec         $rndkey1,@out[2],@out[2]
1039         vaesdec         $rndkey1,@out[3],@out[3]
1040         vaesdec         $rndkey1,@out[4],@out[4]
1041         vaesdec         $rndkey1,@out[5],@out[5]
1042         vaesdec         $rndkey1,@out[6],@out[6]
1043         vaesdec         $rndkey1,@out[7],@out[7]
1044         vmovups         0xb0-0x78($key),$rndkey1
1045
1046         vaesdec         $rndkey0,@out[0],@out[0]
1047         vaesdec         $rndkey0,@out[1],@out[1]
1048         vaesdec         $rndkey0,@out[2],@out[2]
1049         vaesdec         $rndkey0,@out[3],@out[3]
1050         vaesdec         $rndkey0,@out[4],@out[4]
1051         vaesdec         $rndkey0,@out[5],@out[5]
1052         vaesdec         $rndkey0,@out[6],@out[6]
1053         vaesdec         $rndkey0,@out[7],@out[7]
1054         vmovups         0xc0-0x78($key),$rndkey0
1055         je      .Ldec8x_tail
1056
1057         vaesdec         $rndkey1,@out[0],@out[0]
1058         vaesdec         $rndkey1,@out[1],@out[1]
1059         vaesdec         $rndkey1,@out[2],@out[2]
1060         vaesdec         $rndkey1,@out[3],@out[3]
1061         vaesdec         $rndkey1,@out[4],@out[4]
1062         vaesdec         $rndkey1,@out[5],@out[5]
1063         vaesdec         $rndkey1,@out[6],@out[6]
1064         vaesdec         $rndkey1,@out[7],@out[7]
1065         vmovups         0xd0-0x78($key),$rndkey1
1066
1067         vaesdec         $rndkey0,@out[0],@out[0]
1068         vaesdec         $rndkey0,@out[1],@out[1]
1069         vaesdec         $rndkey0,@out[2],@out[2]
1070         vaesdec         $rndkey0,@out[3],@out[3]
1071         vaesdec         $rndkey0,@out[4],@out[4]
1072         vaesdec         $rndkey0,@out[5],@out[5]
1073         vaesdec         $rndkey0,@out[6],@out[6]
1074         vaesdec         $rndkey0,@out[7],@out[7]
1075         vmovups         0xe0-0x78($key),$rndkey0
1076
1077 .Ldec8x_tail:
1078         vaesdec         $rndkey1,@out[0],@out[0]
1079          vpxor          $zero,$zero,$zero
1080         vaesdec         $rndkey1,@out[1],@out[1]
1081         vaesdec         $rndkey1,@out[2],@out[2]
1082          vpcmpgtd       $zero,$counters,$zero
1083         vaesdec         $rndkey1,@out[3],@out[3]
1084         vaesdec         $rndkey1,@out[4],@out[4]
1085          vpaddd         $counters,$zero,$zero           # decrement counters
1086          vmovdqu        48(%rsp),$counters
1087         vaesdec         $rndkey1,@out[5],@out[5]
1088          mov            64(%rsp),$offset                # pre-load 1st offset
1089         vaesdec         $rndkey1,@out[6],@out[6]
1090         vaesdec         $rndkey1,@out[7],@out[7]
1091         vmovups         0x10-0x78($key),$rndkey1
1092
1093         vaesdeclast     $rndkey0,@out[0],@out[0]
1094          vmovdqa        $zero,32(%rsp)                  # update counters
1095          vpxor          $zero,$zero,$zero
1096         vaesdeclast     $rndkey0,@out[1],@out[1]
1097         vpxor           0x00($offload),@out[0],@out[0]  # xor with IV
1098         vaesdeclast     $rndkey0,@out[2],@out[2]
1099         vpxor           0x10($offload),@out[1],@out[1]
1100          vpcmpgtd       $zero,$counters,$zero
1101         vaesdeclast     $rndkey0,@out[3],@out[3]
1102         vpxor           0x20($offload),@out[2],@out[2]
1103         vaesdeclast     $rndkey0,@out[4],@out[4]
1104         vpxor           0x30($offload),@out[3],@out[3]
1105          vpaddd         $zero,$counters,$counters       # decrement counters
1106          vmovdqu        -0x78($key),$zero               # 0-round
1107         vaesdeclast     $rndkey0,@out[5],@out[5]
1108         vpxor           0x40($offload),@out[4],@out[4]
1109         vaesdeclast     $rndkey0,@out[6],@out[6]
1110         vpxor           0x50($offload),@out[5],@out[5]
1111          vmovdqa        $counters,48(%rsp)              # update counters
1112         vaesdeclast     $rndkey0,@out[7],@out[7]
1113         vpxor           0x60($offload),@out[6],@out[6]
1114         vmovups         0x20-0x78($key),$rndkey0
1115
1116         vmovups         @out[0],-16(@ptr[0])            # write output
1117          sub            $offset,@ptr[0]                 # switch to input
1118          vmovdqu        128+0(%rsp),@out[0]
1119         vpxor           0x70($offload),@out[7],@out[7]
1120         vmovups         @out[1],-16(@ptr[1])    
1121          sub            `64+1*8`(%rsp),@ptr[1]
1122          vmovdqu        @out[0],0x00($offload)
1123          vpxor          $zero,@out[0],@out[0]
1124          vmovdqu        128+16(%rsp),@out[1]
1125         vmovups         @out[2],-16(@ptr[2])    
1126          sub            `64+2*8`(%rsp),@ptr[2]
1127          vmovdqu        @out[1],0x10($offload)
1128          vpxor          $zero,@out[1],@out[1]
1129          vmovdqu        128+32(%rsp),@out[2]
1130         vmovups         @out[3],-16(@ptr[3])
1131          sub            `64+3*8`(%rsp),@ptr[3]
1132          vmovdqu        @out[2],0x20($offload)
1133          vpxor          $zero,@out[2],@out[2]
1134          vmovdqu        128+48(%rsp),@out[3]
1135         vmovups         @out[4],-16(@ptr[4])
1136          sub            `64+4*8`(%rsp),@ptr[4]
1137          vmovdqu        @out[3],0x30($offload)
1138          vpxor          $zero,@out[3],@out[3]
1139          vmovdqu        @inp[0],0x40($offload)
1140          vpxor          @inp[0],$zero,@out[4]
1141         vmovups         @out[5],-16(@ptr[5])    
1142          sub            `64+5*8`(%rsp),@ptr[5]
1143          vmovdqu        @inp[1],0x50($offload)
1144          vpxor          @inp[1],$zero,@out[5]
1145         vmovups         @out[6],-16(@ptr[6])    
1146          sub            `64+6*8`(%rsp),@ptr[6]
1147          vmovdqu        @inp[2],0x60($offload)
1148          vpxor          @inp[2],$zero,@out[6]
1149         vmovups         @out[7],-16(@ptr[7])
1150          sub            `64+7*8`(%rsp),@ptr[7]
1151          vmovdqu        @inp[3],0x70($offload)
1152          vpxor          @inp[3],$zero,@out[7]
1153
1154         xor     \$128,$offload
1155         dec     $num
1156         jnz     .Loop_dec8x
1157
1158         mov     16(%rsp),%rax                   # original %rsp
1159         #mov    24(%rsp),$num
1160         #lea    `40*8`($inp),$inp
1161         #dec    $num
1162         #jnz    .Ldec8x_loop_grande
1163
1164 .Ldec8x_done:
1165         vzeroupper
1166 ___
1167 $code.=<<___ if ($win64);
1168         movaps  -0xd8(%rax),%xmm6
1169         movaps  -0xc8(%rax),%xmm7
1170         movaps  -0xb8(%rax),%xmm8
1171         movaps  -0xa8(%rax),%xmm9
1172         movaps  -0x98(%rax),%xmm10
1173         movaps  -0x88(%rax),%xmm11
1174         movaps  -0x78(%rax),%xmm12
1175         movaps  -0x68(%rax),%xmm13
1176         movaps  -0x58(%rax),%xmm14
1177         movaps  -0x48(%rax),%xmm15
1178 ___
1179 $code.=<<___;
1180         mov     -48(%rax),%r15
1181         mov     -40(%rax),%r14
1182         mov     -32(%rax),%r13
1183         mov     -24(%rax),%r12
1184         mov     -16(%rax),%rbp
1185         mov     -8(%rax),%rbx
1186         lea     (%rax),%rsp
1187 .Ldec8x_epilogue:
1188         ret
1189 .size   aesni_multi_cbc_decrypt_avx,.-aesni_multi_cbc_decrypt_avx
1190 ___
1191                                                 }}}
1192
1193 if ($win64) {
1194 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1195 #               CONTEXT *context,DISPATCHER_CONTEXT *disp)
1196 $rec="%rcx";
1197 $frame="%rdx";
1198 $context="%r8";
1199 $disp="%r9";
1200
1201 $code.=<<___;
1202 .extern __imp_RtlVirtualUnwind
1203 .type   se_handler,\@abi-omnipotent
1204 .align  16
1205 se_handler:
1206         push    %rsi
1207         push    %rdi
1208         push    %rbx
1209         push    %rbp
1210         push    %r12
1211         push    %r13
1212         push    %r14
1213         push    %r15
1214         pushfq
1215         sub     \$64,%rsp
1216
1217         mov     120($context),%rax      # pull context->Rax
1218         mov     248($context),%rbx      # pull context->Rip
1219
1220         mov     8($disp),%rsi           # disp->ImageBase
1221         mov     56($disp),%r11          # disp->HandlerData
1222
1223         mov     0(%r11),%r10d           # HandlerData[0]
1224         lea     (%rsi,%r10),%r10        # prologue label
1225         cmp     %r10,%rbx               # context->Rip<.Lprologue
1226         jb      .Lin_prologue
1227
1228         mov     152($context),%rax      # pull context->Rsp
1229
1230         mov     4(%r11),%r10d           # HandlerData[1]
1231         lea     (%rsi,%r10),%r10        # epilogue label
1232         cmp     %r10,%rbx               # context->Rip>=.Lepilogue
1233         jae     .Lin_prologue
1234
1235         mov     16(%rax),%rax           # pull saved stack pointer
1236
1237         mov     -8(%rax),%rbx
1238         mov     -16(%rax),%rbp
1239         mov     -24(%rax),%r12
1240         mov     -32(%rax),%r13
1241         mov     -40(%rax),%r14
1242         mov     -48(%rax),%r15
1243         mov     %rbx,144($context)      # restore context->Rbx
1244         mov     %rbp,160($context)      # restore context->Rbp
1245         mov     %r12,216($context)      # restore cotnext->R12
1246         mov     %r13,224($context)      # restore cotnext->R13
1247         mov     %r14,232($context)      # restore cotnext->R14
1248         mov     %r15,240($context)      # restore cotnext->R15
1249
1250         lea     -56-10*16(%rax),%rsi
1251         lea     512($context),%rdi      # &context.Xmm6
1252         mov     \$20,%ecx
1253         .long   0xa548f3fc              # cld; rep movsq
1254
1255 .Lin_prologue:
1256         mov     8(%rax),%rdi
1257         mov     16(%rax),%rsi
1258         mov     %rax,152($context)      # restore context->Rsp
1259         mov     %rsi,168($context)      # restore context->Rsi
1260         mov     %rdi,176($context)      # restore context->Rdi
1261
1262         mov     40($disp),%rdi          # disp->ContextRecord
1263         mov     $context,%rsi           # context
1264         mov     \$154,%ecx              # sizeof(CONTEXT)
1265         .long   0xa548f3fc              # cld; rep movsq
1266
1267         mov     $disp,%rsi
1268         xor     %rcx,%rcx               # arg1, UNW_FLAG_NHANDLER
1269         mov     8(%rsi),%rdx            # arg2, disp->ImageBase
1270         mov     0(%rsi),%r8             # arg3, disp->ControlPc
1271         mov     16(%rsi),%r9            # arg4, disp->FunctionEntry
1272         mov     40(%rsi),%r10           # disp->ContextRecord
1273         lea     56(%rsi),%r11           # &disp->HandlerData
1274         lea     24(%rsi),%r12           # &disp->EstablisherFrame
1275         mov     %r10,32(%rsp)           # arg5
1276         mov     %r11,40(%rsp)           # arg6
1277         mov     %r12,48(%rsp)           # arg7
1278         mov     %rcx,56(%rsp)           # arg8, (NULL)
1279         call    *__imp_RtlVirtualUnwind(%rip)
1280
1281         mov     \$1,%eax                # ExceptionContinueSearch
1282         add     \$64,%rsp
1283         popfq
1284         pop     %r15
1285         pop     %r14
1286         pop     %r13
1287         pop     %r12
1288         pop     %rbp
1289         pop     %rbx
1290         pop     %rdi
1291         pop     %rsi
1292         ret
1293 .size   se_handler,.-se_handler
1294
1295 .section        .pdata
1296 .align  4
1297         .rva    .LSEH_begin_aesni_multi_cbc_encrypt
1298         .rva    .LSEH_end_aesni_multi_cbc_encrypt
1299         .rva    .LSEH_info_aesni_multi_cbc_encrypt
1300         .rva    .LSEH_begin_aesni_multi_cbc_decrypt
1301         .rva    .LSEH_end_aesni_multi_cbc_decrypt
1302         .rva    .LSEH_info_aesni_multi_cbc_decrypt
1303 ___
1304 $code.=<<___ if ($avx);
1305         .rva    .LSEH_begin_aesni_multi_cbc_encrypt_avx
1306         .rva    .LSEH_end_aesni_multi_cbc_encrypt_avx
1307         .rva    .LSEH_info_aesni_multi_cbc_encrypt_avx
1308         .rva    .LSEH_begin_aesni_multi_cbc_decrypt_avx
1309         .rva    .LSEH_end_aesni_multi_cbc_decrypt_avx
1310         .rva    .LSEH_info_aesni_multi_cbc_decrypt_avx
1311 ___
1312 $code.=<<___;
1313 .section        .xdata
1314 .align  8
1315 .LSEH_info_aesni_multi_cbc_encrypt:
1316         .byte   9,0,0,0
1317         .rva    se_handler
1318         .rva    .Lenc4x_body,.Lenc4x_epilogue           # HandlerData[]
1319 .LSEH_info_aesni_multi_cbc_decrypt:
1320         .byte   9,0,0,0
1321         .rva    se_handler
1322         .rva    .Ldec4x_body,.Ldec4x_epilogue           # HandlerData[]
1323 ___
1324 $code.=<<___ if ($avx);
1325 .LSEH_info_aesni_multi_cbc_encrypt_avx:
1326         .byte   9,0,0,0
1327         .rva    se_handler
1328         .rva    .Lenc8x_body,.Lenc8x_epilogue           # HandlerData[]
1329 .LSEH_info_aesni_multi_cbc_decrypt_avx:
1330         .byte   9,0,0,0
1331         .rva    se_handler
1332         .rva    .Ldec8x_body,.Ldec8x_epilogue           # HandlerData[]
1333 ___
1334 }
1335 ####################################################################
1336
1337 sub rex {
1338   local *opcode=shift;
1339   my ($dst,$src)=@_;
1340   my $rex=0;
1341
1342     $rex|=0x04                  if($dst>=8);
1343     $rex|=0x01                  if($src>=8);
1344     push @opcode,$rex|0x40      if($rex);
1345 }
1346
1347 sub aesni {
1348   my $line=shift;
1349   my @opcode=(0x66);
1350
1351     if ($line=~/(aeskeygenassist)\s+\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1352         rex(\@opcode,$4,$3);
1353         push @opcode,0x0f,0x3a,0xdf;
1354         push @opcode,0xc0|($3&7)|(($4&7)<<3);   # ModR/M
1355         my $c=$2;
1356         push @opcode,$c=~/^0/?oct($c):$c;
1357         return ".byte\t".join(',',@opcode);
1358     }
1359     elsif ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1360         my %opcodelet = (
1361                 "aesimc" => 0xdb,
1362                 "aesenc" => 0xdc,       "aesenclast" => 0xdd,
1363                 "aesdec" => 0xde,       "aesdeclast" => 0xdf
1364         );
1365         return undef if (!defined($opcodelet{$1}));
1366         rex(\@opcode,$3,$2);
1367         push @opcode,0x0f,0x38,$opcodelet{$1};
1368         push @opcode,0xc0|($2&7)|(($3&7)<<3);   # ModR/M
1369         return ".byte\t".join(',',@opcode);
1370     }
1371     elsif ($line=~/(aes[a-z]+)\s+([0x1-9a-fA-F]*)\(%rsp\),\s*%xmm([0-9]+)/) {
1372         my %opcodelet = (
1373                 "aesenc" => 0xdc,       "aesenclast" => 0xdd,
1374                 "aesdec" => 0xde,       "aesdeclast" => 0xdf
1375         );
1376         return undef if (!defined($opcodelet{$1}));
1377         my $off = $2;
1378         push @opcode,0x44 if ($3>=8);
1379         push @opcode,0x0f,0x38,$opcodelet{$1};
1380         push @opcode,0x44|(($3&7)<<3),0x24;     # ModR/M
1381         push @opcode,($off=~/^0/?oct($off):$off)&0xff;
1382         return ".byte\t".join(',',@opcode);
1383     }
1384     return $line;
1385 }
1386
1387 $code =~ s/\`([^\`]*)\`/eval($1)/gem;
1388 $code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem;
1389
1390 print $code;
1391 close STDOUT;