4e3af02db253faf3e22f154a5654359c42fe7c96
[openssl.git] / crypto / aes / asm / aesni-x86.pl
1 #!/usr/bin/env perl
2
3 # ====================================================================
4 # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9 #
10 # This module implements support for Intel AES-NI extension. In
11 # OpenSSL context it's used with Intel engine, but can also be used as
12 # drop-in replacement for crypto/aes/asm/aes-586.pl [see below for
13 # details].
14
15 $PREFIX="aesni";        # if $PREFIX is set to "AES", the script
16                         # generates drop-in replacement for
17                         # crypto/aes/asm/aes-586.pl:-)
18
19 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
20 push(@INC,"${dir}","${dir}../../perlasm");
21 require "x86asm.pl";
22
23 &asm_init($ARGV[0],$0);
24
25 $movekey = eval($RREFIX eq "aseni" ? "*movaps" : "*movups");
26
27 $len="eax";
28 $rounds="ecx";
29 $key="edx";
30 $inp="esi";
31 $out="edi";
32 $rounds_="ebx";
33 $key_="ebp";
34
35 $inout0="xmm0";
36 $inout1="xmm1";
37 $inout2="xmm2";
38 $rndkey0="xmm3";
39 $rndkey1="xmm4";
40 $ivec="xmm5";
41 $in0="xmm6";
42 $in1="xmm7";
43
44 sub _aesni_generate1    # folded loop
45 { my $p=shift;
46
47     &function_begin_B("_aesni_${p}rypt1");
48         &$movekey       ($rndkey0,&QWP(0,$key));
49         &$movekey       ($rndkey1,&QWP(16,$key));
50         &lea            ($key,&DWP(16,$key));
51         &pxor           ($inout0,$rndkey0);
52         &dec            ($rounds);
53     &set_label("${p}1_loop",16);
54         eval"&aes${p}   ($inout0,$rndkey1)";
55         &dec            ($rounds);
56         &lea            ($key,&DWP(16,$key));
57         &$movekey       ($rndkey1,&QWP(0,$key));
58         &jnz            (&label("${p}1_loop"));
59     eval"&aes${p}last   ($inout0,$rndkey1)";
60     &ret();
61     &function_end_B("_aesni_${p}rypt1");
62 }
63
64 sub aesni_generate1     # fully unrolled loop
65 { my $p=shift;
66
67     &function_begin_B("_aesni_${p}rypt1");
68         &$movekey       ($rndkey0,&QWP(0,$key));
69         &$movekey       ($rndkey1,&QWP(0x10,$key));
70         &cmp            ($rounds,12);
71         &pxor           ($inout0,$rndkey0);
72         &$movekey       ($rndkey0,&QWP(0x20,$key));
73         &lea            ($key,&DWP(0x30,$key));
74         &jb             (&label("${p}128"));
75         &lea            ($key,&DWP(0x20,$key));
76         &je             (&label("${p}192"));
77         &lea            ($key,&DWP(0x20,$key));
78         eval"&aes${p}   ($inout0,$rndkey1)";
79         &$movekey       ($rndkey1,&QWP(-0x40,$key));
80         eval"&aes${p}   ($inout0,$rndkey0)";
81         &$movekey       ($rndkey0,&QWP(-0x30,$key));
82     &set_label("${p}192");
83         eval"&aes${p}   ($inout0,$rndkey1)";
84         &$movekey       ($rndkey1,&QWP(-0x20,$key));
85         eval"&aes${p}   ($inout0,$rndkey0)";
86         &$movekey       ($rndkey0,&QWP(-0x10,$key));
87     &set_label("${p}128");
88         eval"&aes${p}   ($inout0,$rndkey1)";
89         &$movekey       ($rndkey1,&QWP(0,$key));
90         eval"&aes${p}   ($inout0,$rndkey0)";
91         &$movekey       ($rndkey0,&QWP(0x10,$key));
92         eval"&aes${p}   ($inout0,$rndkey1)";
93         &$movekey       ($rndkey1,&QWP(0x20,$key));
94         eval"&aes${p}   ($inout0,$rndkey0)";
95         &$movekey       ($rndkey0,&QWP(0x30,$key));
96         eval"&aes${p}   ($inout0,$rndkey1)";
97         &$movekey       ($rndkey1,&QWP(0x40,$key));
98         eval"&aes${p}   ($inout0,$rndkey0)";
99         &$movekey       ($rndkey0,&QWP(0x50,$key));
100         eval"&aes${p}   ($inout0,$rndkey1)";
101         &$movekey       ($rndkey1,&QWP(0x60,$key));
102         eval"&aes${p}   ($inout0,$rndkey0)";
103         &$movekey       ($rndkey0,&QWP(0x70,$key));
104         eval"&aes${p}   ($inout0,$rndkey1)";
105     eval"&aes${p}last   ($inout0,$rndkey0)";
106     &ret();
107     &function_end_B("_aesni_${p}rypt1");
108 }
109
110 &aesni_generate1("enc");
111 # void $PREFIX_encrypt (const void *inp,void *out,const AES_KEY *key);
112 &function_begin_B("${PREFIX}_encrypt");
113         &mov    ("eax",&wparam(0));
114         &mov    ($key,&wparam(2));
115         &movups ($inout0,&QWP(0,"eax"));
116         &mov    ($rounds,&DWP(240,$key));
117         &mov    ("eax",&wparam(1));
118         &call   ("_aesni_encrypt1");
119         &movups (&QWP(0,"eax"),$inout0);
120         &ret    ();
121 &function_end_B("${PREFIX}_encrypt");
122
123 &aesni_generate1("dec");
124 # void $PREFIX_decrypt (const void *inp,void *out,const AES_KEY *key);
125 &function_begin_B("${PREFIX}_decrypt");
126         &mov    ("eax",&wparam(0));
127         &mov    ($key,&wparam(2));
128         &movups ($inout0,&QWP(0,"eax"));
129         &mov    ($rounds,&DWP(240,$key));
130         &mov    ("eax",&wparam(1));
131         &call   ("_aesni_decrypt1");
132         &movups (&QWP(0,"eax"),$inout0);
133         &ret    ();
134 &function_end_B("${PREFIX}_decrypt");
135
136 # _aesni_[en|de]crypt3 are private interfaces, 3 denotes interleave
137 # factor. Why 3x? Even though aes[enc|dec] latency is 6, it turned
138 # out that it can be scheduled only every *second* cycle. Thus 3x
139 # interleave is the one providing optimal utilization, i.e. when
140 # subroutine's throughput is virtually same as of non-interleaved
141 # subroutine for number of input blocks up to 3. This is why it
142 # handles even double-block inputs. Larger interleave factor would
143 # perform suboptimally on shorter inputs... 
144
145 sub aesni_generate3
146 { my $p=shift;
147
148     &function_begin_B("_aesni_${p}rypt3");
149         &$movekey       ($rndkey0,&QWP(0,$key));
150         &$movekey       ($rndkey1,&QWP(16,$key));
151         &shr            ($rounds,1);
152         &lea            ($key,&DWP(32,$key));
153         &pxor           ($inout0,$rndkey0);
154         &pxor           ($inout1,$rndkey0);
155         &dec            ($rounds);
156         &pxor           ($inout2,$rndkey0);
157         &jmp            (&label("${p}3_loop"));
158     &set_label("${p}3_loop",16);
159         eval"&aes${p}   ($inout0,$rndkey1)";
160         &$movekey       ($rndkey0,&QWP(0,$key));
161         eval"&aes${p}   ($inout1,$rndkey1)";
162         &dec            ($rounds);
163         eval"&aes${p}   ($inout2,$rndkey1)";
164         &$movekey       ($rndkey1,&QWP(16,$key));
165         eval"&aes${p}   ($inout0,$rndkey0)";
166         &lea            ($key,&DWP(32,$key));
167         eval"&aes${p}   ($inout1,$rndkey0)";
168         eval"&aes${p}   ($inout2,$rndkey0)";
169         &jnz            (&label("${p}3_loop"));
170     eval"&aes${p}       ($inout0,$rndkey1)";
171     &$movekey           ($rndkey0,&QWP(0,$key));
172     eval"&aes${p}       ($inout1,$rndkey1)";
173     eval"&aes${p}       ($inout2,$rndkey1)";
174     eval"&aes${p}last   ($inout0,$rndkey0)";
175     eval"&aes${p}last   ($inout1,$rndkey0)";
176     eval"&aes${p}last   ($inout2,$rndkey0)";
177     &ret();
178     &function_end_B("_aesni_${p}rypt3");
179 }
180 &aesni_generate3("enc") if ($PREFIX eq "aesni");
181 &aesni_generate3("dec");
182
183 if ($PREFIX eq "aesni") {
184 # void aesni_ecb_encrypt (const void *in, void *out,
185 #                         size_t length, const AES_KEY *key,
186 #                         int enc);
187
188 &function_begin("aesni_ecb_encrypt");
189         &mov    ($inp,&wparam(0));
190         &mov    ($out,&wparam(1));
191         &mov    ($len,&wparam(2));
192         &mov    ($key,&wparam(3));
193         &mov    ($rounds,&wparam(4));
194         &cmp    ($len,16);
195         &jb     (&label("ecb_ret"));
196         &and    ($len,-16);
197         &test   ($rounds,$rounds)
198         &mov    ($rounds,&DWP(240,$key));
199         &mov    ($key_,$key);           # backup $key
200         &mov    ($rounds_,$rounds);     # backup $rounds
201         &jz     (&label("ecb_decrypt"));
202
203         &sub    ($len,0x30);
204         &jc     (&label("ecb_enc_tail"));
205         jmp     (&label("ecb_enc_loop3"));
206
207 &set_label("ecb_enc_loop3",16);
208         &movups ($inout0,&QWP(0,$inp));
209         &movups ($inout1,&QWP(0x10,$inp));
210         &movups ($inout2,&QWP(0x20,$inp));
211         &lea    ($inp,&DWP(0x30,$inp));
212         &call   ("_aesni_encrypt3");
213         &movups (&QWP(0,$out),$inout0);
214         &sub    ($len,0x30);
215         &movups (&QWP(0x10,$out),$inout1);
216         &mov    ($key,$key_);           # restore $key
217         &movups (&QWP(0x20,$out),$inout2);
218         &mov    ($rounds,$rounds_);     # restore $rounds
219         &lea    ($out,&DWP(0x30,$out));
220         &jnc    (&label("ecb_enc_loop3"));
221
222 &set_label("ecb_enc_tail");
223         &add    ($len,0x30);
224         &jz     (&label("ecb_ret"));
225
226         &cmp    ($len,0x10);
227         &movups ($inout0,&QWP(0,$inp));
228         je      (&label("ecb_enc_one"));
229         &movups ($inout1,&QWP(0x10,$inp));
230         &call   ("_aesni_encrypt3");
231         &movups (&QWP(0,$out),$inout0);
232         &movups (&QWP(0x10,$out),$inout1);
233         jmp     (&label("ecb_ret"));
234
235 &set_label("ecb_enc_one",16);
236         &call   ("_aesni_encrypt1");
237         &movups (&QWP(0,$out),$inout0);
238         &jmp    (&label("ecb_ret"));
239
240 &set_label("ecb_decrypt",16);
241         &sub    ($len,0x30);
242         &jc     (&label("ecb_dec_tail"));
243         jmp     (&label("ecb_dec_loop3"));
244
245 &set_label("ecb_dec_loop3",16);
246         &movups ($inout0,&QWP(0,$inp));
247         &movups ($inout1,&QWP(0x10,$inp));
248         &movups ($inout2,&QWP(0x20,$inp));
249         &call   ("_aesni_decrypt3");
250         &movups (&QWP(0,$out),$inout0);
251         &sub    ($len,0x30);
252         &lea    ($inp,&DWP(0x30,$inp));
253         &movups (&QWP(0x10,$out),$inout1);
254         &mov    ($key,$key_);           # restore $key
255         &movups (&QWP(0x20,$out),$inout2);
256         &mov    ($rounds,$rounds_);     # restore $rounds
257         &lea    ($out,&DWP(0x30,$out));
258         &jnc    (&label("ecb_dec_loop3"));
259
260 &set_label("ecb_dec_tail");
261         &add    ($len,0x30);
262         &jz     (&label("ecb_ret"));
263
264         &cmp    ($len,0x10);
265         &movups ($inout0,&QWP(0,$inp));
266         je      (&label("ecb_dec_one"));
267         &movups ($inout1,&QWP(0x10,$inp));
268         &call   ("_aesni_decrypt3");
269         &movups (&QWP(0,$out),$inout0);
270         &movups (&QWP(0x10,$out),$inout1);
271         jmp     (&label("ecb_ret"));
272
273 &set_label("ecb_dec_one",16);
274         &call   ("_aesni_decrypt1");
275         &movups (&QWP(0,$out),$inout0);
276
277 &set_label("ecb_ret");
278 &function_end("aesni_ecb_encrypt");
279 }
280
281 # void $PREFIX_cbc_encrypt (const void *inp, void *out,
282 #                           size_t length, const AES_KEY *key,
283 #                           unsigned char *ivp,const int enc);
284 &function_begin("${PREFIX}_cbc_encrypt");
285         &mov    ($inp,&wparam(0));
286         &mov    ($out,&wparam(1));
287         &mov    ($len,&wparam(2));
288         &mov    ($key,&wparam(3));
289         &test   ($len,$len);
290         &mov    ($key_,&wparam(4));
291         &je     (&label("cbc_ret"));
292
293         &cmp    (&wparam(5),0);
294         &movups ($ivec,&QWP(0,$key_));  # load IV
295         &mov    ($rounds,&DWP(240,$key));
296         &mov    ($key_,$key);           # backup $key
297         &mov    ($rounds_,$rounds);     # backup $rounds
298         &je     (&label("cbc_decrypt"));
299
300         &movaps ($inout0,$ivec);
301         &cmp    ($len,16);
302         &jb     (&label("cbc_enc_tail"));
303         &sub    ($len,16);
304         &jmp    (&label("cbc_enc_loop"));
305
306 &set_label("cbc_enc_loop",16);
307         &movups ($ivec,&QWP(0,$inp));
308         &lea    ($inp,&DWP(16,$inp));
309         &pxor   ($inout0,$ivec);
310         &call   ("_aesni_encrypt1");
311         &sub    ($len,16);
312         &mov    ($rounds,$rounds_);     # restore $rounds
313         &mov    ($key,$key_);           # restore $key
314         &movups (&QWP(0,$out),$inout0);
315         &lea    ($out,&DWP(16,$out));
316         &jnc    (&label("cbc_enc_loop"));
317         &add    ($len,16);
318         &jnz    (&label("cbc_enc_tail"));
319         &movaps ($ivec,$inout0);
320         &jmp    (&label("cbc_ret"));
321
322 &set_label("cbc_enc_tail");
323         &mov    ("ecx",$len);           # zaps $rounds
324         &data_word(0xA4F3F689);         # rep movsb
325         &mov    ("ecx",16);             # zero tail
326         &sub    ("ecx",$len);
327         &xor    ("eax","eax");          # zaps $len
328         &data_word(0xAAF3F689);         # rep stosb
329         &lea    ($out,&DWP(-16,$out));  # rewind $out by 1 block
330         &mov    ($rounds,$rounds_);     # restore $rounds
331         &mov    ($inp,$out);            # $inp and $out are the same
332         &mov    ($key,$key_);           # restore $key
333         &jmp    (&label("cbc_enc_loop"));
334
335 &set_label("cbc_decrypt",16);
336         &sub    ($len,0x30);
337         &jc     (&label("cbc_dec_tail"));
338         &jmp    (&label("cbc_dec_loop3"));
339
340 &set_label("cbc_dec_loop3",16);
341         &movups ($inout0,&QWP(0,$inp));
342         &movups ($inout1,&QWP(0x10,$inp));
343         &movups ($inout2,&QWP(0x20,$inp));
344         &movaps ($in0,$inout0);
345         &movaps ($in1,$inout1);
346         &call   ("_aesni_decrypt3");
347         &sub    ($len,0x30);
348         &lea    ($inp,&DWP(0x30,$inp));
349         &pxor   ($inout0,$ivec);
350         &pxor   ($inout1,$in0);
351         &movups ($ivec,&QWP(0x20,$inp));
352         &pxor   ($inout2,$in1);
353         &movups (&QWP(0,$out),$inout0);
354         &mov    ($rounds,$rounds_)      # restore $rounds
355         &movups (&QWP(0x10,$out),$inout1);
356         &mov    ($key,$key_);           # restore $key
357         &movups (&QWP(0x20,$out),$inout2);
358         &lea    ($out,&DWP(0x30,$out));
359         &jnc    (&label("cbc_dec_loop3"));
360
361 &set_label("cbc_dec_tail");
362         &add    ($len,0x30);
363         &jz     (&label("cbc_ret"));
364
365         &movups ($inout0,&QWP(0,$inp));
366         &cmp    ($len,0x10);
367         &movaps ($in0,$inout0);
368         &jbe    (&label("cbc_dec_one"));
369         &movups ($inout1,&QWP(0x10,$inp));
370         &cmp    ($len,0x20);
371         &movaps ($in1,$inout1);
372         &jbe    (&label("cbc_dec_two"));
373         &movups ($inout2,&QWP(0x20,$inp));
374         &call   ("_aesni_decrypt3");
375         &pxor   ($inout0,$ivec);
376         &movups ($ivec,&QWP(0x20,$inp));
377         &pxor   ($inout1,$in0);
378         &pxor   ($inout2,$in1);
379         &movups (&QWP(0,$out),$inout0);
380         &movups (&QWP(0x10,$out),$inout1);
381         &movaps ($inout0,$inout2);
382         &lea    ($out,&DWP(0x20,$out));
383         &jmp    (&label("cbc_dec_tail_collected"));
384
385 &set_label("cbc_dec_one");
386         &call   ("_aesni_decrypt1");
387         &pxor   ($inout0,$ivec);
388         &movaps ($ivec,$in0);
389         &jmp    (&label("cbc_dec_tail_collected"));
390
391 &set_label("cbc_dec_two");
392         &call   ("_aesni_decrypt3");
393         &pxor   ($inout0,$ivec);
394         &pxor   ($inout1,$in0);
395         &movups (&QWP(0,$out),$inout0);
396         &movaps ($inout0,$inout1);
397         &movaps ($ivec,$in1);
398         &lea    ($out,&DWP(0x10,$out));
399
400 &set_label("cbc_dec_tail_collected");
401         &and    ($len,15);
402         &jnz    (&label("cbc_dec_tail_partial"));
403         &movups (&QWP(0,$out),$inout0);
404         &jmp    (&label("cbc_ret"));
405
406 &set_label("cbc_dec_tail_partial");
407         &mov    ($key_,"esp");
408         &sub    ("esp",16);
409         &and    ("esp",-16);
410         &movaps (&QWP(0,"esp"),$inout0);
411         &mov    ($inp,"esp");
412         &mov    ("ecx",$len);
413         &data_word(0xA4F3F689);         # rep movsb
414         &mov    ("esp",$key_);
415
416 &set_label("cbc_ret");
417         &mov    ($key_,&wparam(4));
418         &movups (&QWP(0,$key_),$ivec);  # output IV
419 &function_end("${PREFIX}_cbc_encrypt");
420
421 # Mechanical port from aesni-x86_64.pl.
422 #
423 # _aesni_set_encrypt_key is private interface,
424 # input:
425 #       "eax"   const unsigned char *userKey
426 #       $rounds int bits
427 #       $key    AES_KEY *key
428 # output:
429 #       "eax"   return code
430 #       $round  rounds
431
432 &function_begin_B("_aesni_set_encrypt_key");
433         &test   ("eax","eax");
434         &jz     (&label("bad_pointer"));
435         &test   ($key,$key);
436         &jz     (&label("bad_pointer"));
437
438         &movups ("xmm0",&QWP(0,"eax")); # pull first 128 bits of *userKey
439         &pxor   ("xmm4","xmm4");        # low dword of xmm4 is assumed 0
440         &lea    ($key,&DWP(16,$key));
441         &cmp    ($rounds,256);
442         &je     (&label("14rounds"));
443         &cmp    ($rounds,192);
444         &je     (&label("12rounds"));
445         &cmp    ($rounds,128);
446         &jne    (&label("bad_keybits"));
447
448 &set_label("10rounds",16);
449         &mov            ($rounds,10);
450         &$movekey       (&QWP(-16,$key),"xmm0");        # round 0
451         &aeskeygenassist("xmm1","xmm0",0x01);           # round 1
452         &call           (&label("key_128_cold"));
453         &aeskeygenassist("xmm1","xmm0",0x2);            # round 2
454         &call           (&label("key_128"));
455         &aeskeygenassist("xmm1","xmm0",0x04);           # round 3
456         &call           (&label("key_128"));
457         &aeskeygenassist("xmm1","xmm0",0x08);           # round 4
458         &call           (&label("key_128"));
459         &aeskeygenassist("xmm1","xmm0",0x10);           # round 5
460         &call           (&label("key_128"));
461         &aeskeygenassist("xmm1","xmm0",0x20);           # round 6
462         &call           (&label("key_128"));
463         &aeskeygenassist("xmm1","xmm0",0x40);           # round 7
464         &call           (&label("key_128"));
465         &aeskeygenassist("xmm1","xmm0",0x80);           # round 8
466         &call           (&label("key_128"));
467         &aeskeygenassist("xmm1","xmm0",0x1b);           # round 9
468         &call           (&label("key_128"));
469         &aeskeygenassist("xmm1","xmm0",0x36);           # round 10
470         &call           (&label("key_128"));
471         &$movekey       (&QWP(0,$key),"xmm0");
472         &mov            (&DWP(80,$key),$rounds);
473         &xor            ("eax","eax");
474         &ret();
475
476 &set_label("key_128",16);
477         &$movekey       (&QWP(0,$key),"xmm0");
478         &lea            ($key,&DWP(16,$key));
479 &set_label("key_128_cold");
480         &shufps         ("xmm4","xmm0",0b00010000);
481         &pxor           ("xmm0","xmm4");
482         &shufps         ("xmm4","xmm0",0b10001100,);
483         &pxor           ("xmm0","xmm4");
484         &pshufd         ("xmm1","xmm1",0b11111111);     # critical path
485         &pxor           ("xmm0","xmm1");
486         &ret();
487
488 &set_label("12rounds",16);
489         &movq           ("xmm2",&QWP(16,"eax"));        # remaining 1/3 of *userKey
490         &mov            ($rounds,12);
491         &$movekey       (&QWP(-16,$key),"xmm0")         # round 0
492         &aeskeygenassist("xmm1","xmm2",0x01);           # round 1,2
493         &call           (&label("key_192a_cold"));
494         &aeskeygenassist("xmm1","xmm2",0x02);           # round 2,3
495         &call           (&label("key_192b"));
496         &aeskeygenassist("xmm1","xmm2",0x04);           # round 4,5
497         &call           (&label("key_192a"));
498         &aeskeygenassist("xmm1","xmm2",0x08);           # round 5,6
499         &call           (&label("key_192b"));
500         &aeskeygenassist("xmm1","xmm2",0x10);           # round 7,8
501         &call           (&label("key_192a"));
502         &aeskeygenassist("xmm1","xmm2",0x20);           # round 8,9
503         &call           (&label("key_192b"));
504         &aeskeygenassist("xmm1","xmm2",0x40);           # round 10,11
505         &call           (&label("key_192a"));
506         &aeskeygenassist("xmm1","xmm2",0x80);           # round 11,12
507         &call           (&label("key_192b"));
508         &$movekey       (&QWP(0,$key),"xmm0");
509         &mov            (&DWP(48,$key),$rounds);
510         &xor            ("eax","eax");
511         &ret();
512
513 &set_label("key_192a",16);
514         &$movekey       (&QWP(0,$key),"xmm0");
515         &lea            ($key,&DWP(16,$key));
516 &set_label("key_192a_cold",16);
517         &movaps         ("xmm5","xmm2");
518 &set_label("key_192b_warm");
519         &shufps         ("xmm4","xmm0",0b00010000);
520         &movaps         ("xmm3","xmm2");
521         &pxor           ("xmm0","xmm4");
522         &shufps         ("xmm4","xmm0",0b10001100);
523         &pslldq         ("xmm3",4);
524         &pxor           ("xmm0","xmm4");
525         &pshufd         ("xmm1","xmm1",0b01010101);     # critical path
526         &pxor           ("xmm2","xmm3");
527         &pxor           ("xmm0","xmm1");
528         &pshufd         ("xmm3","xmm0",0b11111111);
529         &pxor           ("xmm2","xmm3");
530         &ret();
531
532 &set_label("key_192b",16);
533         &movaps         ("xmm3","xmm0");
534         &shufps         ("xmm5","xmm0",0b01000100);
535         &$movekey       (&QWP(0,$key),"xmm5");
536         &shufps         ("xmm3","xmm2",0b01001110);
537         &$movekey       (&QWP(16,$key),"xmm3");
538         &lea            ($key,&DWP(32,$key));
539         &jmp            (&label("key_192b_warm"));
540
541 &set_label("14rounds",16);
542         &movups         ("xmm2",&QWP(16,"eax"));        # remaining half of *userKey
543         &mov            ($rounds,14);
544         &lea            ($key,&DWP(16,$key));
545         &$movekey       (&QWP(-32,$key),"xmm0");        # round 0
546         &$movekey       (&QWP(-16,$key),"xmm2");        # round 1
547         &aeskeygenassist("xmm1","xmm2",0x01);           # round 2
548         &call           (&label("key_256a_cold"));
549         &aeskeygenassist("xmm1","xmm0",0x01);           # round 3
550         &call           (&label("key_256b"));
551         &aeskeygenassist("xmm1","xmm2",0x02);           # round 4
552         &call           (&label("key_256a"));
553         &aeskeygenassist("xmm1","xmm0",0x02);           # round 5
554         &call           (&label("key_256b"));
555         &aeskeygenassist("xmm1","xmm2",0x04);           # round 6
556         &call           (&label("key_256a"));
557         &aeskeygenassist("xmm1","xmm0",0x04);           # round 7
558         &call           (&label("key_256b"));
559         &aeskeygenassist("xmm1","xmm2",0x08);           # round 8
560         &call           (&label("key_256a"));
561         &aeskeygenassist("xmm1","xmm0",0x08);           # round 9
562         &call           (&label("key_256b"));
563         &aeskeygenassist("xmm1","xmm2",0x10);           # round 10
564         &call           (&label("key_256a"));
565         &aeskeygenassist("xmm1","xmm0",0x10);           # round 11
566         &call           (&label("key_256b"));
567         &aeskeygenassist("xmm1","xmm2",0x20);           # round 12
568         &call           (&label("key_256a"));
569         &aeskeygenassist("xmm1","xmm0",0x20);           # round 13
570         &call           (&label("key_256b"));
571         &aeskeygenassist("xmm1","xmm2",0x40);           # round 14
572         &call           (&label("key_256a"));
573         &$movekey       (&QWP(0,$key),"xmm0");
574         &mov            (&DWP(16,$key),$rounds);
575         &xor            ("eax","eax");
576         &ret();
577
578 &set_label("key_256a",16);
579         &$movekey       (&QWP(0,$key),"xmm2");
580         &lea            ($key,&DWP(16,$key));
581 &set_label("key_256a_cold");
582         &shufps         ("xmm4","xmm0",0b00010000);
583         &pxor           ("xmm0","xmm4");
584         &shufps         ("xmm4","xmm0",0b10001100);
585         &pxor           ("xmm0","xmm4");
586         &pshufd         ("xmm1","xmm1",0b11111111);     # critical path
587         &pxor           ("xmm0","xmm1");
588         &ret();
589
590 &set_label("key_256b",16);
591         &$movekey       (&QWP(0,$key),"xmm0");
592         &lea            ($key,&DWP(16,$key));
593
594         &shufps         ("xmm4","xmm2",0b00010000);
595         &pxor           ("xmm2","xmm4");
596         &shufps         ("xmm4","xmm2",0b10001100);
597         &pxor           ("xmm2","xmm4");
598         &pshufd         ("xmm1","xmm1",0b10101010);     # critical path
599         &pxor           ("xmm2","xmm1");
600         &ret();
601
602 &set_label("bad_pointer",4);
603         &mov    ("eax",-1);
604         &ret    ();
605 &set_label("bad_keybits",4);
606         &mov    ("eax",-2);
607         &ret    ();
608 &function_end_B("_aesni_set_encrypt_key");
609
610 # int $PREFIX_set_encrypt_key (const unsigned char *userKey, int bits,
611 #                              AES_KEY *key)
612 &function_begin_B("${PREFIX}_set_encrypt_key");
613         &mov    ("eax",&wparam(0));
614         &mov    ($rounds,&wparam(1));
615         &mov    ($key,&wparam(2));
616         &call   ("_aesni_set_encrypt_key");
617         &ret    ();
618 &function_end_B("${PREFIX}_set_encrypt_key");
619
620 # int $PREFIX_set_decrypt_key (const unsigned char *userKey, int bits,
621 #                              AES_KEY *key)
622 &function_begin_B("${PREFIX}_set_decrypt_key");
623         &mov    ("eax",&wparam(0));
624         &mov    ($rounds,&wparam(1));
625         &mov    ($key,&wparam(2));
626         &call   ("_aesni_set_encrypt_key");
627         &mov    ($key,&wparam(2));
628         &shl    ($rounds,4)     # actually rounds after _aesni_set_encrypt_key
629         &test   ("eax","eax");
630         &jnz    (&label("dec_key_ret"));
631         &lea    ("eax",&DWP(0,$key,$rounds));   # end of key schedule
632
633         &$movekey       ("xmm0",&QWP(0,$key));  # just swap
634         &$movekey       ("xmm1",&QWP(0,"eax"));
635         &$movekey       (&QWP(0,"eax"),"xmm0");
636         &$movekey       (&QWP(0,$key),"xmm1");
637         &lea            ($key,&DWP(16,$key));
638         &lea            ("eax",&DWP(-16,"eax"));
639         &jmp            (&label("dec_key_inverse"));
640
641 &set_label("dec_key_inverse",16);
642         &$movekey       ("xmm0",&QWP(0,$key));  # swap and inverse
643         &$movekey       ("xmm1",&QWP(0,"eax"));
644         &aesimc         ("xmm0","xmm0");
645         &aesimc         ("xmm1","xmm1");
646         &lea            ($key,&DWP(16,$key));
647         &lea            ("eax",&DWP(-16,"eax"));
648         &cmp            ("eax",$key);
649         &$movekey       (&QWP(16,"eax"),"xmm0");
650         &$movekey       (&QWP(-16,$key),"xmm1");
651         &ja             (&label("dec_key_inverse"));
652
653         &$movekey       ("xmm0",&QWP(0,$key));  # inverse middle
654         &aesimc         ("xmm0","xmm0");
655         &$movekey       (&QWP(0,$key),"xmm0");
656
657         &xor            ("eax","eax");          # return success
658 &set_label("dec_key_ret");
659         &ret    ();
660 &function_end_B("${PREFIX}_set_decrypt_key");
661 &asciz("AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>");
662
663 &asm_finish();