7d57edc0eb6c1090838b50ac367a978f64b07118
[openssl.git] / crypto / aes / asm / vpaes-x86.pl
1 #! /usr/bin/env perl
2 # Copyright 2011-2016 The OpenSSL Project Authors. All Rights Reserved.
3 #
4 # Licensed under the OpenSSL license (the "License").  You may not use
5 # this file except in compliance with the License.  You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
8
9
10 ######################################################################
11 ## Constant-time SSSE3 AES core implementation.
12 ## version 0.1
13 ##
14 ## By Mike Hamburg (Stanford University), 2009
15 ## Public domain.
16 ##
17 ## For details see http://shiftleft.org/papers/vector_aes/ and
18 ## http://crypto.stanford.edu/vpaes/.
19
20 ######################################################################
21 # September 2011.
22 #
23 # Port vpaes-x86_64.pl as 32-bit "almost" drop-in replacement for
24 # aes-586.pl. "Almost" refers to the fact that AES_cbc_encrypt
25 # doesn't handle partial vectors (doesn't have to if called from
26 # EVP only). "Drop-in" implies that this module doesn't share key
27 # schedule structure with the original nor does it make assumption
28 # about its alignment...
29 #
30 # Performance summary. aes-586.pl column lists large-block CBC
31 # encrypt/decrypt/with-hyper-threading-off(*) results in cycles per
32 # byte processed with 128-bit key, and vpaes-x86.pl column - [also
33 # large-block CBC] encrypt/decrypt.
34 #
35 #               aes-586.pl              vpaes-x86.pl
36 #
37 # Core 2(**)    28.1/41.4/18.3          21.9/25.2(***)
38 # Nehalem       27.9/40.4/18.1          10.2/11.9
39 # Atom          70.7/92.1/60.1          61.1/75.4(***)
40 # Silvermont    45.4/62.9/24.1          49.2/61.1(***)
41 #
42 # (*)   "Hyper-threading" in the context refers rather to cache shared
43 #       among multiple cores, than to specifically Intel HTT. As vast
44 #       majority of contemporary cores share cache, slower code path
45 #       is common place. In other words "with-hyper-threading-off"
46 #       results are presented mostly for reference purposes.
47 #
48 # (**)  "Core 2" refers to initial 65nm design, a.k.a. Conroe.
49 #
50 # (***) Less impressive improvement on Core 2 and Atom is due to slow
51 #       pshufb, yet it's respectable +28%/64%  improvement on Core 2
52 #       and +15% on Atom (as implied, over "hyper-threading-safe"
53 #       code path).
54 #
55 #                                               <appro@openssl.org>
56
57 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
58 push(@INC,"${dir}","${dir}../../perlasm");
59 require "x86asm.pl";
60
61 $output = pop;
62 open OUT,">$output";
63 *STDOUT=*OUT;
64
65 &asm_init($ARGV[0],$x86only = $ARGV[$#ARGV] eq "386");
66
67 $PREFIX="vpaes";
68
69 my  ($round, $base, $magic, $key, $const, $inp, $out)=
70     ("eax",  "ebx", "ecx",  "edx","ebp",  "esi","edi");
71
72 &static_label("_vpaes_consts");
73 &static_label("_vpaes_schedule_low_round");
74
75 &set_label("_vpaes_consts",64);
76 $k_inv=-0x30;           # inv, inva
77         &data_word(0x0D080180,0x0E05060F,0x0A0B0C02,0x04070309);
78         &data_word(0x0F0B0780,0x01040A06,0x02050809,0x030D0E0C);
79
80 $k_s0F=-0x10;           # s0F
81         &data_word(0x0F0F0F0F,0x0F0F0F0F,0x0F0F0F0F,0x0F0F0F0F);
82
83 $k_ipt=0x00;            # input transform (lo, hi)
84         &data_word(0x5A2A7000,0xC2B2E898,0x52227808,0xCABAE090);
85         &data_word(0x317C4D00,0x4C01307D,0xB0FDCC81,0xCD80B1FC);
86
87 $k_sb1=0x20;            # sb1u, sb1t
88         &data_word(0xCB503E00,0xB19BE18F,0x142AF544,0xA5DF7A6E);
89         &data_word(0xFAE22300,0x3618D415,0x0D2ED9EF,0x3BF7CCC1);
90 $k_sb2=0x40;            # sb2u, sb2t
91         &data_word(0x0B712400,0xE27A93C6,0xBC982FCD,0x5EB7E955);
92         &data_word(0x0AE12900,0x69EB8840,0xAB82234A,0xC2A163C8);
93 $k_sbo=0x60;            # sbou, sbot
94         &data_word(0x6FBDC700,0xD0D26D17,0xC502A878,0x15AABF7A);
95         &data_word(0x5FBB6A00,0xCFE474A5,0x412B35FA,0x8E1E90D1);
96
97 $k_mc_forward=0x80;     # mc_forward
98         &data_word(0x00030201,0x04070605,0x080B0A09,0x0C0F0E0D);
99         &data_word(0x04070605,0x080B0A09,0x0C0F0E0D,0x00030201);
100         &data_word(0x080B0A09,0x0C0F0E0D,0x00030201,0x04070605);
101         &data_word(0x0C0F0E0D,0x00030201,0x04070605,0x080B0A09);
102
103 $k_mc_backward=0xc0;    # mc_backward
104         &data_word(0x02010003,0x06050407,0x0A09080B,0x0E0D0C0F);
105         &data_word(0x0E0D0C0F,0x02010003,0x06050407,0x0A09080B);
106         &data_word(0x0A09080B,0x0E0D0C0F,0x02010003,0x06050407);
107         &data_word(0x06050407,0x0A09080B,0x0E0D0C0F,0x02010003);
108
109 $k_sr=0x100;            # sr
110         &data_word(0x03020100,0x07060504,0x0B0A0908,0x0F0E0D0C);
111         &data_word(0x0F0A0500,0x030E0904,0x07020D08,0x0B06010C);
112         &data_word(0x0B020900,0x0F060D04,0x030A0108,0x070E050C);
113         &data_word(0x070A0D00,0x0B0E0104,0x0F020508,0x0306090C);
114
115 $k_rcon=0x140;          # rcon
116         &data_word(0xAF9DEEB6,0x1F8391B9,0x4D7C7D81,0x702A9808);
117
118 $k_s63=0x150;           # s63: all equal to 0x63 transformed
119         &data_word(0x5B5B5B5B,0x5B5B5B5B,0x5B5B5B5B,0x5B5B5B5B);
120
121 $k_opt=0x160;           # output transform
122         &data_word(0xD6B66000,0xFF9F4929,0xDEBE6808,0xF7974121);
123         &data_word(0x50BCEC00,0x01EDBD51,0xB05C0CE0,0xE10D5DB1);
124
125 $k_deskew=0x180;        # deskew tables: inverts the sbox's "skew"
126         &data_word(0x47A4E300,0x07E4A340,0x5DBEF91A,0x1DFEB95A);
127         &data_word(0x83EA6900,0x5F36B5DC,0xF49D1E77,0x2841C2AB);
128 ##
129 ##  Decryption stuff
130 ##  Key schedule constants
131 ##
132 $k_dksd=0x1a0;          # decryption key schedule: invskew x*D
133         &data_word(0xA3E44700,0xFEB91A5D,0x5A1DBEF9,0x0740E3A4);
134         &data_word(0xB5368300,0x41C277F4,0xAB289D1E,0x5FDC69EA);
135 $k_dksb=0x1c0;          # decryption key schedule: invskew x*B
136         &data_word(0x8550D500,0x9A4FCA1F,0x1CC94C99,0x03D65386);
137         &data_word(0xB6FC4A00,0x115BEDA7,0x7E3482C8,0xD993256F);
138 $k_dkse=0x1e0;          # decryption key schedule: invskew x*E + 0x63
139         &data_word(0x1FC9D600,0xD5031CCA,0x994F5086,0x53859A4C);
140         &data_word(0x4FDC7BE8,0xA2319605,0x20B31487,0xCD5EF96A);
141 $k_dks9=0x200;          # decryption key schedule: invskew x*9
142         &data_word(0x7ED9A700,0xB6116FC8,0x82255BFC,0x4AED9334);
143         &data_word(0x27143300,0x45765162,0xE9DAFDCE,0x8BB89FAC);
144
145 ##
146 ##  Decryption stuff
147 ##  Round function constants
148 ##
149 $k_dipt=0x220;          # decryption input transform
150         &data_word(0x0B545F00,0x0F505B04,0x114E451A,0x154A411E);
151         &data_word(0x60056500,0x86E383E6,0xF491F194,0x12771772);
152
153 $k_dsb9=0x240;          # decryption sbox output *9*u, *9*t
154         &data_word(0x9A86D600,0x851C0353,0x4F994CC9,0xCAD51F50);
155         &data_word(0xECD74900,0xC03B1789,0xB2FBA565,0x725E2C9E);
156 $k_dsbd=0x260;          # decryption sbox output *D*u, *D*t
157         &data_word(0xE6B1A200,0x7D57CCDF,0x882A4439,0xF56E9B13);
158         &data_word(0x24C6CB00,0x3CE2FAF7,0x15DEEFD3,0x2931180D);
159 $k_dsbb=0x280;          # decryption sbox output *B*u, *B*t
160         &data_word(0x96B44200,0xD0226492,0xB0F2D404,0x602646F6);
161         &data_word(0xCD596700,0xC19498A6,0x3255AA6B,0xF3FF0C3E);
162 $k_dsbe=0x2a0;          # decryption sbox output *E*u, *E*t
163         &data_word(0x26D4D000,0x46F29296,0x64B4F6B0,0x22426004);
164         &data_word(0xFFAAC100,0x0C55A6CD,0x98593E32,0x9467F36B);
165 $k_dsbo=0x2c0;          # decryption sbox final output
166         &data_word(0x7EF94000,0x1387EA53,0xD4943E2D,0xC7AA6DB9);
167         &data_word(0x93441D00,0x12D7560F,0xD8C58E9C,0xCA4B8159);
168 &asciz  ("Vector Permutation AES for x86/SSSE3, Mike Hamburg (Stanford University)");
169 &align  (64);
170
171 &function_begin_B("_vpaes_preheat");
172         &add    ($const,&DWP(0,"esp"));
173         &movdqa ("xmm7",&QWP($k_inv,$const));
174         &movdqa ("xmm6",&QWP($k_s0F,$const));
175         &ret    ();
176 &function_end_B("_vpaes_preheat");
177
178 ##
179 ##  _aes_encrypt_core
180 ##
181 ##  AES-encrypt %xmm0.
182 ##
183 ##  Inputs:
184 ##     %xmm0 = input
185 ##     %xmm6-%xmm7 as in _vpaes_preheat
186 ##    (%edx) = scheduled keys
187 ##
188 ##  Output in %xmm0
189 ##  Clobbers  %xmm1-%xmm5, %eax, %ebx, %ecx, %edx
190 ##
191 ##
192 &function_begin_B("_vpaes_encrypt_core");
193         &mov    ($magic,16);
194         &mov    ($round,&DWP(240,$key));
195         &movdqa ("xmm1","xmm6")
196         &movdqa ("xmm2",&QWP($k_ipt,$const));
197         &pandn  ("xmm1","xmm0");
198         &pand   ("xmm0","xmm6");
199         &movdqu ("xmm5",&QWP(0,$key));
200         &pshufb ("xmm2","xmm0");
201         &movdqa ("xmm0",&QWP($k_ipt+16,$const));
202         &pxor   ("xmm2","xmm5");
203         &psrld  ("xmm1",4);
204         &add    ($key,16);
205         &pshufb ("xmm0","xmm1");
206         &lea    ($base,&DWP($k_mc_backward,$const));
207         &pxor   ("xmm0","xmm2");
208         &jmp    (&label("enc_entry"));
209
210
211 &set_label("enc_loop",16);
212         # middle of middle round
213         &movdqa ("xmm4",&QWP($k_sb1,$const));   # 4 : sb1u
214         &movdqa ("xmm0",&QWP($k_sb1+16,$const));# 0 : sb1t
215         &pshufb ("xmm4","xmm2");                # 4 = sb1u
216         &pshufb ("xmm0","xmm3");                # 0 = sb1t
217         &pxor   ("xmm4","xmm5");                # 4 = sb1u + k
218         &movdqa ("xmm5",&QWP($k_sb2,$const));   # 4 : sb2u
219         &pxor   ("xmm0","xmm4");                # 0 = A
220         &movdqa ("xmm1",&QWP(-0x40,$base,$magic));# .Lk_mc_forward[]
221         &pshufb ("xmm5","xmm2");                # 4 = sb2u
222         &movdqa ("xmm2",&QWP($k_sb2+16,$const));# 2 : sb2t
223         &movdqa ("xmm4",&QWP(0,$base,$magic));  # .Lk_mc_backward[]
224         &pshufb ("xmm2","xmm3");                # 2 = sb2t
225         &movdqa ("xmm3","xmm0");                # 3 = A
226         &pxor   ("xmm2","xmm5");                # 2 = 2A
227         &pshufb ("xmm0","xmm1");                # 0 = B
228         &add    ($key,16);                      # next key
229         &pxor   ("xmm0","xmm2");                # 0 = 2A+B
230         &pshufb ("xmm3","xmm4");                # 3 = D
231         &add    ($magic,16);                    # next mc
232         &pxor   ("xmm3","xmm0");                # 3 = 2A+B+D
233         &pshufb ("xmm0","xmm1");                # 0 = 2B+C
234         &and    ($magic,0x30);                  # ... mod 4
235         &sub    ($round,1);                     # nr--
236         &pxor   ("xmm0","xmm3");                # 0 = 2A+3B+C+D
237
238 &set_label("enc_entry");
239         # top of round
240         &movdqa ("xmm1","xmm6");                # 1 : i
241         &movdqa ("xmm5",&QWP($k_inv+16,$const));# 2 : a/k
242         &pandn  ("xmm1","xmm0");                # 1 = i<<4
243         &psrld  ("xmm1",4);                     # 1 = i
244         &pand   ("xmm0","xmm6");                # 0 = k
245         &pshufb ("xmm5","xmm0");                # 2 = a/k
246         &movdqa ("xmm3","xmm7");                # 3 : 1/i
247         &pxor   ("xmm0","xmm1");                # 0 = j
248         &pshufb ("xmm3","xmm1");                # 3 = 1/i
249         &movdqa ("xmm4","xmm7");                # 4 : 1/j
250         &pxor   ("xmm3","xmm5");                # 3 = iak = 1/i + a/k
251         &pshufb ("xmm4","xmm0");                # 4 = 1/j
252         &movdqa ("xmm2","xmm7");                # 2 : 1/iak
253         &pxor   ("xmm4","xmm5");                # 4 = jak = 1/j + a/k
254         &pshufb ("xmm2","xmm3");                # 2 = 1/iak
255         &movdqa ("xmm3","xmm7");                # 3 : 1/jak
256         &pxor   ("xmm2","xmm0");                # 2 = io
257         &pshufb ("xmm3","xmm4");                # 3 = 1/jak
258         &movdqu ("xmm5",&QWP(0,$key));
259         &pxor   ("xmm3","xmm1");                # 3 = jo
260         &jnz    (&label("enc_loop"));
261
262         # middle of last round
263         &movdqa ("xmm4",&QWP($k_sbo,$const));   # 3 : sbou      .Lk_sbo
264         &movdqa ("xmm0",&QWP($k_sbo+16,$const));# 3 : sbot      .Lk_sbo+16
265         &pshufb ("xmm4","xmm2");                # 4 = sbou
266         &pxor   ("xmm4","xmm5");                # 4 = sb1u + k
267         &pshufb ("xmm0","xmm3");                # 0 = sb1t
268         &movdqa ("xmm1",&QWP(0x40,$base,$magic));# .Lk_sr[]
269         &pxor   ("xmm0","xmm4");                # 0 = A
270         &pshufb ("xmm0","xmm1");
271         &ret    ();
272 &function_end_B("_vpaes_encrypt_core");
273
274 ##
275 ##  Decryption core
276 ##
277 ##  Same API as encryption core.
278 ##
279 &function_begin_B("_vpaes_decrypt_core");
280         &lea    ($base,&DWP($k_dsbd,$const));
281         &mov    ($round,&DWP(240,$key));
282         &movdqa ("xmm1","xmm6");
283         &movdqa ("xmm2",&QWP($k_dipt-$k_dsbd,$base));
284         &pandn  ("xmm1","xmm0");
285         &mov    ($magic,$round);
286         &psrld  ("xmm1",4)
287         &movdqu ("xmm5",&QWP(0,$key));
288         &shl    ($magic,4);
289         &pand   ("xmm0","xmm6");
290         &pshufb ("xmm2","xmm0");
291         &movdqa ("xmm0",&QWP($k_dipt-$k_dsbd+16,$base));
292         &xor    ($magic,0x30);
293         &pshufb ("xmm0","xmm1");
294         &and    ($magic,0x30);
295         &pxor   ("xmm2","xmm5");
296         &movdqa ("xmm5",&QWP($k_mc_forward+48,$const));
297         &pxor   ("xmm0","xmm2");
298         &add    ($key,16);
299         &lea    ($magic,&DWP($k_sr-$k_dsbd,$base,$magic));
300         &jmp    (&label("dec_entry"));
301
302 &set_label("dec_loop",16);
303 ##
304 ##  Inverse mix columns
305 ##
306         &movdqa ("xmm4",&QWP(-0x20,$base));     # 4 : sb9u
307         &movdqa ("xmm1",&QWP(-0x10,$base));     # 0 : sb9t
308         &pshufb ("xmm4","xmm2");                # 4 = sb9u
309         &pshufb ("xmm1","xmm3");                # 0 = sb9t
310         &pxor   ("xmm0","xmm4");
311         &movdqa ("xmm4",&QWP(0,$base));         # 4 : sbdu
312         &pxor   ("xmm0","xmm1");                # 0 = ch
313         &movdqa ("xmm1",&QWP(0x10,$base));      # 0 : sbdt
314
315         &pshufb ("xmm4","xmm2");                # 4 = sbdu
316         &pshufb ("xmm0","xmm5");                # MC ch
317         &pshufb ("xmm1","xmm3");                # 0 = sbdt
318         &pxor   ("xmm0","xmm4");                # 4 = ch
319         &movdqa ("xmm4",&QWP(0x20,$base));      # 4 : sbbu
320         &pxor   ("xmm0","xmm1");                # 0 = ch
321         &movdqa ("xmm1",&QWP(0x30,$base));      # 0 : sbbt
322
323         &pshufb ("xmm4","xmm2");                # 4 = sbbu
324         &pshufb ("xmm0","xmm5");                # MC ch
325         &pshufb ("xmm1","xmm3");                # 0 = sbbt
326         &pxor   ("xmm0","xmm4");                # 4 = ch
327         &movdqa ("xmm4",&QWP(0x40,$base));      # 4 : sbeu
328         &pxor   ("xmm0","xmm1");                # 0 = ch
329         &movdqa ("xmm1",&QWP(0x50,$base));      # 0 : sbet
330
331         &pshufb ("xmm4","xmm2");                # 4 = sbeu
332         &pshufb ("xmm0","xmm5");                # MC ch
333         &pshufb ("xmm1","xmm3");                # 0 = sbet
334         &pxor   ("xmm0","xmm4");                # 4 = ch
335         &add    ($key,16);                      # next round key
336         &palignr("xmm5","xmm5",12);
337         &pxor   ("xmm0","xmm1");                # 0 = ch
338         &sub    ($round,1);                     # nr--
339
340 &set_label("dec_entry");
341         # top of round
342         &movdqa ("xmm1","xmm6");                # 1 : i
343         &movdqa ("xmm2",&QWP($k_inv+16,$const));# 2 : a/k
344         &pandn  ("xmm1","xmm0");                # 1 = i<<4
345         &pand   ("xmm0","xmm6");                # 0 = k
346         &psrld  ("xmm1",4);                     # 1 = i
347         &pshufb ("xmm2","xmm0");                # 2 = a/k
348         &movdqa ("xmm3","xmm7");                # 3 : 1/i
349         &pxor   ("xmm0","xmm1");                # 0 = j
350         &pshufb ("xmm3","xmm1");                # 3 = 1/i
351         &movdqa ("xmm4","xmm7");                # 4 : 1/j
352         &pxor   ("xmm3","xmm2");                # 3 = iak = 1/i + a/k
353         &pshufb ("xmm4","xmm0");                # 4 = 1/j
354         &pxor   ("xmm4","xmm2");                # 4 = jak = 1/j + a/k
355         &movdqa ("xmm2","xmm7");                # 2 : 1/iak
356         &pshufb ("xmm2","xmm3");                # 2 = 1/iak
357         &movdqa ("xmm3","xmm7");                # 3 : 1/jak
358         &pxor   ("xmm2","xmm0");                # 2 = io
359         &pshufb ("xmm3","xmm4");                # 3 = 1/jak
360         &movdqu ("xmm0",&QWP(0,$key));
361         &pxor   ("xmm3","xmm1");                # 3 = jo
362         &jnz    (&label("dec_loop"));
363
364         # middle of last round
365         &movdqa ("xmm4",&QWP(0x60,$base));      # 3 : sbou
366         &pshufb ("xmm4","xmm2");                # 4 = sbou
367         &pxor   ("xmm4","xmm0");                # 4 = sb1u + k
368         &movdqa ("xmm0",&QWP(0x70,$base));      # 0 : sbot
369         &movdqa ("xmm2",&QWP(0,$magic));
370         &pshufb ("xmm0","xmm3");                # 0 = sb1t
371         &pxor   ("xmm0","xmm4");                # 0 = A
372         &pshufb ("xmm0","xmm2");
373         &ret    ();
374 &function_end_B("_vpaes_decrypt_core");
375
376 ########################################################
377 ##                                                    ##
378 ##                  AES key schedule                  ##
379 ##                                                    ##
380 ########################################################
381 &function_begin_B("_vpaes_schedule_core");
382         &add    ($const,&DWP(0,"esp"));
383         &movdqu ("xmm0",&QWP(0,$inp));          # load key (unaligned)
384         &movdqa ("xmm2",&QWP($k_rcon,$const));  # load rcon
385
386         # input transform
387         &movdqa ("xmm3","xmm0");
388         &lea    ($base,&DWP($k_ipt,$const));
389         &movdqa (&QWP(4,"esp"),"xmm2");         # xmm8
390         &call   ("_vpaes_schedule_transform");
391         &movdqa ("xmm7","xmm0");
392
393         &test   ($out,$out);
394         &jnz    (&label("schedule_am_decrypting"));
395
396         # encrypting, output zeroth round key after transform
397         &movdqu (&QWP(0,$key),"xmm0");
398         &jmp    (&label("schedule_go"));
399
400 &set_label("schedule_am_decrypting");
401         # decrypting, output zeroth round key after shiftrows
402         &movdqa ("xmm1",&QWP($k_sr,$const,$magic));
403         &pshufb ("xmm3","xmm1");
404         &movdqu (&QWP(0,$key),"xmm3");
405         &xor    ($magic,0x30);
406
407 &set_label("schedule_go");
408         &cmp    ($round,192);
409         &ja     (&label("schedule_256"));
410         &je     (&label("schedule_192"));
411         # 128: fall though
412
413 ##
414 ##  .schedule_128
415 ##
416 ##  128-bit specific part of key schedule.
417 ##
418 ##  This schedule is really simple, because all its parts
419 ##  are accomplished by the subroutines.
420 ##
421 &set_label("schedule_128");
422         &mov    ($round,10);
423
424 &set_label("loop_schedule_128");
425         &call   ("_vpaes_schedule_round");
426         &dec    ($round);
427         &jz     (&label("schedule_mangle_last"));
428         &call   ("_vpaes_schedule_mangle");     # write output
429         &jmp    (&label("loop_schedule_128"));
430
431 ##
432 ##  .aes_schedule_192
433 ##
434 ##  192-bit specific part of key schedule.
435 ##
436 ##  The main body of this schedule is the same as the 128-bit
437 ##  schedule, but with more smearing.  The long, high side is
438 ##  stored in %xmm7 as before, and the short, low side is in
439 ##  the high bits of %xmm6.
440 ##
441 ##  This schedule is somewhat nastier, however, because each
442 ##  round produces 192 bits of key material, or 1.5 round keys.
443 ##  Therefore, on each cycle we do 2 rounds and produce 3 round
444 ##  keys.
445 ##
446 &set_label("schedule_192",16);
447         &movdqu ("xmm0",&QWP(8,$inp));          # load key part 2 (very unaligned)
448         &call   ("_vpaes_schedule_transform");  # input transform
449         &movdqa ("xmm6","xmm0");                # save short part
450         &pxor   ("xmm4","xmm4");                # clear 4
451         &movhlps("xmm6","xmm4");                # clobber low side with zeros
452         &mov    ($round,4);
453
454 &set_label("loop_schedule_192");
455         &call   ("_vpaes_schedule_round");
456         &palignr("xmm0","xmm6",8);
457         &call   ("_vpaes_schedule_mangle");     # save key n
458         &call   ("_vpaes_schedule_192_smear");
459         &call   ("_vpaes_schedule_mangle");     # save key n+1
460         &call   ("_vpaes_schedule_round");
461         &dec    ($round);
462         &jz     (&label("schedule_mangle_last"));
463         &call   ("_vpaes_schedule_mangle");     # save key n+2
464         &call   ("_vpaes_schedule_192_smear");
465         &jmp    (&label("loop_schedule_192"));
466
467 ##
468 ##  .aes_schedule_256
469 ##
470 ##  256-bit specific part of key schedule.
471 ##
472 ##  The structure here is very similar to the 128-bit
473 ##  schedule, but with an additional "low side" in
474 ##  %xmm6.  The low side's rounds are the same as the
475 ##  high side's, except no rcon and no rotation.
476 ##
477 &set_label("schedule_256",16);
478         &movdqu ("xmm0",&QWP(16,$inp));         # load key part 2 (unaligned)
479         &call   ("_vpaes_schedule_transform");  # input transform
480         &mov    ($round,7);
481
482 &set_label("loop_schedule_256");
483         &call   ("_vpaes_schedule_mangle");     # output low result
484         &movdqa ("xmm6","xmm0");                # save cur_lo in xmm6
485
486         # high round
487         &call   ("_vpaes_schedule_round");
488         &dec    ($round);
489         &jz     (&label("schedule_mangle_last"));
490         &call   ("_vpaes_schedule_mangle");
491
492         # low round. swap xmm7 and xmm6
493         &pshufd ("xmm0","xmm0",0xFF);
494         &movdqa (&QWP(20,"esp"),"xmm7");
495         &movdqa ("xmm7","xmm6");
496         &call   ("_vpaes_schedule_low_round");
497         &movdqa ("xmm7",&QWP(20,"esp"));
498
499         &jmp    (&label("loop_schedule_256"));
500
501 ##
502 ##  .aes_schedule_mangle_last
503 ##
504 ##  Mangler for last round of key schedule
505 ##  Mangles %xmm0
506 ##    when encrypting, outputs out(%xmm0) ^ 63
507 ##    when decrypting, outputs unskew(%xmm0)
508 ##
509 ##  Always called right before return... jumps to cleanup and exits
510 ##
511 &set_label("schedule_mangle_last",16);
512         # schedule last round key from xmm0
513         &lea    ($base,&DWP($k_deskew,$const));
514         &test   ($out,$out);
515         &jnz    (&label("schedule_mangle_last_dec"));
516
517         # encrypting
518         &movdqa ("xmm1",&QWP($k_sr,$const,$magic));
519         &pshufb ("xmm0","xmm1");                # output permute
520         &lea    ($base,&DWP($k_opt,$const));    # prepare to output transform
521         &add    ($key,32);
522
523 &set_label("schedule_mangle_last_dec");
524         &add    ($key,-16);
525         &pxor   ("xmm0",&QWP($k_s63,$const));
526         &call   ("_vpaes_schedule_transform");  # output transform
527         &movdqu (&QWP(0,$key),"xmm0");          # save last key
528
529         # cleanup
530         &pxor   ("xmm0","xmm0");
531         &pxor   ("xmm1","xmm1");
532         &pxor   ("xmm2","xmm2");
533         &pxor   ("xmm3","xmm3");
534         &pxor   ("xmm4","xmm4");
535         &pxor   ("xmm5","xmm5");
536         &pxor   ("xmm6","xmm6");
537         &pxor   ("xmm7","xmm7");
538         &ret    ();
539 &function_end_B("_vpaes_schedule_core");
540
541 ##
542 ##  .aes_schedule_192_smear
543 ##
544 ##  Smear the short, low side in the 192-bit key schedule.
545 ##
546 ##  Inputs:
547 ##    %xmm7: high side, b  a  x  y
548 ##    %xmm6:  low side, d  c  0  0
549 ##    %xmm13: 0
550 ##
551 ##  Outputs:
552 ##    %xmm6: b+c+d  b+c  0  0
553 ##    %xmm0: b+c+d  b+c  b  a
554 ##
555 &function_begin_B("_vpaes_schedule_192_smear");
556         &pshufd ("xmm1","xmm6",0x80);           # d c 0 0 -> c 0 0 0
557         &pshufd ("xmm0","xmm7",0xFE);           # b a _ _ -> b b b a
558         &pxor   ("xmm6","xmm1");                # -> c+d c 0 0
559         &pxor   ("xmm1","xmm1");
560         &pxor   ("xmm6","xmm0");                # -> b+c+d b+c b a
561         &movdqa ("xmm0","xmm6");
562         &movhlps("xmm6","xmm1");                # clobber low side with zeros
563         &ret    ();
564 &function_end_B("_vpaes_schedule_192_smear");
565
566 ##
567 ##  .aes_schedule_round
568 ##
569 ##  Runs one main round of the key schedule on %xmm0, %xmm7
570 ##
571 ##  Specifically, runs subbytes on the high dword of %xmm0
572 ##  then rotates it by one byte and xors into the low dword of
573 ##  %xmm7.
574 ##
575 ##  Adds rcon from low byte of %xmm8, then rotates %xmm8 for
576 ##  next rcon.
577 ##
578 ##  Smears the dwords of %xmm7 by xoring the low into the
579 ##  second low, result into third, result into highest.
580 ##
581 ##  Returns results in %xmm7 = %xmm0.
582 ##  Clobbers %xmm1-%xmm5.
583 ##
584 &function_begin_B("_vpaes_schedule_round");
585         # extract rcon from xmm8
586         &movdqa ("xmm2",&QWP(8,"esp"));         # xmm8
587         &pxor   ("xmm1","xmm1");
588         &palignr("xmm1","xmm2",15);
589         &palignr("xmm2","xmm2",15);
590         &pxor   ("xmm7","xmm1");
591
592         # rotate
593         &pshufd ("xmm0","xmm0",0xFF);
594         &palignr("xmm0","xmm0",1);
595
596         # fall through...
597         &movdqa (&QWP(8,"esp"),"xmm2");         # xmm8
598
599         # low round: same as high round, but no rotation and no rcon.
600 &set_label("_vpaes_schedule_low_round");
601         # smear xmm7
602         &movdqa ("xmm1","xmm7");
603         &pslldq ("xmm7",4);
604         &pxor   ("xmm7","xmm1");
605         &movdqa ("xmm1","xmm7");
606         &pslldq ("xmm7",8);
607         &pxor   ("xmm7","xmm1");
608         &pxor   ("xmm7",&QWP($k_s63,$const));
609
610         # subbyte
611         &movdqa ("xmm4",&QWP($k_s0F,$const));
612         &movdqa ("xmm5",&QWP($k_inv,$const));   # 4 : 1/j
613         &movdqa ("xmm1","xmm4");
614         &pandn  ("xmm1","xmm0");
615         &psrld  ("xmm1",4);                     # 1 = i
616         &pand   ("xmm0","xmm4");                # 0 = k
617         &movdqa ("xmm2",&QWP($k_inv+16,$const));# 2 : a/k
618         &pshufb ("xmm2","xmm0");                # 2 = a/k
619         &pxor   ("xmm0","xmm1");                # 0 = j
620         &movdqa ("xmm3","xmm5");                # 3 : 1/i
621         &pshufb ("xmm3","xmm1");                # 3 = 1/i
622         &pxor   ("xmm3","xmm2");                # 3 = iak = 1/i + a/k
623         &movdqa ("xmm4","xmm5");                # 4 : 1/j
624         &pshufb ("xmm4","xmm0");                # 4 = 1/j
625         &pxor   ("xmm4","xmm2");                # 4 = jak = 1/j + a/k
626         &movdqa ("xmm2","xmm5");                # 2 : 1/iak
627         &pshufb ("xmm2","xmm3");                # 2 = 1/iak
628         &pxor   ("xmm2","xmm0");                # 2 = io
629         &movdqa ("xmm3","xmm5");                # 3 : 1/jak
630         &pshufb ("xmm3","xmm4");                # 3 = 1/jak
631         &pxor   ("xmm3","xmm1");                # 3 = jo
632         &movdqa ("xmm4",&QWP($k_sb1,$const));   # 4 : sbou
633         &pshufb ("xmm4","xmm2");                # 4 = sbou
634         &movdqa ("xmm0",&QWP($k_sb1+16,$const));# 0 : sbot
635         &pshufb ("xmm0","xmm3");                # 0 = sb1t
636         &pxor   ("xmm0","xmm4");                # 0 = sbox output
637
638         # add in smeared stuff
639         &pxor   ("xmm0","xmm7");
640         &movdqa ("xmm7","xmm0");
641         &ret    ();
642 &function_end_B("_vpaes_schedule_round");
643
644 ##
645 ##  .aes_schedule_transform
646 ##
647 ##  Linear-transform %xmm0 according to tables at (%ebx)
648 ##
649 ##  Output in %xmm0
650 ##  Clobbers %xmm1, %xmm2
651 ##
652 &function_begin_B("_vpaes_schedule_transform");
653         &movdqa ("xmm2",&QWP($k_s0F,$const));
654         &movdqa ("xmm1","xmm2");
655         &pandn  ("xmm1","xmm0");
656         &psrld  ("xmm1",4);
657         &pand   ("xmm0","xmm2");
658         &movdqa ("xmm2",&QWP(0,$base));
659         &pshufb ("xmm2","xmm0");
660         &movdqa ("xmm0",&QWP(16,$base));
661         &pshufb ("xmm0","xmm1");
662         &pxor   ("xmm0","xmm2");
663         &ret    ();
664 &function_end_B("_vpaes_schedule_transform");
665
666 ##
667 ##  .aes_schedule_mangle
668 ##
669 ##  Mangle xmm0 from (basis-transformed) standard version
670 ##  to our version.
671 ##
672 ##  On encrypt,
673 ##    xor with 0x63
674 ##    multiply by circulant 0,1,1,1
675 ##    apply shiftrows transform
676 ##
677 ##  On decrypt,
678 ##    xor with 0x63
679 ##    multiply by "inverse mixcolumns" circulant E,B,D,9
680 ##    deskew
681 ##    apply shiftrows transform
682 ##
683 ##
684 ##  Writes out to (%edx), and increments or decrements it
685 ##  Keeps track of round number mod 4 in %ecx
686 ##  Preserves xmm0
687 ##  Clobbers xmm1-xmm5
688 ##
689 &function_begin_B("_vpaes_schedule_mangle");
690         &movdqa ("xmm4","xmm0");        # save xmm0 for later
691         &movdqa ("xmm5",&QWP($k_mc_forward,$const));
692         &test   ($out,$out);
693         &jnz    (&label("schedule_mangle_dec"));
694
695         # encrypting
696         &add    ($key,16);
697         &pxor   ("xmm4",&QWP($k_s63,$const));
698         &pshufb ("xmm4","xmm5");
699         &movdqa ("xmm3","xmm4");
700         &pshufb ("xmm4","xmm5");
701         &pxor   ("xmm3","xmm4");
702         &pshufb ("xmm4","xmm5");
703         &pxor   ("xmm3","xmm4");
704
705         &jmp    (&label("schedule_mangle_both"));
706
707 &set_label("schedule_mangle_dec",16);
708         # inverse mix columns
709         &movdqa ("xmm2",&QWP($k_s0F,$const));
710         &lea    ($inp,&DWP($k_dksd,$const));
711         &movdqa ("xmm1","xmm2");
712         &pandn  ("xmm1","xmm4");
713         &psrld  ("xmm1",4);                     # 1 = hi
714         &pand   ("xmm4","xmm2");                # 4 = lo
715
716         &movdqa ("xmm2",&QWP(0,$inp));
717         &pshufb ("xmm2","xmm4");
718         &movdqa ("xmm3",&QWP(0x10,$inp));
719         &pshufb ("xmm3","xmm1");
720         &pxor   ("xmm3","xmm2");
721         &pshufb ("xmm3","xmm5");
722
723         &movdqa ("xmm2",&QWP(0x20,$inp));
724         &pshufb ("xmm2","xmm4");
725         &pxor   ("xmm2","xmm3");
726         &movdqa ("xmm3",&QWP(0x30,$inp));
727         &pshufb ("xmm3","xmm1");
728         &pxor   ("xmm3","xmm2");
729         &pshufb ("xmm3","xmm5");
730
731         &movdqa ("xmm2",&QWP(0x40,$inp));
732         &pshufb ("xmm2","xmm4");
733         &pxor   ("xmm2","xmm3");
734         &movdqa ("xmm3",&QWP(0x50,$inp));
735         &pshufb ("xmm3","xmm1");
736         &pxor   ("xmm3","xmm2");
737         &pshufb ("xmm3","xmm5");
738
739         &movdqa ("xmm2",&QWP(0x60,$inp));
740         &pshufb ("xmm2","xmm4");
741         &pxor   ("xmm2","xmm3");
742         &movdqa ("xmm3",&QWP(0x70,$inp));
743         &pshufb ("xmm3","xmm1");
744         &pxor   ("xmm3","xmm2");
745
746         &add    ($key,-16);
747
748 &set_label("schedule_mangle_both");
749         &movdqa ("xmm1",&QWP($k_sr,$const,$magic));
750         &pshufb ("xmm3","xmm1");
751         &add    ($magic,-16);
752         &and    ($magic,0x30);
753         &movdqu (&QWP(0,$key),"xmm3");
754         &ret    ();
755 &function_end_B("_vpaes_schedule_mangle");
756
757 #
758 # Interface to OpenSSL
759 #
760 &function_begin("${PREFIX}_set_encrypt_key");
761         &mov    ($inp,&wparam(0));              # inp
762         &lea    ($base,&DWP(-56,"esp"));
763         &mov    ($round,&wparam(1));            # bits
764         &and    ($base,-16);
765         &mov    ($key,&wparam(2));              # key
766         &xchg   ($base,"esp");                  # alloca
767         &mov    (&DWP(48,"esp"),$base);
768
769         &mov    ($base,$round);
770         &shr    ($base,5);
771         &add    ($base,5);
772         &mov    (&DWP(240,$key),$base);         # AES_KEY->rounds = nbits/32+5;
773         &mov    ($magic,0x30);
774         &mov    ($out,0);
775
776         &lea    ($const,&DWP(&label("_vpaes_consts")."+0x30-".&label("pic_point")));
777         &call   ("_vpaes_schedule_core");
778 &set_label("pic_point");
779
780         &mov    ("esp",&DWP(48,"esp"));
781         &xor    ("eax","eax");
782 &function_end("${PREFIX}_set_encrypt_key");
783
784 &function_begin("${PREFIX}_set_decrypt_key");
785         &mov    ($inp,&wparam(0));              # inp
786         &lea    ($base,&DWP(-56,"esp"));
787         &mov    ($round,&wparam(1));            # bits
788         &and    ($base,-16);
789         &mov    ($key,&wparam(2));              # key
790         &xchg   ($base,"esp");                  # alloca
791         &mov    (&DWP(48,"esp"),$base);
792
793         &mov    ($base,$round);
794         &shr    ($base,5);
795         &add    ($base,5);
796         &mov    (&DWP(240,$key),$base); # AES_KEY->rounds = nbits/32+5;
797         &shl    ($base,4);
798         &lea    ($key,&DWP(16,$key,$base));
799
800         &mov    ($out,1);
801         &mov    ($magic,$round);
802         &shr    ($magic,1);
803         &and    ($magic,32);
804         &xor    ($magic,32);                    # nbist==192?0:32;
805
806         &lea    ($const,&DWP(&label("_vpaes_consts")."+0x30-".&label("pic_point")));
807         &call   ("_vpaes_schedule_core");
808 &set_label("pic_point");
809
810         &mov    ("esp",&DWP(48,"esp"));
811         &xor    ("eax","eax");
812 &function_end("${PREFIX}_set_decrypt_key");
813
814 &function_begin("${PREFIX}_encrypt");
815         &lea    ($const,&DWP(&label("_vpaes_consts")."+0x30-".&label("pic_point")));
816         &call   ("_vpaes_preheat");
817 &set_label("pic_point");
818         &mov    ($inp,&wparam(0));              # inp
819         &lea    ($base,&DWP(-56,"esp"));
820         &mov    ($out,&wparam(1));              # out
821         &and    ($base,-16);
822         &mov    ($key,&wparam(2));              # key
823         &xchg   ($base,"esp");                  # alloca
824         &mov    (&DWP(48,"esp"),$base);
825
826         &movdqu ("xmm0",&QWP(0,$inp));
827         &call   ("_vpaes_encrypt_core");
828         &movdqu (&QWP(0,$out),"xmm0");
829
830         &mov    ("esp",&DWP(48,"esp"));
831 &function_end("${PREFIX}_encrypt");
832
833 &function_begin("${PREFIX}_decrypt");
834         &lea    ($const,&DWP(&label("_vpaes_consts")."+0x30-".&label("pic_point")));
835         &call   ("_vpaes_preheat");
836 &set_label("pic_point");
837         &mov    ($inp,&wparam(0));              # inp
838         &lea    ($base,&DWP(-56,"esp"));
839         &mov    ($out,&wparam(1));              # out
840         &and    ($base,-16);
841         &mov    ($key,&wparam(2));              # key
842         &xchg   ($base,"esp");                  # alloca
843         &mov    (&DWP(48,"esp"),$base);
844
845         &movdqu ("xmm0",&QWP(0,$inp));
846         &call   ("_vpaes_decrypt_core");
847         &movdqu (&QWP(0,$out),"xmm0");
848
849         &mov    ("esp",&DWP(48,"esp"));
850 &function_end("${PREFIX}_decrypt");
851
852 &function_begin("${PREFIX}_cbc_encrypt");
853         &mov    ($inp,&wparam(0));              # inp
854         &mov    ($out,&wparam(1));              # out
855         &mov    ($round,&wparam(2));            # len
856         &mov    ($key,&wparam(3));              # key
857         &sub    ($round,16);
858         &jc     (&label("cbc_abort"));
859         &lea    ($base,&DWP(-56,"esp"));
860         &mov    ($const,&wparam(4));            # ivp
861         &and    ($base,-16);
862         &mov    ($magic,&wparam(5));            # enc
863         &xchg   ($base,"esp");                  # alloca
864         &movdqu ("xmm1",&QWP(0,$const));        # load IV
865         &sub    ($out,$inp);
866         &mov    (&DWP(48,"esp"),$base);
867
868         &mov    (&DWP(0,"esp"),$out);           # save out
869         &mov    (&DWP(4,"esp"),$key)            # save key
870         &mov    (&DWP(8,"esp"),$const);         # save ivp
871         &mov    ($out,$round);                  # $out works as $len
872
873         &lea    ($const,&DWP(&label("_vpaes_consts")."+0x30-".&label("pic_point")));
874         &call   ("_vpaes_preheat");
875 &set_label("pic_point");
876         &cmp    ($magic,0);
877         &je     (&label("cbc_dec_loop"));
878         &jmp    (&label("cbc_enc_loop"));
879
880 &set_label("cbc_enc_loop",16);
881         &movdqu ("xmm0",&QWP(0,$inp));          # load input
882         &pxor   ("xmm0","xmm1");                # inp^=iv
883         &call   ("_vpaes_encrypt_core");
884         &mov    ($base,&DWP(0,"esp"));          # restore out
885         &mov    ($key,&DWP(4,"esp"));           # restore key
886         &movdqa ("xmm1","xmm0");
887         &movdqu (&QWP(0,$base,$inp),"xmm0");    # write output
888         &lea    ($inp,&DWP(16,$inp));
889         &sub    ($out,16);
890         &jnc    (&label("cbc_enc_loop"));
891         &jmp    (&label("cbc_done"));
892
893 &set_label("cbc_dec_loop",16);
894         &movdqu ("xmm0",&QWP(0,$inp));          # load input
895         &movdqa (&QWP(16,"esp"),"xmm1");        # save IV
896         &movdqa (&QWP(32,"esp"),"xmm0");        # save future IV
897         &call   ("_vpaes_decrypt_core");
898         &mov    ($base,&DWP(0,"esp"));          # restore out
899         &mov    ($key,&DWP(4,"esp"));           # restore key
900         &pxor   ("xmm0",&QWP(16,"esp"));        # out^=iv
901         &movdqa ("xmm1",&QWP(32,"esp"));        # load next IV
902         &movdqu (&QWP(0,$base,$inp),"xmm0");    # write output
903         &lea    ($inp,&DWP(16,$inp));
904         &sub    ($out,16);
905         &jnc    (&label("cbc_dec_loop"));
906
907 &set_label("cbc_done");
908         &mov    ($base,&DWP(8,"esp"));          # restore ivp
909         &mov    ("esp",&DWP(48,"esp"));
910         &movdqu (&QWP(0,$base),"xmm1");         # write IV
911 &set_label("cbc_abort");
912 &function_end("${PREFIX}_cbc_encrypt");
913
914 &asm_finish();
915
916 close STDOUT;