7d76c820da1fb9624c1045855bad61802bf0f9a6
[openssl.git] / crypto / camellia / asm / cmll-x86_64.pl
1 #!/usr/bin/env perl
2
3 # ====================================================================
4 # Copyright (c) 2008 Andy Polyakov <appro@openssl.org>
5 #
6 # This module may be used under the terms of either the GNU General
7 # Public License version 2 or later, the GNU Lesser General Public
8 # License version 2.1 or later, the Mozilla Public License version
9 # 1.1 or the BSD License. The exact terms of either license are
10 # distributed along with this module. For further details see
11 # http://www.openssl.org/~appro/camellia/.
12 # ====================================================================
13
14 # Performance in cycles per processed byte (less is better) in
15 # 'openssl speed ...' benchmark:
16 #
17 #                       AMD64   Core2   EM64T
18 # -evp camellia-128-ecb 16.7    21.0    22.7
19 # + over gcc 3.4.6      +25%    +5%     0%
20 #
21 # camellia-128-cbc      15.7    20.4    21.1
22 #
23 # 128-bit key setup     128     216     205     cycles/key
24 # + over gcc 3.4.6      +54%    +39%    +15%
25 #
26 # Numbers in "+" rows represent performance improvement over compiler
27 # generated code. Key setup timings are impressive on AMD and Core2
28 # thanks to 64-bit operations being covertly deployed. Improvement on
29 # EM64T, pre-Core2 Intel x86_64 CPU, is not as impressive, because it
30 # apparently emulates some of 64-bit operations in [32-bit] microcode.
31
32 $flavour = shift;
33 $output  = shift;
34 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
35
36 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
37
38 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
39 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
40 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
41 die "can't locate x86_64-xlate.pl";
42
43 open STDOUT,"| $^X $xlate $flavour $output";
44
45 sub hi() { my $r=shift; $r =~ s/%[er]([a-d])x/%\1h/;    $r; }
46 sub lo() { my $r=shift; $r =~ s/%[er]([a-d])x/%\1l/;
47                         $r =~ s/%[er]([sd]i)/%\1l/;
48                         $r =~ s/%(r[0-9]+)[d]?/%\1b/;   $r; }
49
50 $t0="%eax";$t1="%ebx";$t2="%ecx";$t3="%edx";
51 @S=("%r8d","%r9d","%r10d","%r11d");
52 $i0="%esi";
53 $i1="%edi";
54 $Tbl="%rbp";    # size optimization
55 $inp="%r12";
56 $out="%r13";
57 $key="%r14";
58 $keyend="%r15";
59 $arg0d=$win64?"%ecx":"%edi";
60
61 # const unsigned int Camellia_SBOX[4][256];
62 # Well, sort of... Camellia_SBOX[0][] is interleaved with [1][],
63 # and [2][] - with [3][]. This is done to minimize code size.
64 $SBOX1_1110=0;          # Camellia_SBOX[0]
65 $SBOX4_4404=4;          # Camellia_SBOX[1]
66 $SBOX2_0222=2048;       # Camellia_SBOX[2]
67 $SBOX3_3033=2052;       # Camellia_SBOX[3]
68
69 sub Camellia_Feistel {
70 my $i=@_[0];
71 my $seed=defined(@_[1])?@_[1]:0;
72 my $scale=$seed<0?-8:8;
73 my $j=($i&1)*2;
74 my $s0=@S[($j)%4],$s1=@S[($j+1)%4],$s2=@S[($j+2)%4],$s3=@S[($j+3)%4];
75
76 $code.=<<___;
77         xor     $s0,$t0                         # t0^=key[0]
78         xor     $s1,$t1                         # t1^=key[1]
79         movz    `&hi("$t0")`,$i0                # (t0>>8)&0xff
80         movz    `&lo("$t1")`,$i1                # (t1>>0)&0xff
81         mov     $SBOX3_3033($Tbl,$i0,8),$t3     # t3=SBOX3_3033[0]
82         mov     $SBOX1_1110($Tbl,$i1,8),$t2     # t2=SBOX1_1110[1]
83         movz    `&lo("$t0")`,$i0                # (t0>>0)&0xff
84         shr     \$16,$t0
85         movz    `&hi("$t1")`,$i1                # (t1>>8)&0xff
86         xor     $SBOX4_4404($Tbl,$i0,8),$t3     # t3^=SBOX4_4404[0]
87         shr     \$16,$t1
88         xor     $SBOX4_4404($Tbl,$i1,8),$t2     # t2^=SBOX4_4404[1]
89         movz    `&hi("$t0")`,$i0                # (t0>>24)&0xff
90         movz    `&lo("$t1")`,$i1                # (t1>>16)&0xff
91         xor     $SBOX1_1110($Tbl,$i0,8),$t3     # t3^=SBOX1_1110[0]
92         xor     $SBOX3_3033($Tbl,$i1,8),$t2     # t2^=SBOX3_3033[1]
93         movz    `&lo("$t0")`,$i0                # (t0>>16)&0xff
94         movz    `&hi("$t1")`,$i1                # (t1>>24)&0xff
95         xor     $SBOX2_0222($Tbl,$i0,8),$t3     # t3^=SBOX2_0222[0]
96         xor     $SBOX2_0222($Tbl,$i1,8),$t2     # t2^=SBOX2_0222[1]
97         mov     `$seed+($i+1)*$scale`($key),$t1 # prefetch key[i+1]
98         mov     `$seed+($i+1)*$scale+4`($key),$t0
99         xor     $t3,$t2                         # t2^=t3
100         ror     \$8,$t3                         # t3=RightRotate(t3,8)
101         xor     $t2,$s2
102         xor     $t2,$s3
103         xor     $t3,$s3
104 ___
105 }
106
107 # void Camellia_EncryptBlock_Rounds(
108 #               int grandRounds,
109 #               const Byte plaintext[],
110 #               const KEY_TABLE_TYPE keyTable,
111 #               Byte ciphertext[])
112 $code=<<___;
113 .text
114
115 # V1.x API
116 .globl  Camellia_EncryptBlock
117 .type   Camellia_EncryptBlock,\@abi-omnipotent
118 .align  16
119 Camellia_EncryptBlock:
120         movl    \$128,%eax
121         subl    $arg0d,%eax
122         movl    \$3,$arg0d
123         adcl    \$0,$arg0d      # keyBitLength==128?3:4
124         jmp     .Lenc_rounds
125 .size   Camellia_EncryptBlock,.-Camellia_EncryptBlock
126 # V2
127 .globl  Camellia_EncryptBlock_Rounds
128 .type   Camellia_EncryptBlock_Rounds,\@function,4
129 .align  16
130 .Lenc_rounds:
131 Camellia_EncryptBlock_Rounds:
132         push    %rbx
133         push    %rbp
134         push    %r13
135         push    %r14
136         push    %r15
137 .Lenc_prologue:
138
139         #mov    %rsi,$inp               # put away arguments
140         mov     %rcx,$out
141         mov     %rdx,$key
142
143         shl     \$6,%edi                # process grandRounds
144         lea     .LCamellia_SBOX(%rip),$Tbl
145         lea     ($key,%rdi),$keyend
146
147         mov     0(%rsi),@S[0]           # load plaintext
148         mov     4(%rsi),@S[1]
149         mov     8(%rsi),@S[2]
150         bswap   @S[0]
151         mov     12(%rsi),@S[3]
152         bswap   @S[1]
153         bswap   @S[2]
154         bswap   @S[3]
155
156         call    _x86_64_Camellia_encrypt
157
158         bswap   @S[0]
159         bswap   @S[1]
160         bswap   @S[2]
161         mov     @S[0],0($out)
162         bswap   @S[3]
163         mov     @S[1],4($out)
164         mov     @S[2],8($out)
165         mov     @S[3],12($out)
166
167         mov     0(%rsp),%r15
168         mov     8(%rsp),%r14
169         mov     16(%rsp),%r13
170         mov     24(%rsp),%rbp
171         mov     32(%rsp),%rbx
172         lea     40(%rsp),%rsp
173 .Lenc_epilogue:
174         ret
175 .size   Camellia_EncryptBlock_Rounds,.-Camellia_EncryptBlock_Rounds
176
177 .type   _x86_64_Camellia_encrypt,\@abi-omnipotent
178 .align  16
179 _x86_64_Camellia_encrypt:
180         xor     0($key),@S[1]
181         xor     4($key),@S[0]           # ^=key[0-3]
182         xor     8($key),@S[3]
183         xor     12($key),@S[2]
184 .align  16
185 .Leloop:
186         mov     16($key),$t1            # prefetch key[4-5]
187         mov     20($key),$t0
188
189 ___
190         for ($i=0;$i<6;$i++) { Camellia_Feistel($i,16); }
191 $code.=<<___;
192         lea     16*4($key),$key
193         cmp     $keyend,$key
194         mov     8($key),$t3             # prefetch key[2-3]
195         mov     12($key),$t2
196         je      .Ledone
197
198         and     @S[0],$t0
199         or      @S[3],$t3
200         rol     \$1,$t0
201         xor     $t3,@S[2]               # s2^=s3|key[3];
202         xor     $t0,@S[1]               # s1^=LeftRotate(s0&key[0],1);
203         and     @S[2],$t2
204         or      @S[1],$t1
205         rol     \$1,$t2
206         xor     $t1,@S[0]               # s0^=s1|key[1];
207         xor     $t2,@S[3]               # s3^=LeftRotate(s2&key[2],1);
208         jmp     .Leloop
209
210 .align  16
211 .Ledone:
212         xor     @S[2],$t0               # SwapHalf
213         xor     @S[3],$t1
214         xor     @S[0],$t2
215         xor     @S[1],$t3
216
217         mov     $t0,@S[0]
218         mov     $t1,@S[1]
219         mov     $t2,@S[2]
220         mov     $t3,@S[3]
221
222         .byte   0xf3,0xc3               # rep ret
223 .size   _x86_64_Camellia_encrypt,.-_x86_64_Camellia_encrypt
224
225 # V1.x API
226 .globl  Camellia_DecryptBlock
227 .type   Camellia_DecryptBlock,\@abi-omnipotent
228 .align  16
229 Camellia_DecryptBlock:
230         movl    \$128,%eax
231         subl    $arg0d,%eax
232         movl    \$3,$arg0d
233         adcl    \$0,$arg0d      # keyBitLength==128?3:4
234         jmp     .Ldec_rounds
235 .size   Camellia_DecryptBlock,.-Camellia_DecryptBlock
236 # V2
237 .globl  Camellia_DecryptBlock_Rounds
238 .type   Camellia_DecryptBlock_Rounds,\@function,4
239 .align  16
240 .Ldec_rounds:
241 Camellia_DecryptBlock_Rounds:
242         push    %rbx
243         push    %rbp
244         push    %r13
245         push    %r14
246         push    %r15
247 .Ldec_prologue:
248
249         #mov    %rsi,$inp               # put away arguments
250         mov     %rcx,$out
251         mov     %rdx,$keyend
252
253         shl     \$6,%edi                # process grandRounds
254         lea     .LCamellia_SBOX(%rip),$Tbl
255         lea     ($keyend,%rdi),$key
256
257         mov     0(%rsi),@S[0]           # load plaintext
258         mov     4(%rsi),@S[1]
259         mov     8(%rsi),@S[2]
260         bswap   @S[0]
261         mov     12(%rsi),@S[3]
262         bswap   @S[1]
263         bswap   @S[2]
264         bswap   @S[3]
265
266         call    _x86_64_Camellia_decrypt
267
268         bswap   @S[0]
269         bswap   @S[1]
270         bswap   @S[2]
271         mov     @S[0],0($out)
272         bswap   @S[3]
273         mov     @S[1],4($out)
274         mov     @S[2],8($out)
275         mov     @S[3],12($out)
276
277         mov     0(%rsp),%r15
278         mov     8(%rsp),%r14
279         mov     16(%rsp),%r13
280         mov     24(%rsp),%rbp
281         mov     32(%rsp),%rbx
282         lea     40(%rsp),%rsp
283 .Ldec_epilogue:
284         ret
285 .size   Camellia_DecryptBlock_Rounds,.-Camellia_DecryptBlock_Rounds
286
287 .type   _x86_64_Camellia_decrypt,\@abi-omnipotent
288 .align  16
289 _x86_64_Camellia_decrypt:
290         xor     0($key),@S[1]
291         xor     4($key),@S[0]           # ^=key[0-3]
292         xor     8($key),@S[3]
293         xor     12($key),@S[2]
294 .align  16
295 .Ldloop:
296         mov     -8($key),$t1            # prefetch key[4-5]
297         mov     -4($key),$t0
298
299 ___
300         for ($i=0;$i<6;$i++) { Camellia_Feistel($i,-8); }
301 $code.=<<___;
302         lea     -16*4($key),$key
303         cmp     $keyend,$key
304         mov     0($key),$t3             # prefetch key[2-3]
305         mov     4($key),$t2
306         je      .Lddone
307
308         and     @S[0],$t0
309         or      @S[3],$t3
310         rol     \$1,$t0
311         xor     $t3,@S[2]               # s2^=s3|key[3];
312         xor     $t0,@S[1]               # s1^=LeftRotate(s0&key[0],1);
313         and     @S[2],$t2
314         or      @S[1],$t1
315         rol     \$1,$t2
316         xor     $t1,@S[0]               # s0^=s1|key[1];
317         xor     $t2,@S[3]               # s3^=LeftRotate(s2&key[2],1);
318
319         jmp     .Ldloop
320
321 .align  16
322 .Lddone:
323         xor     @S[2],$t2
324         xor     @S[3],$t3
325         xor     @S[0],$t0
326         xor     @S[1],$t1
327
328         mov     $t2,@S[0]               # SwapHalf
329         mov     $t3,@S[1]
330         mov     $t0,@S[2]
331         mov     $t1,@S[3]
332
333         .byte   0xf3,0xc3               # rep ret
334 .size   _x86_64_Camellia_decrypt,.-_x86_64_Camellia_decrypt
335 ___
336
337 sub _saveround {
338 my ($rnd,$key,@T)=@_;
339 my $bias=int(@T[0])?shift(@T):0;
340
341     if ($#T==3) {
342         $code.=<<___;
343         mov     @T[1],`$bias+$rnd*8+0`($key)
344         mov     @T[0],`$bias+$rnd*8+4`($key)
345         mov     @T[3],`$bias+$rnd*8+8`($key)
346         mov     @T[2],`$bias+$rnd*8+12`($key)
347 ___
348     } else {
349         $code.="        mov     @T[0],`$bias+$rnd*8+0`($key)\n";
350         $code.="        mov     @T[1],`$bias+$rnd*8+8`($key)\n" if ($#T>=1);
351     }
352 }
353
354 sub _loadround {
355 my ($rnd,$key,@T)=@_;
356 my $bias=int(@T[0])?shift(@T):0;
357
358 $code.="        mov     `$bias+$rnd*8+0`($key),@T[0]\n";
359 $code.="        mov     `$bias+$rnd*8+8`($key),@T[1]\n" if ($#T>=1);
360 }
361
362 # shld is very slow on Intel EM64T family. Even on AMD it limits
363 # instruction decode rate [because it's VectorPath] and consequently
364 # performance...
365 sub __rotl128 {
366 my ($i0,$i1,$rot)=@_;
367
368     if ($rot) {
369         $code.=<<___;
370         mov     $i0,%r11
371         shld    \$$rot,$i1,$i0
372         shld    \$$rot,%r11,$i1
373 ___
374     }
375 }
376
377 # ... Implementing 128-bit rotate without shld gives 80% better
378 # performance EM64T, +15% on AMD64 and only ~7% degradation on
379 # Core2. This is therefore preferred.
380 sub _rotl128 {
381 my ($i0,$i1,$rot)=@_;
382
383     if ($rot) {
384         $code.=<<___;
385         mov     $i0,%r11
386         shl     \$$rot,$i0
387         mov     $i1,%r9
388         shr     \$`64-$rot`,%r9
389         shr     \$`64-$rot`,%r11
390         or      %r9,$i0
391         shl     \$$rot,$i1
392         or      %r11,$i1
393 ___
394     }
395 }
396
397 { my $step=0;
398
399 $code.=<<___;
400 .globl  Camellia_Ekeygen
401 .type   Camellia_Ekeygen,\@function,3
402 .align  16
403 Camellia_Ekeygen:
404         push    %rbx
405         push    %rbp
406         push    %r13
407         push    %r14
408         push    %r15
409 .Lkey_prologue:
410
411         mov     %rdi,$keyend            # put away arguments, keyBitLength
412         mov     %rdx,$out               # keyTable
413
414         mov     0(%rsi),@S[0]           # load 0-127 bits
415         mov     4(%rsi),@S[1]
416         mov     8(%rsi),@S[2]
417         mov     12(%rsi),@S[3]
418
419         bswap   @S[0]
420         bswap   @S[1]
421         bswap   @S[2]
422         bswap   @S[3]
423 ___
424         &_saveround     (0,$out,@S);    # KL<<<0
425 $code.=<<___;
426         cmp     \$128,$keyend           # check keyBitLength
427         je      .L1st128
428
429         mov     16(%rsi),@S[0]          # load 128-191 bits
430         mov     20(%rsi),@S[1]
431         cmp     \$192,$keyend
432         je      .L1st192
433         mov     24(%rsi),@S[2]          # load 192-255 bits
434         mov     28(%rsi),@S[3]
435         jmp     .L1st256
436 .L1st192:
437         mov     @S[0],@S[2]
438         mov     @S[1],@S[3]
439         not     @S[2]
440         not     @S[3]
441 .L1st256:
442         bswap   @S[0]
443         bswap   @S[1]
444         bswap   @S[2]
445         bswap   @S[3]
446 ___
447         &_saveround     (4,$out,@S);    # temp storage for KR!
448 $code.=<<___;
449         xor     0($out),@S[1]           # KR^KL
450         xor     4($out),@S[0]
451         xor     8($out),@S[3]
452         xor     12($out),@S[2]
453
454 .L1st128:
455         lea     .LCamellia_SIGMA(%rip),$key
456         lea     .LCamellia_SBOX(%rip),$Tbl
457
458         mov     0($key),$t1
459         mov     4($key),$t0
460 ___
461         &Camellia_Feistel($step++);
462         &Camellia_Feistel($step++);
463 $code.=<<___;
464         xor     0($out),@S[1]           # ^KL
465         xor     4($out),@S[0]
466         xor     8($out),@S[3]
467         xor     12($out),@S[2]
468 ___
469         &Camellia_Feistel($step++);
470         &Camellia_Feistel($step++);
471 $code.=<<___;
472         cmp     \$128,$keyend
473         jne     .L2nd256
474
475         lea     128($out),$out          # size optimization
476         shl     \$32,%r8                # @S[0]||
477         shl     \$32,%r10               # @S[2]||
478         or      %r9,%r8                 # ||@S[1]
479         or      %r11,%r10               # ||@S[3]
480 ___
481         &_loadround     (0,$out,-128,"%rax","%rbx");    # KL
482         &_saveround     (2,$out,-128,"%r8","%r10");     # KA<<<0
483         &_rotl128       ("%rax","%rbx",15);
484         &_saveround     (4,$out,-128,"%rax","%rbx");    # KL<<<15
485         &_rotl128       ("%r8","%r10",15);
486         &_saveround     (6,$out,-128,"%r8","%r10");     # KA<<<15
487         &_rotl128       ("%r8","%r10",15);              # 15+15=30
488         &_saveround     (8,$out,-128,"%r8","%r10");     # KA<<<30
489         &_rotl128       ("%rax","%rbx",30);             # 15+30=45
490         &_saveround     (10,$out,-128,"%rax","%rbx");   # KL<<<45
491         &_rotl128       ("%r8","%r10",15);              # 30+15=45
492         &_saveround     (12,$out,-128,"%r8");           # KA<<<45
493         &_rotl128       ("%rax","%rbx",15);             # 45+15=60
494         &_saveround     (13,$out,-128,"%rbx");          # KL<<<60
495         &_rotl128       ("%r8","%r10",15);              # 45+15=60
496         &_saveround     (14,$out,-128,"%r8","%r10");    # KA<<<60
497         &_rotl128       ("%rax","%rbx",17);             # 60+17=77
498         &_saveround     (16,$out,-128,"%rax","%rbx");   # KL<<<77
499         &_rotl128       ("%rax","%rbx",17);             # 77+17=94
500         &_saveround     (18,$out,-128,"%rax","%rbx");   # KL<<<94
501         &_rotl128       ("%r8","%r10",34);              # 60+34=94
502         &_saveround     (20,$out,-128,"%r8","%r10");    # KA<<<94
503         &_rotl128       ("%rax","%rbx",17);             # 94+17=111
504         &_saveround     (22,$out,-128,"%rax","%rbx");   # KL<<<111
505         &_rotl128       ("%r8","%r10",17);              # 94+17=111
506         &_saveround     (24,$out,-128,"%r8","%r10");    # KA<<<111
507 $code.=<<___;
508         mov     \$3,%eax
509         jmp     .Ldone
510 .align  16
511 .L2nd256:
512 ___
513         &_saveround     (6,$out,@S);    # temp storage for KA!
514 $code.=<<___;
515         xor     `4*8+0`($out),@S[1]     # KA^KR
516         xor     `4*8+4`($out),@S[0]
517         xor     `5*8+0`($out),@S[3]
518         xor     `5*8+4`($out),@S[2]
519 ___
520         &Camellia_Feistel($step++);
521         &Camellia_Feistel($step++);
522
523         &_loadround     (0,$out,"%rax","%rbx"); # KL
524         &_loadround     (4,$out,"%rcx","%rdx"); # KR
525         &_loadround     (6,$out,"%r14","%r15"); # KA
526 $code.=<<___;
527         lea     128($out),$out          # size optimization
528         shl     \$32,%r8                # @S[0]||
529         shl     \$32,%r10               # @S[2]||
530         or      %r9,%r8                 # ||@S[1]
531         or      %r11,%r10               # ||@S[3]
532 ___
533         &_saveround     (2,$out,-128,"%r8","%r10");     # KB<<<0
534         &_rotl128       ("%rcx","%rdx",15);
535         &_saveround     (4,$out,-128,"%rcx","%rdx");    # KR<<<15
536         &_rotl128       ("%r14","%r15",15);
537         &_saveround     (6,$out,-128,"%r14","%r15");    # KA<<<15
538         &_rotl128       ("%rcx","%rdx",15);             # 15+15=30
539         &_saveround     (8,$out,-128,"%rcx","%rdx");    # KR<<<30
540         &_rotl128       ("%r8","%r10",30);
541         &_saveround     (10,$out,-128,"%r8","%r10");    # KB<<<30
542         &_rotl128       ("%rax","%rbx",45);
543         &_saveround     (12,$out,-128,"%rax","%rbx");   # KL<<<45
544         &_rotl128       ("%r14","%r15",30);             # 15+30=45
545         &_saveround     (14,$out,-128,"%r14","%r15");   # KA<<<45
546         &_rotl128       ("%rax","%rbx",15);             # 45+15=60
547         &_saveround     (16,$out,-128,"%rax","%rbx");   # KL<<<60
548         &_rotl128       ("%rcx","%rdx",30);             # 30+30=60
549         &_saveround     (18,$out,-128,"%rcx","%rdx");   # KR<<<60
550         &_rotl128       ("%r8","%r10",30);              # 30+30=60
551         &_saveround     (20,$out,-128,"%r8","%r10");    # KB<<<60
552         &_rotl128       ("%rax","%rbx",17);             # 60+17=77
553         &_saveround     (22,$out,-128,"%rax","%rbx");   # KL<<<77
554         &_rotl128       ("%r14","%r15",32);             # 45+32=77
555         &_saveround     (24,$out,-128,"%r14","%r15");   # KA<<<77
556         &_rotl128       ("%rcx","%rdx",34);             # 60+34=94
557         &_saveround     (26,$out,-128,"%rcx","%rdx");   # KR<<<94
558         &_rotl128       ("%r14","%r15",17);             # 77+17=94
559         &_saveround     (28,$out,-128,"%r14","%r15");   # KA<<<77
560         &_rotl128       ("%rax","%rbx",34);             # 77+34=111
561         &_saveround     (30,$out,-128,"%rax","%rbx");   # KL<<<111
562         &_rotl128       ("%r8","%r10",51);              # 60+51=111
563         &_saveround     (32,$out,-128,"%r8","%r10");    # KB<<<111
564 $code.=<<___;
565         mov     \$4,%eax
566 .Ldone:
567         mov     0(%rsp),%r15
568         mov     8(%rsp),%r14
569         mov     16(%rsp),%r13
570         mov     24(%rsp),%rbp
571         mov     32(%rsp),%rbx
572         lea     40(%rsp),%rsp
573 .Lkey_epilogue:
574         ret
575 .size   Camellia_Ekeygen,.-Camellia_Ekeygen
576 ___
577 }
578
579 @SBOX=(
580 112,130, 44,236,179, 39,192,229,228,133, 87, 53,234, 12,174, 65,
581  35,239,107,147, 69, 25,165, 33,237, 14, 79, 78, 29,101,146,189,
582 134,184,175,143,124,235, 31,206, 62, 48,220, 95, 94,197, 11, 26,
583 166,225, 57,202,213, 71, 93, 61,217,  1, 90,214, 81, 86,108, 77,
584 139, 13,154,102,251,204,176, 45,116, 18, 43, 32,240,177,132,153,
585 223, 76,203,194, 52,126,118,  5,109,183,169, 49,209, 23,  4,215,
586  20, 88, 58, 97,222, 27, 17, 28, 50, 15,156, 22, 83, 24,242, 34,
587 254, 68,207,178,195,181,122,145, 36,  8,232,168, 96,252,105, 80,
588 170,208,160,125,161,137, 98,151, 84, 91, 30,149,224,255,100,210,
589  16,196,  0, 72,163,247,117,219,138,  3,230,218,  9, 63,221,148,
590 135, 92,131,  2,205, 74,144, 51,115,103,246,243,157,127,191,226,
591  82,155,216, 38,200, 55,198, 59,129,150,111, 75, 19,190, 99, 46,
592 233,121,167,140,159,110,188,142, 41,245,249,182, 47,253,180, 89,
593 120,152,  6,106,231, 70,113,186,212, 37,171, 66,136,162,141,250,
594 114,  7,185, 85,248,238,172, 10, 54, 73, 42,104, 60, 56,241,164,
595  64, 40,211,123,187,201, 67,193, 21,227,173,244,119,199,128,158);
596
597 sub S1110 { my $i=shift; $i=@SBOX[$i]; $i=$i<<24|$i<<16|$i<<8; sprintf("0x%08x",$i); }
598 sub S4404 { my $i=shift; $i=($i<<1|$i>>7)&0xff; $i=@SBOX[$i]; $i=$i<<24|$i<<16|$i; sprintf("0x%08x",$i); }
599 sub S0222 { my $i=shift; $i=@SBOX[$i]; $i=($i<<1|$i>>7)&0xff; $i=$i<<16|$i<<8|$i; sprintf("0x%08x",$i); }
600 sub S3033 { my $i=shift; $i=@SBOX[$i]; $i=($i>>1|$i<<7)&0xff; $i=$i<<24|$i<<8|$i; sprintf("0x%08x",$i); }
601
602 $code.=<<___;
603 .align  64
604 .LCamellia_SIGMA:
605 .long   0x3bcc908b, 0xa09e667f, 0x4caa73b2, 0xb67ae858
606 .long   0xe94f82be, 0xc6ef372f, 0xf1d36f1c, 0x54ff53a5
607 .long   0xde682d1d, 0x10e527fa, 0xb3e6c1fd, 0xb05688c2
608 .long   0,          0,          0,          0
609 .LCamellia_SBOX:
610 ___
611 # tables are interleaved, remember?
612 sub data_word { $code.=".long\t".join(',',@_)."\n"; }
613 for ($i=0;$i<256;$i++) { &data_word(&S1110($i),&S4404($i)); }
614 for ($i=0;$i<256;$i++) { &data_word(&S0222($i),&S3033($i)); }
615
616 # void Camellia_cbc_encrypt (const void char *inp, unsigned char *out,
617 #                       size_t length, const CAMELLIA_KEY *key,
618 #                       unsigned char *ivp,const int enc);
619 {
620 $_key="0(%rsp)";
621 $_end="8(%rsp)";        # inp+len&~15
622 $_res="16(%rsp)";       # len&15
623 $ivec="24(%rsp)";
624 $_ivp="40(%rsp)";
625 $_rsp="48(%rsp)";
626
627 $code.=<<___;
628 .globl  Camellia_cbc_encrypt
629 .type   Camellia_cbc_encrypt,\@function,6
630 .align  16
631 Camellia_cbc_encrypt:
632         cmp     \$0,%rdx
633         je      .Lcbc_abort
634         push    %rbx
635         push    %rbp
636         push    %r12
637         push    %r13
638         push    %r14
639         push    %r15
640 .Lcbc_prologue:
641
642         mov     %rsp,%rbp
643         sub     \$64,%rsp
644         and     \$-64,%rsp
645
646         # place stack frame just "above mod 1024" the key schedule,
647         # this ensures that cache associativity suffices
648         lea     -64-63(%rcx),%r10
649         sub     %rsp,%r10
650         neg     %r10
651         and     \$0x3C0,%r10
652         sub     %r10,%rsp
653         add     \$8,%rsp                # 8 is reserved for callee's ra
654
655         mov     %rdi,$inp               # inp argument
656         mov     %rsi,$out               # out argument
657         mov     %r8,%rbx                # ivp argument
658         mov     %rcx,$key               # key argument
659         mov     272(%rcx),$keyend       # grandRounds
660
661         mov     %r8,$_ivp
662         mov     %rbp,$_rsp
663
664 .Lcbc_body:
665         lea     .LCamellia_SBOX(%rip),$Tbl
666
667         mov     \$32,%ecx
668 .align  4
669 .Lcbc_prefetch_sbox:
670         mov     0($Tbl),%rax
671         mov     32($Tbl),%rsi
672         mov     64($Tbl),%rdi
673         mov     96($Tbl),%r11
674         lea     128($Tbl),$Tbl
675         loop    .Lcbc_prefetch_sbox
676         sub     \$4096,$Tbl
677         shl     \$6,$keyend
678         mov     %rdx,%rcx               # len argument
679         lea     ($key,$keyend),$keyend
680
681         cmp     \$0,%r9d                # enc argument
682         je      .LCBC_DECRYPT
683
684         and     \$-16,%rdx
685         and     \$15,%rcx               # length residue
686         lea     ($inp,%rdx),%rdx
687         mov     $key,$_key
688         mov     %rdx,$_end
689         mov     %rcx,$_res
690
691         cmp     $inp,%rdx
692         mov     0(%rbx),@S[0]           # load IV
693         mov     4(%rbx),@S[1]
694         mov     8(%rbx),@S[2]
695         mov     12(%rbx),@S[3]
696         je      .Lcbc_enc_tail
697         jmp     .Lcbc_eloop
698
699 .align  16
700 .Lcbc_eloop:
701         xor     0($inp),@S[0]
702         xor     4($inp),@S[1]
703         xor     8($inp),@S[2]
704         bswap   @S[0]
705         xor     12($inp),@S[3]
706         bswap   @S[1]
707         bswap   @S[2]
708         bswap   @S[3]
709
710         call    _x86_64_Camellia_encrypt
711
712         mov     $_key,$key              # "rewind" the key
713         bswap   @S[0]
714         mov     $_end,%rdx
715         bswap   @S[1]
716         mov     $_res,%rcx
717         bswap   @S[2]
718         mov     @S[0],0($out)
719         bswap   @S[3]
720         mov     @S[1],4($out)
721         mov     @S[2],8($out)
722         lea     16($inp),$inp
723         mov     @S[3],12($out)
724         cmp     %rdx,$inp
725         lea     16($out),$out
726         jne     .Lcbc_eloop
727
728         cmp     \$0,%rcx
729         jne     .Lcbc_enc_tail
730
731         mov     $_ivp,$out
732         mov     @S[0],0($out)           # write out IV residue
733         mov     @S[1],4($out)
734         mov     @S[2],8($out)
735         mov     @S[3],12($out)
736         jmp     .Lcbc_done
737
738 .align  16
739 .Lcbc_enc_tail:
740         xor     %rax,%rax
741         mov     %rax,0+$ivec
742         mov     %rax,8+$ivec
743         mov     %rax,$_res
744
745         pushfq
746         cld
747         mov     $inp,%rsi
748         lea     $ivec,%rdi
749         .long   0x9066A4F3              # rep movsb
750         popfq
751
752         lea     $ivec,$inp
753         lea     16+$ivec,%rax
754         mov     %rax,$_end
755         jmp     .Lcbc_eloop             # one more time
756
757 .align  16
758 .LCBC_DECRYPT:
759         xchg    $key,$keyend
760         add     \$15,%rdx
761         and     \$15,%rcx               # length residue
762         and     \$-16,%rdx
763         mov     $key,$_key
764         lea     ($inp,%rdx),%rdx
765         mov     %rdx,$_end
766         mov     %rcx,$_res
767
768         mov     (%rbx),%rax             # load IV
769         mov     8(%rbx),%rbx
770         jmp     .Lcbc_dloop
771 .align  16
772 .Lcbc_dloop:
773         mov     0($inp),@S[0]
774         mov     4($inp),@S[1]
775         mov     8($inp),@S[2]
776         bswap   @S[0]
777         mov     12($inp),@S[3]
778         bswap   @S[1]
779         mov     %rax,0+$ivec            # save IV to temporary storage
780         bswap   @S[2]
781         mov     %rbx,8+$ivec
782         bswap   @S[3]
783
784         call    _x86_64_Camellia_decrypt
785
786         mov     $_key,$key              # "rewind" the key
787         mov     $_end,%rdx
788         mov     $_res,%rcx
789
790         bswap   @S[0]
791         mov     ($inp),%rax             # load IV for next iteration
792         bswap   @S[1]
793         mov     8($inp),%rbx
794         bswap   @S[2]
795         xor     0+$ivec,@S[0]
796         bswap   @S[3]
797         xor     4+$ivec,@S[1]
798         xor     8+$ivec,@S[2]
799         lea     16($inp),$inp
800         xor     12+$ivec,@S[3]
801         cmp     %rdx,$inp
802         je      .Lcbc_ddone
803
804         mov     @S[0],0($out)
805         mov     @S[1],4($out)
806         mov     @S[2],8($out)
807         mov     @S[3],12($out)
808
809         lea     16($out),$out
810         jmp     .Lcbc_dloop
811
812 .align  16
813 .Lcbc_ddone:
814         mov     $_ivp,%rdx
815         cmp     \$0,%rcx
816         jne     .Lcbc_dec_tail
817
818         mov     @S[0],0($out)
819         mov     @S[1],4($out)
820         mov     @S[2],8($out)
821         mov     @S[3],12($out)
822
823         mov     %rax,(%rdx)             # write out IV residue
824         mov     %rbx,8(%rdx)
825         jmp     .Lcbc_done
826 .align  16
827 .Lcbc_dec_tail:
828         mov     @S[0],0+$ivec
829         mov     @S[1],4+$ivec
830         mov     @S[2],8+$ivec
831         mov     @S[3],12+$ivec
832
833         pushfq
834         cld
835         lea     $ivec,%rsi
836         lea     ($out),%rdi
837         .long   0x9066A4F3              # rep movsb
838         popfq
839
840         mov     %rax,(%rdx)             # write out IV residue
841         mov     %rbx,8(%rdx)
842         jmp     .Lcbc_done
843
844 .align  16
845 .Lcbc_done:
846         mov     $_rsp,%rcx
847         mov     0(%rcx),%r15
848         mov     8(%rcx),%r14
849         mov     16(%rcx),%r13
850         mov     24(%rcx),%r12
851         mov     32(%rcx),%rbp
852         mov     40(%rcx),%rbx
853         lea     48(%rcx),%rsp
854 .Lcbc_abort:
855         ret
856 .size   Camellia_cbc_encrypt,.-Camellia_cbc_encrypt
857
858 .asciz  "Camellia for x86_64 by <appro@openssl.org>"
859 ___
860 }
861
862 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
863 #               CONTEXT *context,DISPATCHER_CONTEXT *disp)
864 if ($win64) {
865 $rec="%rcx";
866 $frame="%rdx";
867 $context="%r8";
868 $disp="%r9";
869
870 $code.=<<___;
871 .extern __imp_RtlVirtualUnwind
872 .type   common_se_handler,\@abi-omnipotent
873 .align  16
874 common_se_handler:
875         push    %rsi
876         push    %rdi
877         push    %rbx
878         push    %rbp
879         push    %r13
880         push    %r14
881         push    %r15
882         pushfq
883         lea     -72(%rsp),%rsp
884
885         mov     120($context),%rax      # pull context->Rax
886         mov     248($context),%rbx      # pull context->Rip
887
888         mov     8($disp),%rsi           # disp->ImageBase
889         mov     56($disp),%r11          # disp->HandlerData
890
891         mov     0(%r11),%r10d           # HandlerData[0]
892         lea     (%rsi,%r10),%r10        # prologue label
893         cmp     %r10,%rbx               # context->Rip<prologue label
894         jb      .Lin_prologue
895
896         mov     152($context),%rax      # pull context->Rsp
897
898         mov     4(%r11),%r10d           # HandlerData[1]
899         lea     (%rsi,%r10),%r10        # epilogue label
900         cmp     %r10,%rbx               # context->Rip>=epilogue label
901         jae     .Lin_prologue
902
903         lea     40(%rax),%rax
904         mov     -8(%rax),%rbx
905         mov     -16(%rax),%rbp
906         mov     -24(%rax),%r13
907         mov     -32(%rax),%r14
908         mov     -40(%rax),%r15
909         mov     %rbx,144($context)      # restore context->Rbx
910         mov     %rbp,160($context)      # restore context->Rbp
911         mov     %r13,224($context)      # restore context->R13
912         mov     %r14,232($context)      # restore context->R14
913         mov     %r15,240($context)      # restore context->R15
914
915 .Lin_prologue:
916         mov     8(%rax),%rdi
917         mov     16(%rax),%rsi
918         mov     %rax,152($context)      # restore context->Rsp
919         mov     %rsi,168($context)      # restore context->Rsi
920         mov     %rdi,176($context)      # restore context->Rdi
921
922         mov     40($disp),%rdi          # disp->ContextRecord
923         mov     $context,%rsi
924         mov     \$`1232/8`,%ecx
925         .long   0xa548f3fc              # cld; rep movsq
926
927         mov     $disp,%rsi
928         xor     %rcx,%rcx               # arg1, UNW_FLAG_NHANDLER
929         mov     8(%rsi),%rdx            # arg2, disp->ImageBase
930         mov     0(%rsi),%r8             # arg3, disp->ControlPc
931         mov     16(%rsi),%r9            # arg4, disp->FunctionEntry
932         mov     40(%rsi),%r10           # disp->ContextRecord
933         lea     56(%rsi),%r11           # &disp->HandlerData
934         lea     24(%rsi),%r12           # &disp->EstablisherFrame
935         mov     %r10,32(%rsp)           # arg5
936         mov     %r11,40(%rsp)           # arg6
937         mov     %r12,48(%rsp)           # arg7
938         mov     %rcx,56(%rsp)           # arg8, (NULL)
939         call    *__imp_RtlVirtualUnwind(%rip)
940
941         mov     \$1,%eax                # ExceptionContinueSearch
942         lea     72(%rsp),%rsp
943         popfq
944         pop     %r15
945         pop     %r14
946         pop     %r13
947         pop     %rbp
948         pop     %rbx
949         pop     %rdi
950         pop     %rsi
951         ret
952 .size   common_se_handler,.-common_se_handler
953
954 .type   cbc_se_handler,\@abi-omnipotent
955 .align  16
956 cbc_se_handler:
957         push    %rsi
958         push    %rdi
959         push    %rbx
960         push    %rbp
961         push    %r12
962         push    %r13
963         push    %r14
964         push    %r15
965         pushfq
966         lea     -64(%rsp),%rsp
967
968         mov     120($context),%rax      # pull context->Rax
969         mov     248($context),%rbx      # pull context->Rip
970
971         lea     .Lcbc_prologue(%rip),%r10
972         cmp     %r10,%rbx               # context->Rip<.Lcbc_prologue
973         jb      .Lin_cbc_prologue
974
975         lea     .Lcbc_body(%rip),%r10
976         cmp     %r10,%rbx               # context->Rip<.Lcbc_body
977         jb      .Lin_cbc_frame_setup
978
979         mov     152($context),%rax      # pull context->Rsp
980
981         lea     .Lcbc_abort(%rip),%r10
982         cmp     %r10,%rbx               # context->Rip>=.Lcbc_abort
983         jae     .Lin_cbc_prologue
984
985         mov     48(%rax),%rax           # $_rsp
986         lea     48(%rax),%rax
987
988 .Lin_cbc_frame_setup:
989         mov     -8(%rax),%rbx
990         mov     -16(%rax),%rbp
991         mov     -24(%rax),%r12
992         mov     -32(%rax),%r13
993         mov     -40(%rax),%r14
994         mov     -48(%rax),%r15
995         mov     %rbx,144($context)      # restore context->Rbx
996         mov     %rbp,160($context)      # restore context->Rbp
997         mov     %r12,216($context)      # restore context->R12
998         mov     %r13,224($context)      # restore context->R13
999         mov     %r14,232($context)      # restore context->R14
1000         mov     %r15,240($context)      # restore context->R15
1001
1002 .Lin_cbc_prologue:
1003         mov     8(%rax),%rdi
1004         mov     16(%rax),%rsi
1005         mov     %rax,152($context)      # restore context->Rsp
1006         mov     %rsi,168($context)      # restore context->Rsi
1007         mov     %rdi,176($context)      # restore context->Rdi
1008
1009         mov     40($disp),%rdi          # disp->ContextRecord
1010         mov     $context,%rsi           # context
1011         mov     \$`1232/8`,%ecx         # sizeof(CONTEXT)
1012         .long   0xa548f3fc              # cld; rep movsq
1013
1014         mov     $disp,%rsi
1015         xor     %rcx,%rcx               # arg1, UNW_FLAG_NHANDLER
1016         mov     8(%rsi),%rdx            # arg2, disp->ImageBase
1017         mov     0(%rsi),%r8             # arg3, disp->ControlPc
1018         mov     16(%rsi),%r9            # arg4, disp->FunctionEntry
1019         mov     40(%rsi),%r10           # disp->ContextRecord
1020         lea     56(%rsi),%r11           # &disp->HandlerData
1021         lea     24(%rsi),%r12           # &disp->EstablisherFrame
1022         mov     %r10,32(%rsp)           # arg5
1023         mov     %r11,40(%rsp)           # arg6
1024         mov     %r12,48(%rsp)           # arg7
1025         mov     %rcx,56(%rsp)           # arg8, (NULL)
1026         call    *__imp_RtlVirtualUnwind(%rip)
1027
1028         mov     \$1,%eax                # ExceptionContinueSearch
1029         lea     64(%rsp),%rsp
1030         popfq
1031         pop     %r15
1032         pop     %r14
1033         pop     %r13
1034         pop     %r12
1035         pop     %rbp
1036         pop     %rbx
1037         pop     %rdi
1038         pop     %rsi
1039         ret
1040 .size   cbc_se_handler,.-cbc_se_handler
1041
1042 .section        .pdata
1043 .align  4
1044         .rva    .LSEH_begin_Camellia_EncryptBlock_Rounds
1045         .rva    .LSEH_end_Camellia_EncryptBlock_Rounds
1046         .rva    .LSEH_info_Camellia_EncryptBlock_Rounds
1047
1048         .rva    .LSEH_begin_Camellia_DecryptBlock_Rounds
1049         .rva    .LSEH_end_Camellia_DecryptBlock_Rounds
1050         .rva    .LSEH_info_Camellia_DecryptBlock_Rounds
1051
1052         .rva    .LSEH_begin_Camellia_Ekeygen
1053         .rva    .LSEH_end_Camellia_Ekeygen
1054         .rva    .LSEH_info_Camellia_Ekeygen
1055
1056         .rva    .LSEH_begin_Camellia_cbc_encrypt
1057         .rva    .LSEH_end_Camellia_cbc_encrypt
1058         .rva    .LSEH_info_Camellia_cbc_encrypt
1059
1060 .section        .xdata
1061 .align  8
1062 .LSEH_info_Camellia_EncryptBlock_Rounds:
1063         .byte   9,0,0,0
1064         .rva    common_se_handler
1065         .rva    .Lenc_prologue,.Lenc_epilogue   # HandlerData[]
1066 .LSEH_info_Camellia_DecryptBlock_Rounds:
1067         .byte   9,0,0,0
1068         .rva    common_se_handler
1069         .rva    .Ldec_prologue,.Ldec_epilogue   # HandlerData[]
1070 .LSEH_info_Camellia_Ekeygen:
1071         .byte   9,0,0,0
1072         .rva    common_se_handler
1073         .rva    .Lkey_prologue,.Lkey_epilogue   # HandlerData[]
1074 .LSEH_info_Camellia_cbc_encrypt:
1075         .byte   9,0,0,0
1076         .rva    cbc_se_handler
1077 ___
1078 }
1079
1080 $code =~ s/\`([^\`]*)\`/eval $1/gem;
1081 print $code;
1082 close STDOUT;