415dc04a23d387348d165c1a9931ce0c481b238d
[openssl.git] / crypto / aes / asm / aesv8-armx.pl
1 #!/usr/bin/env perl
2 #
3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9 #
10 # This module implements support for ARMv8 AES instructions. The
11 # module is endian-agnostic in sense that it supports both big- and
12 # little-endian cases. As does it support both 32- and 64-bit modes
13 # of operation. Latter is achieved by limiting amount of utilized
14 # registers to 16, which implies additional instructions. This has
15 # no effect on mighty Apple A7, as results are literally equal to
16 # the theoretical estimates based on instruction latencies and issue
17 # rate. It remains to be seen how does it affect other platforms...
18 #
19 # Performance in cycles per byte processed with 128-bit key:
20 #
21 #               CBC enc         CBC dec         CTR
22 # Apple A7      2.39            1.20            1.20
23 # Cortex-A5x    n/a             n/a             n/a
24
25 $flavour = shift;
26 open STDOUT,">".shift;
27
28 $prefix="aes_v8";
29
30 $code=<<___;
31 #include "arm_arch.h"
32
33 #if __ARM_ARCH__>=7
34 .text
35 ___
36 $code.=".arch   armv8-a+crypto\n"       if ($flavour =~ /64/);
37 $code.=".fpu    neon\n.code     32\n"   if ($flavour !~ /64/);
38
39 # Assembler mnemonics are an eclectic mix of 32- and 64-bit syntax,
40 # NEON is mostly 32-bit mnemonics, integer - mostly 64. Goal is to
41 # maintain both 32- and 64-bit codes within single module and
42 # transliterate common code to either flavour with regex vodoo.
43 #
44 {{{
45 my ($inp,$bits,$out,$ptr,$rounds)=("x0","w1","x2","x3","w12");
46 my ($zero,$rcon,$mask,$in0,$in1,$tmp,$key)=
47         $flavour=~/64/? map("q$_",(0..6)) : map("q$_",(0..3,8..10));
48
49
50 $code.=<<___;
51 .align  5
52 rcon:
53 .long   0x01,0x01,0x01,0x01
54 .long   0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d     // rotate-n-splat
55 .long   0x1b,0x1b,0x1b,0x1b
56
57 .globl  ${prefix}_set_encrypt_key
58 .type   ${prefix}_set_encrypt_key,%function
59 .align  5
60 ${prefix}_set_encrypt_key:
61 .Lenc_key:
62 ___
63 $code.=<<___    if ($flavour =~ /64/);
64         stp     x29,x30,[sp,#-16]!
65         add     x29,sp,#0
66 ___
67 $code.=<<___;
68         adr     $ptr,rcon
69         cmp     $bits,#192
70
71         veor    $zero,$zero,$zero
72         vld1.8  {$in0},[$inp],#16
73         mov     $bits,#8                // reuse $bits
74         vld1.32 {$rcon,$mask},[$ptr],#32
75
76         b.lt    .Loop128
77         b.eq    .L192
78         b       .L256
79
80 .align  4
81 .Loop128:
82         vtbl.8  $key,{$in0},$mask
83         vext.8  $tmp,$zero,$in0,#12
84         vst1.32 {$in0},[$out],#16
85         aese    $key,$zero
86         subs    $bits,$bits,#1
87
88         veor    $in0,$in0,$tmp
89         vext.8  $tmp,$zero,$tmp,#12
90         veor    $in0,$in0,$tmp
91         vext.8  $tmp,$zero,$tmp,#12
92          veor   $key,$key,$rcon
93         veor    $in0,$in0,$tmp
94         vshl.u8 $rcon,$rcon,#1
95         veor    $in0,$in0,$key
96         b.ne    .Loop128
97
98         vld1.32 {$rcon},[$ptr]
99
100         vtbl.8  $key,{$in0},$mask
101         vext.8  $tmp,$zero,$in0,#12
102         vst1.32 {$in0},[$out],#16
103         aese    $key,$zero
104
105         veor    $in0,$in0,$tmp
106         vext.8  $tmp,$zero,$tmp,#12
107         veor    $in0,$in0,$tmp
108         vext.8  $tmp,$zero,$tmp,#12
109          veor   $key,$key,$rcon
110         veor    $in0,$in0,$tmp
111         vshl.u8 $rcon,$rcon,#1
112         veor    $in0,$in0,$key
113
114         vtbl.8  $key,{$in0},$mask
115         vext.8  $tmp,$zero,$in0,#12
116         vst1.32 {$in0},[$out],#16
117         aese    $key,$zero
118
119         veor    $in0,$in0,$tmp
120         vext.8  $tmp,$zero,$tmp,#12
121         veor    $in0,$in0,$tmp
122         vext.8  $tmp,$zero,$tmp,#12
123          veor   $key,$key,$rcon
124         veor    $in0,$in0,$tmp
125         veor    $in0,$in0,$key
126         vst1.32 {$in0},[$out]
127         add     $out,$out,#0x50
128
129         mov     $rounds,#10
130         b       .Ldone
131
132 .align  4
133 .L192:
134         vld1.8  {$in1},[$inp],#8
135         vmov.i8 $key,#8                 // borrow $key
136         vst1.32 {$in0},[$out],#16
137         vsub.i8 $mask,$mask,$key        // adjust the mask
138
139 .Loop192:
140         vtbl.8  $key,{$in1},$mask
141         vext.8  $tmp,$zero,$in0,#12
142         vst1.32 {$in1},[$out],#8
143         aese    $key,$zero
144         subs    $bits,$bits,#1
145
146         veor    $in0,$in0,$tmp
147         vext.8  $tmp,$zero,$tmp,#12
148         veor    $in0,$in0,$tmp
149         vext.8  $tmp,$zero,$tmp,#12
150         veor    $in0,$in0,$tmp
151
152         vdup.32 $tmp,${in0}[3]
153         veor    $tmp,$tmp,$in1
154          veor   $key,$key,$rcon
155         vext.8  $in1,$zero,$in1,#12
156         vshl.u8 $rcon,$rcon,#1
157         veor    $in1,$in1,$tmp
158         veor    $in0,$in0,$key
159         veor    $in1,$in1,$key
160         vst1.32 {$in0},[$out],#16
161         b.ne    .Loop192
162
163         mov     $rounds,#12
164         add     $out,$out,#0x20
165         b       .Ldone
166
167 .align  4
168 .L256:
169         vld1.8  {$in1},[$inp]
170         mov     $bits,#7
171         mov     $rounds,#14
172         vst1.32 {$in0},[$out],#16
173
174 .Loop256:
175         vtbl.8  $key,{$in1},$mask
176         vext.8  $tmp,$zero,$in0,#12
177         vst1.32 {$in1},[$out],#16
178         aese    $key,$zero
179         subs    $bits,$bits,#1
180
181         veor    $in0,$in0,$tmp
182         vext.8  $tmp,$zero,$tmp,#12
183         veor    $in0,$in0,$tmp
184         vext.8  $tmp,$zero,$tmp,#12
185          veor   $key,$key,$rcon
186         veor    $in0,$in0,$tmp
187         vshl.u8 $rcon,$rcon,#1
188         veor    $in0,$in0,$key
189         vst1.32 {$in0},[$out],#16
190         b.eq    .Ldone
191
192         vdup.32 $key,${in0}[3]          // just splat
193         vext.8  $tmp,$zero,$in1,#12
194         aese    $key,$zero
195
196         veor    $in1,$in1,$tmp
197         vext.8  $tmp,$zero,$tmp,#12
198         veor    $in1,$in1,$tmp
199         vext.8  $tmp,$zero,$tmp,#12
200         veor    $in1,$in1,$tmp
201
202         veor    $in1,$in1,$key
203         b       .Loop256
204
205 .Ldone:
206         str     $rounds,[$out]
207
208         eor     x0,x0,x0                // return value
209         `"ldr   x29,[sp],#16"           if ($flavour =~ /64/)`
210         ret
211 .size   ${prefix}_set_encrypt_key,.-${prefix}_set_encrypt_key
212
213 .globl  ${prefix}_set_decrypt_key
214 .type   ${prefix}_set_decrypt_key,%function
215 .align  5
216 ${prefix}_set_decrypt_key:
217 ___
218 $code.=<<___    if ($flavour =~ /64/);
219         stp     x29,x30,[sp,#-16]!
220         add     x29,sp,#0
221 ___
222 $code.=<<___    if ($flavour !~ /64/);
223         stmdb   sp!,{r4,lr}
224 ___
225 $code.=<<___;
226         bl      .Lenc_key
227
228         sub     $out,$out,#240          // restore original $out
229         mov     x4,#-16
230         add     $inp,$out,x12,lsl#4     // end of key schedule
231
232         vld1.32 {v0.16b},[$out]
233         vld1.32 {v1.16b},[$inp]
234         vst1.32 {v0.16b},[$inp],x4
235         vst1.32 {v1.16b},[$out],#16
236
237 .Loop_imc:
238         vld1.32 {v0.16b},[$out]
239         vld1.32 {v1.16b},[$inp]
240         aesimc  v0.16b,v0.16b
241         aesimc  v1.16b,v1.16b
242         vst1.32 {v0.16b},[$inp],x4
243         vst1.32 {v1.16b},[$out],#16
244         cmp     $inp,$out
245         b.hi    .Loop_imc
246
247         vld1.32 {v0.16b},[$out]
248         aesimc  v0.16b,v0.16b
249         vst1.32 {v0.16b},[$inp]
250
251         eor     x0,x0,x0                // return value
252 ___
253 $code.=<<___    if ($flavour !~ /64/);
254         ldmia   sp!,{r4,pc}
255 ___
256 $code.=<<___    if ($flavour =~ /64/);
257         ldp     x29,x30,[sp],#16
258         ret
259 ___
260 $code.=<<___;
261 .size   ${prefix}_set_decrypt_key,.-${prefix}_set_decrypt_key
262 ___
263 }}}
264 {{{
265 sub gen_block () {
266 my $dir = shift;
267 my ($e,$mc) = $dir eq "en" ? ("e","mc") : ("d","imc");
268 my ($inp,$out,$key)=map("x$_",(0..2));
269 my $rounds="w3";
270 my ($rndkey0,$rndkey1,$inout)=map("q$_",(0..3));
271
272 $code.=<<___;
273 .globl  ${prefix}_${dir}crypt
274 .type   ${prefix}_${dir}crypt,%function
275 .align  5
276 ${prefix}_${dir}crypt:
277         ldr     $rounds,[$key,#240]
278         vld1.32 {$rndkey0},[$key],#16
279         vld1.8  {$inout},[$inp]
280         sub     $rounds,$rounds,#2
281         vld1.32 {$rndkey1},[$key],#16
282
283 .Loop_${dir}c:
284         aes$e   $inout,$rndkey0
285         vld1.32 {$rndkey0},[$key],#16
286         aes$mc  $inout,$inout
287         subs    $rounds,$rounds,#2
288         aes$e   $inout,$rndkey1
289         vld1.32 {$rndkey1},[$key],#16
290         aes$mc  $inout,$inout
291         b.gt    .Loop_${dir}c
292
293         aes$e   $inout,$rndkey0
294         vld1.32 {$rndkey0},[$key]
295         aes$mc  $inout,$inout
296         aes$e   $inout,$rndkey1
297         veor    $inout,$inout,$rndkey0
298
299         vst1.8  {$inout},[$out]
300         ret
301 .size   ${prefix}_${dir}crypt,.-${prefix}_${dir}crypt
302 ___
303 }
304 &gen_block("en");
305 &gen_block("de");
306 }}}
307 {{{
308 my ($inp,$out,$len,$key,$ivp)=map("x$_",(0..4)); my $enc="w5";
309 my ($rounds,$cnt,$key_,$step,$step1)=($enc,"w6","x7","x8","x12");
310 my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$ivec,$rndlast)=map("q$_",(0..7));
311
312 my ($dat,$tmp,$rndzero_n_last)=($dat0,$tmp0,$tmp1);
313
314 ### q8-q15      preloaded key schedule
315
316 $code.=<<___;
317 .globl  ${prefix}_cbc_encrypt
318 .type   ${prefix}_cbc_encrypt,%function
319 .align  5
320 ${prefix}_cbc_encrypt:
321 ___
322 $code.=<<___    if ($flavour =~ /64/);
323         stp     x29,x30,[sp,#-16]!
324         add     x29,sp,#0
325 ___
326 $code.=<<___    if ($flavour !~ /64/);
327         mov     ip,sp
328         stmdb   sp!,{r4-r8,lr}
329         vstmdb  sp!,{d8-d15}            @ ABI specification says so
330         ldmia   ip,{r4-r5}              @ load remaining args
331 ___
332 $code.=<<___;
333         subs    $len,$len,#16
334         mov     $step,#16
335         b.lo    .Lcbc_abort
336         cclr    $step,eq
337
338         cmp     $enc,#0                 // en- or decrypting?
339         ldr     $rounds,[$key,#240]
340         and     $len,$len,#-16
341         vld1.8  {$ivec},[$ivp]
342         vld1.8  {$dat},[$inp],$step
343
344         vld1.32 {q8-q9},[$key]          // load key schedule...
345         sub     $rounds,$rounds,#6
346         add     $key_,$key,x5,lsl#4     // pointer to last 7 round keys
347         sub     $rounds,$rounds,#2
348         vld1.32 {q10-q11},[$key_],#32
349         vld1.32 {q12-q13},[$key_],#32
350         vld1.32 {q14-q15},[$key_],#32
351         vld1.32 {$rndlast},[$key_]
352
353         add     $key_,$key,#32
354         mov     $cnt,$rounds
355         b.eq    .Lcbc_dec
356
357         cmp     $rounds,#2
358         veor    $dat,$dat,$ivec
359         veor    $rndzero_n_last,q8,$rndlast
360         b.eq    .Lcbc_enc128
361
362 .Loop_cbc_enc:
363         aese    $dat,q8
364         vld1.32 {q8},[$key_],#16
365         aesmc   $dat,$dat
366         subs    $cnt,$cnt,#2
367         aese    $dat,q9
368         vld1.32 {q9},[$key_],#16
369         aesmc   $dat,$dat
370         b.gt    .Loop_cbc_enc
371
372         aese    $dat,q8
373         aesmc   $dat,$dat
374          subs   $len,$len,#16
375         aese    $dat,q9
376         aesmc   $dat,$dat
377          cclr   $step,eq
378         aese    $dat,q10
379         aesmc   $dat,$dat
380          add    $key_,$key,#16
381         aese    $dat,q11
382         aesmc   $dat,$dat
383          vld1.8 {q8},[$inp],$step
384         aese    $dat,q12
385         aesmc   $dat,$dat
386          veor   q8,q8,$rndzero_n_last
387         aese    $dat,q13
388         aesmc   $dat,$dat
389          vld1.32 {q9},[$key_],#16       // re-pre-load rndkey[1]
390         aese    $dat,q14
391         aesmc   $dat,$dat
392         aese    $dat,q15
393
394          mov    $cnt,$rounds
395         veor    $ivec,$dat,$rndlast
396         vst1.8  {$ivec},[$out],#16
397         b.hs    .Loop_cbc_enc
398
399         b       .Lcbc_done
400
401 .align  5
402 .Lcbc_enc128:
403         vld1.32 {$in0-$in1},[$key_]
404         aese    $dat,q8
405         aesmc   $dat,$dat
406         b       .Lenter_cbc_enc128
407 .Loop_cbc_enc128:
408         aese    $dat,q8
409         aesmc   $dat,$dat
410          vst1.8 {$ivec},[$out],#16
411 .Lenter_cbc_enc128:
412         aese    $dat,q9
413         aesmc   $dat,$dat
414          subs   $len,$len,#16
415         aese    $dat,$in0
416         aesmc   $dat,$dat
417          cclr   $step,eq
418         aese    $dat,$in1
419         aesmc   $dat,$dat
420         aese    $dat,q10
421         aesmc   $dat,$dat
422         aese    $dat,q11
423         aesmc   $dat,$dat
424          vld1.8 {q8},[$inp],$step
425         aese    $dat,q12
426         aesmc   $dat,$dat
427         aese    $dat,q13
428         aesmc   $dat,$dat
429         aese    $dat,q14
430         aesmc   $dat,$dat
431          veor   q8,q8,$rndzero_n_last
432         aese    $dat,q15
433         veor    $ivec,$dat,$rndlast
434         b.hs    .Loop_cbc_enc128
435
436         vst1.8  {$ivec},[$out],#16
437         b       .Lcbc_done
438
439 .align  5
440 .Lcbc_dec128:
441         vld1.32 {$tmp0-$tmp1},[$key_]
442         veor    $ivec,$ivec,$rndlast
443         veor    $in0,$dat0,$rndlast
444         mov     $step1,$step
445
446 .Loop2x_cbc_dec128:
447         aesd    $dat0,q8
448         aesd    $dat1,q8
449         aesimc  $dat0,$dat0
450         aesimc  $dat1,$dat1
451          subs   $len,$len,#32
452         aesd    $dat0,q9
453         aesd    $dat1,q9
454         aesimc  $dat0,$dat0
455         aesimc  $dat1,$dat1
456          cclr   $step,lo
457         aesd    $dat0,$tmp0
458         aesd    $dat1,$tmp0
459         aesimc  $dat0,$dat0
460         aesimc  $dat1,$dat1
461          cclr   $step1,ls
462         aesd    $dat0,$tmp1
463         aesd    $dat1,$tmp1
464         aesimc  $dat0,$dat0
465         aesimc  $dat1,$dat1
466         aesd    $dat0,q10
467         aesd    $dat1,q10
468         aesimc  $dat0,$dat0
469         aesimc  $dat1,$dat1
470         aesd    $dat0,q11
471         aesd    $dat1,q11
472         aesimc  $dat0,$dat0
473         aesimc  $dat1,$dat1
474         aesd    $dat0,q12
475         aesd    $dat1,q12
476         aesimc  $dat0,$dat0
477         aesimc  $dat1,$dat1
478         aesd    $dat0,q13
479         aesd    $dat1,q13
480         aesimc  $dat0,$dat0
481         aesimc  $dat1,$dat1
482         aesd    $dat0,q14
483         aesd    $dat1,q14
484         aesimc  $dat0,$dat0
485         aesimc  $dat1,$dat1
486         aesd    $dat0,q15
487         aesd    $dat1,q15
488
489         veor    $ivec,$ivec,$dat0
490         vld1.8  {$dat0},[$inp],$step
491         veor    $in0,$in0,$dat1
492         vld1.8  {$dat1},[$inp],$step1
493         vst1.8  {$ivec},[$out],#16
494         veor    $ivec,$in1,$rndlast
495         vst1.8  {$in0},[$out],#16
496         veor    $in0,$dat0,$rndlast
497         vorr    $in1,$dat1,$dat1
498         b.hs    .Loop2x_cbc_dec128
499
500         adds    $len,$len,#32
501         veor    $ivec,$ivec,$rndlast
502         b.eq    .Lcbc_done
503         veor    $in0,$in0,$rndlast
504         b       .Lcbc_dec_tail
505
506 .align  5
507 .Lcbc_dec:
508         subs    $len,$len,#16
509         vorr    $in0,$dat,$dat
510         b.lo    .Lcbc_dec_tail
511
512         cclr    $step,eq
513         cmp     $rounds,#2
514         vld1.8  {$dat1},[$inp],$step
515         vorr    $in1,$dat1,$dat1
516         b.eq    .Lcbc_dec128
517
518 .Loop2x_cbc_dec:
519         aesd    $dat0,q8
520         aesd    $dat1,q8
521         vld1.32 {q8},[$key_],#16
522         aesimc  $dat0,$dat0
523         aesimc  $dat1,$dat1
524         subs    $cnt,$cnt,#2
525         aesd    $dat0,q9
526         aesd    $dat1,q9
527         vld1.32 {q9},[$key_],#16
528         aesimc  $dat0,$dat0
529         aesimc  $dat1,$dat1
530         b.gt    .Loop2x_cbc_dec
531
532         aesd    $dat0,q8
533         aesd    $dat1,q8
534         aesimc  $dat0,$dat0
535         aesimc  $dat1,$dat1
536          veor   $tmp0,$ivec,$rndlast
537          veor   $tmp1,$in0,$rndlast
538         aesd    $dat0,q9
539         aesd    $dat1,q9
540         aesimc  $dat0,$dat0
541         aesimc  $dat1,$dat1
542          vorr   $ivec,$in1,$in1
543          subs   $len,$len,#32
544         aesd    $dat0,q10
545         aesd    $dat1,q10
546         aesimc  $dat0,$dat0
547          cclr   $step,lo
548         aesimc  $dat1,$dat1
549          mov    $key_,$key
550         aesd    $dat0,q11
551         aesd    $dat1,q11
552         aesimc  $dat0,$dat0
553          vld1.8 {$in0},[$inp],$step
554         aesimc  $dat1,$dat1
555          cclr   $step,ls
556         aesd    $dat0,q12
557         aesd    $dat1,q12
558         aesimc  $dat0,$dat0
559         aesimc  $dat1,$dat1
560          vld1.8 {$in1},[$inp],$step
561         aesd    $dat0,q13
562         aesd    $dat1,q13
563         aesimc  $dat0,$dat0
564         aesimc  $dat1,$dat1
565          vld1.32 {q8},[$key_],#16       // re-pre-load rndkey[0]
566         aesd    $dat0,q14
567         aesd    $dat1,q14
568         aesimc  $dat0,$dat0
569         aesimc  $dat1,$dat1
570          vld1.32 {q9},[$key_],#16       // re-pre-load rndkey[1]
571         aesd    $dat0,q15
572         aesd    $dat1,q15
573
574          mov    $cnt,$rounds
575         veor    $tmp0,$tmp0,$dat0
576         veor    $tmp1,$tmp1,$dat1
577          vorr   $dat0,$in0,$in0
578         vst1.8  {$tmp0},[$out],#16
579          vorr   $dat1,$in1,$in1
580         vst1.8  {$tmp1},[$out],#16
581         b.hs    .Loop2x_cbc_dec
582
583         adds    $len,$len,#32
584         b.eq    .Lcbc_done
585
586 .Lcbc_dec_tail:
587         aesd    $dat,q8
588         vld1.32 {q8},[$key_],#16
589         aesimc  $dat,$dat
590         subs    $cnt,$cnt,#2
591         aesd    $dat,q9
592         vld1.32 {q9},[$key_],#16
593         aesimc  $dat,$dat
594         b.gt    .Lcbc_dec_tail
595
596         aesd    $dat,q8
597         aesimc  $dat,$dat
598         aesd    $dat,q9
599         aesimc  $dat,$dat
600          veor   $tmp,$ivec,$rndlast
601         aesd    $dat,q10
602         aesimc  $dat,$dat
603          vorr   $ivec,$in0,$in0
604         aesd    $dat,q11
605         aesimc  $dat,$dat
606         aesd    $dat,q12
607         aesimc  $dat,$dat
608         aesd    $dat,q13
609         aesimc  $dat,$dat
610         aesd    $dat,q14
611         aesimc  $dat,$dat
612         aesd    $dat,q15
613
614         veor    $tmp,$tmp,$dat
615         vst1.8  {$tmp},[$out],#16
616
617 .Lcbc_done:
618         vst1.8  {$ivec},[$ivp]
619 .Lcbc_abort:
620 ___
621 $code.=<<___    if ($flavour !~ /64/);
622         vldmia  sp!,{d8-d15}
623         ldmia   sp!,{r4-r8,pc}
624 ___
625 $code.=<<___    if ($flavour =~ /64/);
626         ldr     x29,[sp],#16
627         ret
628 ___
629 $code.=<<___;
630 .size   ${prefix}_cbc_encrypt,.-${prefix}_cbc_encrypt
631 ___
632 }}}
633 {{{
634 my ($inp,$out,$len,$key,$ivp)=map("x$_",(0..4));
635 my ($rounds,$cnt,$key_,$ctr,$tctr,$tctr1)=("w5","w6","x7","w8","w9","w10");
636 my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$ivec,$rndlast)=map("q$_",(0..7));
637
638 my ($dat,$tmp)=($dat0,$tmp0);
639
640 ### q8-q15      preloaded key schedule
641
642 $code.=<<___;
643 .globl  ${prefix}_ctr32_encrypt_blocks
644 .type   ${prefix}_ctr32_encrypt_blocks,%function
645 .align  5
646 ${prefix}_ctr32_encrypt_blocks:
647 ___
648 $code.=<<___    if ($flavour =~ /64/);
649         stp             x29,x30,[sp,#-16]!
650         add             x29,sp,#0
651 ___
652 $code.=<<___    if ($flavour !~ /64/);
653         mov             ip,sp
654         stmdb           sp!,{r4-r10,lr}
655         vstmdb          sp!,{d8-d15}            @ ABI specification says so
656         ldr             r4, [ip]                @ load remaining arg
657 ___
658 $code.=<<___;
659         ldr             $rounds,[$key,#240]
660
661         ldr             $ctr, [$ivp, #12]
662         vld1.32         {$dat0},[$ivp]
663
664         vld1.32         {q8-q9},[$key]          // load key schedule...
665         sub             $rounds,$rounds,#6
666         add             $key_,$key,x5,lsl#4     // pointer to last 7 round keys
667         sub             $rounds,$rounds,#2
668         vld1.32         {q10-q11},[$key_],#32
669         vld1.32         {q12-q13},[$key_],#32
670         vld1.32         {q14-q15},[$key_],#32
671         vld1.32         {$rndlast},[$key_]
672
673         add             $key_,$key,#32
674         mov             $cnt,$rounds
675
676         subs            $len,$len,#2
677         b.lo            .Lctr32_tail
678
679 #ifndef __ARMEB__
680         rev             $ctr, $ctr
681 #endif
682         vorr            $dat1,$dat0,$dat0
683         add             $ctr, $ctr, #1
684         vorr            $ivec,$dat0,$dat0
685         rev             $tctr1, $ctr
686         cmp             $rounds,#2
687         vmov.32         ${dat1}[3],$tctr1
688         b.eq            .Lctr32_128
689
690 .Loop2x_ctr32:
691         aese            $dat0,q8
692         aese            $dat1,q8
693         vld1.32         {q8},[$key_],#16
694         aesmc           $dat0,$dat0
695         aesmc           $dat1,$dat1
696         subs            $cnt,$cnt,#2
697         aese            $dat0,q9
698         aese            $dat1,q9
699         vld1.32         {q9},[$key_],#16
700         aesmc           $dat0,$dat0
701         aesmc           $dat1,$dat1
702         b.gt            .Loop2x_ctr32
703
704         aese            $dat0,q8
705         aese            $dat1,q8
706         aesmc           $tmp0,$dat0
707          vorr           $dat0,$ivec,$ivec
708         aesmc           $tmp1,$dat1
709          vorr           $dat1,$ivec,$ivec
710         aese            $tmp0,q9
711         aese            $tmp1,q9
712          vld1.8         {$in0},[$inp],#16
713         aesmc           $tmp0,$tmp0
714          vld1.8         {$in1},[$inp],#16
715         aesmc           $tmp1,$tmp1
716          add            $ctr,$ctr,#1
717         aese            $tmp0,q10
718         aese            $tmp1,q10
719          rev            $tctr,$ctr
720         aesmc           $tmp0,$tmp0
721         aesmc           $tmp1,$tmp1
722          add            $ctr,$ctr,#1
723         aese            $tmp0,q11
724         aese            $tmp1,q11
725          veor           $in0,$in0,$rndlast
726          rev            $tctr1,$ctr
727         aesmc           $tmp0,$tmp0
728         aesmc           $tmp1,$tmp1
729          veor           $in1,$in1,$rndlast
730          mov            $key_,$key
731         aese            $tmp0,q12
732         aese            $tmp1,q12
733          subs           $len,$len,#2
734         aesmc           $tmp0,$tmp0
735         aesmc           $tmp1,$tmp1
736          vld1.32         {q8-q9},[$key_],#32    // re-pre-load rndkey[0-1]
737         aese            $tmp0,q13
738         aese            $tmp1,q13
739         aesmc           $tmp0,$tmp0
740         aesmc           $tmp1,$tmp1
741         aese            $tmp0,q14
742         aese            $tmp1,q14
743          vmov.32        ${dat0}[3], $tctr
744         aesmc           $tmp0,$tmp0
745          vmov.32        ${dat1}[3], $tctr1
746         aesmc           $tmp1,$tmp1
747         aese            $tmp0,q15
748         aese            $tmp1,q15
749
750          mov            $cnt,$rounds
751         veor            $in0,$in0,$tmp0
752         veor            $in1,$in1,$tmp1
753         vst1.8          {$in0},[$out],#16
754         vst1.8          {$in1},[$out],#16
755         b.hs            .Loop2x_ctr32
756
757         adds            $len,$len,#2
758         b.eq            .Lctr32_done
759         b               .Lctr32_tail
760
761 .Lctr32_128:
762         vld1.32         {$tmp0-$tmp1},[$key_]
763
764 .Loop2x_ctr32_128:
765         aese            $dat0,q8
766         aese            $dat1,q8
767         aesmc           $dat0,$dat0
768          vld1.8         {$in0},[$inp],#16
769         aesmc           $dat1,$dat1
770          vld1.8         {$in1},[$inp],#16
771         aese            $dat0,q9
772         aese            $dat1,q9
773          add            $ctr,$ctr,#1
774         aesmc           $dat0,$dat0
775         aesmc           $dat1,$dat1
776          rev            $tctr,$ctr
777         aese            $dat0,$tmp0
778         aese            $dat1,$tmp0
779          add            $ctr,$ctr,#1
780         aesmc           $dat0,$dat0
781         aesmc           $dat1,$dat1
782          rev            $tctr1,$ctr
783         aese            $dat0,$tmp1
784         aese            $dat1,$tmp1
785          subs           $len,$len,#2
786         aesmc           $dat0,$dat0
787         aesmc           $dat1,$dat1
788         aese            $dat0,q10
789         aese            $dat1,q10
790         aesmc           $dat0,$dat0
791         aesmc           $dat1,$dat1
792         aese            $dat0,q11
793         aese            $dat1,q11
794         aesmc           $dat0,$dat0
795         aesmc           $dat1,$dat1
796         aese            $dat0,q12
797         aese            $dat1,q12
798         aesmc           $dat0,$dat0
799         aesmc           $dat1,$dat1
800         aese            $dat0,q13
801         aese            $dat1,q13
802         aesmc           $dat0,$dat0
803         aesmc           $dat1,$dat1
804         aese            $dat0,q14
805         aese            $dat1,q14
806         aesmc           $dat0,$dat0
807         aesmc           $dat1,$dat1
808          veor           $in0,$in0,$rndlast
809         aese            $dat0,q15
810          veor           $in1,$in1,$rndlast
811         aese            $dat1,q15
812
813         veor            $in0,$in0,$dat0
814         vorr            $dat0,$ivec,$ivec
815         veor            $in1,$in1,$dat1
816         vorr            $dat1,$ivec,$ivec
817         vst1.8          {$in0},[$out],#16
818         vmov.32         ${dat0}[3], $tctr
819         vst1.8          {$in1},[$out],#16
820         vmov.32         ${dat1}[3], $tctr1
821         b.hs            .Loop2x_ctr32_128
822
823         adds            $len,$len,#2
824         b.eq            .Lctr32_done
825
826 .Lctr32_tail:
827         aese            $dat,q8
828         vld1.32         {q8},[$key_],#16
829         aesmc           $dat,$dat
830         subs            $cnt,$cnt,#2
831         aese            $dat,q9
832         vld1.32         {q9},[$key_],#16
833         aesmc           $dat,$dat
834         b.gt            .Lctr32_tail
835
836         aese            $dat,q8
837         aesmc           $dat,$dat
838         aese            $dat,q9
839         aesmc           $dat,$dat
840          vld1.8         {$in0},[$inp]
841         aese            $dat,q10
842         aesmc           $dat,$dat
843         aese            $dat,q11
844         aesmc           $dat,$dat
845         aese            $dat,q12
846         aesmc           $dat,$dat
847         aese            $dat,q13
848         aesmc           $dat,$dat
849         aese            $dat,q14
850         aesmc           $dat,$dat
851          veor           $in0,$in0,$rndlast
852         aese            $dat,q15
853
854         veor            $in0,$in0,$dat
855         vst1.8          {$in0},[$out]
856
857 .Lctr32_done:
858 ___
859 $code.=<<___    if ($flavour !~ /64/);
860         vldmia          sp!,{d8-d15}
861         ldmia           sp!,{r4-r10,pc}
862 ___
863 $code.=<<___    if ($flavour =~ /64/);
864         ldr             x29,[sp],#16
865         ret
866 ___
867 $code.=<<___;
868 .size   ${prefix}_ctr32_encrypt_blocks,.-${prefix}_ctr32_encrypt_blocks
869 ___
870 }}}
871 $code.=<<___;
872 #endif
873 ___
874 ########################################
875 if ($flavour =~ /64/) {                 ######## 64-bit code
876     my %opcode = (
877         "aesd"  =>      0x4e285800,     "aese"  =>      0x4e284800,
878         "aesimc"=>      0x4e287800,     "aesmc" =>      0x4e286800      );
879
880     local *unaes = sub {
881         my ($mnemonic,$arg)=@_;
882
883         $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)/o   &&
884         sprintf ".inst\t0x%08x\t//%s %s",
885                         $opcode{$mnemonic}|$1|($2<<5),
886                         $mnemonic,$arg;
887     };
888
889     foreach(split("\n",$code)) {
890         s/\`([^\`]*)\`/eval($1)/geo;
891
892         s/\bq([0-9]+)\b/"v".($1<8?$1:$1+8).".16b"/geo;  # old->new registers
893         s/@\s/\/\//o;                   # old->new style commentary
894
895         #s/[v]?(aes\w+)\s+([qv].*)/unaes($1,$2)/geo     or
896         s/cclr\s+([wx])([^,]+),\s*([a-z]+)/csel $1$2,$1zr,$1$2,$3/o     or
897         s/vmov\.i8/movi/o       or      # fix up legacy mnemonics
898         s/vext\.8/ext/o         or
899         s/vrev32\.8/rev32/o     or
900         s/vtst\.8/cmtst/o       or
901         s/vshr/ushr/o           or
902         s/^(\s+)v/$1/o          or      # strip off v prefix
903         s/\bbx\s+lr\b/ret/o;
904
905         # fix up remainig legacy suffixes
906         s/\.[ui]?8//o;
907         m/\],#8/o and s/\.16b/\.8b/go;
908         s/\.[ui]?32//o and s/\.16b/\.4s/go;
909         s/\.[ui]?64//o and s/\.16b/\.2d/go;
910         s/\.[42]([sd])\[([0-3])\]/\.$1\[$2\]/o;
911
912         print $_,"\n";
913     }
914 } else {                                ######## 32-bit code
915     my %opcode = (
916         "aesd"  =>      0xf3b00340,     "aese"  =>      0xf3b00300,
917         "aesimc"=>      0xf3b003c0,     "aesmc" =>      0xf3b00380      );
918
919     local *unaes = sub {
920         my ($mnemonic,$arg)=@_;
921
922         if ($arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)/o) {
923             my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19)
924                                          |(($2&7)<<1) |(($2&8)<<2);
925             # since ARMv7 instructions are always encoded little-endian.
926             # correct solution is to use .inst directive, but older
927             # assemblers don't implement it:-(
928             sprintf ".byte\t0x%02x,0x%02x,0x%02x,0x%02x\t@ %s %s",
929                         $word&0xff,($word>>8)&0xff,
930                         ($word>>16)&0xff,($word>>24)&0xff,
931                         $mnemonic,$arg;
932         }
933     };
934
935     sub unvtbl {
936         my $arg=shift;
937
938         $arg =~ m/q([0-9]+),\s*\{q([0-9]+)\},\s*q([0-9]+)/o &&
939         sprintf "vtbl.8 d%d,{q%d},d%d\n\t".
940                 "vtbl.8 d%d,{q%d},d%d", 2*$1,$2,2*$3, 2*$1+1,$2,2*$3+1; 
941     }
942
943     sub unvdup32 {
944         my $arg=shift;
945
946         $arg =~ m/q([0-9]+),\s*q([0-9]+)\[([0-3])\]/o &&
947         sprintf "vdup.32        q%d,d%d[%d]",$1,2*$2+($3>>1),$3&1;      
948     }
949
950     sub unvmov32 {
951         my $arg=shift;
952
953         $arg =~ m/q([0-9]+)\[([0-3])\],(.*)/o &&
954         sprintf "vmov.32        d%d[%d],%s",2*$1+($2>>1),$2&1,$3;       
955     }
956
957     foreach(split("\n",$code)) {
958         s/\`([^\`]*)\`/eval($1)/geo;
959
960         s/\b[wx]([0-9]+)\b/r$1/go;              # new->old registers
961         s/\bv([0-9])\.[12468]+[bsd]\b/q$1/go;   # new->old registers
962         s/\/\/\s?/@ /o;                         # new->old style commentary
963
964         # fix up remainig new-style suffixes
965         s/\{q([0-9]+)\},\s*\[(.+)\],#8/sprintf "{d%d},[$2]!",2*$1/eo    or
966         s/\],#[0-9]+/]!/o;
967
968         s/[v]?(aes\w+)\s+([qv].*)/unaes($1,$2)/geo      or
969         s/cclr\s+([^,]+),\s*([a-z]+)/mov$2      $1,#0/o or
970         s/vtbl\.8\s+(.*)/unvtbl($1)/geo                 or
971         s/vdup\.32\s+(.*)/unvdup32($1)/geo              or
972         s/vmov\.32\s+(.*)/unvmov32($1)/geo              or
973         s/^(\s+)b\./$1b/o                               or
974         s/^(\s+)ret/$1bx\tlr/o;
975
976         print $_,"\n";
977     }
978 }
979
980 close STDOUT;