ARM assembly pack: add ChaCha20 and Poly1305 modules.
[openssl.git] / crypto / chacha / asm / chacha-armv8.pl
1 #!/usr/bin/env perl
2 #
3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9 #
10 # June 2015
11
12 # ChaCha20 for ARMv8.
13 #
14 # Performance in cycles per byte out of large buffer.
15 #
16 #                       IALU/gcc-4.9    3xNEON+1xIALU   6xNEON+2xIALU
17 #
18 # Apple A7              5.50/+49%       3.33            1.70
19 # Cortex-A53            8.40/+80%       4.72            4.72(*)
20 # Cortex-A57            8.06/+43%       4.90            4.43(**)
21 # Denver                4.50/+82%       2.63            2.67(*)
22 # X-Gene                9.50/+46%       8.82            8.89(*)
23 #
24 # (*)   it's expected that doubling interleave factor doesn't help
25 #       all processors, only those with higher NEON latency and
26 #       higher instruction issue rate;
27 # (**)  expected improvement was actually higher;
28
29 $flavour=shift;
30 $output=shift;
31
32 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
33 ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
34 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
35 die "can't locate arm-xlate.pl";
36
37 open OUT,"| \"$^X\" $xlate $flavour $output";
38 *STDOUT=*OUT;
39
40 sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
41 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
42   my $arg = pop;
43     $arg = "#$arg" if ($arg*1 eq $arg);
44     $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
45 }
46
47 my ($out,$inp,$len,$key,$ctr) = map("x$_",(0..4));
48
49 my @x=map("x$_",(5..17,19..21));
50 my @d=map("x$_",(22..28,30));
51
52 sub ROUND {
53 my ($a0,$b0,$c0,$d0)=@_;
54 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
55 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
56 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
57
58     (
59         "&add_32        (@x[$a0],@x[$a0],@x[$b0])",
60          "&add_32       (@x[$a1],@x[$a1],@x[$b1])",
61           "&add_32      (@x[$a2],@x[$a2],@x[$b2])",
62            "&add_32     (@x[$a3],@x[$a3],@x[$b3])",
63         "&eor_32        (@x[$d0],@x[$d0],@x[$a0])",
64          "&eor_32       (@x[$d1],@x[$d1],@x[$a1])",
65           "&eor_32      (@x[$d2],@x[$d2],@x[$a2])",
66            "&eor_32     (@x[$d3],@x[$d3],@x[$a3])",
67         "&ror_32        (@x[$d0],@x[$d0],16)",
68          "&ror_32       (@x[$d1],@x[$d1],16)",
69           "&ror_32      (@x[$d2],@x[$d2],16)",
70            "&ror_32     (@x[$d3],@x[$d3],16)",
71
72         "&add_32        (@x[$c0],@x[$c0],@x[$d0])",
73          "&add_32       (@x[$c1],@x[$c1],@x[$d1])",
74           "&add_32      (@x[$c2],@x[$c2],@x[$d2])",
75            "&add_32     (@x[$c3],@x[$c3],@x[$d3])",
76         "&eor_32        (@x[$b0],@x[$b0],@x[$c0])",
77          "&eor_32       (@x[$b1],@x[$b1],@x[$c1])",
78           "&eor_32      (@x[$b2],@x[$b2],@x[$c2])",
79            "&eor_32     (@x[$b3],@x[$b3],@x[$c3])",
80         "&ror_32        (@x[$b0],@x[$b0],20)",
81          "&ror_32       (@x[$b1],@x[$b1],20)",
82           "&ror_32      (@x[$b2],@x[$b2],20)",
83            "&ror_32     (@x[$b3],@x[$b3],20)",
84
85         "&add_32        (@x[$a0],@x[$a0],@x[$b0])",
86          "&add_32       (@x[$a1],@x[$a1],@x[$b1])",
87           "&add_32      (@x[$a2],@x[$a2],@x[$b2])",
88            "&add_32     (@x[$a3],@x[$a3],@x[$b3])",
89         "&eor_32        (@x[$d0],@x[$d0],@x[$a0])",
90          "&eor_32       (@x[$d1],@x[$d1],@x[$a1])",
91           "&eor_32      (@x[$d2],@x[$d2],@x[$a2])",
92            "&eor_32     (@x[$d3],@x[$d3],@x[$a3])",
93         "&ror_32        (@x[$d0],@x[$d0],24)",
94          "&ror_32       (@x[$d1],@x[$d1],24)",
95           "&ror_32      (@x[$d2],@x[$d2],24)",
96            "&ror_32     (@x[$d3],@x[$d3],24)",
97
98         "&add_32        (@x[$c0],@x[$c0],@x[$d0])",
99          "&add_32       (@x[$c1],@x[$c1],@x[$d1])",
100           "&add_32      (@x[$c2],@x[$c2],@x[$d2])",
101            "&add_32     (@x[$c3],@x[$c3],@x[$d3])",
102         "&eor_32        (@x[$b0],@x[$b0],@x[$c0])",
103          "&eor_32       (@x[$b1],@x[$b1],@x[$c1])",
104           "&eor_32      (@x[$b2],@x[$b2],@x[$c2])",
105            "&eor_32     (@x[$b3],@x[$b3],@x[$c3])",
106         "&ror_32        (@x[$b0],@x[$b0],25)",
107          "&ror_32       (@x[$b1],@x[$b1],25)",
108           "&ror_32      (@x[$b2],@x[$b2],25)",
109            "&ror_32     (@x[$b3],@x[$b3],25)"
110     );
111 }
112
113 $code.=<<___;
114 #include "arm_arch.h"
115
116 .text
117
118 .extern OPENSSL_armcap_P
119
120 .align  5
121 .Lsigma:
122 .quad   0x3320646e61707865,0x6b20657479622d32           // endian-neutral
123 .Lone:
124 .long   1,0,0,0
125 .LOPENSSL_armcap_P:
126 #ifdef  __ILP32__
127 .long   OPENSSL_armcap_P-.
128 #else
129 .quad   OPENSSL_armcap_P-.
130 #endif
131 .asciz  "ChaCha20 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
132
133 .globl  ChaCha20_ctr32
134 .type   ChaCha20_ctr32,%function
135 .align  5
136 ChaCha20_ctr32:
137         cbz     $len,.Labort
138         adr     @x[0],.LOPENSSL_armcap_P
139         cmp     $len,#192
140         b.lo    .Lshort
141 #ifdef  __ILP32__
142         ldrsw   @x[1],[@x[0]]
143 #else
144         ldr     @x[1],[@x[0]]
145 #endif
146         ldr     w17,[@x[1],@x[0]]
147         tst     w17,#ARMV7_NEON
148         b.ne    ChaCha20_neon
149
150 .Lshort:
151         stp     x29,x30,[sp,#-96]!
152         add     x29,sp,#0
153
154         adr     @x[0],.Lsigma
155         stp     x19,x20,[sp,#16]
156         stp     x21,x22,[sp,#32]
157         stp     x23,x24,[sp,#48]
158         stp     x25,x26,[sp,#64]
159         stp     x27,x28,[sp,#80]
160         sub     sp,sp,#64
161
162         ldp     @d[0],@d[1],[@x[0]]             // load sigma
163         ldp     @d[2],@d[3],[$key]              // load key
164         ldp     @d[4],@d[5],[$key,#16]
165         ldp     @d[6],@d[7],[$ctr]              // load counter
166 #ifdef  __ARMEB__
167         ror     @d[2],@d[2],#32
168         ror     @d[3],@d[3],#32
169         ror     @d[4],@d[4],#32
170         ror     @d[5],@d[5],#32
171         ror     @d[6],@d[6],#32
172         ror     @d[7],@d[7],#32
173 #endif
174
175 .Loop_outer:
176         mov.32  @x[0],@d[0]                     // unpack key block
177         lsr     @x[1],@d[0],#32
178         mov.32  @x[2],@d[1]
179         lsr     @x[3],@d[1],#32
180         mov.32  @x[4],@d[2]
181         lsr     @x[5],@d[2],#32
182         mov.32  @x[6],@d[3]
183         lsr     @x[7],@d[3],#32
184         mov.32  @x[8],@d[4]
185         lsr     @x[9],@d[4],#32
186         mov.32  @x[10],@d[5]
187         lsr     @x[11],@d[5],#32
188         mov.32  @x[12],@d[6]
189         lsr     @x[13],@d[6],#32
190         mov.32  @x[14],@d[7]
191         lsr     @x[15],@d[7],#32
192
193         mov     $ctr,#10
194         subs    $len,$len,#64
195 .Loop:
196         sub     $ctr,$ctr,#1    
197 ___
198         foreach (&ROUND(0, 4, 8,12)) { eval; }
199         foreach (&ROUND(0, 5,10,15)) { eval; }
200 $code.=<<___;
201         cbnz    $ctr,.Loop
202
203         add.32  @x[0],@x[0],@d[0]               // accumulate key block
204         add     @x[1],@x[1],@d[0],lsr#32
205         add.32  @x[2],@x[2],@d[1]
206         add     @x[3],@x[3],@d[1],lsr#32
207         add.32  @x[4],@x[4],@d[2]
208         add     @x[5],@x[5],@d[2],lsr#32
209         add.32  @x[6],@x[6],@d[3]
210         add     @x[7],@x[7],@d[3],lsr#32
211         add.32  @x[8],@x[8],@d[4]
212         add     @x[9],@x[9],@d[4],lsr#32
213         add.32  @x[10],@x[10],@d[5]
214         add     @x[11],@x[11],@d[5],lsr#32
215         add.32  @x[12],@x[12],@d[6]
216         add     @x[13],@x[13],@d[6],lsr#32
217         add.32  @x[14],@x[14],@d[7]
218         add     @x[15],@x[15],@d[7],lsr#32
219
220         b.lo    .Ltail
221
222         add     @x[0],@x[0],@x[1],lsl#32        // pack
223         add     @x[2],@x[2],@x[3],lsl#32
224         ldp     @x[1],@x[3],[$inp,#0]           // load input
225         add     @x[4],@x[4],@x[5],lsl#32
226         add     @x[6],@x[6],@x[7],lsl#32
227         ldp     @x[5],@x[7],[$inp,#16]
228         add     @x[8],@x[8],@x[9],lsl#32
229         add     @x[10],@x[10],@x[11],lsl#32
230         ldp     @x[9],@x[11],[$inp,#32]
231         add     @x[12],@x[12],@x[13],lsl#32
232         add     @x[14],@x[14],@x[15],lsl#32
233         ldp     @x[13],@x[15],[$inp,#48]
234         add     $inp,$inp,#64
235 #ifdef  __ARMEB__
236         rev     @x[0],@x[0]
237         rev     @x[2],@x[2]
238         rev     @x[4],@x[4]
239         rev     @x[6],@x[6]
240         rev     @x[8],@x[8]
241         rev     @x[10],@x[10]
242         rev     @x[12],@x[12]
243         rev     @x[14],@x[14]
244 #endif
245         eor     @x[0],@x[0],@x[1]
246         eor     @x[2],@x[2],@x[3]
247         eor     @x[4],@x[4],@x[5]
248         eor     @x[6],@x[6],@x[7]
249         eor     @x[8],@x[8],@x[9]
250         eor     @x[10],@x[10],@x[11]
251         eor     @x[12],@x[12],@x[13]
252         eor     @x[14],@x[14],@x[15]
253
254         stp     @x[0],@x[2],[$out,#0]           // store output
255          add    @d[6],@d[6],#1                  // increment counter
256         stp     @x[4],@x[6],[$out,#16]
257         stp     @x[8],@x[10],[$out,#32]
258         stp     @x[12],@x[14],[$out,#48]
259         add     $out,$out,#64
260
261         b.hi    .Loop_outer
262
263         ldp     x19,x20,[x29,#16]
264         add     sp,sp,#64
265         ldp     x21,x22,[x29,#32]
266         ldp     x23,x24,[x29,#48]
267         ldp     x25,x26,[x29,#64]
268         ldp     x27,x28,[x29,#80]
269         ldp     x29,x30,[sp],#96
270 .Labort:
271         ret
272
273 .align  4
274 .Ltail:
275         add     $len,$len,#64
276 .Less_than_64:
277         sub     $out,$out,#1
278         add     $inp,$inp,$len
279         add     $out,$out,$len
280         add     $ctr,sp,$len
281         neg     $len,$len
282
283         add     @x[0],@x[0],@x[1],lsl#32        // pack
284         add     @x[2],@x[2],@x[3],lsl#32
285         add     @x[4],@x[4],@x[5],lsl#32
286         add     @x[6],@x[6],@x[7],lsl#32
287         add     @x[8],@x[8],@x[9],lsl#32
288         add     @x[10],@x[10],@x[11],lsl#32
289         add     @x[12],@x[12],@x[13],lsl#32
290         add     @x[14],@x[14],@x[15],lsl#32
291 #ifdef  __ARMEB__
292         rev     @x[0],@x[0]
293         rev     @x[2],@x[2]
294         rev     @x[4],@x[4]
295         rev     @x[6],@x[6]
296         rev     @x[8],@x[8]
297         rev     @x[10],@x[10]
298         rev     @x[12],@x[12]
299         rev     @x[14],@x[14]
300 #endif
301         stp     @x[0],@x[2],[sp,#0]
302         stp     @x[4],@x[6],[sp,#16]
303         stp     @x[8],@x[10],[sp,#32]
304         stp     @x[12],@x[14],[sp,#48]
305
306 .Loop_tail:
307         ldrb    w10,[$inp,$len]
308         ldrb    w11,[$ctr,$len]
309         add     $len,$len,#1
310         eor     w10,w10,w11
311         strb    w10,[$out,$len]
312         cbnz    $len,.Loop_tail
313
314         stp     xzr,xzr,[sp,#0]
315         stp     xzr,xzr,[sp,#16]
316         stp     xzr,xzr,[sp,#32]
317         stp     xzr,xzr,[sp,#48]
318
319         ldp     x19,x20,[x29,#16]
320         add     sp,sp,#64
321         ldp     x21,x22,[x29,#32]
322         ldp     x23,x24,[x29,#48]
323         ldp     x25,x26,[x29,#64]
324         ldp     x27,x28,[x29,#80]
325         ldp     x29,x30,[sp],#96
326         ret
327 .size   ChaCha20_ctr32,.-ChaCha20_ctr32
328 ___
329
330 {{{
331 my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,$T0,$T1,$T2,$T3) =
332     map("v$_.4s",(0..7,16..23));
333 my (@K)=map("v$_.4s",(24..30));
334 my $ONE="v31.4s";
335
336 sub NEONROUND {
337 my $odd = pop;
338 my ($a,$b,$c,$d,$t)=@_;
339
340         (
341         "&add           ('$a','$a','$b')",
342         "&eor           ('$d','$d','$a')",
343         "&rev32_16      ('$d','$d')",           # vrot ($d,16)
344
345         "&add           ('$c','$c','$d')",
346         "&eor           ('$t','$b','$c')",
347         "&ushr          ('$b','$t',20)",
348         "&sli           ('$b','$t',12)",
349
350         "&add           ('$a','$a','$b')",
351         "&eor           ('$t','$d','$a')",
352         "&ushr          ('$d','$t',24)",
353         "&sli           ('$d','$t',8)",
354
355         "&add           ('$c','$c','$d')",
356         "&eor           ('$t','$b','$c')",
357         "&ushr          ('$b','$t',25)",
358         "&sli           ('$b','$t',7)",
359
360         "&ext           ('$c','$c','$c',8)",
361         "&ext           ('$d','$d','$d',$odd?4:12)",
362         "&ext           ('$b','$b','$b',$odd?12:4)"
363         );
364 }
365
366 $code.=<<___;
367
368 .type   ChaCha20_neon,%function
369 .align  5
370 ChaCha20_neon:
371         stp     x29,x30,[sp,#-96]!
372         add     x29,sp,#0
373
374         adr     @x[0],.Lsigma
375         stp     x19,x20,[sp,#16]
376         stp     x21,x22,[sp,#32]
377         stp     x23,x24,[sp,#48]
378         stp     x25,x26,[sp,#64]
379         stp     x27,x28,[sp,#80]
380         cmp     $len,#512
381         b.hs    .L512_or_more_neon
382
383         sub     sp,sp,#64
384
385         ldp     @d[0],@d[1],[@x[0]]             // load sigma
386         ld1     {@K[0]},[@x[0]],#16
387         ldp     @d[2],@d[3],[$key]              // load key
388         ldp     @d[4],@d[5],[$key,#16]
389         ld1     {@K[1],@K[2]},[$key]
390         ldp     @d[6],@d[7],[$ctr]              // load counter
391         ld1     {@K[3]},[$ctr]
392         ld1     {$ONE},[@x[0]]
393 #ifdef  __ARMEB__
394         rev64   @K[0],@K[0]
395         ror     @d[2],@d[2],#32
396         ror     @d[3],@d[3],#32
397         ror     @d[4],@d[4],#32
398         ror     @d[5],@d[5],#32
399         ror     @d[6],@d[6],#32
400         ror     @d[7],@d[7],#32
401 #endif
402         add     @K[3],@K[3],$ONE                // += 1
403         add     @K[4],@K[3],$ONE
404         add     @K[5],@K[4],$ONE
405         shl     $ONE,$ONE,#2                    // 1 -> 4
406
407 .Loop_outer_neon:
408         mov.32  @x[0],@d[0]                     // unpack key block
409         lsr     @x[1],@d[0],#32
410          mov    $A0,@K[0]
411         mov.32  @x[2],@d[1]
412         lsr     @x[3],@d[1],#32
413          mov    $A1,@K[0]
414         mov.32  @x[4],@d[2]
415         lsr     @x[5],@d[2],#32
416          mov    $A2,@K[0]
417         mov.32  @x[6],@d[3]
418          mov    $B0,@K[1]
419         lsr     @x[7],@d[3],#32
420          mov    $B1,@K[1]
421         mov.32  @x[8],@d[4]
422          mov    $B2,@K[1]
423         lsr     @x[9],@d[4],#32
424          mov    $D0,@K[3]
425         mov.32  @x[10],@d[5]
426          mov    $D1,@K[4]
427         lsr     @x[11],@d[5],#32
428          mov    $D2,@K[5]
429         mov.32  @x[12],@d[6]
430          mov    $C0,@K[2]
431         lsr     @x[13],@d[6],#32
432          mov    $C1,@K[2]
433         mov.32  @x[14],@d[7]
434          mov    $C2,@K[2]
435         lsr     @x[15],@d[7],#32
436
437         mov     $ctr,#10
438         subs    $len,$len,#256
439 .Loop_neon:
440         sub     $ctr,$ctr,#1
441 ___
442         my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0);
443         my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0);
444         my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0);
445         my @thread3=&ROUND(0,4,8,12);
446
447         foreach (@thread0) {
448                 eval;                   eval(shift(@thread3));
449                 eval(shift(@thread1));  eval(shift(@thread3));
450                 eval(shift(@thread2));  eval(shift(@thread3));
451         }
452
453         @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1);
454         @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1);
455         @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1);
456         @thread3=&ROUND(0,5,10,15);
457
458         foreach (@thread0) {
459                 eval;                   eval(shift(@thread3));
460                 eval(shift(@thread1));  eval(shift(@thread3));
461                 eval(shift(@thread2));  eval(shift(@thread3));
462         }
463 $code.=<<___;
464         cbnz    $ctr,.Loop_neon
465
466         add.32  @x[0],@x[0],@d[0]               // accumulate key block
467          add    $A0,$A0,@K[0]
468         add     @x[1],@x[1],@d[0],lsr#32
469          add    $A1,$A1,@K[0]
470         add.32  @x[2],@x[2],@d[1]
471          add    $A2,$A2,@K[0]
472         add     @x[3],@x[3],@d[1],lsr#32
473          add    $C0,$C0,@K[2]
474         add.32  @x[4],@x[4],@d[2]
475          add    $C1,$C1,@K[2]
476         add     @x[5],@x[5],@d[2],lsr#32
477          add    $C2,$C2,@K[2]
478         add.32  @x[6],@x[6],@d[3]
479          add    $D0,$D0,@K[3]
480         add     @x[7],@x[7],@d[3],lsr#32
481         add.32  @x[8],@x[8],@d[4]
482          add    $D1,$D1,@K[4]
483         add     @x[9],@x[9],@d[4],lsr#32
484         add.32  @x[10],@x[10],@d[5]
485          add    $D2,$D2,@K[5]
486         add     @x[11],@x[11],@d[5],lsr#32
487         add.32  @x[12],@x[12],@d[6]
488          add    $B0,$B0,@K[1]
489         add     @x[13],@x[13],@d[6],lsr#32
490         add.32  @x[14],@x[14],@d[7]
491          add    $B1,$B1,@K[1]
492         add     @x[15],@x[15],@d[7],lsr#32
493          add    $B2,$B2,@K[1]
494
495         b.lo    .Ltail_neon
496
497         add     @x[0],@x[0],@x[1],lsl#32        // pack
498         add     @x[2],@x[2],@x[3],lsl#32
499         ldp     @x[1],@x[3],[$inp,#0]           // load input
500         add     @x[4],@x[4],@x[5],lsl#32
501         add     @x[6],@x[6],@x[7],lsl#32
502         ldp     @x[5],@x[7],[$inp,#16]
503         add     @x[8],@x[8],@x[9],lsl#32
504         add     @x[10],@x[10],@x[11],lsl#32
505         ldp     @x[9],@x[11],[$inp,#32]
506         add     @x[12],@x[12],@x[13],lsl#32
507         add     @x[14],@x[14],@x[15],lsl#32
508         ldp     @x[13],@x[15],[$inp,#48]
509         add     $inp,$inp,#64
510 #ifdef  __ARMEB__
511         rev     @x[0],@x[0]
512         rev     @x[2],@x[2]
513         rev     @x[4],@x[4]
514         rev     @x[6],@x[6]
515         rev     @x[8],@x[8]
516         rev     @x[10],@x[10]
517         rev     @x[12],@x[12]
518         rev     @x[14],@x[14]
519 #endif
520         ld1.8   {$T0-$T3},[$inp],#64
521         eor     @x[0],@x[0],@x[1]
522         eor     @x[2],@x[2],@x[3]
523         eor     @x[4],@x[4],@x[5]
524         eor     @x[6],@x[6],@x[7]
525         eor     @x[8],@x[8],@x[9]
526          eor    $A0,$A0,$T0
527         eor     @x[10],@x[10],@x[11]
528          eor    $B0,$B0,$T1
529         eor     @x[12],@x[12],@x[13]
530          eor    $C0,$C0,$T2
531         eor     @x[14],@x[14],@x[15]
532          eor    $D0,$D0,$T3
533          ld1.8  {$T0-$T3},[$inp],#64
534
535         stp     @x[0],@x[2],[$out,#0]           // store output
536          add    @d[6],@d[6],#4                  // increment counter
537         stp     @x[4],@x[6],[$out,#16]
538          add    @K[3],@K[3],$ONE                // += 4
539         stp     @x[8],@x[10],[$out,#32]
540          add    @K[4],@K[4],$ONE
541         stp     @x[12],@x[14],[$out,#48]
542          add    @K[5],@K[5],$ONE
543         add     $out,$out,#64
544
545         st1.8   {$A0-$D0},[$out],#64
546         ld1.8   {$A0-$D0},[$inp],#64
547
548         eor     $A1,$A1,$T0
549         eor     $B1,$B1,$T1
550         eor     $C1,$C1,$T2
551         eor     $D1,$D1,$T3
552         st1.8   {$A1-$D1},[$out],#64
553
554         eor     $A2,$A2,$A0
555         eor     $B2,$B2,$B0
556         eor     $C2,$C2,$C0
557         eor     $D2,$D2,$D0
558         st1.8   {$A2-$D2},[$out],#64
559
560         b.hi    .Loop_outer_neon
561
562         ldp     x19,x20,[x29,#16]
563         add     sp,sp,#64
564         ldp     x21,x22,[x29,#32]
565         ldp     x23,x24,[x29,#48]
566         ldp     x25,x26,[x29,#64]
567         ldp     x27,x28,[x29,#80]
568         ldp     x29,x30,[sp],#96
569         ret
570
571 .Ltail_neon:
572         add     $len,$len,#256
573         cmp     $len,#64
574         b.lo    .Less_than_64
575
576         add     @x[0],@x[0],@x[1],lsl#32        // pack
577         add     @x[2],@x[2],@x[3],lsl#32
578         ldp     @x[1],@x[3],[$inp,#0]           // load input
579         add     @x[4],@x[4],@x[5],lsl#32
580         add     @x[6],@x[6],@x[7],lsl#32
581         ldp     @x[5],@x[7],[$inp,#16]
582         add     @x[8],@x[8],@x[9],lsl#32
583         add     @x[10],@x[10],@x[11],lsl#32
584         ldp     @x[9],@x[11],[$inp,#32]
585         add     @x[12],@x[12],@x[13],lsl#32
586         add     @x[14],@x[14],@x[15],lsl#32
587         ldp     @x[13],@x[15],[$inp,#48]
588         add     $inp,$inp,#64
589 #ifdef  __ARMEB__
590         rev     @x[0],@x[0]
591         rev     @x[2],@x[2]
592         rev     @x[4],@x[4]
593         rev     @x[6],@x[6]
594         rev     @x[8],@x[8]
595         rev     @x[10],@x[10]
596         rev     @x[12],@x[12]
597         rev     @x[14],@x[14]
598 #endif
599         eor     @x[0],@x[0],@x[1]
600         eor     @x[2],@x[2],@x[3]
601         eor     @x[4],@x[4],@x[5]
602         eor     @x[6],@x[6],@x[7]
603         eor     @x[8],@x[8],@x[9]
604         eor     @x[10],@x[10],@x[11]
605         eor     @x[12],@x[12],@x[13]
606         eor     @x[14],@x[14],@x[15]
607
608         stp     @x[0],@x[2],[$out,#0]           // store output
609          add    @d[6],@d[6],#4                  // increment counter
610         stp     @x[4],@x[6],[$out,#16]
611         stp     @x[8],@x[10],[$out,#32]
612         stp     @x[12],@x[14],[$out,#48]
613         add     $out,$out,#64
614         b.eq    .Ldone_neon
615         sub     $len,$len,#64
616         cmp     $len,#64
617         b.lo    .Less_than_128
618
619         ld1.8   {$T0-$T3},[$inp],#64
620         eor     $A0,$A0,$T0
621         eor     $B0,$B0,$T1
622         eor     $C0,$C0,$T2
623         eor     $D0,$D0,$T3
624         st1.8   {$A0-$D0},[$out],#64
625         b.eq    .Ldone_neon
626         sub     $len,$len,#64
627         cmp     $len,#64
628         b.lo    .Less_than_192
629
630         ld1.8   {$T0-$T3},[$inp],#64
631         eor     $A1,$A1,$T0
632         eor     $B1,$B1,$T1
633         eor     $C1,$C1,$T2
634         eor     $D1,$D1,$T3
635         st1.8   {$A1-$D1},[$out],#64
636         b.eq    .Ldone_neon
637         sub     $len,$len,#64
638
639         st1.8   {$A2-$D2},[sp]
640         b       .Last_neon
641
642 .Less_than_128:
643         st1.8   {$A0-$D0},[sp]
644         b       .Last_neon
645 .Less_than_192:
646         st1.8   {$A1-$D1},[sp]
647         b       .Last_neon
648
649 .align  4
650 .Last_neon:
651         sub     $out,$out,#1
652         add     $inp,$inp,$len
653         add     $out,$out,$len
654         add     $ctr,sp,$len
655         neg     $len,$len
656
657 .Loop_tail_neon:
658         ldrb    w10,[$inp,$len]
659         ldrb    w11,[$ctr,$len]
660         add     $len,$len,#1
661         eor     w10,w10,w11
662         strb    w10,[$out,$len]
663         cbnz    $len,.Loop_tail_neon
664
665         stp     xzr,xzr,[sp,#0]
666         stp     xzr,xzr,[sp,#16]
667         stp     xzr,xzr,[sp,#32]
668         stp     xzr,xzr,[sp,#48]
669
670 .Ldone_neon:
671         ldp     x19,x20,[x29,#16]
672         add     sp,sp,#64
673         ldp     x21,x22,[x29,#32]
674         ldp     x23,x24,[x29,#48]
675         ldp     x25,x26,[x29,#64]
676         ldp     x27,x28,[x29,#80]
677         ldp     x29,x30,[sp],#96
678         ret
679 .size   ChaCha20_neon,.-ChaCha20_neon
680 ___
681 {
682 my ($T0,$T1,$T2,$T3,$T4,$T5)=@K;
683 my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,
684     $A3,$B3,$C3,$D3,$A4,$B4,$C4,$D4,$A5,$B5,$C5,$D5) = map("v$_.4s",(0..23));
685
686 $code.=<<___;
687 .type   ChaCha20_512_neon,%function
688 .align  5
689 ChaCha20_512_neon:
690         stp     x29,x30,[sp,#-96]!
691         add     x29,sp,#0
692
693         adr     @x[0],.Lsigma
694         stp     x19,x20,[sp,#16]
695         stp     x21,x22,[sp,#32]
696         stp     x23,x24,[sp,#48]
697         stp     x25,x26,[sp,#64]
698         stp     x27,x28,[sp,#80]
699
700 .L512_or_more_neon:
701         sub     sp,sp,#128+64
702
703         ldp     @d[0],@d[1],[@x[0]]             // load sigma
704         ld1     {@K[0]},[@x[0]],#16
705         ldp     @d[2],@d[3],[$key]              // load key
706         ldp     @d[4],@d[5],[$key,#16]
707         ld1     {@K[1],@K[2]},[$key]
708         ldp     @d[6],@d[7],[$ctr]              // load counter
709         ld1     {@K[3]},[$ctr]
710         ld1     {$ONE},[@x[0]]
711 #ifdef  __ARMEB__
712         rev64   @K[0],@K[0]
713         ror     @d[2],@d[2],#32
714         ror     @d[3],@d[3],#32
715         ror     @d[4],@d[4],#32
716         ror     @d[5],@d[5],#32
717         ror     @d[6],@d[6],#32
718         ror     @d[7],@d[7],#32
719 #endif
720         add     @K[3],@K[3],$ONE                // += 1
721         stp     @K[0],@K[1],[sp,#0]             // off-load key block, invariant part
722         add     @K[3],@K[3],$ONE                // not typo
723         str     @K[2],[sp,#32]
724         add     @K[4],@K[3],$ONE
725         add     @K[5],@K[4],$ONE
726         add     @K[6],@K[5],$ONE
727         shl     $ONE,$ONE,#2                    // 1 -> 4
728
729         stp     d8,d9,[sp,#128+0]               // meet ABI requirements
730         stp     d10,d11,[sp,#128+16]
731         stp     d12,d13,[sp,#128+32]
732         stp     d14,d15,[sp,#128+48]
733
734         sub     $len,$len,#512                  // not typo
735
736 .Loop_outer_512_neon:
737          mov    $A0,@K[0]
738          mov    $A1,@K[0]
739          mov    $A2,@K[0]
740          mov    $A3,@K[0]
741          mov    $A4,@K[0]
742          mov    $A5,@K[0]
743          mov    $B0,@K[1]
744         mov.32  @x[0],@d[0]                     // unpack key block
745          mov    $B1,@K[1]
746         lsr     @x[1],@d[0],#32
747          mov    $B2,@K[1]
748         mov.32  @x[2],@d[1]
749          mov    $B3,@K[1]
750         lsr     @x[3],@d[1],#32
751          mov    $B4,@K[1]
752         mov.32  @x[4],@d[2]
753          mov    $B5,@K[1]
754         lsr     @x[5],@d[2],#32
755          mov    $D0,@K[3]
756         mov.32  @x[6],@d[3]
757          mov    $D1,@K[4]
758         lsr     @x[7],@d[3],#32
759          mov    $D2,@K[5]
760         mov.32  @x[8],@d[4]
761          mov    $D3,@K[6]
762         lsr     @x[9],@d[4],#32
763          mov    $C0,@K[2]
764         mov.32  @x[10],@d[5]
765          mov    $C1,@K[2]
766         lsr     @x[11],@d[5],#32
767          add    $D4,$D0,$ONE                    // +4
768         mov.32  @x[12],@d[6]
769          add    $D5,$D1,$ONE                    // +4
770         lsr     @x[13],@d[6],#32
771          mov    $C2,@K[2]
772         mov.32  @x[14],@d[7]
773          mov    $C3,@K[2]
774         lsr     @x[15],@d[7],#32
775          mov    $C4,@K[2]
776          stp    @K[3],@K[4],[sp,#48]            // off-load key block, variable part
777          mov    $C5,@K[2]
778          str    @K[5],[sp,#80]
779
780         mov     $ctr,#5
781         subs    $len,$len,#512
782 .Loop_upper_neon:
783         sub     $ctr,$ctr,#1
784 ___
785         my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0);
786         my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0);
787         my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0);
788         my @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0);
789         my @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0);
790         my @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0);
791         my @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
792         my $diff = ($#thread0+1)*6 - $#thread67 - 1;
793         my $i = 0;
794
795         foreach (@thread0) {
796                 eval;                   eval(shift(@thread67));
797                 eval(shift(@thread1));  eval(shift(@thread67));
798                 eval(shift(@thread2));  eval(shift(@thread67));
799                 eval(shift(@thread3));  eval(shift(@thread67));
800                 eval(shift(@thread4));  eval(shift(@thread67));
801                 eval(shift(@thread5));  eval(shift(@thread67));
802         }
803
804         @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1);
805         @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1);
806         @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1);
807         @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1);
808         @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1);
809         @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1);
810         @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
811
812         foreach (@thread0) {
813                 eval;                   eval(shift(@thread67));
814                 eval(shift(@thread1));  eval(shift(@thread67));
815                 eval(shift(@thread2));  eval(shift(@thread67));
816                 eval(shift(@thread3));  eval(shift(@thread67));
817                 eval(shift(@thread4));  eval(shift(@thread67));
818                 eval(shift(@thread5));  eval(shift(@thread67));
819         }
820 $code.=<<___;
821         cbnz    $ctr,.Loop_upper_neon
822
823         add.32  @x[0],@x[0],@d[0]               // accumulate key block
824         add     @x[1],@x[1],@d[0],lsr#32
825         add.32  @x[2],@x[2],@d[1]
826         add     @x[3],@x[3],@d[1],lsr#32
827         add.32  @x[4],@x[4],@d[2]
828         add     @x[5],@x[5],@d[2],lsr#32
829         add.32  @x[6],@x[6],@d[3]
830         add     @x[7],@x[7],@d[3],lsr#32
831         add.32  @x[8],@x[8],@d[4]
832         add     @x[9],@x[9],@d[4],lsr#32
833         add.32  @x[10],@x[10],@d[5]
834         add     @x[11],@x[11],@d[5],lsr#32
835         add.32  @x[12],@x[12],@d[6]
836         add     @x[13],@x[13],@d[6],lsr#32
837         add.32  @x[14],@x[14],@d[7]
838         add     @x[15],@x[15],@d[7],lsr#32
839
840         add     @x[0],@x[0],@x[1],lsl#32        // pack
841         add     @x[2],@x[2],@x[3],lsl#32
842         ldp     @x[1],@x[3],[$inp,#0]           // load input
843         add     @x[4],@x[4],@x[5],lsl#32
844         add     @x[6],@x[6],@x[7],lsl#32
845         ldp     @x[5],@x[7],[$inp,#16]
846         add     @x[8],@x[8],@x[9],lsl#32
847         add     @x[10],@x[10],@x[11],lsl#32
848         ldp     @x[9],@x[11],[$inp,#32]
849         add     @x[12],@x[12],@x[13],lsl#32
850         add     @x[14],@x[14],@x[15],lsl#32
851         ldp     @x[13],@x[15],[$inp,#48]
852         add     $inp,$inp,#64
853 #ifdef  __ARMEB__
854         rev     @x[0],@x[0]
855         rev     @x[2],@x[2]
856         rev     @x[4],@x[4]
857         rev     @x[6],@x[6]
858         rev     @x[8],@x[8]
859         rev     @x[10],@x[10]
860         rev     @x[12],@x[12]
861         rev     @x[14],@x[14]
862 #endif
863         eor     @x[0],@x[0],@x[1]
864         eor     @x[2],@x[2],@x[3]
865         eor     @x[4],@x[4],@x[5]
866         eor     @x[6],@x[6],@x[7]
867         eor     @x[8],@x[8],@x[9]
868         eor     @x[10],@x[10],@x[11]
869         eor     @x[12],@x[12],@x[13]
870         eor     @x[14],@x[14],@x[15]
871
872          stp    @x[0],@x[2],[$out,#0]           // store output
873          add    @d[6],@d[6],#1                  // increment counter
874         mov.32  @x[0],@d[0]                     // unpack key block
875         lsr     @x[1],@d[0],#32
876          stp    @x[4],@x[6],[$out,#16]
877         mov.32  @x[2],@d[1]
878         lsr     @x[3],@d[1],#32
879          stp    @x[8],@x[10],[$out,#32]
880         mov.32  @x[4],@d[2]
881         lsr     @x[5],@d[2],#32
882          stp    @x[12],@x[14],[$out,#48]
883          add    $out,$out,#64
884         mov.32  @x[6],@d[3]
885         lsr     @x[7],@d[3],#32
886         mov.32  @x[8],@d[4]
887         lsr     @x[9],@d[4],#32
888         mov.32  @x[10],@d[5]
889         lsr     @x[11],@d[5],#32
890         mov.32  @x[12],@d[6]
891         lsr     @x[13],@d[6],#32
892         mov.32  @x[14],@d[7]
893         lsr     @x[15],@d[7],#32
894
895         mov     $ctr,#5
896 .Loop_lower_neon:
897         sub     $ctr,$ctr,#1
898 ___
899         @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0);
900         @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0);
901         @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0);
902         @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0);
903         @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0);
904         @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0);
905         @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
906
907         foreach (@thread0) {
908                 eval;                   eval(shift(@thread67));
909                 eval(shift(@thread1));  eval(shift(@thread67));
910                 eval(shift(@thread2));  eval(shift(@thread67));
911                 eval(shift(@thread3));  eval(shift(@thread67));
912                 eval(shift(@thread4));  eval(shift(@thread67));
913                 eval(shift(@thread5));  eval(shift(@thread67));
914         }
915
916         @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1);
917         @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1);
918         @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1);
919         @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1);
920         @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1);
921         @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1);
922         @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
923
924         foreach (@thread0) {
925                 eval;                   eval(shift(@thread67));
926                 eval(shift(@thread1));  eval(shift(@thread67));
927                 eval(shift(@thread2));  eval(shift(@thread67));
928                 eval(shift(@thread3));  eval(shift(@thread67));
929                 eval(shift(@thread4));  eval(shift(@thread67));
930                 eval(shift(@thread5));  eval(shift(@thread67));
931         }
932 $code.=<<___;
933         cbnz    $ctr,.Loop_lower_neon
934
935         add.32  @x[0],@x[0],@d[0]               // accumulate key block
936          ldp    @K[0],@K[1],[sp,#0]
937         add     @x[1],@x[1],@d[0],lsr#32
938          ldp    @K[2],@K[3],[sp,#32]
939         add.32  @x[2],@x[2],@d[1]
940          ldp    @K[4],@K[5],[sp,#64]
941         add     @x[3],@x[3],@d[1],lsr#32
942          add    $A0,$A0,@K[0]
943         add.32  @x[4],@x[4],@d[2]
944          add    $A1,$A1,@K[0]
945         add     @x[5],@x[5],@d[2],lsr#32
946          add    $A2,$A2,@K[0]
947         add.32  @x[6],@x[6],@d[3]
948          add    $A3,$A3,@K[0]
949         add     @x[7],@x[7],@d[3],lsr#32
950          add    $A4,$A4,@K[0]
951         add.32  @x[8],@x[8],@d[4]
952          add    $A5,$A5,@K[0]
953         add     @x[9],@x[9],@d[4],lsr#32
954          add    $C0,$C0,@K[2]
955         add.32  @x[10],@x[10],@d[5]
956          add    $C1,$C1,@K[2]
957         add     @x[11],@x[11],@d[5],lsr#32
958          add    $C2,$C2,@K[2]
959         add.32  @x[12],@x[12],@d[6]
960          add    $C3,$C3,@K[2]
961         add     @x[13],@x[13],@d[6],lsr#32
962          add    $C4,$C4,@K[2]
963         add.32  @x[14],@x[14],@d[7]
964          add    $C5,$C5,@K[2]
965         add     @x[15],@x[15],@d[7],lsr#32
966          add    $D4,$D4,$ONE                    // +4
967         add     @x[0],@x[0],@x[1],lsl#32        // pack
968          add    $D5,$D5,$ONE                    // +4
969         add     @x[2],@x[2],@x[3],lsl#32
970          add    $D0,$D0,@K[3]
971         ldp     @x[1],@x[3],[$inp,#0]           // load input
972          add    $D1,$D1,@K[4]
973         add     @x[4],@x[4],@x[5],lsl#32
974          add    $D2,$D2,@K[5]
975         add     @x[6],@x[6],@x[7],lsl#32
976          add    $D3,$D3,@K[6]
977         ldp     @x[5],@x[7],[$inp,#16]
978          add    $D4,$D4,@K[3]
979         add     @x[8],@x[8],@x[9],lsl#32
980          add    $D5,$D5,@K[4]
981         add     @x[10],@x[10],@x[11],lsl#32
982          add    $B0,$B0,@K[1]
983         ldp     @x[9],@x[11],[$inp,#32]
984          add    $B1,$B1,@K[1]
985         add     @x[12],@x[12],@x[13],lsl#32
986          add    $B2,$B2,@K[1]
987         add     @x[14],@x[14],@x[15],lsl#32
988          add    $B3,$B3,@K[1]
989         ldp     @x[13],@x[15],[$inp,#48]
990          add    $B4,$B4,@K[1]
991         add     $inp,$inp,#64
992          add    $B5,$B5,@K[1]
993
994 #ifdef  __ARMEB__
995         rev     @x[0],@x[0]
996         rev     @x[2],@x[2]
997         rev     @x[4],@x[4]
998         rev     @x[6],@x[6]
999         rev     @x[8],@x[8]
1000         rev     @x[10],@x[10]
1001         rev     @x[12],@x[12]
1002         rev     @x[14],@x[14]
1003 #endif
1004         ld1.8   {$T0-$T3},[$inp],#64
1005         eor     @x[0],@x[0],@x[1]
1006         eor     @x[2],@x[2],@x[3]
1007         eor     @x[4],@x[4],@x[5]
1008         eor     @x[6],@x[6],@x[7]
1009         eor     @x[8],@x[8],@x[9]
1010          eor    $A0,$A0,$T0
1011         eor     @x[10],@x[10],@x[11]
1012          eor    $B0,$B0,$T1
1013         eor     @x[12],@x[12],@x[13]
1014          eor    $C0,$C0,$T2
1015         eor     @x[14],@x[14],@x[15]
1016          eor    $D0,$D0,$T3
1017          ld1.8  {$T0-$T3},[$inp],#64
1018
1019         stp     @x[0],@x[2],[$out,#0]           // store output
1020          add    @d[6],@d[6],#7                  // increment counter
1021         stp     @x[4],@x[6],[$out,#16]
1022         stp     @x[8],@x[10],[$out,#32]
1023         stp     @x[12],@x[14],[$out,#48]
1024         add     $out,$out,#64
1025         st1.8   {$A0-$D0},[$out],#64
1026
1027         ld1.8   {$A0-$D0},[$inp],#64
1028         eor     $A1,$A1,$T0
1029         eor     $B1,$B1,$T1
1030         eor     $C1,$C1,$T2
1031         eor     $D1,$D1,$T3
1032         st1.8   {$A1-$D1},[$out],#64
1033
1034         ld1.8   {$A1-$D1},[$inp],#64
1035         eor     $A2,$A2,$A0
1036          ldp    @K[0],@K[1],[sp,#0]
1037         eor     $B2,$B2,$B0
1038          ldp    @K[2],@K[3],[sp,#32]
1039         eor     $C2,$C2,$C0
1040         eor     $D2,$D2,$D0
1041         st1.8   {$A2-$D2},[$out],#64
1042
1043         ld1.8   {$A2-$D2},[$inp],#64
1044         eor     $A3,$A3,$A1
1045         eor     $B3,$B3,$B1
1046         eor     $C3,$C3,$C1
1047         eor     $D3,$D3,$D1
1048         st1.8   {$A3-$D3},[$out],#64
1049
1050         ld1.8   {$A3-$D3},[$inp],#64
1051         eor     $A4,$A4,$A2
1052         eor     $B4,$B4,$B2
1053         eor     $C4,$C4,$C2
1054         eor     $D4,$D4,$D2
1055         st1.8   {$A4-$D4},[$out],#64
1056
1057         shl     $A0,$ONE,#1                     // 4 -> 8
1058         eor     $A5,$A5,$A3
1059         eor     $B5,$B5,$B3
1060         eor     $C5,$C5,$C3
1061         eor     $D5,$D5,$D3
1062         st1.8   {$A5-$D5},[$out],#64
1063
1064         add     @K[3],@K[3],$A0                 // += 8
1065         add     @K[4],@K[4],$A0
1066         add     @K[5],@K[5],$A0
1067         add     @K[6],@K[6],$A0
1068
1069         b.hs    .Loop_outer_512_neon
1070
1071         adds    $len,$len,#512
1072         ushr    $A0,$ONE,#2                     // 4 -> 1
1073
1074         ldp     d8,d9,[sp,#128+0]               // meet ABI requirements
1075         ldp     d10,d11,[sp,#128+16]
1076         ldp     d12,d13,[sp,#128+32]
1077         ldp     d14,d15,[sp,#128+48]
1078
1079         stp     @K[0],$ONE,[sp,#0]              // wipe off-load area
1080         stp     @K[0],$ONE,[sp,#32]
1081         stp     @K[0],$ONE,[sp,#64]
1082
1083         b.eq    .Ldone_512_neon
1084
1085         cmp     $len,#192
1086         sub     @K[3],@K[3],$A0                 // -= 1
1087         sub     @K[4],@K[4],$A0
1088         sub     @K[5],@K[5],$A0
1089         add     sp,sp,#128
1090         b.hs    .Loop_outer_neon
1091
1092         eor     @K[1],@K[1],@K[1]
1093         eor     @K[2],@K[2],@K[2]
1094         eor     @K[3],@K[3],@K[3]
1095         eor     @K[4],@K[4],@K[4]
1096         eor     @K[5],@K[5],@K[5]
1097         eor     @K[6],@K[6],@K[6]
1098         b       .Loop_outer
1099
1100 .Ldone_512_neon:
1101         ldp     x19,x20,[x29,#16]
1102         add     sp,sp,#128+64
1103         ldp     x21,x22,[x29,#32]
1104         ldp     x23,x24,[x29,#48]
1105         ldp     x25,x26,[x29,#64]
1106         ldp     x27,x28,[x29,#80]
1107         ldp     x29,x30,[sp],#96
1108         ret
1109 .size   ChaCha20_512_neon,.-ChaCha20_512_neon
1110 ___
1111 }
1112 }}}
1113
1114 foreach (split("\n",$code)) {
1115         s/\`([^\`]*)\`/eval $1/geo;
1116
1117         (s/\b([a-z]+)\.32\b/$1/ and (s/x([0-9]+)/w$1/g or 1))   or
1118         (m/\b(eor|ext|mov)\b/ and (s/\.4s/\.16b/g or 1))        or
1119         (s/\b((?:ld|st)1)\.8\b/$1/ and (s/\.4s/\.16b/g or 1))   or
1120         (m/\b(ld|st)[rp]\b/ and (s/v([0-9]+)\.4s/q$1/g or 1))   or
1121         (s/\brev32\.16\b/rev32/ and (s/\.4s/\.8h/g or 1));
1122
1123         #s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo;
1124
1125         print $_,"\n";
1126 }