2 # Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the Apache License 2.0 (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
21 # Performance in cycles per byte out of large buffer.
23 # IALU/gcc-4.4 1xNEON 3xNEON+1xIALU
25 # Cortex-A5 19.3(*)/+95% 21.8 14.1
26 # Cortex-A8 10.5(*)/+160% 13.9 6.35
27 # Cortex-A9 12.9(**)/+110% 14.3 6.50
28 # Cortex-A15 11.0/+40% 16.0 5.00
29 # Snapdragon S4 11.5/+125% 13.6 4.90
31 # (*) most "favourable" result for aligned data on little-endian
32 # processor, result for misaligned data is 10-15% lower;
33 # (**) this result is a trade-off: it can be improved by 20%,
34 # but then Snapdragon S4 and Cortex-A8 results get
37 # $output is the last argument if it looks like a file (it has an extension)
38 # $flavour is the first argument if it doesn't look like a file
39 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
40 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
42 if ($flavour && $flavour ne "void") {
43 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
44 ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
45 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
46 die "can't locate arm-xlate.pl";
48 open STDOUT,"| \"$^X\" $xlate $flavour $output"
49 or die "can't call $xlate: $!";
51 $output and open STDOUT,">$output";
54 sub AUTOLOAD() # thunk [simplified] x86-style perlasm
55 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
57 $arg = "#$arg" if ($arg*1 eq $arg);
58 $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
61 my @x=map("r$_",(0..7,"x","x","x","x",12,"x",14,"x"));
62 my @t=map("r$_",(8..11));
65 my ($a0,$b0,$c0,$d0)=@_;
66 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
67 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
68 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
70 my ($xc,$xc_) = (@t[0..1]);
71 my ($xd,$xd_) = $odd ? (@t[2],@x[$d1]) : (@x[$d0],@t[2]);
74 # Consider order in which variables are addressed by their
79 # 0 4 8 12 < even round
83 # 0 5 10 15 < odd round
88 # 'a', 'b' are permanently allocated in registers, @x[0..7],
89 # while 'c's and pair of 'd's are maintained in memory. If
90 # you observe 'c' column, you'll notice that pair of 'c's is
91 # invariant between rounds. This means that we have to reload
92 # them once per round, in the middle. This is why you'll see
93 # bunch of 'c' stores and loads in the middle, but none in
94 # the beginning or end. If you observe 'd' column, you'll
95 # notice that 15 and 13 are reused in next pair of rounds.
96 # This is why these two are chosen for offloading to memory,
97 # to make loads count more.
99 "&add (@x[$a0],@x[$a0],@x[$b0])",
100 "&mov ($xd,$xd,'ror#16')",
101 "&add (@x[$a1],@x[$a1],@x[$b1])",
102 "&mov ($xd_,$xd_,'ror#16')",
103 "&eor ($xd,$xd,@x[$a0],'ror#16')",
104 "&eor ($xd_,$xd_,@x[$a1],'ror#16')",
106 "&add ($xc,$xc,$xd)",
107 "&mov (@x[$b0],@x[$b0],'ror#20')",
108 "&add ($xc_,$xc_,$xd_)",
109 "&mov (@x[$b1],@x[$b1],'ror#20')",
110 "&eor (@x[$b0],@x[$b0],$xc,'ror#20')",
111 "&eor (@x[$b1],@x[$b1],$xc_,'ror#20')",
113 "&add (@x[$a0],@x[$a0],@x[$b0])",
114 "&mov ($xd,$xd,'ror#24')",
115 "&add (@x[$a1],@x[$a1],@x[$b1])",
116 "&mov ($xd_,$xd_,'ror#24')",
117 "&eor ($xd,$xd,@x[$a0],'ror#24')",
118 "&eor ($xd_,$xd_,@x[$a1],'ror#24')",
120 "&add ($xc,$xc,$xd)",
121 "&mov (@x[$b0],@x[$b0],'ror#25')" );
123 "&str ($xd,'[sp,#4*(16+$d0)]')",
124 "&ldr ($xd,'[sp,#4*(16+$d2)]')" ) if ($odd);
126 "&add ($xc_,$xc_,$xd_)",
127 "&mov (@x[$b1],@x[$b1],'ror#25')" );
129 "&str ($xd_,'[sp,#4*(16+$d1)]')",
130 "&ldr ($xd_,'[sp,#4*(16+$d3)]')" ) if (!$odd);
132 "&eor (@x[$b0],@x[$b0],$xc,'ror#25')",
133 "&eor (@x[$b1],@x[$b1],$xc_,'ror#25')" );
135 $xd=@x[$d2] if (!$odd);
136 $xd_=@x[$d3] if ($odd);
138 "&str ($xc,'[sp,#4*(16+$c0)]')",
139 "&ldr ($xc,'[sp,#4*(16+$c2)]')",
140 "&add (@x[$a2],@x[$a2],@x[$b2])",
141 "&mov ($xd,$xd,'ror#16')",
142 "&str ($xc_,'[sp,#4*(16+$c1)]')",
143 "&ldr ($xc_,'[sp,#4*(16+$c3)]')",
144 "&add (@x[$a3],@x[$a3],@x[$b3])",
145 "&mov ($xd_,$xd_,'ror#16')",
146 "&eor ($xd,$xd,@x[$a2],'ror#16')",
147 "&eor ($xd_,$xd_,@x[$a3],'ror#16')",
149 "&add ($xc,$xc,$xd)",
150 "&mov (@x[$b2],@x[$b2],'ror#20')",
151 "&add ($xc_,$xc_,$xd_)",
152 "&mov (@x[$b3],@x[$b3],'ror#20')",
153 "&eor (@x[$b2],@x[$b2],$xc,'ror#20')",
154 "&eor (@x[$b3],@x[$b3],$xc_,'ror#20')",
156 "&add (@x[$a2],@x[$a2],@x[$b2])",
157 "&mov ($xd,$xd,'ror#24')",
158 "&add (@x[$a3],@x[$a3],@x[$b3])",
159 "&mov ($xd_,$xd_,'ror#24')",
160 "&eor ($xd,$xd,@x[$a2],'ror#24')",
161 "&eor ($xd_,$xd_,@x[$a3],'ror#24')",
163 "&add ($xc,$xc,$xd)",
164 "&mov (@x[$b2],@x[$b2],'ror#25')",
165 "&add ($xc_,$xc_,$xd_)",
166 "&mov (@x[$b3],@x[$b3],'ror#25')",
167 "&eor (@x[$b2],@x[$b2],$xc,'ror#25')",
168 "&eor (@x[$b3],@x[$b3],$xc_,'ror#25')" );
174 #include "arm_arch.h"
176 #if defined(__thumb2__) || defined(__clang__)
179 #if defined(__thumb2__)
185 #if defined(__thumb2__) || defined(__clang__)
186 #define ldrhsb ldrbhs
193 .long 0x61707865,0x3320646e,0x79622d32,0x6b206574 @ endian-neutral
196 #if __ARM_MAX_ARCH__>=7
199 .word OPENSSL_armcap_P
201 .word OPENSSL_armcap_P-.LChaCha20_ctr32
207 .globl ChaCha20_ctr32
208 .type ChaCha20_ctr32,%function
212 ldr r12,[sp,#0] @ pull pointer to counter and nonce
213 stmdb sp!,{r0-r2,r4-r11,lr}
214 #if __ARM_ARCH__<7 && !defined(__thumb2__)
215 sub r14,pc,#16 @ ChaCha20_ctr32
217 adr r14,.LChaCha20_ctr32
225 #if __ARM_MAX_ARCH__>=7
226 cmp r2,#192 @ test len
229 # if !defined(_WIN32)
232 # if defined(__APPLE__) || defined(_WIN32)
239 ldmia r12,{r4-r7} @ load counter and nonce
240 sub sp,sp,#4*(16) @ off-load area
241 sub r14,r14,#64 @ .Lsigma
242 stmdb sp!,{r4-r7} @ copy counter and nonce
243 ldmia r3,{r4-r11} @ load key
244 ldmia r14,{r0-r3} @ load sigma
245 stmdb sp!,{r4-r11} @ copy key
246 stmdb sp!,{r0-r3} @ copy sigma
247 str r10,[sp,#4*(16+10)] @ off-load "@x[10]"
248 str r11,[sp,#4*(16+11)] @ off-load "@x[11]"
253 ldmia sp,{r0-r9} @ load key material
254 str @t[3],[sp,#4*(32+2)] @ save len
255 str r12, [sp,#4*(32+1)] @ save inp
256 str r14, [sp,#4*(32+0)] @ save out
258 ldr @t[3], [sp,#4*(15)]
259 ldr @x[12],[sp,#4*(12)] @ modulo-scheduled load
260 ldr @t[2], [sp,#4*(13)]
261 ldr @x[14],[sp,#4*(14)]
262 str @t[3], [sp,#4*(16+15)]
270 foreach (&ROUND(0, 4, 8,12)) { eval; }
271 foreach (&ROUND(0, 5,10,15)) { eval; }
275 ldr @t[3],[sp,#4*(32+2)] @ load len
277 str @t[0], [sp,#4*(16+8)] @ modulo-scheduled store
278 str @t[1], [sp,#4*(16+9)]
279 str @x[12],[sp,#4*(16+12)]
280 str @t[2], [sp,#4*(16+13)]
281 str @x[14],[sp,#4*(16+14)]
283 @ at this point we have first half of 512-bit result in
284 @ @x[0-7] and second half at sp+4*(16+8)
286 cmp @t[3],#64 @ done yet?
290 addlo r12,sp,#4*(0) @ shortcut or ...
291 ldrhs r12,[sp,#4*(32+1)] @ ... load inp
292 addlo r14,sp,#4*(0) @ shortcut or ...
293 ldrhs r14,[sp,#4*(32+0)] @ ... load out
295 ldr @t[0],[sp,#4*(0)] @ load key material
296 ldr @t[1],[sp,#4*(1)]
298 #if __ARM_ARCH__>=6 || !defined(__ARMEB__)
301 tst @t[2],#3 @ are input and output aligned?
302 ldr @t[2],[sp,#4*(2)]
304 cmp @t[3],#64 @ restore flags
306 ldr @t[2],[sp,#4*(2)]
308 ldr @t[3],[sp,#4*(3)]
310 add @x[0],@x[0],@t[0] @ accumulate key material
311 add @x[1],@x[1],@t[1]
315 ldrhs @t[0],[r12],#16 @ load input
316 ldrhs @t[1],[r12,#-12]
318 add @x[2],@x[2],@t[2]
319 add @x[3],@x[3],@t[3]
323 ldrhs @t[2],[r12,#-8]
324 ldrhs @t[3],[r12,#-4]
325 # if __ARM_ARCH__>=6 && defined(__ARMEB__)
334 eorhs @x[0],@x[0],@t[0] @ xor with input
335 eorhs @x[1],@x[1],@t[1]
337 str @x[0],[r14],#16 @ store output
341 eorhs @x[2],@x[2],@t[2]
342 eorhs @x[3],@x[3],@t[3]
343 ldmia @t[0],{@t[0]-@t[3]} @ load key material
348 add @x[4],@x[4],@t[0] @ accumulate key material
349 add @x[5],@x[5],@t[1]
353 ldrhs @t[0],[r12],#16 @ load input
354 ldrhs @t[1],[r12,#-12]
355 add @x[6],@x[6],@t[2]
356 add @x[7],@x[7],@t[3]
360 ldrhs @t[2],[r12,#-8]
361 ldrhs @t[3],[r12,#-4]
362 # if __ARM_ARCH__>=6 && defined(__ARMEB__)
371 eorhs @x[4],@x[4],@t[0]
372 eorhs @x[5],@x[5],@t[1]
374 str @x[4],[r14],#16 @ store output
378 eorhs @x[6],@x[6],@t[2]
379 eorhs @x[7],@x[7],@t[3]
381 ldmia @t[0],{@t[0]-@t[3]} @ load key material
383 add @x[0],sp,#4*(16+8)
386 ldmia @x[0],{@x[0]-@x[7]} @ load second half
388 add @x[0],@x[0],@t[0] @ accumulate key material
389 add @x[1],@x[1],@t[1]
393 ldrhs @t[0],[r12],#16 @ load input
394 ldrhs @t[1],[r12,#-12]
398 strhi @t[2],[sp,#4*(16+10)] @ copy "@x[10]" while at it
399 strhi @t[3],[sp,#4*(16+11)] @ copy "@x[11]" while at it
400 add @x[2],@x[2],@t[2]
401 add @x[3],@x[3],@t[3]
405 ldrhs @t[2],[r12,#-8]
406 ldrhs @t[3],[r12,#-4]
407 # if __ARM_ARCH__>=6 && defined(__ARMEB__)
416 eorhs @x[0],@x[0],@t[0]
417 eorhs @x[1],@x[1],@t[1]
419 str @x[0],[r14],#16 @ store output
423 eorhs @x[2],@x[2],@t[2]
424 eorhs @x[3],@x[3],@t[3]
426 ldmia @t[0],{@t[0]-@t[3]} @ load key material
430 add @x[4],@x[4],@t[0] @ accumulate key material
431 add @x[5],@x[5],@t[1]
435 addhi @t[0],@t[0],#1 @ next counter value
436 strhi @t[0],[sp,#4*(12)] @ save next counter value
440 ldrhs @t[0],[r12],#16 @ load input
441 ldrhs @t[1],[r12,#-12]
442 add @x[6],@x[6],@t[2]
443 add @x[7],@x[7],@t[3]
447 ldrhs @t[2],[r12,#-8]
448 ldrhs @t[3],[r12,#-4]
449 # if __ARM_ARCH__>=6 && defined(__ARMEB__)
458 eorhs @x[4],@x[4],@t[0]
459 eorhs @x[5],@x[5],@t[1]
463 ldrne @t[0],[sp,#4*(32+2)] @ re-load len
467 eorhs @x[6],@x[6],@t[2]
468 eorhs @x[7],@x[7],@t[3]
469 str @x[4],[r14],#16 @ store output
474 subhs @t[3],@t[0],#64 @ len-=64
484 .Lunaligned: @ unaligned endian-neutral path
485 cmp @t[3],#64 @ restore flags
489 ldr @t[3],[sp,#4*(3)]
491 for ($i=0;$i<16;$i+=4) {
494 $code.=<<___ if ($i==4);
495 add @x[0],sp,#4*(16+8)
497 $code.=<<___ if ($i==8);
498 ldmia @x[0],{@x[0]-@x[7]} @ load second half
502 strhi @t[2],[sp,#4*(16+10)] @ copy "@x[10]"
503 strhi @t[3],[sp,#4*(16+11)] @ copy "@x[11]"
506 add @x[$j+0],@x[$j+0],@t[0] @ accumulate key material
508 $code.=<<___ if ($i==12);
512 addhi @t[0],@t[0],#1 @ next counter value
513 strhi @t[0],[sp,#4*(12)] @ save next counter value
516 add @x[$j+1],@x[$j+1],@t[1]
517 add @x[$j+2],@x[$j+2],@t[2]
521 eorlo @t[0],@t[0],@t[0] @ zero or ...
522 ldrhsb @t[0],[r12],#16 @ ... load input
523 eorlo @t[1],@t[1],@t[1]
524 ldrhsb @t[1],[r12,#-12]
526 add @x[$j+3],@x[$j+3],@t[3]
530 eorlo @t[2],@t[2],@t[2]
531 ldrhsb @t[2],[r12,#-8]
532 eorlo @t[3],@t[3],@t[3]
533 ldrhsb @t[3],[r12,#-4]
535 eor @x[$j+0],@t[0],@x[$j+0] @ xor with input (or zero)
536 eor @x[$j+1],@t[1],@x[$j+1]
540 ldrhsb @t[0],[r12,#-15] @ load more input
541 ldrhsb @t[1],[r12,#-11]
542 eor @x[$j+2],@t[2],@x[$j+2]
543 strb @x[$j+0],[r14],#16 @ store output
544 eor @x[$j+3],@t[3],@x[$j+3]
548 ldrhsb @t[2],[r12,#-7]
549 ldrhsb @t[3],[r12,#-3]
550 strb @x[$j+1],[r14,#-12]
551 eor @x[$j+0],@t[0],@x[$j+0],lsr#8
552 strb @x[$j+2],[r14,#-8]
553 eor @x[$j+1],@t[1],@x[$j+1],lsr#8
557 ldrhsb @t[0],[r12,#-14] @ load more input
558 ldrhsb @t[1],[r12,#-10]
559 strb @x[$j+3],[r14,#-4]
560 eor @x[$j+2],@t[2],@x[$j+2],lsr#8
561 strb @x[$j+0],[r14,#-15]
562 eor @x[$j+3],@t[3],@x[$j+3],lsr#8
566 ldrhsb @t[2],[r12,#-6]
567 ldrhsb @t[3],[r12,#-2]
568 strb @x[$j+1],[r14,#-11]
569 eor @x[$j+0],@t[0],@x[$j+0],lsr#8
570 strb @x[$j+2],[r14,#-7]
571 eor @x[$j+1],@t[1],@x[$j+1],lsr#8
575 ldrhsb @t[0],[r12,#-13] @ load more input
576 ldrhsb @t[1],[r12,#-9]
577 strb @x[$j+3],[r14,#-3]
578 eor @x[$j+2],@t[2],@x[$j+2],lsr#8
579 strb @x[$j+0],[r14,#-14]
580 eor @x[$j+3],@t[3],@x[$j+3],lsr#8
584 ldrhsb @t[2],[r12,#-5]
585 ldrhsb @t[3],[r12,#-1]
586 strb @x[$j+1],[r14,#-10]
587 strb @x[$j+2],[r14,#-6]
588 eor @x[$j+0],@t[0],@x[$j+0],lsr#8
589 strb @x[$j+3],[r14,#-2]
590 eor @x[$j+1],@t[1],@x[$j+1],lsr#8
591 strb @x[$j+0],[r14,#-13]
592 eor @x[$j+2],@t[2],@x[$j+2],lsr#8
593 strb @x[$j+1],[r14,#-9]
594 eor @x[$j+3],@t[3],@x[$j+3],lsr#8
595 strb @x[$j+2],[r14,#-5]
596 strb @x[$j+3],[r14,#-1]
598 $code.=<<___ if ($i<12);
599 add @t[0],sp,#4*(4+$i)
600 ldmia @t[0],{@t[0]-@t[3]} @ load key material
607 ldrne @t[0],[sp,#4*(32+2)] @ re-load len
611 subhs @t[3],@t[0],#64 @ len-=64
618 ldr r12,[sp,#4*(32+1)] @ load inp
620 ldr r14,[sp,#4*(32+0)] @ load out
623 ldrb @t[2],[@t[1]],#1 @ read buffer on stack
624 ldrb @t[3],[r12],#1 @ read input
626 eor @t[3],@t[3],@t[2]
627 strb @t[3],[r14],#1 @ store output
633 ldmia sp!,{r4-r11,pc}
634 .size ChaCha20_ctr32,.-ChaCha20_ctr32
638 my ($a0,$b0,$c0,$d0,$a1,$b1,$c1,$d1,$a2,$b2,$c2,$d2,$t0,$t1,$t2,$t3) =
643 my ($a,$b,$c,$d,$t)=@_;
646 "&vadd_i32 ($a,$a,$b)",
648 "&vrev32_16 ($d,$d)", # vrot ($d,16)
650 "&vadd_i32 ($c,$c,$d)",
652 "&vshr_u32 ($b,$t,20)",
653 "&vsli_32 ($b,$t,12)",
655 "&vadd_i32 ($a,$a,$b)",
657 "&vshr_u32 ($d,$t,24)",
658 "&vsli_32 ($d,$t,8)",
660 "&vadd_i32 ($c,$c,$d)",
662 "&vshr_u32 ($b,$t,25)",
663 "&vsli_32 ($b,$t,7)",
665 "&vext_8 ($c,$c,$c,8)",
666 "&vext_8 ($b,$b,$b,$odd?12:4)",
667 "&vext_8 ($d,$d,$d,$odd?4:12)"
672 #if __ARM_MAX_ARCH__>=7
676 .type ChaCha20_neon,%function
679 ldr r12,[sp,#0] @ pull pointer to counter and nonce
680 stmdb sp!,{r0-r2,r4-r11,lr}
683 vstmdb sp!,{d8-d15} @ ABI spec says so
686 vld1.32 {$b0-$c0},[r3] @ load key
687 ldmia r3,{r4-r11} @ load key
690 vld1.32 {$d0},[r12] @ load counter and nonce
692 ldmia r14,{r0-r3} @ load sigma
693 vld1.32 {$a0},[r14]! @ load sigma
694 vld1.32 {$t0},[r14] @ one
695 vst1.32 {$c0-$d0},[r12] @ copy 1/2key|counter|nonce
696 vst1.32 {$a0-$b0},[sp] @ copy sigma|1/2key
698 str r10,[sp,#4*(16+10)] @ off-load "@x[10]"
699 str r11,[sp,#4*(16+11)] @ off-load "@x[11]"
700 vshl.i32 $t1#lo,$t0#lo,#1 @ two
701 vstr $t0#lo,[sp,#4*(16+0)]
702 vshl.i32 $t2#lo,$t0#lo,#2 @ four
703 vstr $t1#lo,[sp,#4*(16+2)]
705 vstr $t2#lo,[sp,#4*(16+4)]
713 ldmia sp,{r0-r9} @ load key material
714 cmp @t[3],#64*2 @ if len<=64*2
715 bls .Lbreak_neon @ switch to integer-only
717 str @t[3],[sp,#4*(32+2)] @ save len
719 str r12, [sp,#4*(32+1)] @ save inp
721 str r14, [sp,#4*(32+0)] @ save out
724 ldr @t[3], [sp,#4*(15)]
725 vadd.i32 $d1,$d0,$t0 @ counter+1
726 ldr @x[12],[sp,#4*(12)] @ modulo-scheduled load
728 ldr @t[2], [sp,#4*(13)]
730 ldr @x[14],[sp,#4*(14)]
731 vadd.i32 $d2,$d1,$t0 @ counter+2
732 str @t[3], [sp,#4*(16+15)]
734 add @x[12],@x[12],#3 @ counter+3
741 my @thread0=&NEONROUND($a0,$b0,$c0,$d0,$t0,0);
742 my @thread1=&NEONROUND($a1,$b1,$c1,$d1,$t1,0);
743 my @thread2=&NEONROUND($a2,$b2,$c2,$d2,$t2,0);
744 my @thread3=&ROUND(0,4,8,12);
747 eval; eval(shift(@thread3));
748 eval(shift(@thread1)); eval(shift(@thread3));
749 eval(shift(@thread2)); eval(shift(@thread3));
752 @thread0=&NEONROUND($a0,$b0,$c0,$d0,$t0,1);
753 @thread1=&NEONROUND($a1,$b1,$c1,$d1,$t1,1);
754 @thread2=&NEONROUND($a2,$b2,$c2,$d2,$t2,1);
755 @thread3=&ROUND(0,5,10,15);
758 eval; eval(shift(@thread3));
759 eval(shift(@thread1)); eval(shift(@thread3));
760 eval(shift(@thread2)); eval(shift(@thread3));
766 vld1.32 {$t0-$t1},[sp] @ load key material
767 vld1.32 {$t2-$t3},[@t[3]]
769 ldr @t[3],[sp,#4*(32+2)] @ load len
771 str @t[0], [sp,#4*(16+8)] @ modulo-scheduled store
772 str @t[1], [sp,#4*(16+9)]
773 str @x[12],[sp,#4*(16+12)]
774 str @t[2], [sp,#4*(16+13)]
775 str @x[14],[sp,#4*(16+14)]
777 @ at this point we have first half of 512-bit result in
778 @ @x[0-7] and second half at sp+4*(16+8)
780 ldr r12,[sp,#4*(32+1)] @ load inp
781 ldr r14,[sp,#4*(32+0)] @ load out
783 vadd.i32 $a0,$a0,$t0 @ accumulate key material
786 vldr $t0#lo,[sp,#4*(16+0)] @ one
791 vldr $t1#lo,[sp,#4*(16+2)] @ two
796 vadd.i32 $d1#lo,$d1#lo,$t0#lo @ counter+1
797 vadd.i32 $d2#lo,$d2#lo,$t1#lo @ counter+2
806 vld1.8 {$t0-$t1},[r12]! @ load input
808 vld1.8 {$t2-$t3},[r12]!
809 veor $a0,$a0,$t0 @ xor with input
811 vld1.8 {$t0-$t1},[r12]!
814 vld1.8 {$t2-$t3},[r12]!
817 vst1.8 {$a0-$b0},[r14]! @ store output
819 vld1.8 {$t0-$t1},[r12]!
821 vst1.8 {$c0-$d0},[r14]!
823 vld1.8 {$t2-$t3},[r12]!
826 vld1.32 {$a0-$b0},[@t[3]]! @ load for next iteration
827 veor $t0#hi,$t0#hi,$t0#hi
828 vldr $t0#lo,[sp,#4*(16+4)] @ four
830 vld1.32 {$c0-$d0},[@t[3]]
832 vst1.8 {$a1-$b1},[r14]!
834 vst1.8 {$c1-$d1},[r14]!
836 vadd.i32 $d0#lo,$d0#lo,$t0#lo @ next counter value
837 vldr $t0#lo,[sp,#4*(16+0)] @ one
839 ldmia sp,{@t[0]-@t[3]} @ load key material
840 add @x[0],@x[0],@t[0] @ accumulate key material
841 ldr @t[0],[r12],#16 @ load input
842 vst1.8 {$a2-$b2},[r14]!
843 add @x[1],@x[1],@t[1]
845 vst1.8 {$c2-$d2},[r14]!
846 add @x[2],@x[2],@t[2]
848 add @x[3],@x[3],@t[3]
856 eor @x[0],@x[0],@t[0] @ xor with input
858 eor @x[1],@x[1],@t[1]
859 str @x[0],[r14],#16 @ store output
860 eor @x[2],@x[2],@t[2]
862 eor @x[3],@x[3],@t[3]
863 ldmia @t[0],{@t[0]-@t[3]} @ load key material
867 add @x[4],@x[4],@t[0] @ accumulate key material
868 ldr @t[0],[r12],#16 @ load input
869 add @x[5],@x[5],@t[1]
871 add @x[6],@x[6],@t[2]
873 add @x[7],@x[7],@t[3]
881 eor @x[4],@x[4],@t[0]
883 eor @x[5],@x[5],@t[1]
884 str @x[4],[r14],#16 @ store output
885 eor @x[6],@x[6],@t[2]
887 eor @x[7],@x[7],@t[3]
888 ldmia @t[0],{@t[0]-@t[3]} @ load key material
890 add @x[0],sp,#4*(16+8)
893 ldmia @x[0],{@x[0]-@x[7]} @ load second half
895 add @x[0],@x[0],@t[0] @ accumulate key material
896 ldr @t[0],[r12],#16 @ load input
897 add @x[1],@x[1],@t[1]
902 strhi @t[2],[sp,#4*(16+10)] @ copy "@x[10]" while at it
903 add @x[2],@x[2],@t[2]
908 strhi @t[3],[sp,#4*(16+11)] @ copy "@x[11]" while at it
909 add @x[3],@x[3],@t[3]
917 eor @x[0],@x[0],@t[0]
919 eor @x[1],@x[1],@t[1]
920 str @x[0],[r14],#16 @ store output
921 eor @x[2],@x[2],@t[2]
923 eor @x[3],@x[3],@t[3]
924 ldmia @t[0],{@t[0]-@t[3]} @ load key material
928 add @x[4],@x[4],@t[0] @ accumulate key material
929 add @t[0],@t[0],#4 @ next counter value
930 add @x[5],@x[5],@t[1]
931 str @t[0],[sp,#4*(12)] @ save next counter value
932 ldr @t[0],[r12],#16 @ load input
933 add @x[6],@x[6],@t[2]
934 add @x[4],@x[4],#3 @ counter+3
936 add @x[7],@x[7],@t[3]
945 eor @x[4],@x[4],@t[0]
949 ldrhi @t[0],[sp,#4*(32+2)] @ re-load len
950 eor @x[5],@x[5],@t[1]
951 eor @x[6],@x[6],@t[2]
952 str @x[4],[r14],#16 @ store output
953 eor @x[7],@x[7],@t[3]
955 sub @t[3],@t[0],#64*4 @ len-=64*4
964 @ harmonize NEON and integer-only stack frames: load data
965 @ from NEON frame, but save to integer-only one; distance
966 @ between the two is 4*(32+4+16-32)=4*(20).
968 str @t[3], [sp,#4*(20+32+2)] @ save len
969 add @t[3],sp,#4*(32+4)
970 str r12, [sp,#4*(20+32+1)] @ save inp
971 str r14, [sp,#4*(20+32+0)] @ save out
973 ldr @x[12],[sp,#4*(16+10)]
974 ldr @x[14],[sp,#4*(16+11)]
975 vldmia @t[3],{d8-d15} @ fulfill ABI requirement
976 str @x[12],[sp,#4*(20+16+10)] @ copy "@x[10]"
977 str @x[14],[sp,#4*(20+16+11)] @ copy "@x[11]"
979 ldr @t[3], [sp,#4*(15)]
980 ldr @x[12],[sp,#4*(12)] @ modulo-scheduled load
981 ldr @t[2], [sp,#4*(13)]
982 ldr @x[14],[sp,#4*(14)]
983 str @t[3], [sp,#4*(20+16+15)]
985 vst1.32 {$a0-$b0},[@t[3]]! @ copy key
986 add sp,sp,#4*(20) @ switch frame
987 vst1.32 {$c0-$d0},[@t[3]]
989 b .Loop @ go integer-only
994 bhs .L192_or_more_neon
996 bhs .L128_or_more_neon
998 bhs .L64_or_more_neon
1001 vst1.8 {$a0-$b0},[sp]
1003 vst1.8 {$c0-$d0},[@t[0]]
1008 vld1.8 {$t0-$t1},[r12]!
1009 vld1.8 {$t2-$t3},[r12]!
1014 vst1.8 {$a0-$b0},[r14]!
1015 vst1.8 {$c0-$d0},[r14]!
1020 vst1.8 {$a1-$b1},[sp]
1022 vst1.8 {$c1-$d1},[@t[0]]
1023 sub @t[3],@t[3],#64*1 @ len-=64*1
1028 vld1.8 {$t0-$t1},[r12]!
1029 vld1.8 {$t2-$t3},[r12]!
1032 vld1.8 {$t0-$t1},[r12]!
1035 vld1.8 {$t2-$t3},[r12]!
1039 vst1.8 {$a0-$b0},[r14]!
1041 vst1.8 {$c0-$d0},[r14]!
1043 vst1.8 {$a1-$b1},[r14]!
1044 vst1.8 {$c1-$d1},[r14]!
1049 vst1.8 {$a2-$b2},[sp]
1051 vst1.8 {$c2-$d2},[@t[0]]
1052 sub @t[3],@t[3],#64*2 @ len-=64*2
1057 vld1.8 {$t0-$t1},[r12]!
1058 vld1.8 {$t2-$t3},[r12]!
1061 vld1.8 {$t0-$t1},[r12]!
1064 vld1.8 {$t2-$t3},[r12]!
1068 vld1.8 {$t0-$t1},[r12]!
1070 vst1.8 {$a0-$b0},[r14]!
1072 vld1.8 {$t2-$t3},[r12]!
1075 vst1.8 {$c0-$d0},[r14]!
1077 vst1.8 {$a1-$b1},[r14]!
1079 vst1.8 {$c1-$d1},[r14]!
1081 vst1.8 {$a2-$b2},[r14]!
1082 vst1.8 {$c2-$d2},[r14]!
1086 ldmia sp,{@t[0]-@t[3]} @ load key material
1087 add @x[0],@x[0],@t[0] @ accumulate key material
1089 add @x[1],@x[1],@t[1]
1090 add @x[2],@x[2],@t[2]
1091 add @x[3],@x[3],@t[3]
1092 ldmia @t[0],{@t[0]-@t[3]} @ load key material
1094 add @x[4],@x[4],@t[0] @ accumulate key material
1096 add @x[5],@x[5],@t[1]
1097 add @x[6],@x[6],@t[2]
1098 add @x[7],@x[7],@t[3]
1099 ldmia @t[0],{@t[0]-@t[3]} @ load key material
1110 stmia sp,{@x[0]-@x[7]}
1111 add @x[0],sp,#4*(16+8)
1113 ldmia @x[0],{@x[0]-@x[7]} @ load second half
1115 add @x[0],@x[0],@t[0] @ accumulate key material
1116 add @t[0],sp,#4*(12)
1117 add @x[1],@x[1],@t[1]
1118 add @x[2],@x[2],@t[2]
1119 add @x[3],@x[3],@t[3]
1120 ldmia @t[0],{@t[0]-@t[3]} @ load key material
1122 add @x[4],@x[4],@t[0] @ accumulate key material
1124 add @x[5],@x[5],@t[1]
1125 add @x[4],@x[4],#3 @ counter+3
1126 add @x[6],@x[6],@t[2]
1127 add @x[7],@x[7],@t[3]
1128 ldr @t[3],[sp,#4*(32+2)] @ re-load len
1139 stmia @t[0],{@x[0]-@x[7]}
1141 sub @t[3],@t[3],#64*3 @ len-=64*3
1144 ldrb @t[0],[@t[2]],#1 @ read buffer on stack
1145 ldrb @t[1],[r12],#1 @ read input
1147 eor @t[0],@t[0],@t[1]
1148 strb @t[0],[r14],#1 @ store output
1155 ldmia sp!,{r4-r11,pc}
1156 .size ChaCha20_neon,.-ChaCha20_neon
1157 .comm OPENSSL_armcap_P,4,4
1162 foreach (split("\n",$code)) {
1163 s/\`([^\`]*)\`/eval $1/geo;
1165 s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo;