Consistently use arm_arch.h constants in armcap assembly code.
[openssl.git] / crypto / bn / asm / armv4-mont.pl
1 #!/usr/bin/env perl
2
3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9
10 # January 2007.
11
12 # Montgomery multiplication for ARMv4.
13 #
14 # Performance improvement naturally varies among CPU implementations
15 # and compilers. The code was observed to provide +65-35% improvement
16 # [depending on key length, less for longer keys] on ARM920T, and
17 # +115-80% on Intel IXP425. This is compared to pre-bn_mul_mont code
18 # base and compiler generated code with in-lined umull and even umlal
19 # instructions. The latter means that this code didn't really have an 
20 # "advantage" of utilizing some "secret" instruction.
21 #
22 # The code is interoperable with Thumb ISA and is rather compact, less
23 # than 1/2KB. Windows CE port would be trivial, as it's exclusively
24 # about decorations, ABI and instruction syntax are identical.
25
26 # November 2013
27 #
28 # Add NEON code path, which handles lengths divisible by 8. RSA/DSA
29 # performance improvement on Cortex-A8 is ~45-100% depending on key
30 # length, more for longer keys. On Cortex-A15 the span is ~10-105%.
31 # On Snapdragon S4 improvement was measured to vary from ~70% to
32 # incredible ~380%, yes, 4.8x faster, for RSA4096 sign. But this is
33 # rather because original integer-only code seems to perform
34 # suboptimally on S4. Situation on Cortex-A9 is unfortunately
35 # different. It's being looked into, but the trouble is that
36 # performance for vectors longer than 256 bits is actually couple
37 # of percent worse than for integer-only code. The code is chosen
38 # for execution on all NEON-capable processors, because gain on
39 # others outweighs the marginal loss on Cortex-A9.
40
41 # September 2015
42 #
43 # Align Cortex-A9 performance with November 2013 improvements, i.e.
44 # NEON code is now ~20-105% faster than integer-only one on this
45 # processor. But this optimization further improved performance even
46 # on other processors: NEON code path is ~45-180% faster than original
47 # integer-only on Cortex-A8, ~10-210% on Cortex-A15, ~70-450% on
48 # Snapdragon S4.
49
50 $flavour = shift;
51 if ($flavour=~/^\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
52 else { while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} }
53
54 if ($flavour && $flavour ne "void") {
55     $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
56     ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
57     ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
58     die "can't locate arm-xlate.pl";
59
60     open STDOUT,"| \"$^X\" $xlate $flavour $output";
61 } else {
62     open STDOUT,">$output";
63 }
64
65 $num="r0";      # starts as num argument, but holds &tp[num-1]
66 $ap="r1";
67 $bp="r2"; $bi="r2"; $rp="r2";
68 $np="r3";
69 $tp="r4";
70 $aj="r5";
71 $nj="r6";
72 $tj="r7";
73 $n0="r8";
74 ###########     # r9 is reserved by ELF as platform specific, e.g. TLS pointer
75 $alo="r10";     # sl, gcc uses it to keep @GOT
76 $ahi="r11";     # fp
77 $nlo="r12";     # ip
78 ###########     # r13 is stack pointer
79 $nhi="r14";     # lr
80 ###########     # r15 is program counter
81
82 #### argument block layout relative to &tp[num-1], a.k.a. $num
83 $_rp="$num,#12*4";
84 # ap permanently resides in r1
85 $_bp="$num,#13*4";
86 # np permanently resides in r3
87 $_n0="$num,#14*4";
88 $_num="$num,#15*4";     $_bpend=$_num;
89
90 $code=<<___;
91 #include "arm_arch.h"
92
93 .text
94 #if defined(__thumb2__)
95 .syntax unified
96 .thumb
97 #else
98 .code   32
99 #endif
100
101 #if __ARM_MAX_ARCH__>=7
102 .align  5
103 .LOPENSSL_armcap:
104 .word   OPENSSL_armcap_P-.Lbn_mul_mont
105 #endif
106
107 .global bn_mul_mont
108 .type   bn_mul_mont,%function
109
110 .align  5
111 bn_mul_mont:
112 .Lbn_mul_mont:
113         ldr     ip,[sp,#4]              @ load num
114         stmdb   sp!,{r0,r2}             @ sp points at argument block
115 #if __ARM_MAX_ARCH__>=7
116         tst     ip,#7
117         bne     .Lialu
118         adr     r0,.Lbn_mul_mont
119         ldr     r2,.LOPENSSL_armcap
120         ldr     r0,[r0,r2]
121 #ifdef  __APPLE__
122         ldr     r0,[r0]
123 #endif
124         tst     r0,#ARMV7_NEON          @ NEON available?
125         ldmia   sp, {r0,r2}
126         beq     .Lialu
127         add     sp,sp,#8
128         b       bn_mul8x_mont_neon
129 .align  4
130 .Lialu:
131 #endif
132         cmp     ip,#2
133         mov     $num,ip                 @ load num
134 #ifdef  __thumb2__
135         ittt    lt
136 #endif
137         movlt   r0,#0
138         addlt   sp,sp,#2*4
139         blt     .Labrt
140
141         stmdb   sp!,{r4-r12,lr}         @ save 10 registers
142
143         mov     $num,$num,lsl#2         @ rescale $num for byte count
144         sub     sp,sp,$num              @ alloca(4*num)
145         sub     sp,sp,#4                @ +extra dword
146         sub     $num,$num,#4            @ "num=num-1"
147         add     $tp,$bp,$num            @ &bp[num-1]
148
149         add     $num,sp,$num            @ $num to point at &tp[num-1]
150         ldr     $n0,[$_n0]              @ &n0
151         ldr     $bi,[$bp]               @ bp[0]
152         ldr     $aj,[$ap],#4            @ ap[0],ap++
153         ldr     $nj,[$np],#4            @ np[0],np++
154         ldr     $n0,[$n0]               @ *n0
155         str     $tp,[$_bpend]           @ save &bp[num]
156
157         umull   $alo,$ahi,$aj,$bi       @ ap[0]*bp[0]
158         str     $n0,[$_n0]              @ save n0 value
159         mul     $n0,$alo,$n0            @ "tp[0]"*n0
160         mov     $nlo,#0
161         umlal   $alo,$nlo,$nj,$n0       @ np[0]*n0+"t[0]"
162         mov     $tp,sp
163
164 .L1st:
165         ldr     $aj,[$ap],#4            @ ap[j],ap++
166         mov     $alo,$ahi
167         ldr     $nj,[$np],#4            @ np[j],np++
168         mov     $ahi,#0
169         umlal   $alo,$ahi,$aj,$bi       @ ap[j]*bp[0]
170         mov     $nhi,#0
171         umlal   $nlo,$nhi,$nj,$n0       @ np[j]*n0
172         adds    $nlo,$nlo,$alo
173         str     $nlo,[$tp],#4           @ tp[j-1]=,tp++
174         adc     $nlo,$nhi,#0
175         cmp     $tp,$num
176         bne     .L1st
177
178         adds    $nlo,$nlo,$ahi
179         ldr     $tp,[$_bp]              @ restore bp
180         mov     $nhi,#0
181         ldr     $n0,[$_n0]              @ restore n0
182         adc     $nhi,$nhi,#0
183         str     $nlo,[$num]             @ tp[num-1]=
184         mov     $tj,sp
185         str     $nhi,[$num,#4]          @ tp[num]=
186 \f
187 .Louter:
188         sub     $tj,$num,$tj            @ "original" $num-1 value
189         sub     $ap,$ap,$tj             @ "rewind" ap to &ap[1]
190         ldr     $bi,[$tp,#4]!           @ *(++bp)
191         sub     $np,$np,$tj             @ "rewind" np to &np[1]
192         ldr     $aj,[$ap,#-4]           @ ap[0]
193         ldr     $alo,[sp]               @ tp[0]
194         ldr     $nj,[$np,#-4]           @ np[0]
195         ldr     $tj,[sp,#4]             @ tp[1]
196
197         mov     $ahi,#0
198         umlal   $alo,$ahi,$aj,$bi       @ ap[0]*bp[i]+tp[0]
199         str     $tp,[$_bp]              @ save bp
200         mul     $n0,$alo,$n0
201         mov     $nlo,#0
202         umlal   $alo,$nlo,$nj,$n0       @ np[0]*n0+"tp[0]"
203         mov     $tp,sp
204
205 .Linner:
206         ldr     $aj,[$ap],#4            @ ap[j],ap++
207         adds    $alo,$ahi,$tj           @ +=tp[j]
208         ldr     $nj,[$np],#4            @ np[j],np++
209         mov     $ahi,#0
210         umlal   $alo,$ahi,$aj,$bi       @ ap[j]*bp[i]
211         mov     $nhi,#0
212         umlal   $nlo,$nhi,$nj,$n0       @ np[j]*n0
213         adc     $ahi,$ahi,#0
214         ldr     $tj,[$tp,#8]            @ tp[j+1]
215         adds    $nlo,$nlo,$alo
216         str     $nlo,[$tp],#4           @ tp[j-1]=,tp++
217         adc     $nlo,$nhi,#0
218         cmp     $tp,$num
219         bne     .Linner
220
221         adds    $nlo,$nlo,$ahi
222         mov     $nhi,#0
223         ldr     $tp,[$_bp]              @ restore bp
224         adc     $nhi,$nhi,#0
225         ldr     $n0,[$_n0]              @ restore n0
226         adds    $nlo,$nlo,$tj
227         ldr     $tj,[$_bpend]           @ restore &bp[num]
228         adc     $nhi,$nhi,#0
229         str     $nlo,[$num]             @ tp[num-1]=
230         str     $nhi,[$num,#4]          @ tp[num]=
231
232         cmp     $tp,$tj
233 #ifdef  __thumb2__
234         itt     ne
235 #endif
236         movne   $tj,sp
237         bne     .Louter
238 \f
239         ldr     $rp,[$_rp]              @ pull rp
240         mov     $aj,sp
241         add     $num,$num,#4            @ $num to point at &tp[num]
242         sub     $aj,$num,$aj            @ "original" num value
243         mov     $tp,sp                  @ "rewind" $tp
244         mov     $ap,$tp                 @ "borrow" $ap
245         sub     $np,$np,$aj             @ "rewind" $np to &np[0]
246
247         subs    $tj,$tj,$tj             @ "clear" carry flag
248 .Lsub:  ldr     $tj,[$tp],#4
249         ldr     $nj,[$np],#4
250         sbcs    $tj,$tj,$nj             @ tp[j]-np[j]
251         str     $tj,[$rp],#4            @ rp[j]=
252         teq     $tp,$num                @ preserve carry
253         bne     .Lsub
254         sbcs    $nhi,$nhi,#0            @ upmost carry
255         mov     $tp,sp                  @ "rewind" $tp
256         sub     $rp,$rp,$aj             @ "rewind" $rp
257
258         and     $ap,$tp,$nhi
259         bic     $np,$rp,$nhi
260         orr     $ap,$ap,$np             @ ap=borrow?tp:rp
261
262 .Lcopy: ldr     $tj,[$ap],#4            @ copy or in-place refresh
263         str     sp,[$tp],#4             @ zap tp
264         str     $tj,[$rp],#4
265         cmp     $tp,$num
266         bne     .Lcopy
267
268         mov     sp,$num
269         add     sp,sp,#4                @ skip over tp[num+1]
270         ldmia   sp!,{r4-r12,lr}         @ restore registers
271         add     sp,sp,#2*4              @ skip over {r0,r2}
272         mov     r0,#1
273 .Labrt:
274 #if __ARM_ARCH__>=5
275         ret                             @ bx lr
276 #else
277         tst     lr,#1
278         moveq   pc,lr                   @ be binary compatible with V4, yet
279         bx      lr                      @ interoperable with Thumb ISA:-)
280 #endif
281 .size   bn_mul_mont,.-bn_mul_mont
282 ___
283 {
284 my ($A0,$A1,$A2,$A3)=map("d$_",(0..3));
285 my ($N0,$N1,$N2,$N3)=map("d$_",(4..7));
286 my ($Z,$Temp)=("q4","q5");
287 my @ACC=map("q$_",(6..13));
288 my ($Bi,$Ni,$M0)=map("d$_",(28..31));
289 my $zero="$Z#lo";
290 my $temp="$Temp#lo";
291
292 my ($rptr,$aptr,$bptr,$nptr,$n0,$num)=map("r$_",(0..5));
293 my ($tinptr,$toutptr,$inner,$outer,$bnptr)=map("r$_",(6..11));
294
295 $code.=<<___;
296 #if __ARM_MAX_ARCH__>=7
297 .arch   armv7-a
298 .fpu    neon
299
300 .type   bn_mul8x_mont_neon,%function
301 .align  5
302 bn_mul8x_mont_neon:
303         mov     ip,sp
304         stmdb   sp!,{r4-r11}
305         vstmdb  sp!,{d8-d15}            @ ABI specification says so
306         ldmia   ip,{r4-r5}              @ load rest of parameter block
307         mov     ip,sp
308
309         cmp     $num,#8
310         bhi     .LNEON_8n
311
312         @ special case for $num==8, everything is in register bank...
313
314         vld1.32         {${Bi}[0]}, [$bptr,:32]!
315         veor            $zero,$zero,$zero
316         sub             $toutptr,sp,$num,lsl#4
317         vld1.32         {$A0-$A3},  [$aptr]!            @ can't specify :32 :-(
318         and             $toutptr,$toutptr,#-64
319         vld1.32         {${M0}[0]}, [$n0,:32]
320         mov             sp,$toutptr                     @ alloca
321         vzip.16         $Bi,$zero
322
323         vmull.u32       @ACC[0],$Bi,${A0}[0]
324         vmull.u32       @ACC[1],$Bi,${A0}[1]
325         vmull.u32       @ACC[2],$Bi,${A1}[0]
326         vshl.i64        $Ni,@ACC[0]#hi,#16
327         vmull.u32       @ACC[3],$Bi,${A1}[1]
328
329         vadd.u64        $Ni,$Ni,@ACC[0]#lo
330         veor            $zero,$zero,$zero
331         vmul.u32        $Ni,$Ni,$M0
332
333         vmull.u32       @ACC[4],$Bi,${A2}[0]
334          vld1.32        {$N0-$N3}, [$nptr]!
335         vmull.u32       @ACC[5],$Bi,${A2}[1]
336         vmull.u32       @ACC[6],$Bi,${A3}[0]
337         vzip.16         $Ni,$zero
338         vmull.u32       @ACC[7],$Bi,${A3}[1]
339
340         vmlal.u32       @ACC[0],$Ni,${N0}[0]
341         sub             $outer,$num,#1
342         vmlal.u32       @ACC[1],$Ni,${N0}[1]
343         vmlal.u32       @ACC[2],$Ni,${N1}[0]
344         vmlal.u32       @ACC[3],$Ni,${N1}[1]
345
346         vmlal.u32       @ACC[4],$Ni,${N2}[0]
347         vmov            $Temp,@ACC[0]
348         vmlal.u32       @ACC[5],$Ni,${N2}[1]
349         vmov            @ACC[0],@ACC[1]
350         vmlal.u32       @ACC[6],$Ni,${N3}[0]
351         vmov            @ACC[1],@ACC[2]
352         vmlal.u32       @ACC[7],$Ni,${N3}[1]
353         vmov            @ACC[2],@ACC[3]
354         vmov            @ACC[3],@ACC[4]
355         vshr.u64        $temp,$temp,#16
356         vmov            @ACC[4],@ACC[5]
357         vmov            @ACC[5],@ACC[6]
358         vadd.u64        $temp,$temp,$Temp#hi
359         vmov            @ACC[6],@ACC[7]
360         veor            @ACC[7],@ACC[7]
361         vshr.u64        $temp,$temp,#16
362
363         b       .LNEON_outer8
364
365 .align  4
366 .LNEON_outer8:
367         vld1.32         {${Bi}[0]}, [$bptr,:32]!
368         veor            $zero,$zero,$zero
369         vzip.16         $Bi,$zero
370         vadd.u64        @ACC[0]#lo,@ACC[0]#lo,$temp
371
372         vmlal.u32       @ACC[0],$Bi,${A0}[0]
373         vmlal.u32       @ACC[1],$Bi,${A0}[1]
374         vmlal.u32       @ACC[2],$Bi,${A1}[0]
375         vshl.i64        $Ni,@ACC[0]#hi,#16
376         vmlal.u32       @ACC[3],$Bi,${A1}[1]
377
378         vadd.u64        $Ni,$Ni,@ACC[0]#lo
379         veor            $zero,$zero,$zero
380         subs            $outer,$outer,#1
381         vmul.u32        $Ni,$Ni,$M0
382
383         vmlal.u32       @ACC[4],$Bi,${A2}[0]
384         vmlal.u32       @ACC[5],$Bi,${A2}[1]
385         vmlal.u32       @ACC[6],$Bi,${A3}[0]
386         vzip.16         $Ni,$zero
387         vmlal.u32       @ACC[7],$Bi,${A3}[1]
388
389         vmlal.u32       @ACC[0],$Ni,${N0}[0]
390         vmlal.u32       @ACC[1],$Ni,${N0}[1]
391         vmlal.u32       @ACC[2],$Ni,${N1}[0]
392         vmlal.u32       @ACC[3],$Ni,${N1}[1]
393
394         vmlal.u32       @ACC[4],$Ni,${N2}[0]
395         vmov            $Temp,@ACC[0]
396         vmlal.u32       @ACC[5],$Ni,${N2}[1]
397         vmov            @ACC[0],@ACC[1]
398         vmlal.u32       @ACC[6],$Ni,${N3}[0]
399         vmov            @ACC[1],@ACC[2]
400         vmlal.u32       @ACC[7],$Ni,${N3}[1]
401         vmov            @ACC[2],@ACC[3]
402         vmov            @ACC[3],@ACC[4]
403         vshr.u64        $temp,$temp,#16
404         vmov            @ACC[4],@ACC[5]
405         vmov            @ACC[5],@ACC[6]
406         vadd.u64        $temp,$temp,$Temp#hi
407         vmov            @ACC[6],@ACC[7]
408         veor            @ACC[7],@ACC[7]
409         vshr.u64        $temp,$temp,#16
410
411         bne     .LNEON_outer8
412
413         vadd.u64        @ACC[0]#lo,@ACC[0]#lo,$temp
414         mov             $toutptr,sp
415         vshr.u64        $temp,@ACC[0]#lo,#16
416         mov             $inner,$num
417         vadd.u64        @ACC[0]#hi,@ACC[0]#hi,$temp
418         add             $tinptr,sp,#96
419         vshr.u64        $temp,@ACC[0]#hi,#16
420         vzip.16         @ACC[0]#lo,@ACC[0]#hi
421
422         b       .LNEON_tail_entry
423
424 .align  4
425 .LNEON_8n:
426         veor            @ACC[0],@ACC[0],@ACC[0]
427          sub            $toutptr,sp,#128
428         veor            @ACC[1],@ACC[1],@ACC[1]
429          sub            $toutptr,$toutptr,$num,lsl#4
430         veor            @ACC[2],@ACC[2],@ACC[2]
431          and            $toutptr,$toutptr,#-64
432         veor            @ACC[3],@ACC[3],@ACC[3]
433          mov            sp,$toutptr                     @ alloca
434         veor            @ACC[4],@ACC[4],@ACC[4]
435          add            $toutptr,$toutptr,#256
436         veor            @ACC[5],@ACC[5],@ACC[5]
437          sub            $inner,$num,#8
438         veor            @ACC[6],@ACC[6],@ACC[6]
439         veor            @ACC[7],@ACC[7],@ACC[7]
440
441 .LNEON_8n_init:
442         vst1.64         {@ACC[0]-@ACC[1]},[$toutptr,:256]!
443         subs            $inner,$inner,#8
444         vst1.64         {@ACC[2]-@ACC[3]},[$toutptr,:256]!
445         vst1.64         {@ACC[4]-@ACC[5]},[$toutptr,:256]!
446         vst1.64         {@ACC[6]-@ACC[7]},[$toutptr,:256]!
447         bne             .LNEON_8n_init
448
449         add             $tinptr,sp,#256
450         vld1.32         {$A0-$A3},[$aptr]!
451         add             $bnptr,sp,#8
452         vld1.32         {${M0}[0]},[$n0,:32]
453         mov             $outer,$num
454         b               .LNEON_8n_outer
455
456 .align  4
457 .LNEON_8n_outer:
458         vld1.32         {${Bi}[0]},[$bptr,:32]! @ *b++
459         veor            $zero,$zero,$zero
460         vzip.16         $Bi,$zero
461         add             $toutptr,sp,#128
462         vld1.32         {$N0-$N3},[$nptr]!
463
464         vmlal.u32       @ACC[0],$Bi,${A0}[0]
465         vmlal.u32       @ACC[1],$Bi,${A0}[1]
466          veor           $zero,$zero,$zero
467         vmlal.u32       @ACC[2],$Bi,${A1}[0]
468          vshl.i64       $Ni,@ACC[0]#hi,#16
469         vmlal.u32       @ACC[3],$Bi,${A1}[1]
470          vadd.u64       $Ni,$Ni,@ACC[0]#lo
471         vmlal.u32       @ACC[4],$Bi,${A2}[0]
472          vmul.u32       $Ni,$Ni,$M0
473         vmlal.u32       @ACC[5],$Bi,${A2}[1]
474         vst1.32         {$Bi},[sp,:64]          @ put aside smashed b[8*i+0]
475         vmlal.u32       @ACC[6],$Bi,${A3}[0]
476          vzip.16        $Ni,$zero
477         vmlal.u32       @ACC[7],$Bi,${A3}[1]
478 ___
479 for ($i=0; $i<7;) {
480 $code.=<<___;
481         vld1.32         {${Bi}[0]},[$bptr,:32]! @ *b++
482         vmlal.u32       @ACC[0],$Ni,${N0}[0]
483         veor            $temp,$temp,$temp
484         vmlal.u32       @ACC[1],$Ni,${N0}[1]
485         vzip.16         $Bi,$temp
486         vmlal.u32       @ACC[2],$Ni,${N1}[0]
487          vshr.u64       @ACC[0]#lo,@ACC[0]#lo,#16
488         vmlal.u32       @ACC[3],$Ni,${N1}[1]
489         vmlal.u32       @ACC[4],$Ni,${N2}[0]
490          vadd.u64       @ACC[0]#lo,@ACC[0]#lo,@ACC[0]#hi
491         vmlal.u32       @ACC[5],$Ni,${N2}[1]
492          vshr.u64       @ACC[0]#lo,@ACC[0]#lo,#16
493         vmlal.u32       @ACC[6],$Ni,${N3}[0]
494         vmlal.u32       @ACC[7],$Ni,${N3}[1]
495          vadd.u64       @ACC[1]#lo,@ACC[1]#lo,@ACC[0]#lo
496         vst1.32         {$Ni},[$bnptr,:64]!     @ put aside smashed m[8*i+$i]
497 ___
498         push(@ACC,shift(@ACC)); $i++;
499 $code.=<<___;
500         vmlal.u32       @ACC[0],$Bi,${A0}[0]
501         vld1.64         {@ACC[7]},[$tinptr,:128]!
502         vmlal.u32       @ACC[1],$Bi,${A0}[1]
503          veor           $zero,$zero,$zero
504         vmlal.u32       @ACC[2],$Bi,${A1}[0]
505          vshl.i64       $Ni,@ACC[0]#hi,#16
506         vmlal.u32       @ACC[3],$Bi,${A1}[1]
507          vadd.u64       $Ni,$Ni,@ACC[0]#lo
508         vmlal.u32       @ACC[4],$Bi,${A2}[0]
509          vmul.u32       $Ni,$Ni,$M0
510         vmlal.u32       @ACC[5],$Bi,${A2}[1]
511         vst1.32         {$Bi},[$bnptr,:64]!     @ put aside smashed b[8*i+$i]
512         vmlal.u32       @ACC[6],$Bi,${A3}[0]
513          vzip.16        $Ni,$zero
514         vmlal.u32       @ACC[7],$Bi,${A3}[1]
515 ___
516 }
517 $code.=<<___;
518         vld1.32         {$Bi},[sp,:64]          @ pull smashed b[8*i+0]
519         vmlal.u32       @ACC[0],$Ni,${N0}[0]
520         vld1.32         {$A0-$A3},[$aptr]!
521         vmlal.u32       @ACC[1],$Ni,${N0}[1]
522         vmlal.u32       @ACC[2],$Ni,${N1}[0]
523          vshr.u64       @ACC[0]#lo,@ACC[0]#lo,#16
524         vmlal.u32       @ACC[3],$Ni,${N1}[1]
525         vmlal.u32       @ACC[4],$Ni,${N2}[0]
526          vadd.u64       @ACC[0]#lo,@ACC[0]#lo,@ACC[0]#hi
527         vmlal.u32       @ACC[5],$Ni,${N2}[1]
528          vshr.u64       @ACC[0]#lo,@ACC[0]#lo,#16
529         vmlal.u32       @ACC[6],$Ni,${N3}[0]
530         vmlal.u32       @ACC[7],$Ni,${N3}[1]
531          vadd.u64       @ACC[1]#lo,@ACC[1]#lo,@ACC[0]#lo
532         vst1.32         {$Ni},[$bnptr,:64]      @ put aside smashed m[8*i+$i]
533         add             $bnptr,sp,#8            @ rewind
534 ___
535         push(@ACC,shift(@ACC));
536 $code.=<<___;
537         sub             $inner,$num,#8
538         b               .LNEON_8n_inner
539
540 .align  4
541 .LNEON_8n_inner:
542         subs            $inner,$inner,#8
543         vmlal.u32       @ACC[0],$Bi,${A0}[0]
544         vld1.64         {@ACC[7]},[$tinptr,:128]
545         vmlal.u32       @ACC[1],$Bi,${A0}[1]
546         vld1.32         {$Ni},[$bnptr,:64]!     @ pull smashed m[8*i+0]
547         vmlal.u32       @ACC[2],$Bi,${A1}[0]
548         vld1.32         {$N0-$N3},[$nptr]!
549         vmlal.u32       @ACC[3],$Bi,${A1}[1]
550         it              ne
551         addne           $tinptr,$tinptr,#16     @ don't advance in last iteration
552         vmlal.u32       @ACC[4],$Bi,${A2}[0]
553         vmlal.u32       @ACC[5],$Bi,${A2}[1]
554         vmlal.u32       @ACC[6],$Bi,${A3}[0]
555         vmlal.u32       @ACC[7],$Bi,${A3}[1]
556 ___
557 for ($i=1; $i<8; $i++) {
558 $code.=<<___;
559         vld1.32         {$Bi},[$bnptr,:64]!     @ pull smashed b[8*i+$i]
560         vmlal.u32       @ACC[0],$Ni,${N0}[0]
561         vmlal.u32       @ACC[1],$Ni,${N0}[1]
562         vmlal.u32       @ACC[2],$Ni,${N1}[0]
563         vmlal.u32       @ACC[3],$Ni,${N1}[1]
564         vmlal.u32       @ACC[4],$Ni,${N2}[0]
565         vmlal.u32       @ACC[5],$Ni,${N2}[1]
566         vmlal.u32       @ACC[6],$Ni,${N3}[0]
567         vmlal.u32       @ACC[7],$Ni,${N3}[1]
568         vst1.64         {@ACC[0]},[$toutptr,:128]!
569 ___
570         push(@ACC,shift(@ACC));
571 $code.=<<___;
572         vmlal.u32       @ACC[0],$Bi,${A0}[0]
573         vld1.64         {@ACC[7]},[$tinptr,:128]
574         vmlal.u32       @ACC[1],$Bi,${A0}[1]
575         vld1.32         {$Ni},[$bnptr,:64]!     @ pull smashed m[8*i+$i]
576         vmlal.u32       @ACC[2],$Bi,${A1}[0]
577         it              ne
578         addne           $tinptr,$tinptr,#16     @ don't advance in last iteration
579         vmlal.u32       @ACC[3],$Bi,${A1}[1]
580         vmlal.u32       @ACC[4],$Bi,${A2}[0]
581         vmlal.u32       @ACC[5],$Bi,${A2}[1]
582         vmlal.u32       @ACC[6],$Bi,${A3}[0]
583         vmlal.u32       @ACC[7],$Bi,${A3}[1]
584 ___
585 }
586 $code.=<<___;
587         it              eq
588         subeq           $aptr,$aptr,$num,lsl#2  @ rewind
589         vmlal.u32       @ACC[0],$Ni,${N0}[0]
590         vld1.32         {$Bi},[sp,:64]          @ pull smashed b[8*i+0]
591         vmlal.u32       @ACC[1],$Ni,${N0}[1]
592         vld1.32         {$A0-$A3},[$aptr]!
593         vmlal.u32       @ACC[2],$Ni,${N1}[0]
594         add             $bnptr,sp,#8            @ rewind
595         vmlal.u32       @ACC[3],$Ni,${N1}[1]
596         vmlal.u32       @ACC[4],$Ni,${N2}[0]
597         vmlal.u32       @ACC[5],$Ni,${N2}[1]
598         vmlal.u32       @ACC[6],$Ni,${N3}[0]
599         vst1.64         {@ACC[0]},[$toutptr,:128]!
600         vmlal.u32       @ACC[7],$Ni,${N3}[1]
601
602         bne             .LNEON_8n_inner
603 ___
604         push(@ACC,shift(@ACC));
605 $code.=<<___;
606         add             $tinptr,sp,#128
607         vst1.64         {@ACC[0]-@ACC[1]},[$toutptr,:256]!
608         veor            q2,q2,q2                @ $N0-$N1
609         vst1.64         {@ACC[2]-@ACC[3]},[$toutptr,:256]!
610         veor            q3,q3,q3                @ $N2-$N3
611         vst1.64         {@ACC[4]-@ACC[5]},[$toutptr,:256]!
612         vst1.64         {@ACC[6]},[$toutptr,:128]
613
614         subs            $outer,$outer,#8
615         vld1.64         {@ACC[0]-@ACC[1]},[$tinptr,:256]!
616         vld1.64         {@ACC[2]-@ACC[3]},[$tinptr,:256]!
617         vld1.64         {@ACC[4]-@ACC[5]},[$tinptr,:256]!
618         vld1.64         {@ACC[6]-@ACC[7]},[$tinptr,:256]!
619
620         itt             ne
621         subne           $nptr,$nptr,$num,lsl#2  @ rewind
622         bne             .LNEON_8n_outer
623
624         add             $toutptr,sp,#128
625         vst1.64         {q2-q3}, [sp,:256]!     @ start wiping stack frame
626         vshr.u64        $temp,@ACC[0]#lo,#16
627         vst1.64         {q2-q3},[sp,:256]!
628         vadd.u64        @ACC[0]#hi,@ACC[0]#hi,$temp
629         vst1.64         {q2-q3}, [sp,:256]!
630         vshr.u64        $temp,@ACC[0]#hi,#16
631         vst1.64         {q2-q3}, [sp,:256]!
632         vzip.16         @ACC[0]#lo,@ACC[0]#hi
633
634         mov             $inner,$num
635         b               .LNEON_tail_entry
636
637 .align  4
638 .LNEON_tail:
639         vadd.u64        @ACC[0]#lo,@ACC[0]#lo,$temp
640         vshr.u64        $temp,@ACC[0]#lo,#16
641         vld1.64         {@ACC[2]-@ACC[3]}, [$tinptr, :256]!
642         vadd.u64        @ACC[0]#hi,@ACC[0]#hi,$temp
643         vld1.64         {@ACC[4]-@ACC[5]}, [$tinptr, :256]!
644         vshr.u64        $temp,@ACC[0]#hi,#16
645         vld1.64         {@ACC[6]-@ACC[7]}, [$tinptr, :256]!
646         vzip.16         @ACC[0]#lo,@ACC[0]#hi
647
648 .LNEON_tail_entry:
649 ___
650 for ($i=1; $i<8; $i++) {
651 $code.=<<___;
652         vadd.u64        @ACC[1]#lo,@ACC[1]#lo,$temp
653         vst1.32         {@ACC[0]#lo[0]}, [$toutptr, :32]!
654         vshr.u64        $temp,@ACC[1]#lo,#16
655         vadd.u64        @ACC[1]#hi,@ACC[1]#hi,$temp
656         vshr.u64        $temp,@ACC[1]#hi,#16
657         vzip.16         @ACC[1]#lo,@ACC[1]#hi
658 ___
659         push(@ACC,shift(@ACC));
660 }
661         push(@ACC,shift(@ACC));
662 $code.=<<___;
663         vld1.64         {@ACC[0]-@ACC[1]}, [$tinptr, :256]!
664         subs            $inner,$inner,#8
665         vst1.32         {@ACC[7]#lo[0]},   [$toutptr, :32]!
666         bne     .LNEON_tail
667
668         vst1.32 {${temp}[0]}, [$toutptr, :32]           @ top-most bit
669         sub     $nptr,$nptr,$num,lsl#2                  @ rewind $nptr
670         subs    $aptr,sp,#0                             @ clear carry flag
671         add     $bptr,sp,$num,lsl#2
672
673 .LNEON_sub:
674         ldmia   $aptr!, {r4-r7}
675         ldmia   $nptr!, {r8-r11}
676         sbcs    r8, r4,r8
677         sbcs    r9, r5,r9
678         sbcs    r10,r6,r10
679         sbcs    r11,r7,r11
680         teq     $aptr,$bptr                             @ preserves carry
681         stmia   $rptr!, {r8-r11}
682         bne     .LNEON_sub
683
684         ldr     r10, [$aptr]                            @ load top-most bit
685         mov     r11,sp
686         veor    q0,q0,q0
687         sub     r11,$bptr,r11                           @ this is num*4
688         veor    q1,q1,q1
689         mov     $aptr,sp
690         sub     $rptr,$rptr,r11                         @ rewind $rptr
691         mov     $nptr,$bptr                             @ second 3/4th of frame
692         sbcs    r10,r10,#0                              @ result is carry flag
693
694 .LNEON_copy_n_zap:
695         ldmia   $aptr!, {r4-r7}
696         ldmia   $rptr,  {r8-r11}
697         it      cc
698         movcc   r8, r4
699         vst1.64 {q0-q1}, [$nptr,:256]!                  @ wipe
700         itt     cc
701         movcc   r9, r5
702         movcc   r10,r6
703         vst1.64 {q0-q1}, [$nptr,:256]!                  @ wipe
704         it      cc
705         movcc   r11,r7
706         ldmia   $aptr, {r4-r7}
707         stmia   $rptr!, {r8-r11}
708         sub     $aptr,$aptr,#16
709         ldmia   $rptr, {r8-r11}
710         it      cc
711         movcc   r8, r4
712         vst1.64 {q0-q1}, [$aptr,:256]!                  @ wipe
713         itt     cc
714         movcc   r9, r5
715         movcc   r10,r6
716         vst1.64 {q0-q1}, [$nptr,:256]!                  @ wipe
717         it      cc
718         movcc   r11,r7
719         teq     $aptr,$bptr                             @ preserves carry
720         stmia   $rptr!, {r8-r11}
721         bne     .LNEON_copy_n_zap
722
723         mov     sp,ip
724         vldmia  sp!,{d8-d15}
725         ldmia   sp!,{r4-r11}
726         ret                                             @ bx lr
727 .size   bn_mul8x_mont_neon,.-bn_mul8x_mont_neon
728 #endif
729 ___
730 }
731 $code.=<<___;
732 .asciz  "Montgomery multiplication for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
733 .align  2
734 #if __ARM_MAX_ARCH__>=7
735 .comm   OPENSSL_armcap_P,4,4
736 #endif
737 ___
738
739 foreach (split("\n",$code)) {
740         s/\`([^\`]*)\`/eval $1/ge;
741
742         s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/ge        or
743         s/\bret\b/bx    lr/g                                            or
744         s/\bbx\s+lr\b/.word\t0xe12fff1e/g;      # make it possible to compile with -march=armv4
745
746         print $_,"\n";
747 }
748
749 close STDOUT;