sha/asm/sha256-armv4.pl: one of "universal" flags combination didn't compile.
[openssl.git] / crypto / sha / asm / sha256-armv4.pl
1 #!/usr/bin/env perl
2
3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 #
9 # Permission to use under GPL terms is granted.
10 # ====================================================================
11
12 # SHA256 block procedure for ARMv4. May 2007.
13
14 # Performance is ~2x better than gcc 3.4 generated code and in "abso-
15 # lute" terms is ~2250 cycles per 64-byte block or ~35 cycles per
16 # byte [on single-issue Xscale PXA250 core].
17
18 # July 2010.
19 #
20 # Rescheduling for dual-issue pipeline resulted in 22% improvement on
21 # Cortex A8 core and ~20 cycles per processed byte.
22
23 # February 2011.
24 #
25 # Profiler-assisted and platform-specific optimization resulted in 16%
26 # improvement on Cortex A8 core and ~15.4 cycles per processed byte.
27
28 # September 2013.
29 #
30 # Add NEON implementation. On Cortex A8 it was measured to process one
31 # byte in 12.5 cycles or 23% faster than integer-only code. Snapdragon
32 # S4 does it in 12.5 cycles too, but it's 50% faster than integer-only
33 # code (meaning that latter performs sub-optimally, nothing was done
34 # about it).
35
36 # May 2014.
37 #
38 # Add ARMv8 code path performing at 2.0 cpb on Apple A7.
39
40 $flavour = shift;
41 if ($flavour=~/^\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
42 else { while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} }
43
44 if ($flavour && $flavour ne "void") {
45     $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
46     ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
47     ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
48     die "can't locate arm-xlate.pl";
49
50     open STDOUT,"| \"$^X\" $xlate $flavour $output";
51 } else {
52     open STDOUT,">$output";
53 }
54
55 $ctx="r0";      $t0="r0";
56 $inp="r1";      $t4="r1";
57 $len="r2";      $t1="r2";
58 $T1="r3";       $t3="r3";
59 $A="r4";
60 $B="r5";
61 $C="r6";
62 $D="r7";
63 $E="r8";
64 $F="r9";
65 $G="r10";
66 $H="r11";
67 @V=($A,$B,$C,$D,$E,$F,$G,$H);
68 $t2="r12";
69 $Ktbl="r14";
70
71 @Sigma0=( 2,13,22);
72 @Sigma1=( 6,11,25);
73 @sigma0=( 7,18, 3);
74 @sigma1=(17,19,10);
75
76 sub BODY_00_15 {
77 my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
78
79 $code.=<<___ if ($i<16);
80 #if __ARM_ARCH__>=7
81         @ ldr   $t1,[$inp],#4                   @ $i
82 # if $i==15
83         str     $inp,[sp,#17*4]                 @ make room for $t4
84 # endif
85         eor     $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
86         add     $a,$a,$t2                       @ h+=Maj(a,b,c) from the past
87         eor     $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]`  @ Sigma1(e)
88 # ifndef __ARMEB__
89         rev     $t1,$t1
90 # endif
91 #else
92         @ ldrb  $t1,[$inp,#3]                   @ $i
93         add     $a,$a,$t2                       @ h+=Maj(a,b,c) from the past
94         ldrb    $t2,[$inp,#2]
95         ldrb    $t0,[$inp,#1]
96         orr     $t1,$t1,$t2,lsl#8
97         ldrb    $t2,[$inp],#4
98         orr     $t1,$t1,$t0,lsl#16
99 # if $i==15
100         str     $inp,[sp,#17*4]                 @ make room for $t4
101 # endif
102         eor     $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
103         orr     $t1,$t1,$t2,lsl#24
104         eor     $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]`  @ Sigma1(e)
105 #endif
106 ___
107 $code.=<<___;
108         ldr     $t2,[$Ktbl],#4                  @ *K256++
109         add     $h,$h,$t1                       @ h+=X[i]
110         str     $t1,[sp,#`$i%16`*4]
111         eor     $t1,$f,$g
112         add     $h,$h,$t0,ror#$Sigma1[0]        @ h+=Sigma1(e)
113         and     $t1,$t1,$e
114         add     $h,$h,$t2                       @ h+=K256[i]
115         eor     $t1,$t1,$g                      @ Ch(e,f,g)
116         eor     $t0,$a,$a,ror#`$Sigma0[1]-$Sigma0[0]`
117         add     $h,$h,$t1                       @ h+=Ch(e,f,g)
118 #if $i==31
119         and     $t2,$t2,#0xff
120         cmp     $t2,#0xf2                       @ done?
121 #endif
122 #if $i<15
123 # if __ARM_ARCH__>=7
124         ldr     $t1,[$inp],#4                   @ prefetch
125 # else
126         ldrb    $t1,[$inp,#3]
127 # endif
128         eor     $t2,$a,$b                       @ a^b, b^c in next round
129 #else
130         ldr     $t1,[sp,#`($i+2)%16`*4]         @ from future BODY_16_xx
131         eor     $t2,$a,$b                       @ a^b, b^c in next round
132         ldr     $t4,[sp,#`($i+15)%16`*4]        @ from future BODY_16_xx
133 #endif
134         eor     $t0,$t0,$a,ror#`$Sigma0[2]-$Sigma0[0]`  @ Sigma0(a)
135         and     $t3,$t3,$t2                     @ (b^c)&=(a^b)
136         add     $d,$d,$h                        @ d+=h
137         eor     $t3,$t3,$b                      @ Maj(a,b,c)
138         add     $h,$h,$t0,ror#$Sigma0[0]        @ h+=Sigma0(a)
139         @ add   $h,$h,$t3                       @ h+=Maj(a,b,c)
140 ___
141         ($t2,$t3)=($t3,$t2);
142 }
143
144 sub BODY_16_XX {
145 my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
146
147 $code.=<<___;
148         @ ldr   $t1,[sp,#`($i+1)%16`*4]         @ $i
149         @ ldr   $t4,[sp,#`($i+14)%16`*4]
150         mov     $t0,$t1,ror#$sigma0[0]
151         add     $a,$a,$t2                       @ h+=Maj(a,b,c) from the past
152         mov     $t2,$t4,ror#$sigma1[0]
153         eor     $t0,$t0,$t1,ror#$sigma0[1]
154         eor     $t2,$t2,$t4,ror#$sigma1[1]
155         eor     $t0,$t0,$t1,lsr#$sigma0[2]      @ sigma0(X[i+1])
156         ldr     $t1,[sp,#`($i+0)%16`*4]
157         eor     $t2,$t2,$t4,lsr#$sigma1[2]      @ sigma1(X[i+14])
158         ldr     $t4,[sp,#`($i+9)%16`*4]
159
160         add     $t2,$t2,$t0
161         eor     $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`   @ from BODY_00_15
162         add     $t1,$t1,$t2
163         eor     $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]`  @ Sigma1(e)
164         add     $t1,$t1,$t4                     @ X[i]
165 ___
166         &BODY_00_15(@_);
167 }
168
169 $code=<<___;
170 #ifndef __KERNEL__
171 # include "arm_arch.h"
172 #else
173 # define __ARM_ARCH__ __LINUX_ARM_ARCH__
174 # define __ARM_MAX_ARCH__ 7
175 #endif
176
177 .text
178 #if defined(__thumb2__)
179 .syntax unified
180 .thumb
181 #else
182 .code   32
183 #endif
184
185 .type   K256,%object
186 .align  5
187 K256:
188 .word   0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
189 .word   0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
190 .word   0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
191 .word   0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
192 .word   0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
193 .word   0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
194 .word   0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
195 .word   0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
196 .word   0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
197 .word   0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
198 .word   0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
199 .word   0xd192e819,0xd6990624,0xf40e3585,0x106aa070
200 .word   0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
201 .word   0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
202 .word   0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
203 .word   0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
204 .size   K256,.-K256
205 .word   0                               @ terminator
206 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
207 .LOPENSSL_armcap:
208 .word   OPENSSL_armcap_P-.Lsha256_block_data_order
209 #endif
210 .align  5
211
212 .global sha256_block_data_order
213 .type   sha256_block_data_order,%function
214 sha256_block_data_order:
215 .Lsha256_block_data_order:
216 #if __ARM_ARCH__<7 && !defined(__thumb2__)
217         sub     r3,pc,#8                @ sha256_block_data_order
218 #else
219         adr     r3,.Lsha256_block_data_order
220 #endif
221 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
222         ldr     r12,.LOPENSSL_armcap
223         ldr     r12,[r3,r12]            @ OPENSSL_armcap_P
224 #ifdef  __APPLE__
225         ldr     r12,[r12]
226 #endif
227         tst     r12,#ARMV8_SHA256
228         bne     .LARMv8
229         tst     r12,#ARMV7_NEON
230         bne     .LNEON
231 #endif
232         add     $len,$inp,$len,lsl#6    @ len to point at the end of inp
233         stmdb   sp!,{$ctx,$inp,$len,r4-r11,lr}
234         ldmia   $ctx,{$A,$B,$C,$D,$E,$F,$G,$H}
235         sub     $Ktbl,r3,#256+32        @ K256
236         sub     sp,sp,#16*4             @ alloca(X[16])
237 .Loop:
238 # if __ARM_ARCH__>=7
239         ldr     $t1,[$inp],#4
240 # else
241         ldrb    $t1,[$inp,#3]
242 # endif
243         eor     $t3,$B,$C               @ magic
244         eor     $t2,$t2,$t2
245 ___
246 for($i=0;$i<16;$i++)    { &BODY_00_15($i,@V); unshift(@V,pop(@V)); }
247 $code.=".Lrounds_16_xx:\n";
248 for (;$i<32;$i++)       { &BODY_16_XX($i,@V); unshift(@V,pop(@V)); }
249 $code.=<<___;
250 #if __ARM_ARCH__>=7
251         ite     eq                      @ Thumb2 thing, sanity check in ARM
252 #endif
253         ldreq   $t3,[sp,#16*4]          @ pull ctx
254         bne     .Lrounds_16_xx
255
256         add     $A,$A,$t2               @ h+=Maj(a,b,c) from the past
257         ldr     $t0,[$t3,#0]
258         ldr     $t1,[$t3,#4]
259         ldr     $t2,[$t3,#8]
260         add     $A,$A,$t0
261         ldr     $t0,[$t3,#12]
262         add     $B,$B,$t1
263         ldr     $t1,[$t3,#16]
264         add     $C,$C,$t2
265         ldr     $t2,[$t3,#20]
266         add     $D,$D,$t0
267         ldr     $t0,[$t3,#24]
268         add     $E,$E,$t1
269         ldr     $t1,[$t3,#28]
270         add     $F,$F,$t2
271         ldr     $inp,[sp,#17*4]         @ pull inp
272         ldr     $t2,[sp,#18*4]          @ pull inp+len
273         add     $G,$G,$t0
274         add     $H,$H,$t1
275         stmia   $t3,{$A,$B,$C,$D,$E,$F,$G,$H}
276         cmp     $inp,$t2
277         sub     $Ktbl,$Ktbl,#256        @ rewind Ktbl
278         bne     .Loop
279
280         add     sp,sp,#`16+3`*4 @ destroy frame
281 #if __ARM_ARCH__>=5
282         ldmia   sp!,{r4-r11,pc}
283 #else
284         ldmia   sp!,{r4-r11,lr}
285         tst     lr,#1
286         moveq   pc,lr                   @ be binary compatible with V4, yet
287         bx      lr                      @ interoperable with Thumb ISA:-)
288 #endif
289 .size   sha256_block_data_order,.-sha256_block_data_order
290 ___
291 ######################################################################
292 # NEON stuff
293 #
294 {{{
295 my @X=map("q$_",(0..3));
296 my ($T0,$T1,$T2,$T3,$T4,$T5)=("q8","q9","q10","q11","d24","d25");
297 my $Xfer=$t4;
298 my $j=0;
299
300 sub Dlo()   { shift=~m|q([1]?[0-9])|?"d".($1*2):"";     }
301 sub Dhi()   { shift=~m|q([1]?[0-9])|?"d".($1*2+1):"";   }
302
303 sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
304 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
305   my $arg = pop;
306     $arg = "#$arg" if ($arg*1 eq $arg);
307     $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
308 }
309
310 sub Xupdate()
311 { use integer;
312   my $body = shift;
313   my @insns = (&$body,&$body,&$body,&$body);
314   my ($a,$b,$c,$d,$e,$f,$g,$h);
315
316         &vext_8         ($T0,@X[0],@X[1],4);    # X[1..4]
317          eval(shift(@insns));
318          eval(shift(@insns));
319          eval(shift(@insns));
320         &vext_8         ($T1,@X[2],@X[3],4);    # X[9..12]
321          eval(shift(@insns));
322          eval(shift(@insns));
323          eval(shift(@insns));
324         &vshr_u32       ($T2,$T0,$sigma0[0]);
325          eval(shift(@insns));
326          eval(shift(@insns));
327         &vadd_i32       (@X[0],@X[0],$T1);      # X[0..3] += X[9..12]
328          eval(shift(@insns));
329          eval(shift(@insns));
330         &vshr_u32       ($T1,$T0,$sigma0[2]);
331          eval(shift(@insns));
332          eval(shift(@insns));
333         &vsli_32        ($T2,$T0,32-$sigma0[0]);
334          eval(shift(@insns));
335          eval(shift(@insns));
336         &vshr_u32       ($T3,$T0,$sigma0[1]);
337          eval(shift(@insns));
338          eval(shift(@insns));
339         &veor           ($T1,$T1,$T2);
340          eval(shift(@insns));
341          eval(shift(@insns));
342         &vsli_32        ($T3,$T0,32-$sigma0[1]);
343          eval(shift(@insns));
344          eval(shift(@insns));
345           &vshr_u32     ($T4,&Dhi(@X[3]),$sigma1[0]);
346          eval(shift(@insns));
347          eval(shift(@insns));
348         &veor           ($T1,$T1,$T3);          # sigma0(X[1..4])
349          eval(shift(@insns));
350          eval(shift(@insns));
351           &vsli_32      ($T4,&Dhi(@X[3]),32-$sigma1[0]);
352          eval(shift(@insns));
353          eval(shift(@insns));
354           &vshr_u32     ($T5,&Dhi(@X[3]),$sigma1[2]);
355          eval(shift(@insns));
356          eval(shift(@insns));
357         &vadd_i32       (@X[0],@X[0],$T1);      # X[0..3] += sigma0(X[1..4])
358          eval(shift(@insns));
359          eval(shift(@insns));
360           &veor         ($T5,$T5,$T4);
361          eval(shift(@insns));
362          eval(shift(@insns));
363           &vshr_u32     ($T4,&Dhi(@X[3]),$sigma1[1]);
364          eval(shift(@insns));
365          eval(shift(@insns));
366           &vsli_32      ($T4,&Dhi(@X[3]),32-$sigma1[1]);
367          eval(shift(@insns));
368          eval(shift(@insns));
369           &veor         ($T5,$T5,$T4);          # sigma1(X[14..15])
370          eval(shift(@insns));
371          eval(shift(@insns));
372         &vadd_i32       (&Dlo(@X[0]),&Dlo(@X[0]),$T5);# X[0..1] += sigma1(X[14..15])
373          eval(shift(@insns));
374          eval(shift(@insns));
375           &vshr_u32     ($T4,&Dlo(@X[0]),$sigma1[0]);
376          eval(shift(@insns));
377          eval(shift(@insns));
378           &vsli_32      ($T4,&Dlo(@X[0]),32-$sigma1[0]);
379          eval(shift(@insns));
380          eval(shift(@insns));
381           &vshr_u32     ($T5,&Dlo(@X[0]),$sigma1[2]);
382          eval(shift(@insns));
383          eval(shift(@insns));
384           &veor         ($T5,$T5,$T4);
385          eval(shift(@insns));
386          eval(shift(@insns));
387           &vshr_u32     ($T4,&Dlo(@X[0]),$sigma1[1]);
388          eval(shift(@insns));
389          eval(shift(@insns));
390         &vld1_32        ("{$T0}","[$Ktbl,:128]!");
391          eval(shift(@insns));
392          eval(shift(@insns));
393           &vsli_32      ($T4,&Dlo(@X[0]),32-$sigma1[1]);
394          eval(shift(@insns));
395          eval(shift(@insns));
396           &veor         ($T5,$T5,$T4);          # sigma1(X[16..17])
397          eval(shift(@insns));
398          eval(shift(@insns));
399         &vadd_i32       (&Dhi(@X[0]),&Dhi(@X[0]),$T5);# X[2..3] += sigma1(X[16..17])
400          eval(shift(@insns));
401          eval(shift(@insns));
402         &vadd_i32       ($T0,$T0,@X[0]);
403          while($#insns>=2) { eval(shift(@insns)); }
404         &vst1_32        ("{$T0}","[$Xfer,:128]!");
405          eval(shift(@insns));
406          eval(shift(@insns));
407
408         push(@X,shift(@X));             # "rotate" X[]
409 }
410
411 sub Xpreload()
412 { use integer;
413   my $body = shift;
414   my @insns = (&$body,&$body,&$body,&$body);
415   my ($a,$b,$c,$d,$e,$f,$g,$h);
416
417          eval(shift(@insns));
418          eval(shift(@insns));
419          eval(shift(@insns));
420          eval(shift(@insns));
421         &vld1_32        ("{$T0}","[$Ktbl,:128]!");
422          eval(shift(@insns));
423          eval(shift(@insns));
424          eval(shift(@insns));
425          eval(shift(@insns));
426         &vrev32_8       (@X[0],@X[0]);
427          eval(shift(@insns));
428          eval(shift(@insns));
429          eval(shift(@insns));
430          eval(shift(@insns));
431         &vadd_i32       ($T0,$T0,@X[0]);
432          foreach (@insns) { eval; }     # remaining instructions
433         &vst1_32        ("{$T0}","[$Xfer,:128]!");
434
435         push(@X,shift(@X));             # "rotate" X[]
436 }
437
438 sub body_00_15 () {
439         (
440         '($a,$b,$c,$d,$e,$f,$g,$h)=@V;'.
441         '&add   ($h,$h,$t1)',                   # h+=X[i]+K[i]
442         '&eor   ($t1,$f,$g)',
443         '&eor   ($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))',
444         '&add   ($a,$a,$t2)',                   # h+=Maj(a,b,c) from the past
445         '&and   ($t1,$t1,$e)',
446         '&eor   ($t2,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))',   # Sigma1(e)
447         '&eor   ($t0,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))',
448         '&eor   ($t1,$t1,$g)',                  # Ch(e,f,g)
449         '&add   ($h,$h,$t2,"ror#$Sigma1[0]")',  # h+=Sigma1(e)
450         '&eor   ($t2,$a,$b)',                   # a^b, b^c in next round
451         '&eor   ($t0,$t0,$a,"ror#".($Sigma0[2]-$Sigma0[0]))',   # Sigma0(a)
452         '&add   ($h,$h,$t1)',                   # h+=Ch(e,f,g)
453         '&ldr   ($t1,sprintf "[sp,#%d]",4*(($j+1)&15))  if (($j&15)!=15);'.
454         '&ldr   ($t1,"[$Ktbl]")                         if ($j==15);'.
455         '&ldr   ($t1,"[sp,#64]")                        if ($j==31)',
456         '&and   ($t3,$t3,$t2)',                 # (b^c)&=(a^b)
457         '&add   ($d,$d,$h)',                    # d+=h
458         '&add   ($h,$h,$t0,"ror#$Sigma0[0]");'. # h+=Sigma0(a)
459         '&eor   ($t3,$t3,$b)',                  # Maj(a,b,c)
460         '$j++;  unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);'
461         )
462 }
463
464 $code.=<<___;
465 #if __ARM_MAX_ARCH__>=7
466 .arch   armv7-a
467 .fpu    neon
468
469 .global sha256_block_data_order_neon
470 .type   sha256_block_data_order_neon,%function
471 .align  5
472 .skip   16
473 sha256_block_data_order_neon:
474 .LNEON:
475         stmdb   sp!,{r4-r12,lr}
476
477         sub     $H,sp,#16*4+16
478         adr     $Ktbl,K256
479         bic     $H,$H,#15               @ align for 128-bit stores
480         mov     $t2,sp
481         mov     sp,$H                   @ alloca
482         add     $len,$inp,$len,lsl#6    @ len to point at the end of inp
483
484         vld1.8          {@X[0]},[$inp]!
485         vld1.8          {@X[1]},[$inp]!
486         vld1.8          {@X[2]},[$inp]!
487         vld1.8          {@X[3]},[$inp]!
488         vld1.32         {$T0},[$Ktbl,:128]!
489         vld1.32         {$T1},[$Ktbl,:128]!
490         vld1.32         {$T2},[$Ktbl,:128]!
491         vld1.32         {$T3},[$Ktbl,:128]!
492         vrev32.8        @X[0],@X[0]             @ yes, even on
493         str             $ctx,[sp,#64]
494         vrev32.8        @X[1],@X[1]             @ big-endian
495         str             $inp,[sp,#68]
496         mov             $Xfer,sp
497         vrev32.8        @X[2],@X[2]
498         str             $len,[sp,#72]
499         vrev32.8        @X[3],@X[3]
500         str             $t2,[sp,#76]            @ save original sp
501         vadd.i32        $T0,$T0,@X[0]
502         vadd.i32        $T1,$T1,@X[1]
503         vst1.32         {$T0},[$Xfer,:128]!
504         vadd.i32        $T2,$T2,@X[2]
505         vst1.32         {$T1},[$Xfer,:128]!
506         vadd.i32        $T3,$T3,@X[3]
507         vst1.32         {$T2},[$Xfer,:128]!
508         vst1.32         {$T3},[$Xfer,:128]!
509
510         ldmia           $ctx,{$A-$H}
511         sub             $Xfer,$Xfer,#64
512         ldr             $t1,[sp,#0]
513         eor             $t2,$t2,$t2
514         eor             $t3,$B,$C
515         b               .L_00_48
516
517 .align  4
518 .L_00_48:
519 ___
520         &Xupdate(\&body_00_15);
521         &Xupdate(\&body_00_15);
522         &Xupdate(\&body_00_15);
523         &Xupdate(\&body_00_15);
524 $code.=<<___;
525         teq     $t1,#0                          @ check for K256 terminator
526         ldr     $t1,[sp,#0]
527         sub     $Xfer,$Xfer,#64
528         bne     .L_00_48
529
530         ldr             $inp,[sp,#68]
531         ldr             $t0,[sp,#72]
532         sub             $Ktbl,$Ktbl,#256        @ rewind $Ktbl
533         teq             $inp,$t0
534         it              eq
535         subeq           $inp,$inp,#64           @ avoid SEGV
536         vld1.8          {@X[0]},[$inp]!         @ load next input block
537         vld1.8          {@X[1]},[$inp]!
538         vld1.8          {@X[2]},[$inp]!
539         vld1.8          {@X[3]},[$inp]!
540         it              ne
541         strne           $inp,[sp,#68]
542         mov             $Xfer,sp
543 ___
544         &Xpreload(\&body_00_15);
545         &Xpreload(\&body_00_15);
546         &Xpreload(\&body_00_15);
547         &Xpreload(\&body_00_15);
548 $code.=<<___;
549         ldr     $t0,[$t1,#0]
550         add     $A,$A,$t2                       @ h+=Maj(a,b,c) from the past
551         ldr     $t2,[$t1,#4]
552         ldr     $t3,[$t1,#8]
553         ldr     $t4,[$t1,#12]
554         add     $A,$A,$t0                       @ accumulate
555         ldr     $t0,[$t1,#16]
556         add     $B,$B,$t2
557         ldr     $t2,[$t1,#20]
558         add     $C,$C,$t3
559         ldr     $t3,[$t1,#24]
560         add     $D,$D,$t4
561         ldr     $t4,[$t1,#28]
562         add     $E,$E,$t0
563         str     $A,[$t1],#4
564         add     $F,$F,$t2
565         str     $B,[$t1],#4
566         add     $G,$G,$t3
567         str     $C,[$t1],#4
568         add     $H,$H,$t4
569         str     $D,[$t1],#4
570         stmia   $t1,{$E-$H}
571
572         ittte   ne
573         movne   $Xfer,sp
574         ldrne   $t1,[sp,#0]
575         eorne   $t2,$t2,$t2
576         ldreq   sp,[sp,#76]                     @ restore original sp
577         itt     ne
578         eorne   $t3,$B,$C
579         bne     .L_00_48
580
581         ldmia   sp!,{r4-r12,pc}
582 .size   sha256_block_data_order_neon,.-sha256_block_data_order_neon
583 #endif
584 ___
585 }}}
586 ######################################################################
587 # ARMv8 stuff
588 #
589 {{{
590 my ($ABCD,$EFGH,$abcd)=map("q$_",(0..2));
591 my @MSG=map("q$_",(8..11));
592 my ($W0,$W1,$ABCD_SAVE,$EFGH_SAVE)=map("q$_",(12..15));
593 my $Ktbl="r3";
594
595 $code.=<<___;
596 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
597
598 # if defined(__thumb2__)
599 #  define INST(a,b,c,d) .byte   c,d|0xc,a,b
600 # else
601 #  define INST(a,b,c,d) .byte   a,b,c,d
602 # endif
603
604 .type   sha256_block_data_order_armv8,%function
605 .align  5
606 sha256_block_data_order_armv8:
607 .LARMv8:
608         vld1.32 {$ABCD,$EFGH},[$ctx]
609         sub     $Ktbl,$Ktbl,#256+32
610         add     $len,$inp,$len,lsl#6    @ len to point at the end of inp
611         b       .Loop_v8
612
613 .align  4
614 .Loop_v8:
615         vld1.8          {@MSG[0]-@MSG[1]},[$inp]!
616         vld1.8          {@MSG[2]-@MSG[3]},[$inp]!
617         vld1.32         {$W0},[$Ktbl]!
618         vrev32.8        @MSG[0],@MSG[0]
619         vrev32.8        @MSG[1],@MSG[1]
620         vrev32.8        @MSG[2],@MSG[2]
621         vrev32.8        @MSG[3],@MSG[3]
622         vmov            $ABCD_SAVE,$ABCD        @ offload
623         vmov            $EFGH_SAVE,$EFGH
624         teq             $inp,$len
625 ___
626 for($i=0;$i<12;$i++) {
627 $code.=<<___;
628         vld1.32         {$W1},[$Ktbl]!
629         vadd.i32        $W0,$W0,@MSG[0]
630         sha256su0       @MSG[0],@MSG[1]
631         vmov            $abcd,$ABCD
632         sha256h         $ABCD,$EFGH,$W0
633         sha256h2        $EFGH,$abcd,$W0
634         sha256su1       @MSG[0],@MSG[2],@MSG[3]
635 ___
636         ($W0,$W1)=($W1,$W0);    push(@MSG,shift(@MSG));
637 }
638 $code.=<<___;
639         vld1.32         {$W1},[$Ktbl]!
640         vadd.i32        $W0,$W0,@MSG[0]
641         vmov            $abcd,$ABCD
642         sha256h         $ABCD,$EFGH,$W0
643         sha256h2        $EFGH,$abcd,$W0
644
645         vld1.32         {$W0},[$Ktbl]!
646         vadd.i32        $W1,$W1,@MSG[1]
647         vmov            $abcd,$ABCD
648         sha256h         $ABCD,$EFGH,$W1
649         sha256h2        $EFGH,$abcd,$W1
650
651         vld1.32         {$W1},[$Ktbl]
652         vadd.i32        $W0,$W0,@MSG[2]
653         sub             $Ktbl,$Ktbl,#256-16     @ rewind
654         vmov            $abcd,$ABCD
655         sha256h         $ABCD,$EFGH,$W0
656         sha256h2        $EFGH,$abcd,$W0
657
658         vadd.i32        $W1,$W1,@MSG[3]
659         vmov            $abcd,$ABCD
660         sha256h         $ABCD,$EFGH,$W1
661         sha256h2        $EFGH,$abcd,$W1
662
663         vadd.i32        $ABCD,$ABCD,$ABCD_SAVE
664         vadd.i32        $EFGH,$EFGH,$EFGH_SAVE
665         it              ne
666         bne             .Loop_v8
667
668         vst1.32         {$ABCD,$EFGH},[$ctx]
669
670         ret             @ bx lr
671 .size   sha256_block_data_order_armv8,.-sha256_block_data_order_armv8
672 #endif
673 ___
674 }}}
675 $code.=<<___;
676 .asciz  "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
677 .align  2
678 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
679 .comm   OPENSSL_armcap_P,4,4
680 #endif
681 ___
682
683 open SELF,$0;
684 while(<SELF>) {
685         next if (/^#!/);
686         last if (!s/^#/@/ and !/^$/);
687         print;
688 }
689 close SELF;
690
691 {   my  %opcode = (
692         "sha256h"       => 0xf3000c40,  "sha256h2"      => 0xf3100c40,
693         "sha256su0"     => 0xf3ba03c0,  "sha256su1"     => 0xf3200c40   );
694
695     sub unsha256 {
696         my ($mnemonic,$arg)=@_;
697
698         if ($arg =~ m/q([0-9]+)(?:,\s*q([0-9]+))?,\s*q([0-9]+)/o) {
699             my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19)
700                                          |(($2&7)<<17)|(($2&8)<<4)
701                                          |(($3&7)<<1) |(($3&8)<<2);
702             # since ARMv7 instructions are always encoded little-endian.
703             # correct solution is to use .inst directive, but older
704             # assemblers don't implement it:-(
705             sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s",
706                         $word&0xff,($word>>8)&0xff,
707                         ($word>>16)&0xff,($word>>24)&0xff,
708                         $mnemonic,$arg;
709         }
710     }
711 }
712
713 foreach (split($/,$code)) {
714
715         s/\`([^\`]*)\`/eval $1/geo;
716
717         s/\b(sha256\w+)\s+(q.*)/unsha256($1,$2)/geo;
718
719         s/\bret\b/bx    lr/go           or
720         s/\bbx\s+lr\b/.word\t0xe12fff1e/go;     # make it possible to compile with -march=armv4
721
722         print $_,"\n";
723 }
724
725 close STDOUT; # enforce flush