crypto/sha/asm/sha256-armv4.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2007-2018 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the OpenSSL license (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9
  10 # ====================================================================
  11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  12 # project. The module is, however, dual licensed under OpenSSL and
  13 # CRYPTOGAMS licenses depending on where you obtain it. For further
  14 # details see http://www.openssl.org/~appro/cryptogams/.
  15 #
  16 # Permission to use under GPL terms is granted.
  17 # ====================================================================
  18
  19 # SHA256 block procedure for ARMv4. May 2007.
  20
  21 # Performance is ~2x better than gcc 3.4 generated code and in "abso-
  22 # lute" terms is ~2250 cycles per 64-byte block or ~35 cycles per
  23 # byte [on single-issue Xscale PXA250 core].
  24
  25 # July 2010.
  26 #
  27 # Rescheduling for dual-issue pipeline resulted in 22% improvement on
  28 # Cortex A8 core and ~20 cycles per processed byte.
  29
  30 # February 2011.
  31 #
  32 # Profiler-assisted and platform-specific optimization resulted in 16%
  33 # improvement on Cortex A8 core and ~15.4 cycles per processed byte.
  34
  35 # September 2013.
  36 #
  37 # Add NEON implementation. On Cortex A8 it was measured to process one
  38 # byte in 12.5 cycles or 23% faster than integer-only code. Snapdragon
  39 # S4 does it in 12.5 cycles too, but it's 50% faster than integer-only
  40 # code (meaning that latter performs sub-optimally, nothing was done
  41 # about it).
  42
  43 # May 2014.
  44 #
  45 # Add ARMv8 code path performing at 2.0 cpb on Apple A7.
  46
  47 $flavour = shift;
  48 if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
  49 else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }
  50
  51 if ($flavour && $flavour ne "void") {
  52     $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  53     ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
  54     ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
  55     die "can't locate arm-xlate.pl";
  56
  57     open STDOUT,"| \"$^X\" $xlate $flavour $output";
  58 } else {
  59     open STDOUT,">$output";
  60 }
  61
  62 $ctx="r0";      $t0="r0";
  63 $inp="r1";      $t4="r1";
  64 $len="r2";      $t1="r2";
  65 $T1="r3";       $t3="r3";
  66 $A="r4";
  67 $B="r5";
  68 $C="r6";
  69 $D="r7";
  70 $E="r8";
  71 $F="r9";
  72 $G="r10";
  73 $H="r11";
  74 @V=($A,$B,$C,$D,$E,$F,$G,$H);
  75 $t2="r12";
  76 $Ktbl="r14";
  77
  78 @Sigma0=( 2,13,22);
  79 @Sigma1=( 6,11,25);
  80 @sigma0=( 7,18, 3);
  81 @sigma1=(17,19,10);
  82
  83 sub BODY_00_15 {
  84 my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
  85
  86 $code.=<<___ if ($i<16);
  87 #if __ARM_ARCH__>=7
  88         @ ldr   $t1,[$inp],#4                   @ $i
  89 # if $i==15
  90         str     $inp,[sp,#17*4]                 @ make room for $t4
  91 # endif
  92         eor     $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
  93         add     $a,$a,$t2                       @ h+=Maj(a,b,c) from the past
  94         eor     $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]`  @ Sigma1(e)
  95 # ifndef __ARMEB__
  96         rev     $t1,$t1
  97 # endif
  98 #else
  99         @ ldrb  $t1,[$inp,#3]                   @ $i
 100         add     $a,$a,$t2                       @ h+=Maj(a,b,c) from the past
 101         ldrb    $t2,[$inp,#2]
 102         ldrb    $t0,[$inp,#1]
 103         orr     $t1,$t1,$t2,lsl#8
 104         ldrb    $t2,[$inp],#4
 105         orr     $t1,$t1,$t0,lsl#16
 106 # if $i==15
 107         str     $inp,[sp,#17*4]                 @ make room for $t4
 108 # endif
 109         eor     $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
 110         orr     $t1,$t1,$t2,lsl#24
 111         eor     $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]`  @ Sigma1(e)
 112 #endif
 113 ___
 114 $code.=<<___;
 115         ldr     $t2,[$Ktbl],#4                  @ *K256++
 116         add     $h,$h,$t1                       @ h+=X[i]
 117         str     $t1,[sp,#`$i%16`*4]
 118         eor     $t1,$f,$g
 119         add     $h,$h,$t0,ror#$Sigma1[0]        @ h+=Sigma1(e)
 120         and     $t1,$t1,$e
 121         add     $h,$h,$t2                       @ h+=K256[i]
 122         eor     $t1,$t1,$g                      @ Ch(e,f,g)
 123         eor     $t0,$a,$a,ror#`$Sigma0[1]-$Sigma0[0]`
 124         add     $h,$h,$t1                       @ h+=Ch(e,f,g)
 125 #if $i==31
 126         and     $t2,$t2,#0xff
 127         cmp     $t2,#0xf2                       @ done?
 128 #endif
 129 #if $i<15
 130 # if __ARM_ARCH__>=7
 131         ldr     $t1,[$inp],#4                   @ prefetch
 132 # else
 133         ldrb    $t1,[$inp,#3]
 134 # endif
 135         eor     $t2,$a,$b                       @ a^b, b^c in next round
 136 #else
 137         ldr     $t1,[sp,#`($i+2)%16`*4]         @ from future BODY_16_xx
 138         eor     $t2,$a,$b                       @ a^b, b^c in next round
 139         ldr     $t4,[sp,#`($i+15)%16`*4]        @ from future BODY_16_xx
 140 #endif
 141         eor     $t0,$t0,$a,ror#`$Sigma0[2]-$Sigma0[0]`  @ Sigma0(a)
 142         and     $t3,$t3,$t2                     @ (b^c)&=(a^b)
 143         add     $d,$d,$h                        @ d+=h
 144         eor     $t3,$t3,$b                      @ Maj(a,b,c)
 145         add     $h,$h,$t0,ror#$Sigma0[0]        @ h+=Sigma0(a)
 146         @ add   $h,$h,$t3                       @ h+=Maj(a,b,c)
 147 ___
 148         ($t2,$t3)=($t3,$t2);
 149 }
 150
 151 sub BODY_16_XX {
 152 my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
 153
 154 $code.=<<___;
 155         @ ldr   $t1,[sp,#`($i+1)%16`*4]         @ $i
 156         @ ldr   $t4,[sp,#`($i+14)%16`*4]
 157         mov     $t0,$t1,ror#$sigma0[0]
 158         add     $a,$a,$t2                       @ h+=Maj(a,b,c) from the past
 159         mov     $t2,$t4,ror#$sigma1[0]
 160         eor     $t0,$t0,$t1,ror#$sigma0[1]
 161         eor     $t2,$t2,$t4,ror#$sigma1[1]
 162         eor     $t0,$t0,$t1,lsr#$sigma0[2]      @ sigma0(X[i+1])
 163         ldr     $t1,[sp,#`($i+0)%16`*4]
 164         eor     $t2,$t2,$t4,lsr#$sigma1[2]      @ sigma1(X[i+14])
 165         ldr     $t4,[sp,#`($i+9)%16`*4]
 166
 167         add     $t2,$t2,$t0
 168         eor     $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`   @ from BODY_00_15
 169         add     $t1,$t1,$t2
 170         eor     $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]`  @ Sigma1(e)
 171         add     $t1,$t1,$t4                     @ X[i]
 172 ___
 173         &BODY_00_15(@_);
 174 }
 175
 176 $code=<<___;
 177 #ifndef __KERNEL__
 178 # include "arm_arch.h"
 179 #else
 180 # define __ARM_ARCH__ __LINUX_ARM_ARCH__
 181 # define __ARM_MAX_ARCH__ 7
 182 #endif
 183
 184 .text
 185 #if defined(__thumb2__)
 186 .syntax unified
 187 .thumb
 188 #else
 189 .code   32
 190 #endif
 191
 192 .type   K256,%object
 193 .align  5
 194 K256:
 195 .word   0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
 196 .word   0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
 197 .word   0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
 198 .word   0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
 199 .word   0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
 200 .word   0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
 201 .word   0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
 202 .word   0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
 203 .word   0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
 204 .word   0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
 205 .word   0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
 206 .word   0xd192e819,0xd6990624,0xf40e3585,0x106aa070
 207 .word   0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
 208 .word   0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
 209 .word   0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
 210 .word   0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
 211 .size   K256,.-K256
 212 .word   0                               @ terminator
 213 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
 214 .LOPENSSL_armcap:
 215 .word   OPENSSL_armcap_P-.Lsha256_block_data_order
 216 #endif
 217 .align  5
 218
 219 .global sha256_block_data_order
 220 .type   sha256_block_data_order,%function
 221 sha256_block_data_order:
 222 .Lsha256_block_data_order:
 223 #if __ARM_ARCH__<7 && !defined(__thumb2__)
 224         sub     r3,pc,#8                @ sha256_block_data_order
 225 #else
 226         adr     r3,.Lsha256_block_data_order
 227 #endif
 228 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
 229         ldr     r12,.LOPENSSL_armcap
 230         ldr     r12,[r3,r12]            @ OPENSSL_armcap_P
 231 #ifdef  __APPLE__
 232         ldr     r12,[r12]
 233 #endif
 234         tst     r12,#ARMV8_SHA256
 235         bne     .LARMv8
 236         tst     r12,#ARMV7_NEON
 237         bne     .LNEON
 238 #endif
 239         add     $len,$inp,$len,lsl#6    @ len to point at the end of inp
 240         stmdb   sp!,{$ctx,$inp,$len,r4-r11,lr}
 241         ldmia   $ctx,{$A,$B,$C,$D,$E,$F,$G,$H}
 242         sub     $Ktbl,r3,#256+32        @ K256
 243         sub     sp,sp,#16*4             @ alloca(X[16])
 244 .Loop:
 245 # if __ARM_ARCH__>=7
 246         ldr     $t1,[$inp],#4
 247 # else
 248         ldrb    $t1,[$inp,#3]
 249 # endif
 250         eor     $t3,$B,$C               @ magic
 251         eor     $t2,$t2,$t2
 252 ___
 253 for($i=0;$i<16;$i++)    { &BODY_00_15($i,@V); unshift(@V,pop(@V)); }
 254 $code.=".Lrounds_16_xx:\n";
 255 for (;$i<32;$i++)       { &BODY_16_XX($i,@V); unshift(@V,pop(@V)); }
 256 $code.=<<___;
 257 #ifdef  __thumb2__
 258         ite     eq                      @ Thumb2 thing, sanity check in ARM
 259 #endif
 260         ldreq   $t3,[sp,#16*4]          @ pull ctx
 261         bne     .Lrounds_16_xx
 262
 263         add     $A,$A,$t2               @ h+=Maj(a,b,c) from the past
 264         ldr     $t0,[$t3,#0]
 265         ldr     $t1,[$t3,#4]
 266         ldr     $t2,[$t3,#8]
 267         add     $A,$A,$t0
 268         ldr     $t0,[$t3,#12]
 269         add     $B,$B,$t1
 270         ldr     $t1,[$t3,#16]
 271         add     $C,$C,$t2
 272         ldr     $t2,[$t3,#20]
 273         add     $D,$D,$t0
 274         ldr     $t0,[$t3,#24]
 275         add     $E,$E,$t1
 276         ldr     $t1,[$t3,#28]
 277         add     $F,$F,$t2
 278         ldr     $inp,[sp,#17*4]         @ pull inp
 279         ldr     $t2,[sp,#18*4]          @ pull inp+len
 280         add     $G,$G,$t0
 281         add     $H,$H,$t1
 282         stmia   $t3,{$A,$B,$C,$D,$E,$F,$G,$H}
 283         cmp     $inp,$t2
 284         sub     $Ktbl,$Ktbl,#256        @ rewind Ktbl
 285         bne     .Loop
 286
 287         add     sp,sp,#`16+3`*4 @ destroy frame
 288 #if __ARM_ARCH__>=5
 289         ldmia   sp!,{r4-r11,pc}
 290 #else
 291         ldmia   sp!,{r4-r11,lr}
 292         tst     lr,#1
 293         moveq   pc,lr                   @ be binary compatible with V4, yet
 294         bx      lr                      @ interoperable with Thumb ISA:-)
 295 #endif
 296 .size   sha256_block_data_order,.-sha256_block_data_order
 297 ___
 298 ######################################################################
 299 # NEON stuff
 300 #
 301 {{{
 302 my @X=map("q$_",(0..3));
 303 my ($T0,$T1,$T2,$T3,$T4,$T5)=("q8","q9","q10","q11","d24","d25");
 304 my $Xfer=$t4;
 305 my $j=0;
 306
 307 sub Dlo()   { shift=~m|q([1]?[0-9])|?"d".($1*2):"";     }
 308 sub Dhi()   { shift=~m|q([1]?[0-9])|?"d".($1*2+1):"";   }
 309
 310 sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
 311 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
 312   my $arg = pop;
 313     $arg = "#$arg" if ($arg*1 eq $arg);
 314     $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
 315 }
 316
 317 sub Xupdate()
 318 { use integer;
 319   my $body = shift;
 320   my @insns = (&$body,&$body,&$body,&$body);
 321   my ($a,$b,$c,$d,$e,$f,$g,$h);
 322
 323         &vext_8         ($T0,@X[0],@X[1],4);    # X[1..4]
 324          eval(shift(@insns));
 325          eval(shift(@insns));
 326          eval(shift(@insns));
 327         &vext_8         ($T1,@X[2],@X[3],4);    # X[9..12]
 328          eval(shift(@insns));
 329          eval(shift(@insns));
 330          eval(shift(@insns));
 331         &vshr_u32       ($T2,$T0,$sigma0[0]);
 332          eval(shift(@insns));
 333          eval(shift(@insns));
 334         &vadd_i32       (@X[0],@X[0],$T1);      # X[0..3] += X[9..12]
 335          eval(shift(@insns));
 336          eval(shift(@insns));
 337         &vshr_u32       ($T1,$T0,$sigma0[2]);
 338          eval(shift(@insns));
 339          eval(shift(@insns));
 340         &vsli_32        ($T2,$T0,32-$sigma0[0]);
 341          eval(shift(@insns));
 342          eval(shift(@insns));
 343         &vshr_u32       ($T3,$T0,$sigma0[1]);
 344          eval(shift(@insns));
 345          eval(shift(@insns));
 346         &veor           ($T1,$T1,$T2);
 347          eval(shift(@insns));
 348          eval(shift(@insns));
 349         &vsli_32        ($T3,$T0,32-$sigma0[1]);
 350          eval(shift(@insns));
 351          eval(shift(@insns));
 352           &vshr_u32     ($T4,&Dhi(@X[3]),$sigma1[0]);
 353          eval(shift(@insns));
 354          eval(shift(@insns));
 355         &veor           ($T1,$T1,$T3);          # sigma0(X[1..4])
 356          eval(shift(@insns));
 357          eval(shift(@insns));
 358           &vsli_32      ($T4,&Dhi(@X[3]),32-$sigma1[0]);
 359          eval(shift(@insns));
 360          eval(shift(@insns));
 361           &vshr_u32     ($T5,&Dhi(@X[3]),$sigma1[2]);
 362          eval(shift(@insns));
 363          eval(shift(@insns));
 364         &vadd_i32       (@X[0],@X[0],$T1);      # X[0..3] += sigma0(X[1..4])
 365          eval(shift(@insns));
 366          eval(shift(@insns));
 367           &veor         ($T5,$T5,$T4);
 368          eval(shift(@insns));
 369          eval(shift(@insns));
 370           &vshr_u32     ($T4,&Dhi(@X[3]),$sigma1[1]);
 371          eval(shift(@insns));
 372          eval(shift(@insns));
 373           &vsli_32      ($T4,&Dhi(@X[3]),32-$sigma1[1]);
 374          eval(shift(@insns));
 375          eval(shift(@insns));
 376           &veor         ($T5,$T5,$T4);          # sigma1(X[14..15])
 377          eval(shift(@insns));
 378          eval(shift(@insns));
 379         &vadd_i32       (&Dlo(@X[0]),&Dlo(@X[0]),$T5);# X[0..1] += sigma1(X[14..15])
 380          eval(shift(@insns));
 381          eval(shift(@insns));
 382           &vshr_u32     ($T4,&Dlo(@X[0]),$sigma1[0]);
 383          eval(shift(@insns));
 384          eval(shift(@insns));
 385           &vsli_32      ($T4,&Dlo(@X[0]),32-$sigma1[0]);
 386          eval(shift(@insns));
 387          eval(shift(@insns));
 388           &vshr_u32     ($T5,&Dlo(@X[0]),$sigma1[2]);
 389          eval(shift(@insns));
 390          eval(shift(@insns));
 391           &veor         ($T5,$T5,$T4);
 392          eval(shift(@insns));
 393          eval(shift(@insns));
 394           &vshr_u32     ($T4,&Dlo(@X[0]),$sigma1[1]);
 395          eval(shift(@insns));
 396          eval(shift(@insns));
 397         &vld1_32        ("{$T0}","[$Ktbl,:128]!");
 398          eval(shift(@insns));
 399          eval(shift(@insns));
 400           &vsli_32      ($T4,&Dlo(@X[0]),32-$sigma1[1]);
 401          eval(shift(@insns));
 402          eval(shift(@insns));
 403           &veor         ($T5,$T5,$T4);          # sigma1(X[16..17])
 404          eval(shift(@insns));
 405          eval(shift(@insns));
 406         &vadd_i32       (&Dhi(@X[0]),&Dhi(@X[0]),$T5);# X[2..3] += sigma1(X[16..17])
 407          eval(shift(@insns));
 408          eval(shift(@insns));
 409         &vadd_i32       ($T0,$T0,@X[0]);
 410          while($#insns>=2) { eval(shift(@insns)); }
 411         &vst1_32        ("{$T0}","[$Xfer,:128]!");
 412          eval(shift(@insns));
 413          eval(shift(@insns));
 414
 415         push(@X,shift(@X));             # "rotate" X[]
 416 }
 417
 418 sub Xpreload()
 419 { use integer;
 420   my $body = shift;
 421   my @insns = (&$body,&$body,&$body,&$body);
 422   my ($a,$b,$c,$d,$e,$f,$g,$h);
 423
 424          eval(shift(@insns));
 425          eval(shift(@insns));
 426          eval(shift(@insns));
 427          eval(shift(@insns));
 428         &vld1_32        ("{$T0}","[$Ktbl,:128]!");
 429          eval(shift(@insns));
 430          eval(shift(@insns));
 431          eval(shift(@insns));
 432          eval(shift(@insns));
 433         &vrev32_8       (@X[0],@X[0]);
 434          eval(shift(@insns));
 435          eval(shift(@insns));
 436          eval(shift(@insns));
 437          eval(shift(@insns));
 438         &vadd_i32       ($T0,$T0,@X[0]);
 439          foreach (@insns) { eval; }     # remaining instructions
 440         &vst1_32        ("{$T0}","[$Xfer,:128]!");
 441
 442         push(@X,shift(@X));             # "rotate" X[]
 443 }
 444
 445 sub body_00_15 () {
 446         (
 447         '($a,$b,$c,$d,$e,$f,$g,$h)=@V;'.
 448         '&add   ($h,$h,$t1)',                   # h+=X[i]+K[i]
 449         '&eor   ($t1,$f,$g)',
 450         '&eor   ($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))',
 451         '&add   ($a,$a,$t2)',                   # h+=Maj(a,b,c) from the past
 452         '&and   ($t1,$t1,$e)',
 453         '&eor   ($t2,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))',   # Sigma1(e)
 454         '&eor   ($t0,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))',
 455         '&eor   ($t1,$t1,$g)',                  # Ch(e,f,g)
 456         '&add   ($h,$h,$t2,"ror#$Sigma1[0]")',  # h+=Sigma1(e)
 457         '&eor   ($t2,$a,$b)',                   # a^b, b^c in next round
 458         '&eor   ($t0,$t0,$a,"ror#".($Sigma0[2]-$Sigma0[0]))',   # Sigma0(a)
 459         '&add   ($h,$h,$t1)',                   # h+=Ch(e,f,g)
 460         '&ldr   ($t1,sprintf "[sp,#%d]",4*(($j+1)&15))  if (($j&15)!=15);'.
 461         '&ldr   ($t1,"[$Ktbl]")                         if ($j==15);'.
 462         '&ldr   ($t1,"[sp,#64]")                        if ($j==31)',
 463         '&and   ($t3,$t3,$t2)',                 # (b^c)&=(a^b)
 464         '&add   ($d,$d,$h)',                    # d+=h
 465         '&add   ($h,$h,$t0,"ror#$Sigma0[0]");'. # h+=Sigma0(a)
 466         '&eor   ($t3,$t3,$b)',                  # Maj(a,b,c)
 467         '$j++;  unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);'
 468         )
 469 }
 470
 471 $code.=<<___;
 472 #if __ARM_MAX_ARCH__>=7
 473 .arch   armv7-a
 474 .fpu    neon
 475
 476 .global sha256_block_data_order_neon
 477 .type   sha256_block_data_order_neon,%function
 478 .align  5
 479 .skip   16
 480 sha256_block_data_order_neon:
 481 .LNEON:
 482         stmdb   sp!,{r4-r12,lr}
 483
 484         sub     $H,sp,#16*4+16
 485         adr     $Ktbl,K256
 486         bic     $H,$H,#15               @ align for 128-bit stores
 487         mov     $t2,sp
 488         mov     sp,$H                   @ alloca
 489         add     $len,$inp,$len,lsl#6    @ len to point at the end of inp
 490
 491         vld1.8          {@X[0]},[$inp]!
 492         vld1.8          {@X[1]},[$inp]!
 493         vld1.8          {@X[2]},[$inp]!
 494         vld1.8          {@X[3]},[$inp]!
 495         vld1.32         {$T0},[$Ktbl,:128]!
 496         vld1.32         {$T1},[$Ktbl,:128]!
 497         vld1.32         {$T2},[$Ktbl,:128]!
 498         vld1.32         {$T3},[$Ktbl,:128]!
 499         vrev32.8        @X[0],@X[0]             @ yes, even on
 500         str             $ctx,[sp,#64]
 501         vrev32.8        @X[1],@X[1]             @ big-endian
 502         str             $inp,[sp,#68]
 503         mov             $Xfer,sp
 504         vrev32.8        @X[2],@X[2]
 505         str             $len,[sp,#72]
 506         vrev32.8        @X[3],@X[3]
 507         str             $t2,[sp,#76]            @ save original sp
 508         vadd.i32        $T0,$T0,@X[0]
 509         vadd.i32        $T1,$T1,@X[1]
 510         vst1.32         {$T0},[$Xfer,:128]!
 511         vadd.i32        $T2,$T2,@X[2]
 512         vst1.32         {$T1},[$Xfer,:128]!
 513         vadd.i32        $T3,$T3,@X[3]
 514         vst1.32         {$T2},[$Xfer,:128]!
 515         vst1.32         {$T3},[$Xfer,:128]!
 516
 517         ldmia           $ctx,{$A-$H}
 518         sub             $Xfer,$Xfer,#64
 519         ldr             $t1,[sp,#0]
 520         eor             $t2,$t2,$t2
 521         eor             $t3,$B,$C
 522         b               .L_00_48
 523
 524 .align  4
 525 .L_00_48:
 526 ___
 527         &Xupdate(\&body_00_15);
 528         &Xupdate(\&body_00_15);
 529         &Xupdate(\&body_00_15);
 530         &Xupdate(\&body_00_15);
 531 $code.=<<___;
 532         teq     $t1,#0                          @ check for K256 terminator
 533         ldr     $t1,[sp,#0]
 534         sub     $Xfer,$Xfer,#64
 535         bne     .L_00_48
 536
 537         ldr             $inp,[sp,#68]
 538         ldr             $t0,[sp,#72]
 539         sub             $Ktbl,$Ktbl,#256        @ rewind $Ktbl
 540         teq             $inp,$t0
 541         it              eq
 542         subeq           $inp,$inp,#64           @ avoid SEGV
 543         vld1.8          {@X[0]},[$inp]!         @ load next input block
 544         vld1.8          {@X[1]},[$inp]!
 545         vld1.8          {@X[2]},[$inp]!
 546         vld1.8          {@X[3]},[$inp]!
 547         it              ne
 548         strne           $inp,[sp,#68]
 549         mov             $Xfer,sp
 550 ___
 551         &Xpreload(\&body_00_15);
 552         &Xpreload(\&body_00_15);
 553         &Xpreload(\&body_00_15);
 554         &Xpreload(\&body_00_15);
 555 $code.=<<___;
 556         ldr     $t0,[$t1,#0]
 557         add     $A,$A,$t2                       @ h+=Maj(a,b,c) from the past
 558         ldr     $t2,[$t1,#4]
 559         ldr     $t3,[$t1,#8]
 560         ldr     $t4,[$t1,#12]
 561         add     $A,$A,$t0                       @ accumulate
 562         ldr     $t0,[$t1,#16]
 563         add     $B,$B,$t2
 564         ldr     $t2,[$t1,#20]
 565         add     $C,$C,$t3
 566         ldr     $t3,[$t1,#24]
 567         add     $D,$D,$t4
 568         ldr     $t4,[$t1,#28]
 569         add     $E,$E,$t0
 570         str     $A,[$t1],#4
 571         add     $F,$F,$t2
 572         str     $B,[$t1],#4
 573         add     $G,$G,$t3
 574         str     $C,[$t1],#4
 575         add     $H,$H,$t4
 576         str     $D,[$t1],#4
 577         stmia   $t1,{$E-$H}
 578
 579         ittte   ne
 580         movne   $Xfer,sp
 581         ldrne   $t1,[sp,#0]
 582         eorne   $t2,$t2,$t2
 583         ldreq   sp,[sp,#76]                     @ restore original sp
 584         itt     ne
 585         eorne   $t3,$B,$C
 586         bne     .L_00_48
 587
 588         ldmia   sp!,{r4-r12,pc}
 589 .size   sha256_block_data_order_neon,.-sha256_block_data_order_neon
 590 #endif
 591 ___
 592 }}}
 593 ######################################################################
 594 # ARMv8 stuff
 595 #
 596 {{{
 597 my ($ABCD,$EFGH,$abcd)=map("q$_",(0..2));
 598 my @MSG=map("q$_",(8..11));
 599 my ($W0,$W1,$ABCD_SAVE,$EFGH_SAVE)=map("q$_",(12..15));
 600 my $Ktbl="r3";
 601
 602 $code.=<<___;
 603 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
 604
 605 # if defined(__thumb2__)
 606 #  define INST(a,b,c,d) .byte   c,d|0xc,a,b
 607 # else
 608 #  define INST(a,b,c,d) .byte   a,b,c,d
 609 # endif
 610
 611 .type   sha256_block_data_order_armv8,%function
 612 .align  5
 613 sha256_block_data_order_armv8:
 614 .LARMv8:
 615         vld1.32 {$ABCD,$EFGH},[$ctx]
 616         sub     $Ktbl,$Ktbl,#256+32
 617         add     $len,$inp,$len,lsl#6    @ len to point at the end of inp
 618         b       .Loop_v8
 619
 620 .align  4
 621 .Loop_v8:
 622         vld1.8          {@MSG[0]-@MSG[1]},[$inp]!
 623         vld1.8          {@MSG[2]-@MSG[3]},[$inp]!
 624         vld1.32         {$W0},[$Ktbl]!
 625         vrev32.8        @MSG[0],@MSG[0]
 626         vrev32.8        @MSG[1],@MSG[1]
 627         vrev32.8        @MSG[2],@MSG[2]
 628         vrev32.8        @MSG[3],@MSG[3]
 629         vmov            $ABCD_SAVE,$ABCD        @ offload
 630         vmov            $EFGH_SAVE,$EFGH
 631         teq             $inp,$len
 632 ___
 633 for($i=0;$i<12;$i++) {
 634 $code.=<<___;
 635         vld1.32         {$W1},[$Ktbl]!
 636         vadd.i32        $W0,$W0,@MSG[0]
 637         sha256su0       @MSG[0],@MSG[1]
 638         vmov            $abcd,$ABCD
 639         sha256h         $ABCD,$EFGH,$W0
 640         sha256h2        $EFGH,$abcd,$W0
 641         sha256su1       @MSG[0],@MSG[2],@MSG[3]
 642 ___
 643         ($W0,$W1)=($W1,$W0);    push(@MSG,shift(@MSG));
 644 }
 645 $code.=<<___;
 646         vld1.32         {$W1},[$Ktbl]!
 647         vadd.i32        $W0,$W0,@MSG[0]
 648         vmov            $abcd,$ABCD
 649         sha256h         $ABCD,$EFGH,$W0
 650         sha256h2        $EFGH,$abcd,$W0
 651
 652         vld1.32         {$W0},[$Ktbl]!
 653         vadd.i32        $W1,$W1,@MSG[1]
 654         vmov            $abcd,$ABCD
 655         sha256h         $ABCD,$EFGH,$W1
 656         sha256h2        $EFGH,$abcd,$W1
 657
 658         vld1.32         {$W1},[$Ktbl]
 659         vadd.i32        $W0,$W0,@MSG[2]
 660         sub             $Ktbl,$Ktbl,#256-16     @ rewind
 661         vmov            $abcd,$ABCD
 662         sha256h         $ABCD,$EFGH,$W0
 663         sha256h2        $EFGH,$abcd,$W0
 664
 665         vadd.i32        $W1,$W1,@MSG[3]
 666         vmov            $abcd,$ABCD
 667         sha256h         $ABCD,$EFGH,$W1
 668         sha256h2        $EFGH,$abcd,$W1
 669
 670         vadd.i32        $ABCD,$ABCD,$ABCD_SAVE
 671         vadd.i32        $EFGH,$EFGH,$EFGH_SAVE
 672         it              ne
 673         bne             .Loop_v8
 674
 675         vst1.32         {$ABCD,$EFGH},[$ctx]
 676
 677         ret             @ bx lr
 678 .size   sha256_block_data_order_armv8,.-sha256_block_data_order_armv8
 679 #endif
 680 ___
 681 }}}
 682 $code.=<<___;
 683 .asciz  "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
 684 .align  2
 685 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
 686 .comm   OPENSSL_armcap_P,4,4
 687 #endif
 688 ___
 689
 690 open SELF,$0;
 691 while(<SELF>) {
 692         next if (/^#!/);
 693         last if (!s/^#/@/ and !/^$/);
 694         print;
 695 }
 696 close SELF;
 697
 698 {   my  %opcode = (
 699         "sha256h"       => 0xf3000c40,  "sha256h2"      => 0xf3100c40,
 700         "sha256su0"     => 0xf3ba03c0,  "sha256su1"     => 0xf3200c40   );
 701
 702     sub unsha256 {
 703         my ($mnemonic,$arg)=@_;
 704
 705         if ($arg =~ m/q([0-9]+)(?:,\s*q([0-9]+))?,\s*q([0-9]+)/o) {
 706             my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19)
 707                                          |(($2&7)<<17)|(($2&8)<<4)
 708                                          |(($3&7)<<1) |(($3&8)<<2);
 709             # since ARMv7 instructions are always encoded little-endian.
 710             # correct solution is to use .inst directive, but older
 711             # assemblers don't implement it:-(
 712             sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s",
 713                         $word&0xff,($word>>8)&0xff,
 714                         ($word>>16)&0xff,($word>>24)&0xff,
 715                         $mnemonic,$arg;
 716         }
 717     }
 718 }
 719
 720 foreach (split($/,$code)) {
 721
 722         s/\`([^\`]*)\`/eval $1/geo;
 723
 724         s/\b(sha256\w+)\s+(q.*)/unsha256($1,$2)/geo;
 725
 726         s/\bret\b/bx    lr/go           or
 727         s/\bbx\s+lr\b/.word\t0xe12fff1e/go;     # make it possible to compile with -march=armv4
 728
 729         print $_,"\n";
 730 }
 731
 732 close STDOUT; # enforce flush