crypto/chacha/asm/chacha-ppc.pl

   1 #!/usr/bin/env perl
   2 #
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9 #
  10 # October 2015
  11 #
  12 # ChaCha20 for PowerPC/AltiVec.
  13 #
  14 # Performance in cycles per byte out of large buffer.
  15 #
  16 #                       IALU/gcc-4.x    3xAltiVec+1xIALU
  17 #
  18 # Freescale e300        13.6/+115%      -
  19 # PPC74x0               6.81/+310%      4.66
  20 # POWER7                8.62/+61%       4.27
  21 # POWER8                8.70/+51%       3.96
  22
  23 $flavour = shift;
  24
  25 if ($flavour =~ /64/) {
  26         $SIZE_T =8;
  27         $LRSAVE =2*$SIZE_T;
  28         $STU    ="stdu";
  29         $POP    ="ld";
  30         $PUSH   ="std";
  31         $UCMP   ="cmpld";
  32 } elsif ($flavour =~ /32/) {
  33         $SIZE_T =4;
  34         $LRSAVE =$SIZE_T;
  35         $STU    ="stwu";
  36         $POP    ="lwz";
  37         $PUSH   ="stw";
  38         $UCMP   ="cmplw";
  39 } else { die "nonsense $flavour"; }
  40
  41 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
  42
  43 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  44 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
  45 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
  46 die "can't locate ppc-xlate.pl";
  47
  48 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
  49
  50 $LOCALS=6*$SIZE_T;
  51 $FRAME=$LOCALS+64+18*$SIZE_T;   # 64 is for local variables
  52
  53 sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
  54 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
  55     $code .= "\t$opcode\t".join(',',@_)."\n";
  56 }
  57
  58 my $sp = "r1";
  59
  60 my ($out,$inp,$len,$key,$ctr) = map("r$_",(3..7));
  61
  62 my @x=map("r$_",(16..31));
  63 my @d=map("r$_",(11,12,14,15));
  64 my @t=map("r$_",(7..10));
  65
  66 sub ROUND {
  67 my ($a0,$b0,$c0,$d0)=@_;
  68 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
  69 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
  70 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
  71
  72     (
  73         "&add           (@x[$a0],@x[$a0],@x[$b0])",
  74          "&add          (@x[$a1],@x[$a1],@x[$b1])",
  75           "&add         (@x[$a2],@x[$a2],@x[$b2])",
  76            "&add        (@x[$a3],@x[$a3],@x[$b3])",
  77         "&xor           (@x[$d0],@x[$d0],@x[$a0])",
  78          "&xor          (@x[$d1],@x[$d1],@x[$a1])",
  79           "&xor         (@x[$d2],@x[$d2],@x[$a2])",
  80            "&xor        (@x[$d3],@x[$d3],@x[$a3])",
  81         "&rotlwi        (@x[$d0],@x[$d0],16)",
  82          "&rotlwi       (@x[$d1],@x[$d1],16)",
  83           "&rotlwi      (@x[$d2],@x[$d2],16)",
  84            "&rotlwi     (@x[$d3],@x[$d3],16)",
  85
  86         "&add           (@x[$c0],@x[$c0],@x[$d0])",
  87          "&add          (@x[$c1],@x[$c1],@x[$d1])",
  88           "&add         (@x[$c2],@x[$c2],@x[$d2])",
  89            "&add        (@x[$c3],@x[$c3],@x[$d3])",
  90         "&xor           (@x[$b0],@x[$b0],@x[$c0])",
  91          "&xor          (@x[$b1],@x[$b1],@x[$c1])",
  92           "&xor         (@x[$b2],@x[$b2],@x[$c2])",
  93            "&xor        (@x[$b3],@x[$b3],@x[$c3])",
  94         "&rotlwi        (@x[$b0],@x[$b0],12)",
  95          "&rotlwi       (@x[$b1],@x[$b1],12)",
  96           "&rotlwi      (@x[$b2],@x[$b2],12)",
  97            "&rotlwi     (@x[$b3],@x[$b3],12)",
  98
  99         "&add           (@x[$a0],@x[$a0],@x[$b0])",
 100          "&add          (@x[$a1],@x[$a1],@x[$b1])",
 101           "&add         (@x[$a2],@x[$a2],@x[$b2])",
 102            "&add        (@x[$a3],@x[$a3],@x[$b3])",
 103         "&xor           (@x[$d0],@x[$d0],@x[$a0])",
 104          "&xor          (@x[$d1],@x[$d1],@x[$a1])",
 105           "&xor         (@x[$d2],@x[$d2],@x[$a2])",
 106            "&xor        (@x[$d3],@x[$d3],@x[$a3])",
 107         "&rotlwi        (@x[$d0],@x[$d0],8)",
 108          "&rotlwi       (@x[$d1],@x[$d1],8)",
 109           "&rotlwi      (@x[$d2],@x[$d2],8)",
 110            "&rotlwi     (@x[$d3],@x[$d3],8)",
 111
 112         "&add           (@x[$c0],@x[$c0],@x[$d0])",
 113          "&add          (@x[$c1],@x[$c1],@x[$d1])",
 114           "&add         (@x[$c2],@x[$c2],@x[$d2])",
 115            "&add        (@x[$c3],@x[$c3],@x[$d3])",
 116         "&xor           (@x[$b0],@x[$b0],@x[$c0])",
 117          "&xor          (@x[$b1],@x[$b1],@x[$c1])",
 118           "&xor         (@x[$b2],@x[$b2],@x[$c2])",
 119            "&xor        (@x[$b3],@x[$b3],@x[$c3])",
 120         "&rotlwi        (@x[$b0],@x[$b0],7)",
 121          "&rotlwi       (@x[$b1],@x[$b1],7)",
 122           "&rotlwi      (@x[$b2],@x[$b2],7)",
 123            "&rotlwi     (@x[$b3],@x[$b3],7)"
 124     );
 125 }
 126
 127 $code.=<<___;
 128 .machine        "any"
 129
 130 .globl  .ChaCha20_ctr32_int
 131 .align  5
 132 .ChaCha20_ctr32_int:
 133 __ChaCha20_ctr32_int:
 134         ${UCMP}i $len,0
 135         beqlr-
 136
 137         $STU    $sp,-$FRAME($sp)
 138         mflr    r0
 139
 140         $PUSH   r14,`$FRAME-$SIZE_T*18`($sp)
 141         $PUSH   r15,`$FRAME-$SIZE_T*17`($sp)
 142         $PUSH   r16,`$FRAME-$SIZE_T*16`($sp)
 143         $PUSH   r17,`$FRAME-$SIZE_T*15`($sp)
 144         $PUSH   r18,`$FRAME-$SIZE_T*14`($sp)
 145         $PUSH   r19,`$FRAME-$SIZE_T*13`($sp)
 146         $PUSH   r20,`$FRAME-$SIZE_T*12`($sp)
 147         $PUSH   r21,`$FRAME-$SIZE_T*11`($sp)
 148         $PUSH   r22,`$FRAME-$SIZE_T*10`($sp)
 149         $PUSH   r23,`$FRAME-$SIZE_T*9`($sp)
 150         $PUSH   r24,`$FRAME-$SIZE_T*8`($sp)
 151         $PUSH   r25,`$FRAME-$SIZE_T*7`($sp)
 152         $PUSH   r26,`$FRAME-$SIZE_T*6`($sp)
 153         $PUSH   r27,`$FRAME-$SIZE_T*5`($sp)
 154         $PUSH   r28,`$FRAME-$SIZE_T*4`($sp)
 155         $PUSH   r29,`$FRAME-$SIZE_T*3`($sp)
 156         $PUSH   r30,`$FRAME-$SIZE_T*2`($sp)
 157         $PUSH   r31,`$FRAME-$SIZE_T*1`($sp)
 158         $PUSH   r0,`$FRAME+$LRSAVE`($sp)
 159
 160         lwz     @d[0],0($ctr)                   # load counter
 161         lwz     @d[1],4($ctr)
 162         lwz     @d[2],8($ctr)
 163         lwz     @d[3],12($ctr)
 164
 165         bl      __ChaCha20_1x
 166
 167         $POP    r0,`$FRAME+$LRSAVE`($sp)
 168         $POP    r14,`$FRAME-$SIZE_T*18`($sp)
 169         $POP    r15,`$FRAME-$SIZE_T*17`($sp)
 170         $POP    r16,`$FRAME-$SIZE_T*16`($sp)
 171         $POP    r17,`$FRAME-$SIZE_T*15`($sp)
 172         $POP    r18,`$FRAME-$SIZE_T*14`($sp)
 173         $POP    r19,`$FRAME-$SIZE_T*13`($sp)
 174         $POP    r20,`$FRAME-$SIZE_T*12`($sp)
 175         $POP    r21,`$FRAME-$SIZE_T*11`($sp)
 176         $POP    r22,`$FRAME-$SIZE_T*10`($sp)
 177         $POP    r23,`$FRAME-$SIZE_T*9`($sp)
 178         $POP    r24,`$FRAME-$SIZE_T*8`($sp)
 179         $POP    r25,`$FRAME-$SIZE_T*7`($sp)
 180         $POP    r26,`$FRAME-$SIZE_T*6`($sp)
 181         $POP    r27,`$FRAME-$SIZE_T*5`($sp)
 182         $POP    r28,`$FRAME-$SIZE_T*4`($sp)
 183         $POP    r29,`$FRAME-$SIZE_T*3`($sp)
 184         $POP    r30,`$FRAME-$SIZE_T*2`($sp)
 185         $POP    r31,`$FRAME-$SIZE_T*1`($sp)
 186         mtlr    r0
 187         addi    $sp,$sp,$FRAME
 188         blr
 189         .long   0
 190         .byte   0,12,4,1,0x80,18,5,0
 191         .long   0
 192 .size   .ChaCha20_ctr32_int,.-.ChaCha20_ctr32_int
 193
 194 .align  5
 195 __ChaCha20_1x:
 196 Loop_outer:
 197         lis     @x[0],0x6170                    # synthesize sigma
 198         lis     @x[1],0x3320
 199         lis     @x[2],0x7962
 200         lis     @x[3],0x6b20
 201         ori     @x[0],@x[0],0x7865
 202         ori     @x[1],@x[1],0x646e
 203         ori     @x[2],@x[2],0x2d32
 204         ori     @x[3],@x[3],0x6574
 205
 206         li      r0,10                           # inner loop counter
 207         lwz     @x[4],0($key)                   # load key
 208         lwz     @x[5],4($key)
 209         lwz     @x[6],8($key)
 210         lwz     @x[7],12($key)
 211         lwz     @x[8],16($key)
 212         mr      @x[12],@d[0]                    # copy counter
 213         lwz     @x[9],20($key)
 214         mr      @x[13],@d[1]
 215         lwz     @x[10],24($key)
 216         mr      @x[14],@d[2]
 217         lwz     @x[11],28($key)
 218         mr      @x[15],@d[3]
 219
 220         mr      @t[0],@x[4]
 221         mr      @t[1],@x[5]
 222         mr      @t[2],@x[6]
 223         mr      @t[3],@x[7]
 224
 225         mtctr   r0
 226 Loop:
 227 ___
 228         foreach (&ROUND(0, 4, 8,12)) { eval; }
 229         foreach (&ROUND(0, 5,10,15)) { eval; }
 230 $code.=<<___;
 231         bdnz    Loop
 232
 233         subic   $len,$len,64                    # $len-=64
 234         addi    @x[0],@x[0],0x7865              # accumulate key block
 235         addi    @x[1],@x[1],0x646e
 236         addi    @x[2],@x[2],0x2d32
 237         addi    @x[3],@x[3],0x6574
 238         addis   @x[0],@x[0],0x6170
 239         addis   @x[1],@x[1],0x3320
 240         addis   @x[2],@x[2],0x7962
 241         addis   @x[3],@x[3],0x6b20
 242
 243         subfe.  r0,r0,r0                        # borrow?-1:0
 244         add     @x[4],@x[4],@t[0]
 245         lwz     @t[0],16($key)
 246         add     @x[5],@x[5],@t[1]
 247         lwz     @t[1],20($key)
 248         add     @x[6],@x[6],@t[2]
 249         lwz     @t[2],24($key)
 250         add     @x[7],@x[7],@t[3]
 251         lwz     @t[3],28($key)
 252         add     @x[8],@x[8],@t[0]
 253         add     @x[9],@x[9],@t[1]
 254         add     @x[10],@x[10],@t[2]
 255         add     @x[11],@x[11],@t[3]
 256
 257         add     @x[12],@x[12],@d[0]
 258         add     @x[13],@x[13],@d[1]
 259         add     @x[14],@x[14],@d[2]
 260         add     @x[15],@x[15],@d[3]
 261         addi    @d[0],@d[0],1                   # increment counter
 262 ___
 263 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {   # flip byte order
 264 $code.=<<___;
 265         mr      @t[$i&3],@x[$i]
 266         rotlwi  @x[$i],@x[$i],8
 267         rlwimi  @x[$i],@t[$i&3],24,0,7
 268         rlwimi  @x[$i],@t[$i&3],24,16,23
 269 ___
 270 } }
 271 $code.=<<___;
 272         bne     Ltail                           # $len-=64 borrowed
 273
 274         lwz     @t[0],0($inp)                   # load input, aligned or not
 275         lwz     @t[1],4($inp)
 276         ${UCMP}i $len,0                         # done already?
 277         lwz     @t[2],8($inp)
 278         lwz     @t[3],12($inp)
 279         xor     @x[0],@x[0],@t[0]               # xor with input
 280         lwz     @t[0],16($inp)
 281         xor     @x[1],@x[1],@t[1]
 282         lwz     @t[1],20($inp)
 283         xor     @x[2],@x[2],@t[2]
 284         lwz     @t[2],24($inp)
 285         xor     @x[3],@x[3],@t[3]
 286         lwz     @t[3],28($inp)
 287         xor     @x[4],@x[4],@t[0]
 288         lwz     @t[0],32($inp)
 289         xor     @x[5],@x[5],@t[1]
 290         lwz     @t[1],36($inp)
 291         xor     @x[6],@x[6],@t[2]
 292         lwz     @t[2],40($inp)
 293         xor     @x[7],@x[7],@t[3]
 294         lwz     @t[3],44($inp)
 295         xor     @x[8],@x[8],@t[0]
 296         lwz     @t[0],48($inp)
 297         xor     @x[9],@x[9],@t[1]
 298         lwz     @t[1],52($inp)
 299         xor     @x[10],@x[10],@t[2]
 300         lwz     @t[2],56($inp)
 301         xor     @x[11],@x[11],@t[3]
 302         lwz     @t[3],60($inp)
 303         xor     @x[12],@x[12],@t[0]
 304         stw     @x[0],0($out)                   # store output, aligned or not
 305         xor     @x[13],@x[13],@t[1]
 306         stw     @x[1],4($out)
 307         xor     @x[14],@x[14],@t[2]
 308         stw     @x[2],8($out)
 309         xor     @x[15],@x[15],@t[3]
 310         stw     @x[3],12($out)
 311         stw     @x[4],16($out)
 312         stw     @x[5],20($out)
 313         stw     @x[6],24($out)
 314         stw     @x[7],28($out)
 315         stw     @x[8],32($out)
 316         stw     @x[9],36($out)
 317         stw     @x[10],40($out)
 318         stw     @x[11],44($out)
 319         stw     @x[12],48($out)
 320         stw     @x[13],52($out)
 321         stw     @x[14],56($out)
 322         addi    $inp,$inp,64
 323         stw     @x[15],60($out)
 324         addi    $out,$out,64
 325
 326         bne     Loop_outer
 327
 328         blr
 329
 330 .align  4
 331 Ltail:
 332         addi    $len,$len,64                    # restore tail length
 333         subi    $inp,$inp,1                     # prepare for *++ptr
 334         subi    $out,$out,1
 335         addi    @t[0],$sp,$LOCALS-1
 336         mtctr   $len
 337
 338         stw     @x[0],`$LOCALS+0`($sp)          # save whole block to stack
 339         stw     @x[1],`$LOCALS+4`($sp)
 340         stw     @x[2],`$LOCALS+8`($sp)
 341         stw     @x[3],`$LOCALS+12`($sp)
 342         stw     @x[4],`$LOCALS+16`($sp)
 343         stw     @x[5],`$LOCALS+20`($sp)
 344         stw     @x[6],`$LOCALS+24`($sp)
 345         stw     @x[7],`$LOCALS+28`($sp)
 346         stw     @x[8],`$LOCALS+32`($sp)
 347         stw     @x[9],`$LOCALS+36`($sp)
 348         stw     @x[10],`$LOCALS+40`($sp)
 349         stw     @x[11],`$LOCALS+44`($sp)
 350         stw     @x[12],`$LOCALS+48`($sp)
 351         stw     @x[13],`$LOCALS+52`($sp)
 352         stw     @x[14],`$LOCALS+56`($sp)
 353         stw     @x[15],`$LOCALS+60`($sp)
 354
 355 Loop_tail:                                      # byte-by-byte loop
 356         lbzu    @d[0],1($inp)
 357         lbzu    @x[0],1(@t[0])
 358         xor     @d[1],@d[0],@x[0]
 359         stbu    @d[1],1($out)
 360         bdnz    Loop_tail
 361
 362         stw     $sp,`$LOCALS+0`($sp)            # wipe block on stack
 363         stw     $sp,`$LOCALS+4`($sp)
 364         stw     $sp,`$LOCALS+8`($sp)
 365         stw     $sp,`$LOCALS+12`($sp)
 366         stw     $sp,`$LOCALS+16`($sp)
 367         stw     $sp,`$LOCALS+20`($sp)
 368         stw     $sp,`$LOCALS+24`($sp)
 369         stw     $sp,`$LOCALS+28`($sp)
 370         stw     $sp,`$LOCALS+32`($sp)
 371         stw     $sp,`$LOCALS+36`($sp)
 372         stw     $sp,`$LOCALS+40`($sp)
 373         stw     $sp,`$LOCALS+44`($sp)
 374         stw     $sp,`$LOCALS+48`($sp)
 375         stw     $sp,`$LOCALS+52`($sp)
 376         stw     $sp,`$LOCALS+56`($sp)
 377         stw     $sp,`$LOCALS+60`($sp)
 378
 379         blr
 380         .long   0
 381         .byte   0,12,0x14,0,0,0,0,0
 382 ___
 383
 384 {{{
 385 my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,$T0,$T1,$T2) =
 386     map("v$_",(0..14));
 387 my (@K)=map("v$_",(15..20));
 388 my ($FOUR,$sixteen,$twenty4,$twenty,$twelve,$twenty5,$seven) =
 389     map("v$_",(21..27));
 390 my ($inpperm,$outperm,$outmask) = map("v$_",(28..30));
 391 my @D=("v31",$seven,$T0,$T1,$T2);
 392
 393 my $FRAME=$LOCALS+64+13*16+18*$SIZE_T;  # 13*16 is for v20-v31 offload
 394
 395 sub VMXROUND {
 396 my $odd = pop;
 397 my ($a,$b,$c,$d,$t)=@_;
 398
 399         (
 400         "&vadduwm       ('$a','$a','$b')",
 401         "&vxor          ('$d','$d','$a')",
 402         "&vperm         ('$d','$d','$d','$sixteen')",
 403
 404         "&vadduwm       ('$c','$c','$d')",
 405         "&vxor          ('$t','$b','$c')",
 406         "&vsrw          ('$b','$t','$twenty')",
 407         "&vslw          ('$t','$t','$twelve')",
 408         "&vor           ('$b','$b','$t')",
 409
 410         "&vadduwm       ('$a','$a','$b')",
 411         "&vxor          ('$d','$d','$a')",
 412         "&vperm         ('$d','$d','$d','$twenty4')",
 413
 414         "&vadduwm       ('$c','$c','$d')",
 415         "&vxor          ('$t','$b','$c')",
 416         "&vsrw          ('$b','$t','$twenty5')",
 417         "&vslw          ('$t','$t','$seven')",
 418         "&vor           ('$b','$b','$t')",
 419
 420         "&vsldoi        ('$c','$c','$c',8)",
 421         "&vsldoi        ('$b','$b','$b',$odd?4:12)",
 422         "&vsldoi        ('$d','$d','$d',$odd?12:4)"
 423         );
 424 }
 425
 426 $code.=<<___;
 427
 428 .globl  .ChaCha20_ctr32_vmx
 429 .align  5
 430 .ChaCha20_ctr32_vmx:
 431         ${UCMP}i $len,256
 432         blt     __ChaCha20_ctr32_int
 433
 434         $STU    $sp,-$FRAME($sp)
 435         mflr    r0
 436         li      r10,`15+$LOCALS+64`
 437         li      r11,`31+$LOCALS+64`
 438         mfspr   r12,256
 439         stvx    v20,r10,$sp
 440         addi    r10,r10,32
 441         stvx    v21,r11,$sp
 442         addi    r11,r11,32
 443         stvx    v22,r10,$sp
 444         addi    r10,r10,32
 445         stvx    v23,r11,$sp
 446         addi    r11,r11,32
 447         stvx    v24,r10,$sp
 448         addi    r10,r10,32
 449         stvx    v25,r11,$sp
 450         addi    r11,r11,32
 451         stvx    v26,r10,$sp
 452         addi    r10,r10,32
 453         stvx    v27,r11,$sp
 454         addi    r11,r11,32
 455         stvx    v28,r10,$sp
 456         addi    r10,r10,32
 457         stvx    v29,r11,$sp
 458         addi    r11,r11,32
 459         stvx    v30,r10,$sp
 460         stvx    v31,r11,$sp
 461         stw     r12,`$FRAME-$SIZE_T*18-4`($sp)  # save vrsave
 462         $PUSH   r14,`$FRAME-$SIZE_T*18`($sp)
 463         $PUSH   r15,`$FRAME-$SIZE_T*17`($sp)
 464         $PUSH   r16,`$FRAME-$SIZE_T*16`($sp)
 465         $PUSH   r17,`$FRAME-$SIZE_T*15`($sp)
 466         $PUSH   r18,`$FRAME-$SIZE_T*14`($sp)
 467         $PUSH   r19,`$FRAME-$SIZE_T*13`($sp)
 468         $PUSH   r20,`$FRAME-$SIZE_T*12`($sp)
 469         $PUSH   r21,`$FRAME-$SIZE_T*11`($sp)
 470         $PUSH   r22,`$FRAME-$SIZE_T*10`($sp)
 471         $PUSH   r23,`$FRAME-$SIZE_T*9`($sp)
 472         $PUSH   r24,`$FRAME-$SIZE_T*8`($sp)
 473         $PUSH   r25,`$FRAME-$SIZE_T*7`($sp)
 474         $PUSH   r26,`$FRAME-$SIZE_T*6`($sp)
 475         $PUSH   r27,`$FRAME-$SIZE_T*5`($sp)
 476         $PUSH   r28,`$FRAME-$SIZE_T*4`($sp)
 477         $PUSH   r29,`$FRAME-$SIZE_T*3`($sp)
 478         $PUSH   r30,`$FRAME-$SIZE_T*2`($sp)
 479         $PUSH   r31,`$FRAME-$SIZE_T*1`($sp)
 480         li      r12,-1
 481         $PUSH   r0, `$FRAME+$LRSAVE`($sp)
 482         mtspr   256,r12                         # preserve all AltiVec registers
 483
 484         bl      Lconsts                         # returns pointer Lsigma in r12
 485         li      @x[0],16
 486         li      @x[1],32
 487         li      @x[2],48
 488         li      @x[3],64
 489         li      @x[4],31                        # 31 is not a typo
 490         li      @x[5],15                        # nor is 15
 491
 492         lvx     @K[1],0,$key                    # load key
 493         ?lvsr   $T0,0,$key                      # prepare unaligned load
 494         lvx     @K[2],@x[0],$key
 495         lvx     @D[0],@x[4],$key
 496
 497         lvx     @K[3],0,$ctr                    # load counter
 498         ?lvsr   $T1,0,$ctr                      # prepare unaligned load
 499         lvx     @D[1],@x[5],$ctr
 500
 501         lvx     @K[0],0,r12                     # load constants
 502         lvx     @K[5],@x[0],r12                 # one
 503         lvx     $FOUR,@x[1],r12
 504         lvx     $sixteen,@x[2],r12
 505         lvx     $twenty4,@x[3],r12
 506
 507         ?vperm  @K[1],@K[2],@K[1],$T0           # align key
 508         ?vperm  @K[2],@D[0],@K[2],$T0
 509         ?vperm  @K[3],@D[1],@K[3],$T1           # align counter
 510
 511         lwz     @d[0],0($ctr)                   # load counter to GPR
 512         lwz     @d[1],4($ctr)
 513         vadduwm @K[3],@K[3],@K[5]               # adjust AltiVec counter
 514         lwz     @d[2],8($ctr)
 515         vadduwm @K[4],@K[3],@K[5]
 516         lwz     @d[3],12($ctr)
 517         vadduwm @K[5],@K[4],@K[5]
 518
 519         vspltisw $twenty,-12                    # synthesize constants
 520         vspltisw $twelve,12
 521         vspltisw $twenty5,-7
 522         #vspltisw $seven,7                      # synthesized in the loop
 523
 524         vxor    $T0,$T0,$T0                     # 0x00..00
 525         vspltisw $outmask,-1                    # 0xff..ff
 526         ?lvsr   $inpperm,0,$inp                 # prepare for unaligned load
 527         ?lvsl   $outperm,0,$out                 # prepare for unaligned store
 528         ?vperm  $outmask,$outmask,$T0,$outperm
 529
 530         be?lvsl $T0,0,@x[0]                     # 0x00..0f
 531         be?vspltisb $T1,3                       # 0x03..03
 532         be?vxor $T0,$T0,$T1                     # swap bytes within words
 533         be?vxor $outperm,$outperm,$T1
 534         be?vperm $inpperm,$inpperm,$inpperm,$T0
 535
 536         b       Loop_outer_vmx
 537
 538 .align  4
 539 Loop_outer_vmx:
 540         lis     @x[0],0x6170                    # synthesize sigma
 541         lis     @x[1],0x3320
 542          vmr    $A0,@K[0]
 543         lis     @x[2],0x7962
 544         lis     @x[3],0x6b20
 545          vmr    $A1,@K[0]
 546         ori     @x[0],@x[0],0x7865
 547         ori     @x[1],@x[1],0x646e
 548          vmr    $A2,@K[0]
 549         ori     @x[2],@x[2],0x2d32
 550         ori     @x[3],@x[3],0x6574
 551          vmr    $B0,@K[1]
 552
 553         li      r0,10                           # inner loop counter
 554         lwz     @x[4],0($key)                   # load key to GPR
 555          vmr    $B1,@K[1]
 556         lwz     @x[5],4($key)
 557          vmr    $B2,@K[1]
 558         lwz     @x[6],8($key)
 559          vmr    $C0,@K[2]
 560         lwz     @x[7],12($key)
 561          vmr    $C1,@K[2]
 562         lwz     @x[8],16($key)
 563          vmr    $C2,@K[2]
 564         mr      @x[12],@d[0]                    # copy GPR counter
 565         lwz     @x[9],20($key)
 566          vmr    $D0,@K[3]
 567         mr      @x[13],@d[1]
 568         lwz     @x[10],24($key)
 569          vmr    $D1,@K[4]
 570         mr      @x[14],@d[2]
 571         lwz     @x[11],28($key)
 572          vmr    $D2,@K[5]
 573         mr      @x[15],@d[3]
 574
 575         mr      @t[0],@x[4]
 576         mr      @t[1],@x[5]
 577         mr      @t[2],@x[6]
 578         mr      @t[3],@x[7]
 579         vspltisw $seven,7
 580
 581         mtctr   r0
 582         nop
 583 Loop_vmx:
 584 ___
 585         my @thread0=&VMXROUND($A0,$B0,$C0,$D0,$T0,0);
 586         my @thread1=&VMXROUND($A1,$B1,$C1,$D1,$T1,0);
 587         my @thread2=&VMXROUND($A2,$B2,$C2,$D2,$T2,0);
 588         my @thread3=&ROUND(0,4,8,12);
 589
 590         foreach (@thread0) {
 591                 eval;                   eval(shift(@thread3));
 592                 eval(shift(@thread1));  eval(shift(@thread3));
 593                 eval(shift(@thread2));  eval(shift(@thread3));
 594         }
 595
 596         @thread0=&VMXROUND($A0,$B0,$C0,$D0,$T0,1);
 597         @thread1=&VMXROUND($A1,$B1,$C1,$D1,$T1,1);
 598         @thread2=&VMXROUND($A2,$B2,$C2,$D2,$T2,1);
 599         @thread3=&ROUND(0,5,10,15);
 600
 601         foreach (@thread0) {
 602                 eval;                   eval(shift(@thread3));
 603                 eval(shift(@thread1));  eval(shift(@thread3));
 604                 eval(shift(@thread2));  eval(shift(@thread3));
 605         }
 606 $code.=<<___;
 607         bdnz    Loop_vmx
 608
 609         subi    $len,$len,256                   # $len-=256
 610         addi    @x[0],@x[0],0x7865              # accumulate key block
 611         addi    @x[1],@x[1],0x646e
 612         addi    @x[2],@x[2],0x2d32
 613         addi    @x[3],@x[3],0x6574
 614         addis   @x[0],@x[0],0x6170
 615         addis   @x[1],@x[1],0x3320
 616         addis   @x[2],@x[2],0x7962
 617         addis   @x[3],@x[3],0x6b20
 618         add     @x[4],@x[4],@t[0]
 619         lwz     @t[0],16($key)
 620         add     @x[5],@x[5],@t[1]
 621         lwz     @t[1],20($key)
 622         add     @x[6],@x[6],@t[2]
 623         lwz     @t[2],24($key)
 624         add     @x[7],@x[7],@t[3]
 625         lwz     @t[3],28($key)
 626         add     @x[8],@x[8],@t[0]
 627         add     @x[9],@x[9],@t[1]
 628         add     @x[10],@x[10],@t[2]
 629         add     @x[11],@x[11],@t[3]
 630         add     @x[12],@x[12],@d[0]
 631         add     @x[13],@x[13],@d[1]
 632         add     @x[14],@x[14],@d[2]
 633         add     @x[15],@x[15],@d[3]
 634
 635         vadduwm $A0,$A0,@K[0]                   # accumulate key block
 636         vadduwm $A1,$A1,@K[0]
 637         vadduwm $A2,$A2,@K[0]
 638         vadduwm $B0,$B0,@K[1]
 639         vadduwm $B1,$B1,@K[1]
 640         vadduwm $B2,$B2,@K[1]
 641         vadduwm $C0,$C0,@K[2]
 642         vadduwm $C1,$C1,@K[2]
 643         vadduwm $C2,$C2,@K[2]
 644         vadduwm $D0,$D0,@K[3]
 645         vadduwm $D1,$D1,@K[4]
 646         vadduwm $D2,$D2,@K[5]
 647
 648         addi    @d[0],@d[0],4                   # increment counter
 649         vadduwm @K[3],@K[3],$FOUR
 650         vadduwm @K[4],@K[4],$FOUR
 651         vadduwm @K[5],@K[5],$FOUR
 652
 653 ___
 654 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {   # flip byte order
 655 $code.=<<___;
 656         mr      @t[$i&3],@x[$i]
 657         rotlwi  @x[$i],@x[$i],8
 658         rlwimi  @x[$i],@t[$i&3],24,0,7
 659         rlwimi  @x[$i],@t[$i&3],24,16,23
 660 ___
 661 } }
 662 $code.=<<___;
 663         lwz     @t[0],0($inp)                   # load input, aligned or not
 664         lwz     @t[1],4($inp)
 665         lwz     @t[2],8($inp)
 666         lwz     @t[3],12($inp)
 667         xor     @x[0],@x[0],@t[0]               # xor with input
 668         lwz     @t[0],16($inp)
 669         xor     @x[1],@x[1],@t[1]
 670         lwz     @t[1],20($inp)
 671         xor     @x[2],@x[2],@t[2]
 672         lwz     @t[2],24($inp)
 673         xor     @x[3],@x[3],@t[3]
 674         lwz     @t[3],28($inp)
 675         xor     @x[4],@x[4],@t[0]
 676         lwz     @t[0],32($inp)
 677         xor     @x[5],@x[5],@t[1]
 678         lwz     @t[1],36($inp)
 679         xor     @x[6],@x[6],@t[2]
 680         lwz     @t[2],40($inp)
 681         xor     @x[7],@x[7],@t[3]
 682         lwz     @t[3],44($inp)
 683         xor     @x[8],@x[8],@t[0]
 684         lwz     @t[0],48($inp)
 685         xor     @x[9],@x[9],@t[1]
 686         lwz     @t[1],52($inp)
 687         xor     @x[10],@x[10],@t[2]
 688         lwz     @t[2],56($inp)
 689         xor     @x[11],@x[11],@t[3]
 690         lwz     @t[3],60($inp)
 691         xor     @x[12],@x[12],@t[0]
 692         stw     @x[0],0($out)                   # store output, aligned or not
 693         xor     @x[13],@x[13],@t[1]
 694         stw     @x[1],4($out)
 695         xor     @x[14],@x[14],@t[2]
 696         stw     @x[2],8($out)
 697         xor     @x[15],@x[15],@t[3]
 698         stw     @x[3],12($out)
 699         addi    $inp,$inp,64
 700         stw     @x[4],16($out)
 701         li      @t[0],16
 702         stw     @x[5],20($out)
 703         li      @t[1],32
 704         stw     @x[6],24($out)
 705         li      @t[2],48
 706         stw     @x[7],28($out)
 707         li      @t[3],64
 708         stw     @x[8],32($out)
 709         stw     @x[9],36($out)
 710         stw     @x[10],40($out)
 711         stw     @x[11],44($out)
 712         stw     @x[12],48($out)
 713         stw     @x[13],52($out)
 714         stw     @x[14],56($out)
 715         stw     @x[15],60($out)
 716         addi    $out,$out,64
 717
 718         lvx     @D[0],0,$inp                    # load input
 719         lvx     @D[1],@t[0],$inp
 720         lvx     @D[2],@t[1],$inp
 721         lvx     @D[3],@t[2],$inp
 722         lvx     @D[4],@t[3],$inp
 723         addi    $inp,$inp,64
 724
 725         ?vperm  @D[0],@D[1],@D[0],$inpperm      # align input
 726         ?vperm  @D[1],@D[2],@D[1],$inpperm
 727         ?vperm  @D[2],@D[3],@D[2],$inpperm
 728         ?vperm  @D[3],@D[4],@D[3],$inpperm
 729         vxor    $A0,$A0,@D[0]                   # xor with input
 730         vxor    $B0,$B0,@D[1]
 731         lvx     @D[1],@t[0],$inp                # keep loading input
 732         vxor    $C0,$C0,@D[2]
 733         lvx     @D[2],@t[1],$inp
 734         vxor    $D0,$D0,@D[3]
 735         lvx     @D[3],@t[2],$inp
 736         lvx     @D[0],@t[3],$inp
 737         addi    $inp,$inp,64
 738         li      @t[3],63                        # 63 is not a typo
 739         vperm   $A0,$A0,$A0,$outperm            # pre-misalign output
 740         vperm   $B0,$B0,$B0,$outperm
 741         vperm   $C0,$C0,$C0,$outperm
 742         vperm   $D0,$D0,$D0,$outperm
 743
 744         ?vperm  @D[4],@D[1],@D[4],$inpperm      # align input
 745         ?vperm  @D[1],@D[2],@D[1],$inpperm
 746         ?vperm  @D[2],@D[3],@D[2],$inpperm
 747         ?vperm  @D[3],@D[0],@D[3],$inpperm
 748         vxor    $A1,$A1,@D[4]
 749         vxor    $B1,$B1,@D[1]
 750         lvx     @D[1],@t[0],$inp                # keep loading input
 751         vxor    $C1,$C1,@D[2]
 752         lvx     @D[2],@t[1],$inp
 753         vxor    $D1,$D1,@D[3]
 754         lvx     @D[3],@t[2],$inp
 755         lvx     @D[4],@t[3],$inp                # redundant in aligned case
 756         addi    $inp,$inp,64
 757         vperm   $A1,$A1,$A1,$outperm            # pre-misalign output
 758         vperm   $B1,$B1,$B1,$outperm
 759         vperm   $C1,$C1,$C1,$outperm
 760         vperm   $D1,$D1,$D1,$outperm
 761
 762         ?vperm  @D[0],@D[1],@D[0],$inpperm      # align input
 763         ?vperm  @D[1],@D[2],@D[1],$inpperm
 764         ?vperm  @D[2],@D[3],@D[2],$inpperm
 765         ?vperm  @D[3],@D[4],@D[3],$inpperm
 766         vxor    $A2,$A2,@D[0]
 767         vxor    $B2,$B2,@D[1]
 768         vxor    $C2,$C2,@D[2]
 769         vxor    $D2,$D2,@D[3]
 770         vperm   $A2,$A2,$A2,$outperm            # pre-misalign output
 771         vperm   $B2,$B2,$B2,$outperm
 772         vperm   $C2,$C2,$C2,$outperm
 773         vperm   $D2,$D2,$D2,$outperm
 774
 775         andi.   @x[1],$out,15                   # is $out aligned?
 776         mr      @x[0],$out
 777
 778         vsel    @D[0],$A0,$B0,$outmask          # collect pre-misaligned output
 779         vsel    @D[1],$B0,$C0,$outmask
 780         vsel    @D[2],$C0,$D0,$outmask
 781         vsel    @D[3],$D0,$A1,$outmask
 782         vsel    $B0,$A1,$B1,$outmask
 783         vsel    $C0,$B1,$C1,$outmask
 784         vsel    $D0,$C1,$D1,$outmask
 785         vsel    $A1,$D1,$A2,$outmask
 786         vsel    $B1,$A2,$B2,$outmask
 787         vsel    $C1,$B2,$C2,$outmask
 788         vsel    $D1,$C2,$D2,$outmask
 789
 790         #stvx   $A0,0,$out                      # take it easy on the edges
 791         stvx    @D[0],@t[0],$out                # store output
 792         stvx    @D[1],@t[1],$out
 793         stvx    @D[2],@t[2],$out
 794         addi    $out,$out,64
 795         stvx    @D[3],0,$out
 796         stvx    $B0,@t[0],$out
 797         stvx    $C0,@t[1],$out
 798         stvx    $D0,@t[2],$out
 799         addi    $out,$out,64
 800         stvx    $A1,0,$out
 801         stvx    $B1,@t[0],$out
 802         stvx    $C1,@t[1],$out
 803         stvx    $D1,@t[2],$out
 804         addi    $out,$out,64
 805
 806         beq     Laligned_vmx
 807
 808         sub     @x[2],$out,@x[1]                # in misaligned case edges
 809         li      @x[3],0                         # are written byte-by-byte
 810 Lunaligned_tail_vmx:
 811         stvebx  $D2,@x[3],@x[2]
 812         addi    @x[3],@x[3],1
 813         cmpw    @x[3],@x[1]
 814         bne     Lunaligned_tail_vmx
 815
 816         sub     @x[2],@x[0],@x[1]
 817 Lunaligned_head_vmx:
 818         stvebx  $A0,@x[1],@x[2]
 819         cmpwi   @x[1],15
 820         addi    @x[1],@x[1],1
 821         bne     Lunaligned_head_vmx
 822
 823         ${UCMP}i $len,255                       # done with 256-byte blocks yet?
 824         bgt     Loop_outer_vmx
 825
 826         b       Ldone_vmx
 827
 828 .align  4
 829 Laligned_vmx:
 830         stvx    $A0,0,@x[0]                     # head hexaword was not stored
 831
 832         ${UCMP}i $len,255                       # done with 256-byte blocks yet?
 833         bgt     Loop_outer_vmx
 834         nop
 835
 836 Ldone_vmx:
 837         ${UCMP}i $len,0                         # done yet?
 838         bnel    __ChaCha20_1x
 839
 840         lwz     r12,`$FRAME-$SIZE_T*18-4`($sp)  # pull vrsave
 841         li      r10,`15+$LOCALS+64`
 842         li      r11,`31+$LOCALS+64`
 843         mtspr   256,r12                         # restore vrsave
 844         lvx     v20,r10,$sp
 845         addi    r10,r10,32
 846         lvx     v21,r11,$sp
 847         addi    r11,r11,32
 848         lvx     v22,r10,$sp
 849         addi    r10,r10,32
 850         lvx     v23,r11,$sp
 851         addi    r11,r11,32
 852         lvx     v24,r10,$sp
 853         addi    r10,r10,32
 854         lvx     v25,r11,$sp
 855         addi    r11,r11,32
 856         lvx     v26,r10,$sp
 857         addi    r10,r10,32
 858         lvx     v27,r11,$sp
 859         addi    r11,r11,32
 860         lvx     v28,r10,$sp
 861         addi    r10,r10,32
 862         lvx     v29,r11,$sp
 863         addi    r11,r11,32
 864         lvx     v30,r10,$sp
 865         lvx     v31,r11,$sp
 866         $POP    r0, `$FRAME+$LRSAVE`($sp)
 867         $POP    r14,`$FRAME-$SIZE_T*18`($sp)
 868         $POP    r15,`$FRAME-$SIZE_T*17`($sp)
 869         $POP    r16,`$FRAME-$SIZE_T*16`($sp)
 870         $POP    r17,`$FRAME-$SIZE_T*15`($sp)
 871         $POP    r18,`$FRAME-$SIZE_T*14`($sp)
 872         $POP    r19,`$FRAME-$SIZE_T*13`($sp)
 873         $POP    r20,`$FRAME-$SIZE_T*12`($sp)
 874         $POP    r21,`$FRAME-$SIZE_T*11`($sp)
 875         $POP    r22,`$FRAME-$SIZE_T*10`($sp)
 876         $POP    r23,`$FRAME-$SIZE_T*9`($sp)
 877         $POP    r24,`$FRAME-$SIZE_T*8`($sp)
 878         $POP    r25,`$FRAME-$SIZE_T*7`($sp)
 879         $POP    r26,`$FRAME-$SIZE_T*6`($sp)
 880         $POP    r27,`$FRAME-$SIZE_T*5`($sp)
 881         $POP    r28,`$FRAME-$SIZE_T*4`($sp)
 882         $POP    r29,`$FRAME-$SIZE_T*3`($sp)
 883         $POP    r30,`$FRAME-$SIZE_T*2`($sp)
 884         $POP    r31,`$FRAME-$SIZE_T*1`($sp)
 885         mtlr    r0
 886         addi    $sp,$sp,$FRAME
 887         blr
 888         .long   0
 889         .byte   0,12,0x04,1,0x80,18,5,0
 890         .long   0
 891 .size   .ChaCha20_ctr32_vmx,.-.ChaCha20_ctr32_vmx
 892
 893 .align  5
 894 Lconsts:
 895         mflr    r0
 896         bcl     20,31,\$+4
 897         mflr    r12     #vvvvv "distance between . and _vpaes_consts
 898         addi    r12,r12,`64-8`
 899         mtlr    r0
 900         blr
 901         .long   0
 902         .byte   0,12,0x14,0,0,0,0,0
 903         .space  `64-9*4`
 904 Lsigma:
 905         .long   0x61707865,0x3320646e,0x79622d32,0x6b206574
 906         .long   1,0,0,0
 907         .long   4,0,0,0
 908 ___
 909 $code.=<<___    if ($LITTLE_ENDIAN);
 910         .long   0x0e0f0c0d,0x0a0b0809,0x06070405,0x02030001
 911         .long   0x0d0e0f0c,0x090a0b08,0x05060704,0x01020300
 912 ___
 913 $code.=<<___    if (!$LITTLE_ENDIAN);   # flipped words
 914         .long   0x02030001,0x06070405,0x0a0b0809,0x0e0f0c0d
 915         .long   0x01020300,0x05060704,0x090a0b08,0x0d0e0f0c
 916 ___
 917 $code.=<<___;
 918 .asciz  "ChaCha20 for PowerPC/AltiVec, CRYPTOGAMS by <appro\@openssl.org>"
 919 .align  2
 920 ___
 921 }}}
 922
 923 foreach (split("\n",$code)) {
 924         s/\`([^\`]*)\`/eval $1/ge;
 925
 926         # instructions prefixed with '?' are endian-specific and need
 927         # to be adjusted accordingly...
 928         if ($flavour !~ /le$/) {        # big-endian
 929             s/be\?//            or
 930             s/le\?/#le#/        or
 931             s/\?lvsr/lvsl/      or
 932             s/\?lvsl/lvsr/      or
 933             s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/ or
 934             s/(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/;
 935         } else {                        # little-endian
 936             s/le\?//            or
 937             s/be\?/#be#/        or
 938             s/\?([a-z]+)/$1/;
 939         }
 940
 941         print $_,"\n";
 942 }
 943
 944 close STDOUT;