crypto/chacha/asm/chacha-ppc.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the OpenSSL license (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9 #
  10 # ====================================================================
  11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  12 # project. The module is, however, dual licensed under OpenSSL and
  13 # CRYPTOGAMS licenses depending on where you obtain it. For further
  14 # details see http://www.openssl.org/~appro/cryptogams/.
  15 # ====================================================================
  16 #
  17 # October 2015
  18 #
  19 # ChaCha20 for PowerPC/AltiVec.
  20 #
  21 # Performance in cycles per byte out of large buffer.
  22 #
  23 #                       IALU/gcc-4.x    3xAltiVec+1xIALU
  24 #
  25 # Freescale e300        13.6/+115%      -
  26 # PPC74x0               6.81/+310%      4.66
  27 # POWER7                8.62/+61%       4.27
  28 # POWER8                8.70/+51%       3.96
  29
  30 $flavour = shift;
  31
  32 if ($flavour =~ /64/) {
  33         $SIZE_T =8;
  34         $LRSAVE =2*$SIZE_T;
  35         $STU    ="stdu";
  36         $POP    ="ld";
  37         $PUSH   ="std";
  38         $UCMP   ="cmpld";
  39 } elsif ($flavour =~ /32/) {
  40         $SIZE_T =4;
  41         $LRSAVE =$SIZE_T;
  42         $STU    ="stwu";
  43         $POP    ="lwz";
  44         $PUSH   ="stw";
  45         $UCMP   ="cmplw";
  46 } else { die "nonsense $flavour"; }
  47
  48 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
  49
  50 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  51 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
  52 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
  53 die "can't locate ppc-xlate.pl";
  54
  55 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
  56
  57 $LOCALS=6*$SIZE_T;
  58 $FRAME=$LOCALS+64+18*$SIZE_T;   # 64 is for local variables
  59
  60 sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
  61 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
  62     $code .= "\t$opcode\t".join(',',@_)."\n";
  63 }
  64
  65 my $sp = "r1";
  66
  67 my ($out,$inp,$len,$key,$ctr) = map("r$_",(3..7));
  68
  69 my @x=map("r$_",(16..31));
  70 my @d=map("r$_",(11,12,14,15));
  71 my @t=map("r$_",(7..10));
  72
  73 sub ROUND {
  74 my ($a0,$b0,$c0,$d0)=@_;
  75 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
  76 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
  77 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
  78
  79     (
  80         "&add           (@x[$a0],@x[$a0],@x[$b0])",
  81          "&add          (@x[$a1],@x[$a1],@x[$b1])",
  82           "&add         (@x[$a2],@x[$a2],@x[$b2])",
  83            "&add        (@x[$a3],@x[$a3],@x[$b3])",
  84         "&xor           (@x[$d0],@x[$d0],@x[$a0])",
  85          "&xor          (@x[$d1],@x[$d1],@x[$a1])",
  86           "&xor         (@x[$d2],@x[$d2],@x[$a2])",
  87            "&xor        (@x[$d3],@x[$d3],@x[$a3])",
  88         "&rotlwi        (@x[$d0],@x[$d0],16)",
  89          "&rotlwi       (@x[$d1],@x[$d1],16)",
  90           "&rotlwi      (@x[$d2],@x[$d2],16)",
  91            "&rotlwi     (@x[$d3],@x[$d3],16)",
  92
  93         "&add           (@x[$c0],@x[$c0],@x[$d0])",
  94          "&add          (@x[$c1],@x[$c1],@x[$d1])",
  95           "&add         (@x[$c2],@x[$c2],@x[$d2])",
  96            "&add        (@x[$c3],@x[$c3],@x[$d3])",
  97         "&xor           (@x[$b0],@x[$b0],@x[$c0])",
  98          "&xor          (@x[$b1],@x[$b1],@x[$c1])",
  99           "&xor         (@x[$b2],@x[$b2],@x[$c2])",
 100            "&xor        (@x[$b3],@x[$b3],@x[$c3])",
 101         "&rotlwi        (@x[$b0],@x[$b0],12)",
 102          "&rotlwi       (@x[$b1],@x[$b1],12)",
 103           "&rotlwi      (@x[$b2],@x[$b2],12)",
 104            "&rotlwi     (@x[$b3],@x[$b3],12)",
 105
 106         "&add           (@x[$a0],@x[$a0],@x[$b0])",
 107          "&add          (@x[$a1],@x[$a1],@x[$b1])",
 108           "&add         (@x[$a2],@x[$a2],@x[$b2])",
 109            "&add        (@x[$a3],@x[$a3],@x[$b3])",
 110         "&xor           (@x[$d0],@x[$d0],@x[$a0])",
 111          "&xor          (@x[$d1],@x[$d1],@x[$a1])",
 112           "&xor         (@x[$d2],@x[$d2],@x[$a2])",
 113            "&xor        (@x[$d3],@x[$d3],@x[$a3])",
 114         "&rotlwi        (@x[$d0],@x[$d0],8)",
 115          "&rotlwi       (@x[$d1],@x[$d1],8)",
 116           "&rotlwi      (@x[$d2],@x[$d2],8)",
 117            "&rotlwi     (@x[$d3],@x[$d3],8)",
 118
 119         "&add           (@x[$c0],@x[$c0],@x[$d0])",
 120          "&add          (@x[$c1],@x[$c1],@x[$d1])",
 121           "&add         (@x[$c2],@x[$c2],@x[$d2])",
 122            "&add        (@x[$c3],@x[$c3],@x[$d3])",
 123         "&xor           (@x[$b0],@x[$b0],@x[$c0])",
 124          "&xor          (@x[$b1],@x[$b1],@x[$c1])",
 125           "&xor         (@x[$b2],@x[$b2],@x[$c2])",
 126            "&xor        (@x[$b3],@x[$b3],@x[$c3])",
 127         "&rotlwi        (@x[$b0],@x[$b0],7)",
 128          "&rotlwi       (@x[$b1],@x[$b1],7)",
 129           "&rotlwi      (@x[$b2],@x[$b2],7)",
 130            "&rotlwi     (@x[$b3],@x[$b3],7)"
 131     );
 132 }
 133
 134 $code.=<<___;
 135 .machine        "any"
 136
 137 .globl  .ChaCha20_ctr32_int
 138 .align  5
 139 .ChaCha20_ctr32_int:
 140 __ChaCha20_ctr32_int:
 141         ${UCMP}i $len,0
 142         beqlr-
 143
 144         $STU    $sp,-$FRAME($sp)
 145         mflr    r0
 146
 147         $PUSH   r14,`$FRAME-$SIZE_T*18`($sp)
 148         $PUSH   r15,`$FRAME-$SIZE_T*17`($sp)
 149         $PUSH   r16,`$FRAME-$SIZE_T*16`($sp)
 150         $PUSH   r17,`$FRAME-$SIZE_T*15`($sp)
 151         $PUSH   r18,`$FRAME-$SIZE_T*14`($sp)
 152         $PUSH   r19,`$FRAME-$SIZE_T*13`($sp)
 153         $PUSH   r20,`$FRAME-$SIZE_T*12`($sp)
 154         $PUSH   r21,`$FRAME-$SIZE_T*11`($sp)
 155         $PUSH   r22,`$FRAME-$SIZE_T*10`($sp)
 156         $PUSH   r23,`$FRAME-$SIZE_T*9`($sp)
 157         $PUSH   r24,`$FRAME-$SIZE_T*8`($sp)
 158         $PUSH   r25,`$FRAME-$SIZE_T*7`($sp)
 159         $PUSH   r26,`$FRAME-$SIZE_T*6`($sp)
 160         $PUSH   r27,`$FRAME-$SIZE_T*5`($sp)
 161         $PUSH   r28,`$FRAME-$SIZE_T*4`($sp)
 162         $PUSH   r29,`$FRAME-$SIZE_T*3`($sp)
 163         $PUSH   r30,`$FRAME-$SIZE_T*2`($sp)
 164         $PUSH   r31,`$FRAME-$SIZE_T*1`($sp)
 165         $PUSH   r0,`$FRAME+$LRSAVE`($sp)
 166
 167         lwz     @d[0],0($ctr)                   # load counter
 168         lwz     @d[1],4($ctr)
 169         lwz     @d[2],8($ctr)
 170         lwz     @d[3],12($ctr)
 171
 172         bl      __ChaCha20_1x
 173
 174         $POP    r0,`$FRAME+$LRSAVE`($sp)
 175         $POP    r14,`$FRAME-$SIZE_T*18`($sp)
 176         $POP    r15,`$FRAME-$SIZE_T*17`($sp)
 177         $POP    r16,`$FRAME-$SIZE_T*16`($sp)
 178         $POP    r17,`$FRAME-$SIZE_T*15`($sp)
 179         $POP    r18,`$FRAME-$SIZE_T*14`($sp)
 180         $POP    r19,`$FRAME-$SIZE_T*13`($sp)
 181         $POP    r20,`$FRAME-$SIZE_T*12`($sp)
 182         $POP    r21,`$FRAME-$SIZE_T*11`($sp)
 183         $POP    r22,`$FRAME-$SIZE_T*10`($sp)
 184         $POP    r23,`$FRAME-$SIZE_T*9`($sp)
 185         $POP    r24,`$FRAME-$SIZE_T*8`($sp)
 186         $POP    r25,`$FRAME-$SIZE_T*7`($sp)
 187         $POP    r26,`$FRAME-$SIZE_T*6`($sp)
 188         $POP    r27,`$FRAME-$SIZE_T*5`($sp)
 189         $POP    r28,`$FRAME-$SIZE_T*4`($sp)
 190         $POP    r29,`$FRAME-$SIZE_T*3`($sp)
 191         $POP    r30,`$FRAME-$SIZE_T*2`($sp)
 192         $POP    r31,`$FRAME-$SIZE_T*1`($sp)
 193         mtlr    r0
 194         addi    $sp,$sp,$FRAME
 195         blr
 196         .long   0
 197         .byte   0,12,4,1,0x80,18,5,0
 198         .long   0
 199 .size   .ChaCha20_ctr32_int,.-.ChaCha20_ctr32_int
 200
 201 .align  5
 202 __ChaCha20_1x:
 203 Loop_outer:
 204         lis     @x[0],0x6170                    # synthesize sigma
 205         lis     @x[1],0x3320
 206         lis     @x[2],0x7962
 207         lis     @x[3],0x6b20
 208         ori     @x[0],@x[0],0x7865
 209         ori     @x[1],@x[1],0x646e
 210         ori     @x[2],@x[2],0x2d32
 211         ori     @x[3],@x[3],0x6574
 212
 213         li      r0,10                           # inner loop counter
 214         lwz     @x[4],0($key)                   # load key
 215         lwz     @x[5],4($key)
 216         lwz     @x[6],8($key)
 217         lwz     @x[7],12($key)
 218         lwz     @x[8],16($key)
 219         mr      @x[12],@d[0]                    # copy counter
 220         lwz     @x[9],20($key)
 221         mr      @x[13],@d[1]
 222         lwz     @x[10],24($key)
 223         mr      @x[14],@d[2]
 224         lwz     @x[11],28($key)
 225         mr      @x[15],@d[3]
 226
 227         mr      @t[0],@x[4]
 228         mr      @t[1],@x[5]
 229         mr      @t[2],@x[6]
 230         mr      @t[3],@x[7]
 231
 232         mtctr   r0
 233 Loop:
 234 ___
 235         foreach (&ROUND(0, 4, 8,12)) { eval; }
 236         foreach (&ROUND(0, 5,10,15)) { eval; }
 237 $code.=<<___;
 238         bdnz    Loop
 239
 240         subic   $len,$len,64                    # $len-=64
 241         addi    @x[0],@x[0],0x7865              # accumulate key block
 242         addi    @x[1],@x[1],0x646e
 243         addi    @x[2],@x[2],0x2d32
 244         addi    @x[3],@x[3],0x6574
 245         addis   @x[0],@x[0],0x6170
 246         addis   @x[1],@x[1],0x3320
 247         addis   @x[2],@x[2],0x7962
 248         addis   @x[3],@x[3],0x6b20
 249
 250         subfe.  r0,r0,r0                        # borrow?-1:0
 251         add     @x[4],@x[4],@t[0]
 252         lwz     @t[0],16($key)
 253         add     @x[5],@x[5],@t[1]
 254         lwz     @t[1],20($key)
 255         add     @x[6],@x[6],@t[2]
 256         lwz     @t[2],24($key)
 257         add     @x[7],@x[7],@t[3]
 258         lwz     @t[3],28($key)
 259         add     @x[8],@x[8],@t[0]
 260         add     @x[9],@x[9],@t[1]
 261         add     @x[10],@x[10],@t[2]
 262         add     @x[11],@x[11],@t[3]
 263
 264         add     @x[12],@x[12],@d[0]
 265         add     @x[13],@x[13],@d[1]
 266         add     @x[14],@x[14],@d[2]
 267         add     @x[15],@x[15],@d[3]
 268         addi    @d[0],@d[0],1                   # increment counter
 269 ___
 270 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {   # flip byte order
 271 $code.=<<___;
 272         mr      @t[$i&3],@x[$i]
 273         rotlwi  @x[$i],@x[$i],8
 274         rlwimi  @x[$i],@t[$i&3],24,0,7
 275         rlwimi  @x[$i],@t[$i&3],24,16,23
 276 ___
 277 } }
 278 $code.=<<___;
 279         bne     Ltail                           # $len-=64 borrowed
 280
 281         lwz     @t[0],0($inp)                   # load input, aligned or not
 282         lwz     @t[1],4($inp)
 283         ${UCMP}i $len,0                         # done already?
 284         lwz     @t[2],8($inp)
 285         lwz     @t[3],12($inp)
 286         xor     @x[0],@x[0],@t[0]               # xor with input
 287         lwz     @t[0],16($inp)
 288         xor     @x[1],@x[1],@t[1]
 289         lwz     @t[1],20($inp)
 290         xor     @x[2],@x[2],@t[2]
 291         lwz     @t[2],24($inp)
 292         xor     @x[3],@x[3],@t[3]
 293         lwz     @t[3],28($inp)
 294         xor     @x[4],@x[4],@t[0]
 295         lwz     @t[0],32($inp)
 296         xor     @x[5],@x[5],@t[1]
 297         lwz     @t[1],36($inp)
 298         xor     @x[6],@x[6],@t[2]
 299         lwz     @t[2],40($inp)
 300         xor     @x[7],@x[7],@t[3]
 301         lwz     @t[3],44($inp)
 302         xor     @x[8],@x[8],@t[0]
 303         lwz     @t[0],48($inp)
 304         xor     @x[9],@x[9],@t[1]
 305         lwz     @t[1],52($inp)
 306         xor     @x[10],@x[10],@t[2]
 307         lwz     @t[2],56($inp)
 308         xor     @x[11],@x[11],@t[3]
 309         lwz     @t[3],60($inp)
 310         xor     @x[12],@x[12],@t[0]
 311         stw     @x[0],0($out)                   # store output, aligned or not
 312         xor     @x[13],@x[13],@t[1]
 313         stw     @x[1],4($out)
 314         xor     @x[14],@x[14],@t[2]
 315         stw     @x[2],8($out)
 316         xor     @x[15],@x[15],@t[3]
 317         stw     @x[3],12($out)
 318         stw     @x[4],16($out)
 319         stw     @x[5],20($out)
 320         stw     @x[6],24($out)
 321         stw     @x[7],28($out)
 322         stw     @x[8],32($out)
 323         stw     @x[9],36($out)
 324         stw     @x[10],40($out)
 325         stw     @x[11],44($out)
 326         stw     @x[12],48($out)
 327         stw     @x[13],52($out)
 328         stw     @x[14],56($out)
 329         addi    $inp,$inp,64
 330         stw     @x[15],60($out)
 331         addi    $out,$out,64
 332
 333         bne     Loop_outer
 334
 335         blr
 336
 337 .align  4
 338 Ltail:
 339         addi    $len,$len,64                    # restore tail length
 340         subi    $inp,$inp,1                     # prepare for *++ptr
 341         subi    $out,$out,1
 342         addi    @t[0],$sp,$LOCALS-1
 343         mtctr   $len
 344
 345         stw     @x[0],`$LOCALS+0`($sp)          # save whole block to stack
 346         stw     @x[1],`$LOCALS+4`($sp)
 347         stw     @x[2],`$LOCALS+8`($sp)
 348         stw     @x[3],`$LOCALS+12`($sp)
 349         stw     @x[4],`$LOCALS+16`($sp)
 350         stw     @x[5],`$LOCALS+20`($sp)
 351         stw     @x[6],`$LOCALS+24`($sp)
 352         stw     @x[7],`$LOCALS+28`($sp)
 353         stw     @x[8],`$LOCALS+32`($sp)
 354         stw     @x[9],`$LOCALS+36`($sp)
 355         stw     @x[10],`$LOCALS+40`($sp)
 356         stw     @x[11],`$LOCALS+44`($sp)
 357         stw     @x[12],`$LOCALS+48`($sp)
 358         stw     @x[13],`$LOCALS+52`($sp)
 359         stw     @x[14],`$LOCALS+56`($sp)
 360         stw     @x[15],`$LOCALS+60`($sp)
 361
 362 Loop_tail:                                      # byte-by-byte loop
 363         lbzu    @d[0],1($inp)
 364         lbzu    @x[0],1(@t[0])
 365         xor     @d[1],@d[0],@x[0]
 366         stbu    @d[1],1($out)
 367         bdnz    Loop_tail
 368
 369         stw     $sp,`$LOCALS+0`($sp)            # wipe block on stack
 370         stw     $sp,`$LOCALS+4`($sp)
 371         stw     $sp,`$LOCALS+8`($sp)
 372         stw     $sp,`$LOCALS+12`($sp)
 373         stw     $sp,`$LOCALS+16`($sp)
 374         stw     $sp,`$LOCALS+20`($sp)
 375         stw     $sp,`$LOCALS+24`($sp)
 376         stw     $sp,`$LOCALS+28`($sp)
 377         stw     $sp,`$LOCALS+32`($sp)
 378         stw     $sp,`$LOCALS+36`($sp)
 379         stw     $sp,`$LOCALS+40`($sp)
 380         stw     $sp,`$LOCALS+44`($sp)
 381         stw     $sp,`$LOCALS+48`($sp)
 382         stw     $sp,`$LOCALS+52`($sp)
 383         stw     $sp,`$LOCALS+56`($sp)
 384         stw     $sp,`$LOCALS+60`($sp)
 385
 386         blr
 387         .long   0
 388         .byte   0,12,0x14,0,0,0,0,0
 389 ___
 390
 391 {{{
 392 my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,$T0,$T1,$T2) =
 393     map("v$_",(0..14));
 394 my (@K)=map("v$_",(15..20));
 395 my ($FOUR,$sixteen,$twenty4,$twenty,$twelve,$twenty5,$seven) =
 396     map("v$_",(21..27));
 397 my ($inpperm,$outperm,$outmask) = map("v$_",(28..30));
 398 my @D=("v31",$seven,$T0,$T1,$T2);
 399
 400 my $FRAME=$LOCALS+64+13*16+18*$SIZE_T;  # 13*16 is for v20-v31 offload
 401
 402 sub VMXROUND {
 403 my $odd = pop;
 404 my ($a,$b,$c,$d,$t)=@_;
 405
 406         (
 407         "&vadduwm       ('$a','$a','$b')",
 408         "&vxor          ('$d','$d','$a')",
 409         "&vperm         ('$d','$d','$d','$sixteen')",
 410
 411         "&vadduwm       ('$c','$c','$d')",
 412         "&vxor          ('$t','$b','$c')",
 413         "&vsrw          ('$b','$t','$twenty')",
 414         "&vslw          ('$t','$t','$twelve')",
 415         "&vor           ('$b','$b','$t')",
 416
 417         "&vadduwm       ('$a','$a','$b')",
 418         "&vxor          ('$d','$d','$a')",
 419         "&vperm         ('$d','$d','$d','$twenty4')",
 420
 421         "&vadduwm       ('$c','$c','$d')",
 422         "&vxor          ('$t','$b','$c')",
 423         "&vsrw          ('$b','$t','$twenty5')",
 424         "&vslw          ('$t','$t','$seven')",
 425         "&vor           ('$b','$b','$t')",
 426
 427         "&vsldoi        ('$c','$c','$c',8)",
 428         "&vsldoi        ('$b','$b','$b',$odd?4:12)",
 429         "&vsldoi        ('$d','$d','$d',$odd?12:4)"
 430         );
 431 }
 432
 433 $code.=<<___;
 434
 435 .globl  .ChaCha20_ctr32_vmx
 436 .align  5
 437 .ChaCha20_ctr32_vmx:
 438         ${UCMP}i $len,256
 439         blt     __ChaCha20_ctr32_int
 440
 441         $STU    $sp,-$FRAME($sp)
 442         mflr    r0
 443         li      r10,`15+$LOCALS+64`
 444         li      r11,`31+$LOCALS+64`
 445         mfspr   r12,256
 446         stvx    v20,r10,$sp
 447         addi    r10,r10,32
 448         stvx    v21,r11,$sp
 449         addi    r11,r11,32
 450         stvx    v22,r10,$sp
 451         addi    r10,r10,32
 452         stvx    v23,r11,$sp
 453         addi    r11,r11,32
 454         stvx    v24,r10,$sp
 455         addi    r10,r10,32
 456         stvx    v25,r11,$sp
 457         addi    r11,r11,32
 458         stvx    v26,r10,$sp
 459         addi    r10,r10,32
 460         stvx    v27,r11,$sp
 461         addi    r11,r11,32
 462         stvx    v28,r10,$sp
 463         addi    r10,r10,32
 464         stvx    v29,r11,$sp
 465         addi    r11,r11,32
 466         stvx    v30,r10,$sp
 467         stvx    v31,r11,$sp
 468         stw     r12,`$FRAME-$SIZE_T*18-4`($sp)  # save vrsave
 469         $PUSH   r14,`$FRAME-$SIZE_T*18`($sp)
 470         $PUSH   r15,`$FRAME-$SIZE_T*17`($sp)
 471         $PUSH   r16,`$FRAME-$SIZE_T*16`($sp)
 472         $PUSH   r17,`$FRAME-$SIZE_T*15`($sp)
 473         $PUSH   r18,`$FRAME-$SIZE_T*14`($sp)
 474         $PUSH   r19,`$FRAME-$SIZE_T*13`($sp)
 475         $PUSH   r20,`$FRAME-$SIZE_T*12`($sp)
 476         $PUSH   r21,`$FRAME-$SIZE_T*11`($sp)
 477         $PUSH   r22,`$FRAME-$SIZE_T*10`($sp)
 478         $PUSH   r23,`$FRAME-$SIZE_T*9`($sp)
 479         $PUSH   r24,`$FRAME-$SIZE_T*8`($sp)
 480         $PUSH   r25,`$FRAME-$SIZE_T*7`($sp)
 481         $PUSH   r26,`$FRAME-$SIZE_T*6`($sp)
 482         $PUSH   r27,`$FRAME-$SIZE_T*5`($sp)
 483         $PUSH   r28,`$FRAME-$SIZE_T*4`($sp)
 484         $PUSH   r29,`$FRAME-$SIZE_T*3`($sp)
 485         $PUSH   r30,`$FRAME-$SIZE_T*2`($sp)
 486         $PUSH   r31,`$FRAME-$SIZE_T*1`($sp)
 487         li      r12,-1
 488         $PUSH   r0, `$FRAME+$LRSAVE`($sp)
 489         mtspr   256,r12                         # preserve all AltiVec registers
 490
 491         bl      Lconsts                         # returns pointer Lsigma in r12
 492         li      @x[0],16
 493         li      @x[1],32
 494         li      @x[2],48
 495         li      @x[3],64
 496         li      @x[4],31                        # 31 is not a typo
 497         li      @x[5],15                        # nor is 15
 498
 499         lvx     @K[1],0,$key                    # load key
 500         ?lvsr   $T0,0,$key                      # prepare unaligned load
 501         lvx     @K[2],@x[0],$key
 502         lvx     @D[0],@x[4],$key
 503
 504         lvx     @K[3],0,$ctr                    # load counter
 505         ?lvsr   $T1,0,$ctr                      # prepare unaligned load
 506         lvx     @D[1],@x[5],$ctr
 507
 508         lvx     @K[0],0,r12                     # load constants
 509         lvx     @K[5],@x[0],r12                 # one
 510         lvx     $FOUR,@x[1],r12
 511         lvx     $sixteen,@x[2],r12
 512         lvx     $twenty4,@x[3],r12
 513
 514         ?vperm  @K[1],@K[2],@K[1],$T0           # align key
 515         ?vperm  @K[2],@D[0],@K[2],$T0
 516         ?vperm  @K[3],@D[1],@K[3],$T1           # align counter
 517
 518         lwz     @d[0],0($ctr)                   # load counter to GPR
 519         lwz     @d[1],4($ctr)
 520         vadduwm @K[3],@K[3],@K[5]               # adjust AltiVec counter
 521         lwz     @d[2],8($ctr)
 522         vadduwm @K[4],@K[3],@K[5]
 523         lwz     @d[3],12($ctr)
 524         vadduwm @K[5],@K[4],@K[5]
 525
 526         vspltisw $twenty,-12                    # synthesize constants
 527         vspltisw $twelve,12
 528         vspltisw $twenty5,-7
 529         #vspltisw $seven,7                      # synthesized in the loop
 530
 531         vxor    $T0,$T0,$T0                     # 0x00..00
 532         vspltisw $outmask,-1                    # 0xff..ff
 533         ?lvsr   $inpperm,0,$inp                 # prepare for unaligned load
 534         ?lvsl   $outperm,0,$out                 # prepare for unaligned store
 535         ?vperm  $outmask,$outmask,$T0,$outperm
 536
 537         be?lvsl $T0,0,@x[0]                     # 0x00..0f
 538         be?vspltisb $T1,3                       # 0x03..03
 539         be?vxor $T0,$T0,$T1                     # swap bytes within words
 540         be?vxor $outperm,$outperm,$T1
 541         be?vperm $inpperm,$inpperm,$inpperm,$T0
 542
 543         b       Loop_outer_vmx
 544
 545 .align  4
 546 Loop_outer_vmx:
 547         lis     @x[0],0x6170                    # synthesize sigma
 548         lis     @x[1],0x3320
 549          vmr    $A0,@K[0]
 550         lis     @x[2],0x7962
 551         lis     @x[3],0x6b20
 552          vmr    $A1,@K[0]
 553         ori     @x[0],@x[0],0x7865
 554         ori     @x[1],@x[1],0x646e
 555          vmr    $A2,@K[0]
 556         ori     @x[2],@x[2],0x2d32
 557         ori     @x[3],@x[3],0x6574
 558          vmr    $B0,@K[1]
 559
 560         li      r0,10                           # inner loop counter
 561         lwz     @x[4],0($key)                   # load key to GPR
 562          vmr    $B1,@K[1]
 563         lwz     @x[5],4($key)
 564          vmr    $B2,@K[1]
 565         lwz     @x[6],8($key)
 566          vmr    $C0,@K[2]
 567         lwz     @x[7],12($key)
 568          vmr    $C1,@K[2]
 569         lwz     @x[8],16($key)
 570          vmr    $C2,@K[2]
 571         mr      @x[12],@d[0]                    # copy GPR counter
 572         lwz     @x[9],20($key)
 573          vmr    $D0,@K[3]
 574         mr      @x[13],@d[1]
 575         lwz     @x[10],24($key)
 576          vmr    $D1,@K[4]
 577         mr      @x[14],@d[2]
 578         lwz     @x[11],28($key)
 579          vmr    $D2,@K[5]
 580         mr      @x[15],@d[3]
 581
 582         mr      @t[0],@x[4]
 583         mr      @t[1],@x[5]
 584         mr      @t[2],@x[6]
 585         mr      @t[3],@x[7]
 586         vspltisw $seven,7
 587
 588         mtctr   r0
 589         nop
 590 Loop_vmx:
 591 ___
 592         my @thread0=&VMXROUND($A0,$B0,$C0,$D0,$T0,0);
 593         my @thread1=&VMXROUND($A1,$B1,$C1,$D1,$T1,0);
 594         my @thread2=&VMXROUND($A2,$B2,$C2,$D2,$T2,0);
 595         my @thread3=&ROUND(0,4,8,12);
 596
 597         foreach (@thread0) {
 598                 eval;                   eval(shift(@thread3));
 599                 eval(shift(@thread1));  eval(shift(@thread3));
 600                 eval(shift(@thread2));  eval(shift(@thread3));
 601         }
 602
 603         @thread0=&VMXROUND($A0,$B0,$C0,$D0,$T0,1);
 604         @thread1=&VMXROUND($A1,$B1,$C1,$D1,$T1,1);
 605         @thread2=&VMXROUND($A2,$B2,$C2,$D2,$T2,1);
 606         @thread3=&ROUND(0,5,10,15);
 607
 608         foreach (@thread0) {
 609                 eval;                   eval(shift(@thread3));
 610                 eval(shift(@thread1));  eval(shift(@thread3));
 611                 eval(shift(@thread2));  eval(shift(@thread3));
 612         }
 613 $code.=<<___;
 614         bdnz    Loop_vmx
 615
 616         subi    $len,$len,256                   # $len-=256
 617         addi    @x[0],@x[0],0x7865              # accumulate key block
 618         addi    @x[1],@x[1],0x646e
 619         addi    @x[2],@x[2],0x2d32
 620         addi    @x[3],@x[3],0x6574
 621         addis   @x[0],@x[0],0x6170
 622         addis   @x[1],@x[1],0x3320
 623         addis   @x[2],@x[2],0x7962
 624         addis   @x[3],@x[3],0x6b20
 625         add     @x[4],@x[4],@t[0]
 626         lwz     @t[0],16($key)
 627         add     @x[5],@x[5],@t[1]
 628         lwz     @t[1],20($key)
 629         add     @x[6],@x[6],@t[2]
 630         lwz     @t[2],24($key)
 631         add     @x[7],@x[7],@t[3]
 632         lwz     @t[3],28($key)
 633         add     @x[8],@x[8],@t[0]
 634         add     @x[9],@x[9],@t[1]
 635         add     @x[10],@x[10],@t[2]
 636         add     @x[11],@x[11],@t[3]
 637         add     @x[12],@x[12],@d[0]
 638         add     @x[13],@x[13],@d[1]
 639         add     @x[14],@x[14],@d[2]
 640         add     @x[15],@x[15],@d[3]
 641
 642         vadduwm $A0,$A0,@K[0]                   # accumulate key block
 643         vadduwm $A1,$A1,@K[0]
 644         vadduwm $A2,$A2,@K[0]
 645         vadduwm $B0,$B0,@K[1]
 646         vadduwm $B1,$B1,@K[1]
 647         vadduwm $B2,$B2,@K[1]
 648         vadduwm $C0,$C0,@K[2]
 649         vadduwm $C1,$C1,@K[2]
 650         vadduwm $C2,$C2,@K[2]
 651         vadduwm $D0,$D0,@K[3]
 652         vadduwm $D1,$D1,@K[4]
 653         vadduwm $D2,$D2,@K[5]
 654
 655         addi    @d[0],@d[0],4                   # increment counter
 656         vadduwm @K[3],@K[3],$FOUR
 657         vadduwm @K[4],@K[4],$FOUR
 658         vadduwm @K[5],@K[5],$FOUR
 659
 660 ___
 661 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {   # flip byte order
 662 $code.=<<___;
 663         mr      @t[$i&3],@x[$i]
 664         rotlwi  @x[$i],@x[$i],8
 665         rlwimi  @x[$i],@t[$i&3],24,0,7
 666         rlwimi  @x[$i],@t[$i&3],24,16,23
 667 ___
 668 } }
 669 $code.=<<___;
 670         lwz     @t[0],0($inp)                   # load input, aligned or not
 671         lwz     @t[1],4($inp)
 672         lwz     @t[2],8($inp)
 673         lwz     @t[3],12($inp)
 674         xor     @x[0],@x[0],@t[0]               # xor with input
 675         lwz     @t[0],16($inp)
 676         xor     @x[1],@x[1],@t[1]
 677         lwz     @t[1],20($inp)
 678         xor     @x[2],@x[2],@t[2]
 679         lwz     @t[2],24($inp)
 680         xor     @x[3],@x[3],@t[3]
 681         lwz     @t[3],28($inp)
 682         xor     @x[4],@x[4],@t[0]
 683         lwz     @t[0],32($inp)
 684         xor     @x[5],@x[5],@t[1]
 685         lwz     @t[1],36($inp)
 686         xor     @x[6],@x[6],@t[2]
 687         lwz     @t[2],40($inp)
 688         xor     @x[7],@x[7],@t[3]
 689         lwz     @t[3],44($inp)
 690         xor     @x[8],@x[8],@t[0]
 691         lwz     @t[0],48($inp)
 692         xor     @x[9],@x[9],@t[1]
 693         lwz     @t[1],52($inp)
 694         xor     @x[10],@x[10],@t[2]
 695         lwz     @t[2],56($inp)
 696         xor     @x[11],@x[11],@t[3]
 697         lwz     @t[3],60($inp)
 698         xor     @x[12],@x[12],@t[0]
 699         stw     @x[0],0($out)                   # store output, aligned or not
 700         xor     @x[13],@x[13],@t[1]
 701         stw     @x[1],4($out)
 702         xor     @x[14],@x[14],@t[2]
 703         stw     @x[2],8($out)
 704         xor     @x[15],@x[15],@t[3]
 705         stw     @x[3],12($out)
 706         addi    $inp,$inp,64
 707         stw     @x[4],16($out)
 708         li      @t[0],16
 709         stw     @x[5],20($out)
 710         li      @t[1],32
 711         stw     @x[6],24($out)
 712         li      @t[2],48
 713         stw     @x[7],28($out)
 714         li      @t[3],64
 715         stw     @x[8],32($out)
 716         stw     @x[9],36($out)
 717         stw     @x[10],40($out)
 718         stw     @x[11],44($out)
 719         stw     @x[12],48($out)
 720         stw     @x[13],52($out)
 721         stw     @x[14],56($out)
 722         stw     @x[15],60($out)
 723         addi    $out,$out,64
 724
 725         lvx     @D[0],0,$inp                    # load input
 726         lvx     @D[1],@t[0],$inp
 727         lvx     @D[2],@t[1],$inp
 728         lvx     @D[3],@t[2],$inp
 729         lvx     @D[4],@t[3],$inp
 730         addi    $inp,$inp,64
 731
 732         ?vperm  @D[0],@D[1],@D[0],$inpperm      # align input
 733         ?vperm  @D[1],@D[2],@D[1],$inpperm
 734         ?vperm  @D[2],@D[3],@D[2],$inpperm
 735         ?vperm  @D[3],@D[4],@D[3],$inpperm
 736         vxor    $A0,$A0,@D[0]                   # xor with input
 737         vxor    $B0,$B0,@D[1]
 738         lvx     @D[1],@t[0],$inp                # keep loading input
 739         vxor    $C0,$C0,@D[2]
 740         lvx     @D[2],@t[1],$inp
 741         vxor    $D0,$D0,@D[3]
 742         lvx     @D[3],@t[2],$inp
 743         lvx     @D[0],@t[3],$inp
 744         addi    $inp,$inp,64
 745         li      @t[3],63                        # 63 is not a typo
 746         vperm   $A0,$A0,$A0,$outperm            # pre-misalign output
 747         vperm   $B0,$B0,$B0,$outperm
 748         vperm   $C0,$C0,$C0,$outperm
 749         vperm   $D0,$D0,$D0,$outperm
 750
 751         ?vperm  @D[4],@D[1],@D[4],$inpperm      # align input
 752         ?vperm  @D[1],@D[2],@D[1],$inpperm
 753         ?vperm  @D[2],@D[3],@D[2],$inpperm
 754         ?vperm  @D[3],@D[0],@D[3],$inpperm
 755         vxor    $A1,$A1,@D[4]
 756         vxor    $B1,$B1,@D[1]
 757         lvx     @D[1],@t[0],$inp                # keep loading input
 758         vxor    $C1,$C1,@D[2]
 759         lvx     @D[2],@t[1],$inp
 760         vxor    $D1,$D1,@D[3]
 761         lvx     @D[3],@t[2],$inp
 762         lvx     @D[4],@t[3],$inp                # redundant in aligned case
 763         addi    $inp,$inp,64
 764         vperm   $A1,$A1,$A1,$outperm            # pre-misalign output
 765         vperm   $B1,$B1,$B1,$outperm
 766         vperm   $C1,$C1,$C1,$outperm
 767         vperm   $D1,$D1,$D1,$outperm
 768
 769         ?vperm  @D[0],@D[1],@D[0],$inpperm      # align input
 770         ?vperm  @D[1],@D[2],@D[1],$inpperm
 771         ?vperm  @D[2],@D[3],@D[2],$inpperm
 772         ?vperm  @D[3],@D[4],@D[3],$inpperm
 773         vxor    $A2,$A2,@D[0]
 774         vxor    $B2,$B2,@D[1]
 775         vxor    $C2,$C2,@D[2]
 776         vxor    $D2,$D2,@D[3]
 777         vperm   $A2,$A2,$A2,$outperm            # pre-misalign output
 778         vperm   $B2,$B2,$B2,$outperm
 779         vperm   $C2,$C2,$C2,$outperm
 780         vperm   $D2,$D2,$D2,$outperm
 781
 782         andi.   @x[1],$out,15                   # is $out aligned?
 783         mr      @x[0],$out
 784
 785         vsel    @D[0],$A0,$B0,$outmask          # collect pre-misaligned output
 786         vsel    @D[1],$B0,$C0,$outmask
 787         vsel    @D[2],$C0,$D0,$outmask
 788         vsel    @D[3],$D0,$A1,$outmask
 789         vsel    $B0,$A1,$B1,$outmask
 790         vsel    $C0,$B1,$C1,$outmask
 791         vsel    $D0,$C1,$D1,$outmask
 792         vsel    $A1,$D1,$A2,$outmask
 793         vsel    $B1,$A2,$B2,$outmask
 794         vsel    $C1,$B2,$C2,$outmask
 795         vsel    $D1,$C2,$D2,$outmask
 796
 797         #stvx   $A0,0,$out                      # take it easy on the edges
 798         stvx    @D[0],@t[0],$out                # store output
 799         stvx    @D[1],@t[1],$out
 800         stvx    @D[2],@t[2],$out
 801         addi    $out,$out,64
 802         stvx    @D[3],0,$out
 803         stvx    $B0,@t[0],$out
 804         stvx    $C0,@t[1],$out
 805         stvx    $D0,@t[2],$out
 806         addi    $out,$out,64
 807         stvx    $A1,0,$out
 808         stvx    $B1,@t[0],$out
 809         stvx    $C1,@t[1],$out
 810         stvx    $D1,@t[2],$out
 811         addi    $out,$out,64
 812
 813         beq     Laligned_vmx
 814
 815         sub     @x[2],$out,@x[1]                # in misaligned case edges
 816         li      @x[3],0                         # are written byte-by-byte
 817 Lunaligned_tail_vmx:
 818         stvebx  $D2,@x[3],@x[2]
 819         addi    @x[3],@x[3],1
 820         cmpw    @x[3],@x[1]
 821         bne     Lunaligned_tail_vmx
 822
 823         sub     @x[2],@x[0],@x[1]
 824 Lunaligned_head_vmx:
 825         stvebx  $A0,@x[1],@x[2]
 826         cmpwi   @x[1],15
 827         addi    @x[1],@x[1],1
 828         bne     Lunaligned_head_vmx
 829
 830         ${UCMP}i $len,255                       # done with 256-byte blocks yet?
 831         bgt     Loop_outer_vmx
 832
 833         b       Ldone_vmx
 834
 835 .align  4
 836 Laligned_vmx:
 837         stvx    $A0,0,@x[0]                     # head hexaword was not stored
 838
 839         ${UCMP}i $len,255                       # done with 256-byte blocks yet?
 840         bgt     Loop_outer_vmx
 841         nop
 842
 843 Ldone_vmx:
 844         ${UCMP}i $len,0                         # done yet?
 845         bnel    __ChaCha20_1x
 846
 847         lwz     r12,`$FRAME-$SIZE_T*18-4`($sp)  # pull vrsave
 848         li      r10,`15+$LOCALS+64`
 849         li      r11,`31+$LOCALS+64`
 850         mtspr   256,r12                         # restore vrsave
 851         lvx     v20,r10,$sp
 852         addi    r10,r10,32
 853         lvx     v21,r11,$sp
 854         addi    r11,r11,32
 855         lvx     v22,r10,$sp
 856         addi    r10,r10,32
 857         lvx     v23,r11,$sp
 858         addi    r11,r11,32
 859         lvx     v24,r10,$sp
 860         addi    r10,r10,32
 861         lvx     v25,r11,$sp
 862         addi    r11,r11,32
 863         lvx     v26,r10,$sp
 864         addi    r10,r10,32
 865         lvx     v27,r11,$sp
 866         addi    r11,r11,32
 867         lvx     v28,r10,$sp
 868         addi    r10,r10,32
 869         lvx     v29,r11,$sp
 870         addi    r11,r11,32
 871         lvx     v30,r10,$sp
 872         lvx     v31,r11,$sp
 873         $POP    r0, `$FRAME+$LRSAVE`($sp)
 874         $POP    r14,`$FRAME-$SIZE_T*18`($sp)
 875         $POP    r15,`$FRAME-$SIZE_T*17`($sp)
 876         $POP    r16,`$FRAME-$SIZE_T*16`($sp)
 877         $POP    r17,`$FRAME-$SIZE_T*15`($sp)
 878         $POP    r18,`$FRAME-$SIZE_T*14`($sp)
 879         $POP    r19,`$FRAME-$SIZE_T*13`($sp)
 880         $POP    r20,`$FRAME-$SIZE_T*12`($sp)
 881         $POP    r21,`$FRAME-$SIZE_T*11`($sp)
 882         $POP    r22,`$FRAME-$SIZE_T*10`($sp)
 883         $POP    r23,`$FRAME-$SIZE_T*9`($sp)
 884         $POP    r24,`$FRAME-$SIZE_T*8`($sp)
 885         $POP    r25,`$FRAME-$SIZE_T*7`($sp)
 886         $POP    r26,`$FRAME-$SIZE_T*6`($sp)
 887         $POP    r27,`$FRAME-$SIZE_T*5`($sp)
 888         $POP    r28,`$FRAME-$SIZE_T*4`($sp)
 889         $POP    r29,`$FRAME-$SIZE_T*3`($sp)
 890         $POP    r30,`$FRAME-$SIZE_T*2`($sp)
 891         $POP    r31,`$FRAME-$SIZE_T*1`($sp)
 892         mtlr    r0
 893         addi    $sp,$sp,$FRAME
 894         blr
 895         .long   0
 896         .byte   0,12,0x04,1,0x80,18,5,0
 897         .long   0
 898 .size   .ChaCha20_ctr32_vmx,.-.ChaCha20_ctr32_vmx
 899
 900 .align  5
 901 Lconsts:
 902         mflr    r0
 903         bcl     20,31,\$+4
 904         mflr    r12     #vvvvv "distance between . and _vpaes_consts
 905         addi    r12,r12,`64-8`
 906         mtlr    r0
 907         blr
 908         .long   0
 909         .byte   0,12,0x14,0,0,0,0,0
 910         .space  `64-9*4`
 911 Lsigma:
 912         .long   0x61707865,0x3320646e,0x79622d32,0x6b206574
 913         .long   1,0,0,0
 914         .long   4,0,0,0
 915 ___
 916 $code.=<<___    if ($LITTLE_ENDIAN);
 917         .long   0x0e0f0c0d,0x0a0b0809,0x06070405,0x02030001
 918         .long   0x0d0e0f0c,0x090a0b08,0x05060704,0x01020300
 919 ___
 920 $code.=<<___    if (!$LITTLE_ENDIAN);   # flipped words
 921         .long   0x02030001,0x06070405,0x0a0b0809,0x0e0f0c0d
 922         .long   0x01020300,0x05060704,0x090a0b08,0x0d0e0f0c
 923 ___
 924 $code.=<<___;
 925 .asciz  "ChaCha20 for PowerPC/AltiVec, CRYPTOGAMS by <appro\@openssl.org>"
 926 .align  2
 927 ___
 928 }}}
 929
 930 foreach (split("\n",$code)) {
 931         s/\`([^\`]*)\`/eval $1/ge;
 932
 933         # instructions prefixed with '?' are endian-specific and need
 934         # to be adjusted accordingly...
 935         if ($flavour !~ /le$/) {        # big-endian
 936             s/be\?//            or
 937             s/le\?/#le#/        or
 938             s/\?lvsr/lvsl/      or
 939             s/\?lvsl/lvsr/      or
 940             s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/ or
 941             s/(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/;
 942         } else {                        # little-endian
 943             s/le\?//            or
 944             s/be\?/#be#/        or
 945             s/\?([a-z]+)/$1/;
 946         }
 947
 948         print $_,"\n";
 949 }
 950
 951 close STDOUT;