crypto/chacha/asm/chacha-ppc.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the OpenSSL license (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9 #
  10 # ====================================================================
  11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  12 # project. The module is, however, dual licensed under OpenSSL and
  13 # CRYPTOGAMS licenses depending on where you obtain it. For further
  14 # details see http://www.openssl.org/~appro/cryptogams/.
  15 # ====================================================================
  16 #
  17 # October 2015
  18 #
  19 # ChaCha20 for PowerPC/AltiVec.
  20 #
  21 # Performance in cycles per byte out of large buffer.
  22 #
  23 #                       IALU/gcc-4.x    3xAltiVec+1xIALU
  24 #
  25 # Freescale e300        13.6/+115%      -
  26 # PPC74x0               6.81/+310%      4.66
  27 # POWER7                8.62/+61%       4.27
  28 # POWER8                8.70/+51%       3.96
  29
  30 $flavour = shift;
  31
  32 if ($flavour =~ /64/) {
  33         $SIZE_T =8;
  34         $LRSAVE =2*$SIZE_T;
  35         $STU    ="stdu";
  36         $POP    ="ld";
  37         $PUSH   ="std";
  38         $UCMP   ="cmpld";
  39 } elsif ($flavour =~ /32/) {
  40         $SIZE_T =4;
  41         $LRSAVE =$SIZE_T;
  42         $STU    ="stwu";
  43         $POP    ="lwz";
  44         $PUSH   ="stw";
  45         $UCMP   ="cmplw";
  46 } else { die "nonsense $flavour"; }
  47
  48 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
  49
  50 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  51 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
  52 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
  53 die "can't locate ppc-xlate.pl";
  54
  55 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
  56
  57 $LOCALS=6*$SIZE_T;
  58 $FRAME=$LOCALS+64+18*$SIZE_T;   # 64 is for local variables
  59
  60 sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
  61 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
  62     $code .= "\t$opcode\t".join(',',@_)."\n";
  63 }
  64
  65 my $sp = "r1";
  66
  67 my ($out,$inp,$len,$key,$ctr) = map("r$_",(3..7));
  68
  69 my @x=map("r$_",(16..31));
  70 my @d=map("r$_",(11,12,14,15));
  71 my @t=map("r$_",(7..10));
  72
  73 sub ROUND {
  74 my ($a0,$b0,$c0,$d0)=@_;
  75 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
  76 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
  77 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
  78
  79     (
  80         "&add           (@x[$a0],@x[$a0],@x[$b0])",
  81          "&add          (@x[$a1],@x[$a1],@x[$b1])",
  82           "&add         (@x[$a2],@x[$a2],@x[$b2])",
  83            "&add        (@x[$a3],@x[$a3],@x[$b3])",
  84         "&xor           (@x[$d0],@x[$d0],@x[$a0])",
  85          "&xor          (@x[$d1],@x[$d1],@x[$a1])",
  86           "&xor         (@x[$d2],@x[$d2],@x[$a2])",
  87            "&xor        (@x[$d3],@x[$d3],@x[$a3])",
  88         "&rotlwi        (@x[$d0],@x[$d0],16)",
  89          "&rotlwi       (@x[$d1],@x[$d1],16)",
  90           "&rotlwi      (@x[$d2],@x[$d2],16)",
  91            "&rotlwi     (@x[$d3],@x[$d3],16)",
  92
  93         "&add           (@x[$c0],@x[$c0],@x[$d0])",
  94          "&add          (@x[$c1],@x[$c1],@x[$d1])",
  95           "&add         (@x[$c2],@x[$c2],@x[$d2])",
  96            "&add        (@x[$c3],@x[$c3],@x[$d3])",
  97         "&xor           (@x[$b0],@x[$b0],@x[$c0])",
  98          "&xor          (@x[$b1],@x[$b1],@x[$c1])",
  99           "&xor         (@x[$b2],@x[$b2],@x[$c2])",
 100            "&xor        (@x[$b3],@x[$b3],@x[$c3])",
 101         "&rotlwi        (@x[$b0],@x[$b0],12)",
 102          "&rotlwi       (@x[$b1],@x[$b1],12)",
 103           "&rotlwi      (@x[$b2],@x[$b2],12)",
 104            "&rotlwi     (@x[$b3],@x[$b3],12)",
 105
 106         "&add           (@x[$a0],@x[$a0],@x[$b0])",
 107          "&add          (@x[$a1],@x[$a1],@x[$b1])",
 108           "&add         (@x[$a2],@x[$a2],@x[$b2])",
 109            "&add        (@x[$a3],@x[$a3],@x[$b3])",
 110         "&xor           (@x[$d0],@x[$d0],@x[$a0])",
 111          "&xor          (@x[$d1],@x[$d1],@x[$a1])",
 112           "&xor         (@x[$d2],@x[$d2],@x[$a2])",
 113            "&xor        (@x[$d3],@x[$d3],@x[$a3])",
 114         "&rotlwi        (@x[$d0],@x[$d0],8)",
 115          "&rotlwi       (@x[$d1],@x[$d1],8)",
 116           "&rotlwi      (@x[$d2],@x[$d2],8)",
 117            "&rotlwi     (@x[$d3],@x[$d3],8)",
 118
 119         "&add           (@x[$c0],@x[$c0],@x[$d0])",
 120          "&add          (@x[$c1],@x[$c1],@x[$d1])",
 121           "&add         (@x[$c2],@x[$c2],@x[$d2])",
 122            "&add        (@x[$c3],@x[$c3],@x[$d3])",
 123         "&xor           (@x[$b0],@x[$b0],@x[$c0])",
 124          "&xor          (@x[$b1],@x[$b1],@x[$c1])",
 125           "&xor         (@x[$b2],@x[$b2],@x[$c2])",
 126            "&xor        (@x[$b3],@x[$b3],@x[$c3])",
 127         "&rotlwi        (@x[$b0],@x[$b0],7)",
 128          "&rotlwi       (@x[$b1],@x[$b1],7)",
 129           "&rotlwi      (@x[$b2],@x[$b2],7)",
 130            "&rotlwi     (@x[$b3],@x[$b3],7)"
 131     );
 132 }
 133
 134 $code.=<<___;
 135 .machine        "any"
 136 .text
 137
 138 .globl  .ChaCha20_ctr32_int
 139 .align  5
 140 .ChaCha20_ctr32_int:
 141 __ChaCha20_ctr32_int:
 142         ${UCMP}i $len,0
 143         beqlr-
 144
 145         $STU    $sp,-$FRAME($sp)
 146         mflr    r0
 147
 148         $PUSH   r14,`$FRAME-$SIZE_T*18`($sp)
 149         $PUSH   r15,`$FRAME-$SIZE_T*17`($sp)
 150         $PUSH   r16,`$FRAME-$SIZE_T*16`($sp)
 151         $PUSH   r17,`$FRAME-$SIZE_T*15`($sp)
 152         $PUSH   r18,`$FRAME-$SIZE_T*14`($sp)
 153         $PUSH   r19,`$FRAME-$SIZE_T*13`($sp)
 154         $PUSH   r20,`$FRAME-$SIZE_T*12`($sp)
 155         $PUSH   r21,`$FRAME-$SIZE_T*11`($sp)
 156         $PUSH   r22,`$FRAME-$SIZE_T*10`($sp)
 157         $PUSH   r23,`$FRAME-$SIZE_T*9`($sp)
 158         $PUSH   r24,`$FRAME-$SIZE_T*8`($sp)
 159         $PUSH   r25,`$FRAME-$SIZE_T*7`($sp)
 160         $PUSH   r26,`$FRAME-$SIZE_T*6`($sp)
 161         $PUSH   r27,`$FRAME-$SIZE_T*5`($sp)
 162         $PUSH   r28,`$FRAME-$SIZE_T*4`($sp)
 163         $PUSH   r29,`$FRAME-$SIZE_T*3`($sp)
 164         $PUSH   r30,`$FRAME-$SIZE_T*2`($sp)
 165         $PUSH   r31,`$FRAME-$SIZE_T*1`($sp)
 166         $PUSH   r0,`$FRAME+$LRSAVE`($sp)
 167
 168         lwz     @d[0],0($ctr)                   # load counter
 169         lwz     @d[1],4($ctr)
 170         lwz     @d[2],8($ctr)
 171         lwz     @d[3],12($ctr)
 172
 173         bl      __ChaCha20_1x
 174
 175         $POP    r0,`$FRAME+$LRSAVE`($sp)
 176         $POP    r14,`$FRAME-$SIZE_T*18`($sp)
 177         $POP    r15,`$FRAME-$SIZE_T*17`($sp)
 178         $POP    r16,`$FRAME-$SIZE_T*16`($sp)
 179         $POP    r17,`$FRAME-$SIZE_T*15`($sp)
 180         $POP    r18,`$FRAME-$SIZE_T*14`($sp)
 181         $POP    r19,`$FRAME-$SIZE_T*13`($sp)
 182         $POP    r20,`$FRAME-$SIZE_T*12`($sp)
 183         $POP    r21,`$FRAME-$SIZE_T*11`($sp)
 184         $POP    r22,`$FRAME-$SIZE_T*10`($sp)
 185         $POP    r23,`$FRAME-$SIZE_T*9`($sp)
 186         $POP    r24,`$FRAME-$SIZE_T*8`($sp)
 187         $POP    r25,`$FRAME-$SIZE_T*7`($sp)
 188         $POP    r26,`$FRAME-$SIZE_T*6`($sp)
 189         $POP    r27,`$FRAME-$SIZE_T*5`($sp)
 190         $POP    r28,`$FRAME-$SIZE_T*4`($sp)
 191         $POP    r29,`$FRAME-$SIZE_T*3`($sp)
 192         $POP    r30,`$FRAME-$SIZE_T*2`($sp)
 193         $POP    r31,`$FRAME-$SIZE_T*1`($sp)
 194         mtlr    r0
 195         addi    $sp,$sp,$FRAME
 196         blr
 197         .long   0
 198         .byte   0,12,4,1,0x80,18,5,0
 199         .long   0
 200 .size   .ChaCha20_ctr32_int,.-.ChaCha20_ctr32_int
 201
 202 .align  5
 203 __ChaCha20_1x:
 204 Loop_outer:
 205         lis     @x[0],0x6170                    # synthesize sigma
 206         lis     @x[1],0x3320
 207         lis     @x[2],0x7962
 208         lis     @x[3],0x6b20
 209         ori     @x[0],@x[0],0x7865
 210         ori     @x[1],@x[1],0x646e
 211         ori     @x[2],@x[2],0x2d32
 212         ori     @x[3],@x[3],0x6574
 213
 214         li      r0,10                           # inner loop counter
 215         lwz     @x[4],0($key)                   # load key
 216         lwz     @x[5],4($key)
 217         lwz     @x[6],8($key)
 218         lwz     @x[7],12($key)
 219         lwz     @x[8],16($key)
 220         mr      @x[12],@d[0]                    # copy counter
 221         lwz     @x[9],20($key)
 222         mr      @x[13],@d[1]
 223         lwz     @x[10],24($key)
 224         mr      @x[14],@d[2]
 225         lwz     @x[11],28($key)
 226         mr      @x[15],@d[3]
 227
 228         mr      @t[0],@x[4]
 229         mr      @t[1],@x[5]
 230         mr      @t[2],@x[6]
 231         mr      @t[3],@x[7]
 232
 233         mtctr   r0
 234 Loop:
 235 ___
 236         foreach (&ROUND(0, 4, 8,12)) { eval; }
 237         foreach (&ROUND(0, 5,10,15)) { eval; }
 238 $code.=<<___;
 239         bdnz    Loop
 240
 241         subic   $len,$len,64                    # $len-=64
 242         addi    @x[0],@x[0],0x7865              # accumulate key block
 243         addi    @x[1],@x[1],0x646e
 244         addi    @x[2],@x[2],0x2d32
 245         addi    @x[3],@x[3],0x6574
 246         addis   @x[0],@x[0],0x6170
 247         addis   @x[1],@x[1],0x3320
 248         addis   @x[2],@x[2],0x7962
 249         addis   @x[3],@x[3],0x6b20
 250
 251         subfe.  r0,r0,r0                        # borrow?-1:0
 252         add     @x[4],@x[4],@t[0]
 253         lwz     @t[0],16($key)
 254         add     @x[5],@x[5],@t[1]
 255         lwz     @t[1],20($key)
 256         add     @x[6],@x[6],@t[2]
 257         lwz     @t[2],24($key)
 258         add     @x[7],@x[7],@t[3]
 259         lwz     @t[3],28($key)
 260         add     @x[8],@x[8],@t[0]
 261         add     @x[9],@x[9],@t[1]
 262         add     @x[10],@x[10],@t[2]
 263         add     @x[11],@x[11],@t[3]
 264
 265         add     @x[12],@x[12],@d[0]
 266         add     @x[13],@x[13],@d[1]
 267         add     @x[14],@x[14],@d[2]
 268         add     @x[15],@x[15],@d[3]
 269         addi    @d[0],@d[0],1                   # increment counter
 270 ___
 271 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {   # flip byte order
 272 $code.=<<___;
 273         mr      @t[$i&3],@x[$i]
 274         rotlwi  @x[$i],@x[$i],8
 275         rlwimi  @x[$i],@t[$i&3],24,0,7
 276         rlwimi  @x[$i],@t[$i&3],24,16,23
 277 ___
 278 } }
 279 $code.=<<___;
 280         bne     Ltail                           # $len-=64 borrowed
 281
 282         lwz     @t[0],0($inp)                   # load input, aligned or not
 283         lwz     @t[1],4($inp)
 284         ${UCMP}i $len,0                         # done already?
 285         lwz     @t[2],8($inp)
 286         lwz     @t[3],12($inp)
 287         xor     @x[0],@x[0],@t[0]               # xor with input
 288         lwz     @t[0],16($inp)
 289         xor     @x[1],@x[1],@t[1]
 290         lwz     @t[1],20($inp)
 291         xor     @x[2],@x[2],@t[2]
 292         lwz     @t[2],24($inp)
 293         xor     @x[3],@x[3],@t[3]
 294         lwz     @t[3],28($inp)
 295         xor     @x[4],@x[4],@t[0]
 296         lwz     @t[0],32($inp)
 297         xor     @x[5],@x[5],@t[1]
 298         lwz     @t[1],36($inp)
 299         xor     @x[6],@x[6],@t[2]
 300         lwz     @t[2],40($inp)
 301         xor     @x[7],@x[7],@t[3]
 302         lwz     @t[3],44($inp)
 303         xor     @x[8],@x[8],@t[0]
 304         lwz     @t[0],48($inp)
 305         xor     @x[9],@x[9],@t[1]
 306         lwz     @t[1],52($inp)
 307         xor     @x[10],@x[10],@t[2]
 308         lwz     @t[2],56($inp)
 309         xor     @x[11],@x[11],@t[3]
 310         lwz     @t[3],60($inp)
 311         xor     @x[12],@x[12],@t[0]
 312         stw     @x[0],0($out)                   # store output, aligned or not
 313         xor     @x[13],@x[13],@t[1]
 314         stw     @x[1],4($out)
 315         xor     @x[14],@x[14],@t[2]
 316         stw     @x[2],8($out)
 317         xor     @x[15],@x[15],@t[3]
 318         stw     @x[3],12($out)
 319         stw     @x[4],16($out)
 320         stw     @x[5],20($out)
 321         stw     @x[6],24($out)
 322         stw     @x[7],28($out)
 323         stw     @x[8],32($out)
 324         stw     @x[9],36($out)
 325         stw     @x[10],40($out)
 326         stw     @x[11],44($out)
 327         stw     @x[12],48($out)
 328         stw     @x[13],52($out)
 329         stw     @x[14],56($out)
 330         addi    $inp,$inp,64
 331         stw     @x[15],60($out)
 332         addi    $out,$out,64
 333
 334         bne     Loop_outer
 335
 336         blr
 337
 338 .align  4
 339 Ltail:
 340         addi    $len,$len,64                    # restore tail length
 341         subi    $inp,$inp,1                     # prepare for *++ptr
 342         subi    $out,$out,1
 343         addi    @t[0],$sp,$LOCALS-1
 344         mtctr   $len
 345
 346         stw     @x[0],`$LOCALS+0`($sp)          # save whole block to stack
 347         stw     @x[1],`$LOCALS+4`($sp)
 348         stw     @x[2],`$LOCALS+8`($sp)
 349         stw     @x[3],`$LOCALS+12`($sp)
 350         stw     @x[4],`$LOCALS+16`($sp)
 351         stw     @x[5],`$LOCALS+20`($sp)
 352         stw     @x[6],`$LOCALS+24`($sp)
 353         stw     @x[7],`$LOCALS+28`($sp)
 354         stw     @x[8],`$LOCALS+32`($sp)
 355         stw     @x[9],`$LOCALS+36`($sp)
 356         stw     @x[10],`$LOCALS+40`($sp)
 357         stw     @x[11],`$LOCALS+44`($sp)
 358         stw     @x[12],`$LOCALS+48`($sp)
 359         stw     @x[13],`$LOCALS+52`($sp)
 360         stw     @x[14],`$LOCALS+56`($sp)
 361         stw     @x[15],`$LOCALS+60`($sp)
 362
 363 Loop_tail:                                      # byte-by-byte loop
 364         lbzu    @d[0],1($inp)
 365         lbzu    @x[0],1(@t[0])
 366         xor     @d[1],@d[0],@x[0]
 367         stbu    @d[1],1($out)
 368         bdnz    Loop_tail
 369
 370         stw     $sp,`$LOCALS+0`($sp)            # wipe block on stack
 371         stw     $sp,`$LOCALS+4`($sp)
 372         stw     $sp,`$LOCALS+8`($sp)
 373         stw     $sp,`$LOCALS+12`($sp)
 374         stw     $sp,`$LOCALS+16`($sp)
 375         stw     $sp,`$LOCALS+20`($sp)
 376         stw     $sp,`$LOCALS+24`($sp)
 377         stw     $sp,`$LOCALS+28`($sp)
 378         stw     $sp,`$LOCALS+32`($sp)
 379         stw     $sp,`$LOCALS+36`($sp)
 380         stw     $sp,`$LOCALS+40`($sp)
 381         stw     $sp,`$LOCALS+44`($sp)
 382         stw     $sp,`$LOCALS+48`($sp)
 383         stw     $sp,`$LOCALS+52`($sp)
 384         stw     $sp,`$LOCALS+56`($sp)
 385         stw     $sp,`$LOCALS+60`($sp)
 386
 387         blr
 388         .long   0
 389         .byte   0,12,0x14,0,0,0,0,0
 390 ___
 391
 392 {{{
 393 my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,$T0,$T1,$T2) =
 394     map("v$_",(0..14));
 395 my (@K)=map("v$_",(15..20));
 396 my ($FOUR,$sixteen,$twenty4,$twenty,$twelve,$twenty5,$seven) =
 397     map("v$_",(21..27));
 398 my ($inpperm,$outperm,$outmask) = map("v$_",(28..30));
 399 my @D=("v31",$seven,$T0,$T1,$T2);
 400
 401 my $FRAME=$LOCALS+64+13*16+18*$SIZE_T;  # 13*16 is for v20-v31 offload
 402
 403 sub VMXROUND {
 404 my $odd = pop;
 405 my ($a,$b,$c,$d,$t)=@_;
 406
 407         (
 408         "&vadduwm       ('$a','$a','$b')",
 409         "&vxor          ('$d','$d','$a')",
 410         "&vperm         ('$d','$d','$d','$sixteen')",
 411
 412         "&vadduwm       ('$c','$c','$d')",
 413         "&vxor          ('$t','$b','$c')",
 414         "&vsrw          ('$b','$t','$twenty')",
 415         "&vslw          ('$t','$t','$twelve')",
 416         "&vor           ('$b','$b','$t')",
 417
 418         "&vadduwm       ('$a','$a','$b')",
 419         "&vxor          ('$d','$d','$a')",
 420         "&vperm         ('$d','$d','$d','$twenty4')",
 421
 422         "&vadduwm       ('$c','$c','$d')",
 423         "&vxor          ('$t','$b','$c')",
 424         "&vsrw          ('$b','$t','$twenty5')",
 425         "&vslw          ('$t','$t','$seven')",
 426         "&vor           ('$b','$b','$t')",
 427
 428         "&vsldoi        ('$c','$c','$c',8)",
 429         "&vsldoi        ('$b','$b','$b',$odd?4:12)",
 430         "&vsldoi        ('$d','$d','$d',$odd?12:4)"
 431         );
 432 }
 433
 434 $code.=<<___;
 435
 436 .globl  .ChaCha20_ctr32_vmx
 437 .align  5
 438 .ChaCha20_ctr32_vmx:
 439         ${UCMP}i $len,256
 440         blt     __ChaCha20_ctr32_int
 441
 442         $STU    $sp,-$FRAME($sp)
 443         mflr    r0
 444         li      r10,`15+$LOCALS+64`
 445         li      r11,`31+$LOCALS+64`
 446         mfspr   r12,256
 447         stvx    v20,r10,$sp
 448         addi    r10,r10,32
 449         stvx    v21,r11,$sp
 450         addi    r11,r11,32
 451         stvx    v22,r10,$sp
 452         addi    r10,r10,32
 453         stvx    v23,r11,$sp
 454         addi    r11,r11,32
 455         stvx    v24,r10,$sp
 456         addi    r10,r10,32
 457         stvx    v25,r11,$sp
 458         addi    r11,r11,32
 459         stvx    v26,r10,$sp
 460         addi    r10,r10,32
 461         stvx    v27,r11,$sp
 462         addi    r11,r11,32
 463         stvx    v28,r10,$sp
 464         addi    r10,r10,32
 465         stvx    v29,r11,$sp
 466         addi    r11,r11,32
 467         stvx    v30,r10,$sp
 468         stvx    v31,r11,$sp
 469         stw     r12,`$FRAME-$SIZE_T*18-4`($sp)  # save vrsave
 470         $PUSH   r14,`$FRAME-$SIZE_T*18`($sp)
 471         $PUSH   r15,`$FRAME-$SIZE_T*17`($sp)
 472         $PUSH   r16,`$FRAME-$SIZE_T*16`($sp)
 473         $PUSH   r17,`$FRAME-$SIZE_T*15`($sp)
 474         $PUSH   r18,`$FRAME-$SIZE_T*14`($sp)
 475         $PUSH   r19,`$FRAME-$SIZE_T*13`($sp)
 476         $PUSH   r20,`$FRAME-$SIZE_T*12`($sp)
 477         $PUSH   r21,`$FRAME-$SIZE_T*11`($sp)
 478         $PUSH   r22,`$FRAME-$SIZE_T*10`($sp)
 479         $PUSH   r23,`$FRAME-$SIZE_T*9`($sp)
 480         $PUSH   r24,`$FRAME-$SIZE_T*8`($sp)
 481         $PUSH   r25,`$FRAME-$SIZE_T*7`($sp)
 482         $PUSH   r26,`$FRAME-$SIZE_T*6`($sp)
 483         $PUSH   r27,`$FRAME-$SIZE_T*5`($sp)
 484         $PUSH   r28,`$FRAME-$SIZE_T*4`($sp)
 485         $PUSH   r29,`$FRAME-$SIZE_T*3`($sp)
 486         $PUSH   r30,`$FRAME-$SIZE_T*2`($sp)
 487         $PUSH   r31,`$FRAME-$SIZE_T*1`($sp)
 488         li      r12,-1
 489         $PUSH   r0, `$FRAME+$LRSAVE`($sp)
 490         mtspr   256,r12                         # preserve all AltiVec registers
 491
 492         bl      Lconsts                         # returns pointer Lsigma in r12
 493         li      @x[0],16
 494         li      @x[1],32
 495         li      @x[2],48
 496         li      @x[3],64
 497         li      @x[4],31                        # 31 is not a typo
 498         li      @x[5],15                        # nor is 15
 499
 500         lvx     @K[1],0,$key                    # load key
 501         ?lvsr   $T0,0,$key                      # prepare unaligned load
 502         lvx     @K[2],@x[0],$key
 503         lvx     @D[0],@x[4],$key
 504
 505         lvx     @K[3],0,$ctr                    # load counter
 506         ?lvsr   $T1,0,$ctr                      # prepare unaligned load
 507         lvx     @D[1],@x[5],$ctr
 508
 509         lvx     @K[0],0,r12                     # load constants
 510         lvx     @K[5],@x[0],r12                 # one
 511         lvx     $FOUR,@x[1],r12
 512         lvx     $sixteen,@x[2],r12
 513         lvx     $twenty4,@x[3],r12
 514
 515         ?vperm  @K[1],@K[2],@K[1],$T0           # align key
 516         ?vperm  @K[2],@D[0],@K[2],$T0
 517         ?vperm  @K[3],@D[1],@K[3],$T1           # align counter
 518
 519         lwz     @d[0],0($ctr)                   # load counter to GPR
 520         lwz     @d[1],4($ctr)
 521         vadduwm @K[3],@K[3],@K[5]               # adjust AltiVec counter
 522         lwz     @d[2],8($ctr)
 523         vadduwm @K[4],@K[3],@K[5]
 524         lwz     @d[3],12($ctr)
 525         vadduwm @K[5],@K[4],@K[5]
 526
 527         vspltisw $twenty,-12                    # synthesize constants
 528         vspltisw $twelve,12
 529         vspltisw $twenty5,-7
 530         #vspltisw $seven,7                      # synthesized in the loop
 531
 532         vxor    $T0,$T0,$T0                     # 0x00..00
 533         vspltisw $outmask,-1                    # 0xff..ff
 534         ?lvsr   $inpperm,0,$inp                 # prepare for unaligned load
 535         ?lvsl   $outperm,0,$out                 # prepare for unaligned store
 536         ?vperm  $outmask,$outmask,$T0,$outperm
 537
 538         be?lvsl $T0,0,@x[0]                     # 0x00..0f
 539         be?vspltisb $T1,3                       # 0x03..03
 540         be?vxor $T0,$T0,$T1                     # swap bytes within words
 541         be?vxor $outperm,$outperm,$T1
 542         be?vperm $inpperm,$inpperm,$inpperm,$T0
 543
 544         b       Loop_outer_vmx
 545
 546 .align  4
 547 Loop_outer_vmx:
 548         lis     @x[0],0x6170                    # synthesize sigma
 549         lis     @x[1],0x3320
 550          vmr    $A0,@K[0]
 551         lis     @x[2],0x7962
 552         lis     @x[3],0x6b20
 553          vmr    $A1,@K[0]
 554         ori     @x[0],@x[0],0x7865
 555         ori     @x[1],@x[1],0x646e
 556          vmr    $A2,@K[0]
 557         ori     @x[2],@x[2],0x2d32
 558         ori     @x[3],@x[3],0x6574
 559          vmr    $B0,@K[1]
 560
 561         li      r0,10                           # inner loop counter
 562         lwz     @x[4],0($key)                   # load key to GPR
 563          vmr    $B1,@K[1]
 564         lwz     @x[5],4($key)
 565          vmr    $B2,@K[1]
 566         lwz     @x[6],8($key)
 567          vmr    $C0,@K[2]
 568         lwz     @x[7],12($key)
 569          vmr    $C1,@K[2]
 570         lwz     @x[8],16($key)
 571          vmr    $C2,@K[2]
 572         mr      @x[12],@d[0]                    # copy GPR counter
 573         lwz     @x[9],20($key)
 574          vmr    $D0,@K[3]
 575         mr      @x[13],@d[1]
 576         lwz     @x[10],24($key)
 577          vmr    $D1,@K[4]
 578         mr      @x[14],@d[2]
 579         lwz     @x[11],28($key)
 580          vmr    $D2,@K[5]
 581         mr      @x[15],@d[3]
 582
 583         mr      @t[0],@x[4]
 584         mr      @t[1],@x[5]
 585         mr      @t[2],@x[6]
 586         mr      @t[3],@x[7]
 587         vspltisw $seven,7
 588
 589         mtctr   r0
 590         nop
 591 Loop_vmx:
 592 ___
 593         my @thread0=&VMXROUND($A0,$B0,$C0,$D0,$T0,0);
 594         my @thread1=&VMXROUND($A1,$B1,$C1,$D1,$T1,0);
 595         my @thread2=&VMXROUND($A2,$B2,$C2,$D2,$T2,0);
 596         my @thread3=&ROUND(0,4,8,12);
 597
 598         foreach (@thread0) {
 599                 eval;                   eval(shift(@thread3));
 600                 eval(shift(@thread1));  eval(shift(@thread3));
 601                 eval(shift(@thread2));  eval(shift(@thread3));
 602         }
 603
 604         @thread0=&VMXROUND($A0,$B0,$C0,$D0,$T0,1);
 605         @thread1=&VMXROUND($A1,$B1,$C1,$D1,$T1,1);
 606         @thread2=&VMXROUND($A2,$B2,$C2,$D2,$T2,1);
 607         @thread3=&ROUND(0,5,10,15);
 608
 609         foreach (@thread0) {
 610                 eval;                   eval(shift(@thread3));
 611                 eval(shift(@thread1));  eval(shift(@thread3));
 612                 eval(shift(@thread2));  eval(shift(@thread3));
 613         }
 614 $code.=<<___;
 615         bdnz    Loop_vmx
 616
 617         subi    $len,$len,256                   # $len-=256
 618         addi    @x[0],@x[0],0x7865              # accumulate key block
 619         addi    @x[1],@x[1],0x646e
 620         addi    @x[2],@x[2],0x2d32
 621         addi    @x[3],@x[3],0x6574
 622         addis   @x[0],@x[0],0x6170
 623         addis   @x[1],@x[1],0x3320
 624         addis   @x[2],@x[2],0x7962
 625         addis   @x[3],@x[3],0x6b20
 626         add     @x[4],@x[4],@t[0]
 627         lwz     @t[0],16($key)
 628         add     @x[5],@x[5],@t[1]
 629         lwz     @t[1],20($key)
 630         add     @x[6],@x[6],@t[2]
 631         lwz     @t[2],24($key)
 632         add     @x[7],@x[7],@t[3]
 633         lwz     @t[3],28($key)
 634         add     @x[8],@x[8],@t[0]
 635         add     @x[9],@x[9],@t[1]
 636         add     @x[10],@x[10],@t[2]
 637         add     @x[11],@x[11],@t[3]
 638         add     @x[12],@x[12],@d[0]
 639         add     @x[13],@x[13],@d[1]
 640         add     @x[14],@x[14],@d[2]
 641         add     @x[15],@x[15],@d[3]
 642
 643         vadduwm $A0,$A0,@K[0]                   # accumulate key block
 644         vadduwm $A1,$A1,@K[0]
 645         vadduwm $A2,$A2,@K[0]
 646         vadduwm $B0,$B0,@K[1]
 647         vadduwm $B1,$B1,@K[1]
 648         vadduwm $B2,$B2,@K[1]
 649         vadduwm $C0,$C0,@K[2]
 650         vadduwm $C1,$C1,@K[2]
 651         vadduwm $C2,$C2,@K[2]
 652         vadduwm $D0,$D0,@K[3]
 653         vadduwm $D1,$D1,@K[4]
 654         vadduwm $D2,$D2,@K[5]
 655
 656         addi    @d[0],@d[0],4                   # increment counter
 657         vadduwm @K[3],@K[3],$FOUR
 658         vadduwm @K[4],@K[4],$FOUR
 659         vadduwm @K[5],@K[5],$FOUR
 660
 661 ___
 662 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {   # flip byte order
 663 $code.=<<___;
 664         mr      @t[$i&3],@x[$i]
 665         rotlwi  @x[$i],@x[$i],8
 666         rlwimi  @x[$i],@t[$i&3],24,0,7
 667         rlwimi  @x[$i],@t[$i&3],24,16,23
 668 ___
 669 } }
 670 $code.=<<___;
 671         lwz     @t[0],0($inp)                   # load input, aligned or not
 672         lwz     @t[1],4($inp)
 673         lwz     @t[2],8($inp)
 674         lwz     @t[3],12($inp)
 675         xor     @x[0],@x[0],@t[0]               # xor with input
 676         lwz     @t[0],16($inp)
 677         xor     @x[1],@x[1],@t[1]
 678         lwz     @t[1],20($inp)
 679         xor     @x[2],@x[2],@t[2]
 680         lwz     @t[2],24($inp)
 681         xor     @x[3],@x[3],@t[3]
 682         lwz     @t[3],28($inp)
 683         xor     @x[4],@x[4],@t[0]
 684         lwz     @t[0],32($inp)
 685         xor     @x[5],@x[5],@t[1]
 686         lwz     @t[1],36($inp)
 687         xor     @x[6],@x[6],@t[2]
 688         lwz     @t[2],40($inp)
 689         xor     @x[7],@x[7],@t[3]
 690         lwz     @t[3],44($inp)
 691         xor     @x[8],@x[8],@t[0]
 692         lwz     @t[0],48($inp)
 693         xor     @x[9],@x[9],@t[1]
 694         lwz     @t[1],52($inp)
 695         xor     @x[10],@x[10],@t[2]
 696         lwz     @t[2],56($inp)
 697         xor     @x[11],@x[11],@t[3]
 698         lwz     @t[3],60($inp)
 699         xor     @x[12],@x[12],@t[0]
 700         stw     @x[0],0($out)                   # store output, aligned or not
 701         xor     @x[13],@x[13],@t[1]
 702         stw     @x[1],4($out)
 703         xor     @x[14],@x[14],@t[2]
 704         stw     @x[2],8($out)
 705         xor     @x[15],@x[15],@t[3]
 706         stw     @x[3],12($out)
 707         addi    $inp,$inp,64
 708         stw     @x[4],16($out)
 709         li      @t[0],16
 710         stw     @x[5],20($out)
 711         li      @t[1],32
 712         stw     @x[6],24($out)
 713         li      @t[2],48
 714         stw     @x[7],28($out)
 715         li      @t[3],64
 716         stw     @x[8],32($out)
 717         stw     @x[9],36($out)
 718         stw     @x[10],40($out)
 719         stw     @x[11],44($out)
 720         stw     @x[12],48($out)
 721         stw     @x[13],52($out)
 722         stw     @x[14],56($out)
 723         stw     @x[15],60($out)
 724         addi    $out,$out,64
 725
 726         lvx     @D[0],0,$inp                    # load input
 727         lvx     @D[1],@t[0],$inp
 728         lvx     @D[2],@t[1],$inp
 729         lvx     @D[3],@t[2],$inp
 730         lvx     @D[4],@t[3],$inp
 731         addi    $inp,$inp,64
 732
 733         ?vperm  @D[0],@D[1],@D[0],$inpperm      # align input
 734         ?vperm  @D[1],@D[2],@D[1],$inpperm
 735         ?vperm  @D[2],@D[3],@D[2],$inpperm
 736         ?vperm  @D[3],@D[4],@D[3],$inpperm
 737         vxor    $A0,$A0,@D[0]                   # xor with input
 738         vxor    $B0,$B0,@D[1]
 739         lvx     @D[1],@t[0],$inp                # keep loading input
 740         vxor    $C0,$C0,@D[2]
 741         lvx     @D[2],@t[1],$inp
 742         vxor    $D0,$D0,@D[3]
 743         lvx     @D[3],@t[2],$inp
 744         lvx     @D[0],@t[3],$inp
 745         addi    $inp,$inp,64
 746         li      @t[3],63                        # 63 is not a typo
 747         vperm   $A0,$A0,$A0,$outperm            # pre-misalign output
 748         vperm   $B0,$B0,$B0,$outperm
 749         vperm   $C0,$C0,$C0,$outperm
 750         vperm   $D0,$D0,$D0,$outperm
 751
 752         ?vperm  @D[4],@D[1],@D[4],$inpperm      # align input
 753         ?vperm  @D[1],@D[2],@D[1],$inpperm
 754         ?vperm  @D[2],@D[3],@D[2],$inpperm
 755         ?vperm  @D[3],@D[0],@D[3],$inpperm
 756         vxor    $A1,$A1,@D[4]
 757         vxor    $B1,$B1,@D[1]
 758         lvx     @D[1],@t[0],$inp                # keep loading input
 759         vxor    $C1,$C1,@D[2]
 760         lvx     @D[2],@t[1],$inp
 761         vxor    $D1,$D1,@D[3]
 762         lvx     @D[3],@t[2],$inp
 763         lvx     @D[4],@t[3],$inp                # redundant in aligned case
 764         addi    $inp,$inp,64
 765         vperm   $A1,$A1,$A1,$outperm            # pre-misalign output
 766         vperm   $B1,$B1,$B1,$outperm
 767         vperm   $C1,$C1,$C1,$outperm
 768         vperm   $D1,$D1,$D1,$outperm
 769
 770         ?vperm  @D[0],@D[1],@D[0],$inpperm      # align input
 771         ?vperm  @D[1],@D[2],@D[1],$inpperm
 772         ?vperm  @D[2],@D[3],@D[2],$inpperm
 773         ?vperm  @D[3],@D[4],@D[3],$inpperm
 774         vxor    $A2,$A2,@D[0]
 775         vxor    $B2,$B2,@D[1]
 776         vxor    $C2,$C2,@D[2]
 777         vxor    $D2,$D2,@D[3]
 778         vperm   $A2,$A2,$A2,$outperm            # pre-misalign output
 779         vperm   $B2,$B2,$B2,$outperm
 780         vperm   $C2,$C2,$C2,$outperm
 781         vperm   $D2,$D2,$D2,$outperm
 782
 783         andi.   @x[1],$out,15                   # is $out aligned?
 784         mr      @x[0],$out
 785
 786         vsel    @D[0],$A0,$B0,$outmask          # collect pre-misaligned output
 787         vsel    @D[1],$B0,$C0,$outmask
 788         vsel    @D[2],$C0,$D0,$outmask
 789         vsel    @D[3],$D0,$A1,$outmask
 790         vsel    $B0,$A1,$B1,$outmask
 791         vsel    $C0,$B1,$C1,$outmask
 792         vsel    $D0,$C1,$D1,$outmask
 793         vsel    $A1,$D1,$A2,$outmask
 794         vsel    $B1,$A2,$B2,$outmask
 795         vsel    $C1,$B2,$C2,$outmask
 796         vsel    $D1,$C2,$D2,$outmask
 797
 798         #stvx   $A0,0,$out                      # take it easy on the edges
 799         stvx    @D[0],@t[0],$out                # store output
 800         stvx    @D[1],@t[1],$out
 801         stvx    @D[2],@t[2],$out
 802         addi    $out,$out,64
 803         stvx    @D[3],0,$out
 804         stvx    $B0,@t[0],$out
 805         stvx    $C0,@t[1],$out
 806         stvx    $D0,@t[2],$out
 807         addi    $out,$out,64
 808         stvx    $A1,0,$out
 809         stvx    $B1,@t[0],$out
 810         stvx    $C1,@t[1],$out
 811         stvx    $D1,@t[2],$out
 812         addi    $out,$out,64
 813
 814         beq     Laligned_vmx
 815
 816         sub     @x[2],$out,@x[1]                # in misaligned case edges
 817         li      @x[3],0                         # are written byte-by-byte
 818 Lunaligned_tail_vmx:
 819         stvebx  $D2,@x[3],@x[2]
 820         addi    @x[3],@x[3],1
 821         cmpw    @x[3],@x[1]
 822         bne     Lunaligned_tail_vmx
 823
 824         sub     @x[2],@x[0],@x[1]
 825 Lunaligned_head_vmx:
 826         stvebx  $A0,@x[1],@x[2]
 827         cmpwi   @x[1],15
 828         addi    @x[1],@x[1],1
 829         bne     Lunaligned_head_vmx
 830
 831         ${UCMP}i $len,255                       # done with 256-byte blocks yet?
 832         bgt     Loop_outer_vmx
 833
 834         b       Ldone_vmx
 835
 836 .align  4
 837 Laligned_vmx:
 838         stvx    $A0,0,@x[0]                     # head hexaword was not stored
 839
 840         ${UCMP}i $len,255                       # done with 256-byte blocks yet?
 841         bgt     Loop_outer_vmx
 842         nop
 843
 844 Ldone_vmx:
 845         ${UCMP}i $len,0                         # done yet?
 846         bnel    __ChaCha20_1x
 847
 848         lwz     r12,`$FRAME-$SIZE_T*18-4`($sp)  # pull vrsave
 849         li      r10,`15+$LOCALS+64`
 850         li      r11,`31+$LOCALS+64`
 851         mtspr   256,r12                         # restore vrsave
 852         lvx     v20,r10,$sp
 853         addi    r10,r10,32
 854         lvx     v21,r11,$sp
 855         addi    r11,r11,32
 856         lvx     v22,r10,$sp
 857         addi    r10,r10,32
 858         lvx     v23,r11,$sp
 859         addi    r11,r11,32
 860         lvx     v24,r10,$sp
 861         addi    r10,r10,32
 862         lvx     v25,r11,$sp
 863         addi    r11,r11,32
 864         lvx     v26,r10,$sp
 865         addi    r10,r10,32
 866         lvx     v27,r11,$sp
 867         addi    r11,r11,32
 868         lvx     v28,r10,$sp
 869         addi    r10,r10,32
 870         lvx     v29,r11,$sp
 871         addi    r11,r11,32
 872         lvx     v30,r10,$sp
 873         lvx     v31,r11,$sp
 874         $POP    r0, `$FRAME+$LRSAVE`($sp)
 875         $POP    r14,`$FRAME-$SIZE_T*18`($sp)
 876         $POP    r15,`$FRAME-$SIZE_T*17`($sp)
 877         $POP    r16,`$FRAME-$SIZE_T*16`($sp)
 878         $POP    r17,`$FRAME-$SIZE_T*15`($sp)
 879         $POP    r18,`$FRAME-$SIZE_T*14`($sp)
 880         $POP    r19,`$FRAME-$SIZE_T*13`($sp)
 881         $POP    r20,`$FRAME-$SIZE_T*12`($sp)
 882         $POP    r21,`$FRAME-$SIZE_T*11`($sp)
 883         $POP    r22,`$FRAME-$SIZE_T*10`($sp)
 884         $POP    r23,`$FRAME-$SIZE_T*9`($sp)
 885         $POP    r24,`$FRAME-$SIZE_T*8`($sp)
 886         $POP    r25,`$FRAME-$SIZE_T*7`($sp)
 887         $POP    r26,`$FRAME-$SIZE_T*6`($sp)
 888         $POP    r27,`$FRAME-$SIZE_T*5`($sp)
 889         $POP    r28,`$FRAME-$SIZE_T*4`($sp)
 890         $POP    r29,`$FRAME-$SIZE_T*3`($sp)
 891         $POP    r30,`$FRAME-$SIZE_T*2`($sp)
 892         $POP    r31,`$FRAME-$SIZE_T*1`($sp)
 893         mtlr    r0
 894         addi    $sp,$sp,$FRAME
 895         blr
 896         .long   0
 897         .byte   0,12,0x04,1,0x80,18,5,0
 898         .long   0
 899 .size   .ChaCha20_ctr32_vmx,.-.ChaCha20_ctr32_vmx
 900
 901 .align  5
 902 Lconsts:
 903         mflr    r0
 904         bcl     20,31,\$+4
 905         mflr    r12     #vvvvv "distance between . and _vpaes_consts
 906         addi    r12,r12,`64-8`
 907         mtlr    r0
 908         blr
 909         .long   0
 910         .byte   0,12,0x14,0,0,0,0,0
 911         .space  `64-9*4`
 912 Lsigma:
 913         .long   0x61707865,0x3320646e,0x79622d32,0x6b206574
 914         .long   1,0,0,0
 915         .long   4,0,0,0
 916 ___
 917 $code.=<<___    if ($LITTLE_ENDIAN);
 918         .long   0x0e0f0c0d,0x0a0b0809,0x06070405,0x02030001
 919         .long   0x0d0e0f0c,0x090a0b08,0x05060704,0x01020300
 920 ___
 921 $code.=<<___    if (!$LITTLE_ENDIAN);   # flipped words
 922         .long   0x02030001,0x06070405,0x0a0b0809,0x0e0f0c0d
 923         .long   0x01020300,0x05060704,0x090a0b08,0x0d0e0f0c
 924 ___
 925 $code.=<<___;
 926 .asciz  "ChaCha20 for PowerPC/AltiVec, CRYPTOGAMS by <appro\@openssl.org>"
 927 .align  2
 928 ___
 929 }}}
 930
 931 foreach (split("\n",$code)) {
 932         s/\`([^\`]*)\`/eval $1/ge;
 933
 934         # instructions prefixed with '?' are endian-specific and need
 935         # to be adjusted accordingly...
 936         if ($flavour !~ /le$/) {        # big-endian
 937             s/be\?//            or
 938             s/le\?/#le#/        or
 939             s/\?lvsr/lvsl/      or
 940             s/\?lvsl/lvsr/      or
 941             s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/ or
 942             s/(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/;
 943         } else {                        # little-endian
 944             s/le\?//            or
 945             s/be\?/#be#/        or
 946             s/\?([a-z]+)/$1/;
 947         }
 948
 949         print $_,"\n";
 950 }
 951
 952 close STDOUT;