crypto/aes/asm/aesp8-ppc.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2014-2018 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License 2.0 (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9 #
  10 # ====================================================================
  11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  12 # project. The module is, however, dual licensed under OpenSSL and
  13 # CRYPTOGAMS licenses depending on where you obtain it. For further
  14 # details see http://www.openssl.org/~appro/cryptogams/.
  15 # ====================================================================
  16 #
  17 # This module implements support for AES instructions as per PowerISA
  18 # specification version 2.07, first implemented by POWER8 processor.
  19 # The module is endian-agnostic in sense that it supports both big-
  20 # and little-endian cases. Data alignment in parallelizable modes is
  21 # handled with VSX loads and stores, which implies MSR.VSX flag being
  22 # set. It should also be noted that ISA specification doesn't prohibit
  23 # alignment exceptions for these instructions on page boundaries.
  24 # Initially alignment was handled in pure AltiVec/VMX way [when data
  25 # is aligned programmatically, which in turn guarantees exception-
  26 # free execution], but it turned to hamper performance when vcipher
  27 # instructions are interleaved. It's reckoned that eventual
  28 # misalignment penalties at page boundaries are in average lower
  29 # than additional overhead in pure AltiVec approach.
  30 #
  31 # May 2016
  32 #
  33 # Add XTS subroutine, 9x on little- and 12x improvement on big-endian
  34 # systems were measured.
  35 #
  36 ######################################################################
  37 # Current large-block performance in cycles per byte processed with
  38 # 128-bit key (less is better).
  39 #
  40 #               CBC en-/decrypt CTR     XTS
  41 # POWER8[le]    3.96/0.72       0.74    1.1
  42 # POWER8[be]    3.75/0.65       0.66    1.0
  43 # POWER9[le]    4.02/0.86       0.84    1.05
  44 # POWER9[be]    3.99/0.78       0.79    0.97
  45
  46 # $output is the last argument if it looks like a file (it has an extension)
  47 # $flavour is the first argument if it doesn't look like a file
  48 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
  49 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
  50
  51 if ($flavour =~ /64/) {
  52         $SIZE_T =8;
  53         $LRSAVE =2*$SIZE_T;
  54         $STU    ="stdu";
  55         $POP    ="ld";
  56         $PUSH   ="std";
  57         $UCMP   ="cmpld";
  58         $SHL    ="sldi";
  59 } elsif ($flavour =~ /32/) {
  60         $SIZE_T =4;
  61         $LRSAVE =$SIZE_T;
  62         $STU    ="stwu";
  63         $POP    ="lwz";
  64         $PUSH   ="stw";
  65         $UCMP   ="cmplw";
  66         $SHL    ="slwi";
  67 } else { die "nonsense $flavour"; }
  68
  69 $LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;
  70
  71 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  72 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
  73 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
  74 die "can't locate ppc-xlate.pl";
  75
  76 open STDOUT,"| $^X $xlate $flavour \"$output\""
  77     or die "can't call $xlate: $!";
  78
  79 $FRAME=8*$SIZE_T;
  80 $prefix="aes_p8";
  81
  82 $sp="r1";
  83 $vrsave="r12";
  84
  85 #########################################################################
  86 {{{     # Key setup procedures                                          #
  87 my ($inp,$bits,$out,$ptr,$cnt,$rounds)=map("r$_",(3..8));
  88 my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6));
  89 my ($stage,$outperm,$outmask,$outhead,$outtail)=map("v$_",(7..11));
  90
  91 $code.=<<___;
  92 .machine        "any"
  93
  94 .text
  95
  96 .align  7
  97 rcon:
  98 .long   0x01000000, 0x01000000, 0x01000000, 0x01000000  ?rev
  99 .long   0x1b000000, 0x1b000000, 0x1b000000, 0x1b000000  ?rev
 100 .long   0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c  ?rev
 101 .long   0,0,0,0                                         ?asis
 102 Lconsts:
 103         mflr    r0
 104         bcl     20,31,\$+4
 105         mflr    $ptr     #vvvvv "distance between . and rcon
 106         addi    $ptr,$ptr,-0x48
 107         mtlr    r0
 108         blr
 109         .long   0
 110         .byte   0,12,0x14,0,0,0,0,0
 111 .asciz  "AES for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
 112
 113 .globl  .${prefix}_set_encrypt_key
 114 .align  5
 115 .${prefix}_set_encrypt_key:
 116 Lset_encrypt_key:
 117         mflr            r11
 118         $PUSH           r11,$LRSAVE($sp)
 119
 120         li              $ptr,-1
 121         ${UCMP}i        $inp,0
 122         beq-            Lenc_key_abort          # if ($inp==0) return -1;
 123         ${UCMP}i        $out,0
 124         beq-            Lenc_key_abort          # if ($out==0) return -1;
 125         li              $ptr,-2
 126         cmpwi           $bits,128
 127         blt-            Lenc_key_abort
 128         cmpwi           $bits,256
 129         bgt-            Lenc_key_abort
 130         andi.           r0,$bits,0x3f
 131         bne-            Lenc_key_abort
 132
 133         lis             r0,0xfff0
 134         mfspr           $vrsave,256
 135         mtspr           256,r0
 136
 137         bl              Lconsts
 138         mtlr            r11
 139
 140         neg             r9,$inp
 141         lvx             $in0,0,$inp
 142         addi            $inp,$inp,15            # 15 is not typo
 143         lvsr            $key,0,r9               # borrow $key
 144         li              r8,0x20
 145         cmpwi           $bits,192
 146         lvx             $in1,0,$inp
 147         le?vspltisb     $mask,0x0f              # borrow $mask
 148         lvx             $rcon,0,$ptr
 149         le?vxor         $key,$key,$mask         # adjust for byte swap
 150         lvx             $mask,r8,$ptr
 151         addi            $ptr,$ptr,0x10
 152         vperm           $in0,$in0,$in1,$key     # align [and byte swap in LE]
 153         li              $cnt,8
 154         vxor            $zero,$zero,$zero
 155         mtctr           $cnt
 156
 157         ?lvsr           $outperm,0,$out
 158         vspltisb        $outmask,-1
 159         lvx             $outhead,0,$out
 160         ?vperm          $outmask,$zero,$outmask,$outperm
 161
 162         blt             Loop128
 163         addi            $inp,$inp,8
 164         beq             L192
 165         addi            $inp,$inp,8
 166         b               L256
 167
 168 .align  4
 169 Loop128:
 170         vperm           $key,$in0,$in0,$mask    # rotate-n-splat
 171         vsldoi          $tmp,$zero,$in0,12      # >>32
 172          vperm          $outtail,$in0,$in0,$outperm     # rotate
 173          vsel           $stage,$outhead,$outtail,$outmask
 174          vmr            $outhead,$outtail
 175         vcipherlast     $key,$key,$rcon
 176          stvx           $stage,0,$out
 177          addi           $out,$out,16
 178
 179         vxor            $in0,$in0,$tmp
 180         vsldoi          $tmp,$zero,$tmp,12      # >>32
 181         vxor            $in0,$in0,$tmp
 182         vsldoi          $tmp,$zero,$tmp,12      # >>32
 183         vxor            $in0,$in0,$tmp
 184          vadduwm        $rcon,$rcon,$rcon
 185         vxor            $in0,$in0,$key
 186         bdnz            Loop128
 187
 188         lvx             $rcon,0,$ptr            # last two round keys
 189
 190         vperm           $key,$in0,$in0,$mask    # rotate-n-splat
 191         vsldoi          $tmp,$zero,$in0,12      # >>32
 192          vperm          $outtail,$in0,$in0,$outperm     # rotate
 193          vsel           $stage,$outhead,$outtail,$outmask
 194          vmr            $outhead,$outtail
 195         vcipherlast     $key,$key,$rcon
 196          stvx           $stage,0,$out
 197          addi           $out,$out,16
 198
 199         vxor            $in0,$in0,$tmp
 200         vsldoi          $tmp,$zero,$tmp,12      # >>32
 201         vxor            $in0,$in0,$tmp
 202         vsldoi          $tmp,$zero,$tmp,12      # >>32
 203         vxor            $in0,$in0,$tmp
 204          vadduwm        $rcon,$rcon,$rcon
 205         vxor            $in0,$in0,$key
 206
 207         vperm           $key,$in0,$in0,$mask    # rotate-n-splat
 208         vsldoi          $tmp,$zero,$in0,12      # >>32
 209          vperm          $outtail,$in0,$in0,$outperm     # rotate
 210          vsel           $stage,$outhead,$outtail,$outmask
 211          vmr            $outhead,$outtail
 212         vcipherlast     $key,$key,$rcon
 213          stvx           $stage,0,$out
 214          addi           $out,$out,16
 215
 216         vxor            $in0,$in0,$tmp
 217         vsldoi          $tmp,$zero,$tmp,12      # >>32
 218         vxor            $in0,$in0,$tmp
 219         vsldoi          $tmp,$zero,$tmp,12      # >>32
 220         vxor            $in0,$in0,$tmp
 221         vxor            $in0,$in0,$key
 222          vperm          $outtail,$in0,$in0,$outperm     # rotate
 223          vsel           $stage,$outhead,$outtail,$outmask
 224          vmr            $outhead,$outtail
 225          stvx           $stage,0,$out
 226
 227         addi            $inp,$out,15            # 15 is not typo
 228         addi            $out,$out,0x50
 229
 230         li              $rounds,10
 231         b               Ldone
 232
 233 .align  4
 234 L192:
 235         lvx             $tmp,0,$inp
 236         li              $cnt,4
 237          vperm          $outtail,$in0,$in0,$outperm     # rotate
 238          vsel           $stage,$outhead,$outtail,$outmask
 239          vmr            $outhead,$outtail
 240          stvx           $stage,0,$out
 241          addi           $out,$out,16
 242         vperm           $in1,$in1,$tmp,$key     # align [and byte swap in LE]
 243         vspltisb        $key,8                  # borrow $key
 244         mtctr           $cnt
 245         vsububm         $mask,$mask,$key        # adjust the mask
 246
 247 Loop192:
 248         vperm           $key,$in1,$in1,$mask    # roate-n-splat
 249         vsldoi          $tmp,$zero,$in0,12      # >>32
 250         vcipherlast     $key,$key,$rcon
 251
 252         vxor            $in0,$in0,$tmp
 253         vsldoi          $tmp,$zero,$tmp,12      # >>32
 254         vxor            $in0,$in0,$tmp
 255         vsldoi          $tmp,$zero,$tmp,12      # >>32
 256         vxor            $in0,$in0,$tmp
 257
 258          vsldoi         $stage,$zero,$in1,8
 259         vspltw          $tmp,$in0,3
 260         vxor            $tmp,$tmp,$in1
 261         vsldoi          $in1,$zero,$in1,12      # >>32
 262          vadduwm        $rcon,$rcon,$rcon
 263         vxor            $in1,$in1,$tmp
 264         vxor            $in0,$in0,$key
 265         vxor            $in1,$in1,$key
 266          vsldoi         $stage,$stage,$in0,8
 267
 268         vperm           $key,$in1,$in1,$mask    # rotate-n-splat
 269         vsldoi          $tmp,$zero,$in0,12      # >>32
 270          vperm          $outtail,$stage,$stage,$outperm # rotate
 271          vsel           $stage,$outhead,$outtail,$outmask
 272          vmr            $outhead,$outtail
 273         vcipherlast     $key,$key,$rcon
 274          stvx           $stage,0,$out
 275          addi           $out,$out,16
 276
 277          vsldoi         $stage,$in0,$in1,8
 278         vxor            $in0,$in0,$tmp
 279         vsldoi          $tmp,$zero,$tmp,12      # >>32
 280          vperm          $outtail,$stage,$stage,$outperm # rotate
 281          vsel           $stage,$outhead,$outtail,$outmask
 282          vmr            $outhead,$outtail
 283         vxor            $in0,$in0,$tmp
 284         vsldoi          $tmp,$zero,$tmp,12      # >>32
 285         vxor            $in0,$in0,$tmp
 286          stvx           $stage,0,$out
 287          addi           $out,$out,16
 288
 289         vspltw          $tmp,$in0,3
 290         vxor            $tmp,$tmp,$in1
 291         vsldoi          $in1,$zero,$in1,12      # >>32
 292          vadduwm        $rcon,$rcon,$rcon
 293         vxor            $in1,$in1,$tmp
 294         vxor            $in0,$in0,$key
 295         vxor            $in1,$in1,$key
 296          vperm          $outtail,$in0,$in0,$outperm     # rotate
 297          vsel           $stage,$outhead,$outtail,$outmask
 298          vmr            $outhead,$outtail
 299          stvx           $stage,0,$out
 300          addi           $inp,$out,15            # 15 is not typo
 301          addi           $out,$out,16
 302         bdnz            Loop192
 303
 304         li              $rounds,12
 305         addi            $out,$out,0x20
 306         b               Ldone
 307
 308 .align  4
 309 L256:
 310         lvx             $tmp,0,$inp
 311         li              $cnt,7
 312         li              $rounds,14
 313          vperm          $outtail,$in0,$in0,$outperm     # rotate
 314          vsel           $stage,$outhead,$outtail,$outmask
 315          vmr            $outhead,$outtail
 316          stvx           $stage,0,$out
 317          addi           $out,$out,16
 318         vperm           $in1,$in1,$tmp,$key     # align [and byte swap in LE]
 319         mtctr           $cnt
 320
 321 Loop256:
 322         vperm           $key,$in1,$in1,$mask    # rotate-n-splat
 323         vsldoi          $tmp,$zero,$in0,12      # >>32
 324          vperm          $outtail,$in1,$in1,$outperm     # rotate
 325          vsel           $stage,$outhead,$outtail,$outmask
 326          vmr            $outhead,$outtail
 327         vcipherlast     $key,$key,$rcon
 328          stvx           $stage,0,$out
 329          addi           $out,$out,16
 330
 331         vxor            $in0,$in0,$tmp
 332         vsldoi          $tmp,$zero,$tmp,12      # >>32
 333         vxor            $in0,$in0,$tmp
 334         vsldoi          $tmp,$zero,$tmp,12      # >>32
 335         vxor            $in0,$in0,$tmp
 336          vadduwm        $rcon,$rcon,$rcon
 337         vxor            $in0,$in0,$key
 338          vperm          $outtail,$in0,$in0,$outperm     # rotate
 339          vsel           $stage,$outhead,$outtail,$outmask
 340          vmr            $outhead,$outtail
 341          stvx           $stage,0,$out
 342          addi           $inp,$out,15            # 15 is not typo
 343          addi           $out,$out,16
 344         bdz             Ldone
 345
 346         vspltw          $key,$in0,3             # just splat
 347         vsldoi          $tmp,$zero,$in1,12      # >>32
 348         vsbox           $key,$key
 349
 350         vxor            $in1,$in1,$tmp
 351         vsldoi          $tmp,$zero,$tmp,12      # >>32
 352         vxor            $in1,$in1,$tmp
 353         vsldoi          $tmp,$zero,$tmp,12      # >>32
 354         vxor            $in1,$in1,$tmp
 355
 356         vxor            $in1,$in1,$key
 357         b               Loop256
 358
 359 .align  4
 360 Ldone:
 361         lvx             $in1,0,$inp             # redundant in aligned case
 362         vsel            $in1,$outhead,$in1,$outmask
 363         stvx            $in1,0,$inp
 364         li              $ptr,0
 365         mtspr           256,$vrsave
 366         stw             $rounds,0($out)
 367
 368 Lenc_key_abort:
 369         mr              r3,$ptr
 370         blr
 371         .long           0
 372         .byte           0,12,0x14,1,0,0,3,0
 373         .long           0
 374 .size   .${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key
 375
 376 .globl  .${prefix}_set_decrypt_key
 377 .align  5
 378 .${prefix}_set_decrypt_key:
 379         $STU            $sp,-$FRAME($sp)
 380         mflr            r10
 381         $PUSH           r10,$FRAME+$LRSAVE($sp)
 382         bl              Lset_encrypt_key
 383         mtlr            r10
 384
 385         cmpwi           r3,0
 386         bne-            Ldec_key_abort
 387
 388         slwi            $cnt,$rounds,4
 389         subi            $inp,$out,240           # first round key
 390         srwi            $rounds,$rounds,1
 391         add             $out,$inp,$cnt          # last round key
 392         mtctr           $rounds
 393
 394 Ldeckey:
 395         lwz             r0, 0($inp)
 396         lwz             r6, 4($inp)
 397         lwz             r7, 8($inp)
 398         lwz             r8, 12($inp)
 399         addi            $inp,$inp,16
 400         lwz             r9, 0($out)
 401         lwz             r10,4($out)
 402         lwz             r11,8($out)
 403         lwz             r12,12($out)
 404         stw             r0, 0($out)
 405         stw             r6, 4($out)
 406         stw             r7, 8($out)
 407         stw             r8, 12($out)
 408         subi            $out,$out,16
 409         stw             r9, -16($inp)
 410         stw             r10,-12($inp)
 411         stw             r11,-8($inp)
 412         stw             r12,-4($inp)
 413         bdnz            Ldeckey
 414
 415         xor             r3,r3,r3                # return value
 416 Ldec_key_abort:
 417         addi            $sp,$sp,$FRAME
 418         blr
 419         .long           0
 420         .byte           0,12,4,1,0x80,0,3,0
 421         .long           0
 422 .size   .${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key
 423 ___
 424 }}}
 425 #########################################################################
 426 {{{     # Single block en- and decrypt procedures                       #
 427 sub gen_block () {
 428 my $dir = shift;
 429 my $n   = $dir eq "de" ? "n" : "";
 430 my ($inp,$out,$key,$rounds,$idx)=map("r$_",(3..7));
 431
 432 $code.=<<___;
 433 .globl  .${prefix}_${dir}crypt
 434 .align  5
 435 .${prefix}_${dir}crypt:
 436         lwz             $rounds,240($key)
 437         lis             r0,0xfc00
 438         mfspr           $vrsave,256
 439         li              $idx,15                 # 15 is not typo
 440         mtspr           256,r0
 441
 442         lvx             v0,0,$inp
 443         neg             r11,$out
 444         lvx             v1,$idx,$inp
 445         lvsl            v2,0,$inp               # inpperm
 446         le?vspltisb     v4,0x0f
 447         ?lvsl           v3,0,r11                # outperm
 448         le?vxor         v2,v2,v4
 449         li              $idx,16
 450         vperm           v0,v0,v1,v2             # align [and byte swap in LE]
 451         lvx             v1,0,$key
 452         ?lvsl           v5,0,$key               # keyperm
 453         srwi            $rounds,$rounds,1
 454         lvx             v2,$idx,$key
 455         addi            $idx,$idx,16
 456         subi            $rounds,$rounds,1
 457         ?vperm          v1,v1,v2,v5             # align round key
 458
 459         vxor            v0,v0,v1
 460         lvx             v1,$idx,$key
 461         addi            $idx,$idx,16
 462         mtctr           $rounds
 463
 464 Loop_${dir}c:
 465         ?vperm          v2,v2,v1,v5
 466         v${n}cipher     v0,v0,v2
 467         lvx             v2,$idx,$key
 468         addi            $idx,$idx,16
 469         ?vperm          v1,v1,v2,v5
 470         v${n}cipher     v0,v0,v1
 471         lvx             v1,$idx,$key
 472         addi            $idx,$idx,16
 473         bdnz            Loop_${dir}c
 474
 475         ?vperm          v2,v2,v1,v5
 476         v${n}cipher     v0,v0,v2
 477         lvx             v2,$idx,$key
 478         ?vperm          v1,v1,v2,v5
 479         v${n}cipherlast v0,v0,v1
 480
 481         vspltisb        v2,-1
 482         vxor            v1,v1,v1
 483         li              $idx,15                 # 15 is not typo
 484         ?vperm          v2,v1,v2,v3             # outmask
 485         le?vxor         v3,v3,v4
 486         lvx             v1,0,$out               # outhead
 487         vperm           v0,v0,v0,v3             # rotate [and byte swap in LE]
 488         vsel            v1,v1,v0,v2
 489         lvx             v4,$idx,$out
 490         stvx            v1,0,$out
 491         vsel            v0,v0,v4,v2
 492         stvx            v0,$idx,$out
 493
 494         mtspr           256,$vrsave
 495         blr
 496         .long           0
 497         .byte           0,12,0x14,0,0,0,3,0
 498         .long           0
 499 .size   .${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt
 500 ___
 501 }
 502 &gen_block("en");
 503 &gen_block("de");
 504 }}}
 505 #########################################################################
 506 {{{     # CBC en- and decrypt procedures                                #
 507 my ($inp,$out,$len,$key,$ivp,$enc,$rounds,$idx)=map("r$_",(3..10));
 508 my ($rndkey0,$rndkey1,$inout,$tmp)=             map("v$_",(0..3));
 509 my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)=
 510                                                 map("v$_",(4..10));
 511 $code.=<<___;
 512 .globl  .${prefix}_cbc_encrypt
 513 .align  5
 514 .${prefix}_cbc_encrypt:
 515         ${UCMP}i        $len,16
 516         bltlr-
 517
 518         cmpwi           $enc,0                  # test direction
 519         lis             r0,0xffe0
 520         mfspr           $vrsave,256
 521         mtspr           256,r0
 522
 523         li              $idx,15
 524         vxor            $rndkey0,$rndkey0,$rndkey0
 525         le?vspltisb     $tmp,0x0f
 526
 527         lvx             $ivec,0,$ivp            # load [unaligned] iv
 528         lvsl            $inpperm,0,$ivp
 529         lvx             $inptail,$idx,$ivp
 530         le?vxor         $inpperm,$inpperm,$tmp
 531         vperm           $ivec,$ivec,$inptail,$inpperm
 532
 533         neg             r11,$inp
 534         ?lvsl           $keyperm,0,$key         # prepare for unaligned key
 535         lwz             $rounds,240($key)
 536
 537         lvsr            $inpperm,0,r11          # prepare for unaligned load
 538         lvx             $inptail,0,$inp
 539         addi            $inp,$inp,15            # 15 is not typo
 540         le?vxor         $inpperm,$inpperm,$tmp
 541
 542         ?lvsr           $outperm,0,$out         # prepare for unaligned store
 543         vspltisb        $outmask,-1
 544         lvx             $outhead,0,$out
 545         ?vperm          $outmask,$rndkey0,$outmask,$outperm
 546         le?vxor         $outperm,$outperm,$tmp
 547
 548         srwi            $rounds,$rounds,1
 549         li              $idx,16
 550         subi            $rounds,$rounds,1
 551         beq             Lcbc_dec
 552
 553 Lcbc_enc:
 554         vmr             $inout,$inptail
 555         lvx             $inptail,0,$inp
 556         addi            $inp,$inp,16
 557         mtctr           $rounds
 558         subi            $len,$len,16            # len-=16
 559
 560         lvx             $rndkey0,0,$key
 561          vperm          $inout,$inout,$inptail,$inpperm
 562         lvx             $rndkey1,$idx,$key
 563         addi            $idx,$idx,16
 564         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
 565         vxor            $inout,$inout,$rndkey0
 566         lvx             $rndkey0,$idx,$key
 567         addi            $idx,$idx,16
 568         vxor            $inout,$inout,$ivec
 569
 570 Loop_cbc_enc:
 571         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
 572         vcipher         $inout,$inout,$rndkey1
 573         lvx             $rndkey1,$idx,$key
 574         addi            $idx,$idx,16
 575         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
 576         vcipher         $inout,$inout,$rndkey0
 577         lvx             $rndkey0,$idx,$key
 578         addi            $idx,$idx,16
 579         bdnz            Loop_cbc_enc
 580
 581         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
 582         vcipher         $inout,$inout,$rndkey1
 583         lvx             $rndkey1,$idx,$key
 584         li              $idx,16
 585         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
 586         vcipherlast     $ivec,$inout,$rndkey0
 587         ${UCMP}i        $len,16
 588
 589         vperm           $tmp,$ivec,$ivec,$outperm
 590         vsel            $inout,$outhead,$tmp,$outmask
 591         vmr             $outhead,$tmp
 592         stvx            $inout,0,$out
 593         addi            $out,$out,16
 594         bge             Lcbc_enc
 595
 596         b               Lcbc_done
 597
 598 .align  4
 599 Lcbc_dec:
 600         ${UCMP}i        $len,128
 601         bge             _aesp8_cbc_decrypt8x
 602         vmr             $tmp,$inptail
 603         lvx             $inptail,0,$inp
 604         addi            $inp,$inp,16
 605         mtctr           $rounds
 606         subi            $len,$len,16            # len-=16
 607
 608         lvx             $rndkey0,0,$key
 609          vperm          $tmp,$tmp,$inptail,$inpperm
 610         lvx             $rndkey1,$idx,$key
 611         addi            $idx,$idx,16
 612         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
 613         vxor            $inout,$tmp,$rndkey0
 614         lvx             $rndkey0,$idx,$key
 615         addi            $idx,$idx,16
 616
 617 Loop_cbc_dec:
 618         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
 619         vncipher        $inout,$inout,$rndkey1
 620         lvx             $rndkey1,$idx,$key
 621         addi            $idx,$idx,16
 622         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
 623         vncipher        $inout,$inout,$rndkey0
 624         lvx             $rndkey0,$idx,$key
 625         addi            $idx,$idx,16
 626         bdnz            Loop_cbc_dec
 627
 628         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
 629         vncipher        $inout,$inout,$rndkey1
 630         lvx             $rndkey1,$idx,$key
 631         li              $idx,16
 632         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
 633         vncipherlast    $inout,$inout,$rndkey0
 634         ${UCMP}i        $len,16
 635
 636         vxor            $inout,$inout,$ivec
 637         vmr             $ivec,$tmp
 638         vperm           $tmp,$inout,$inout,$outperm
 639         vsel            $inout,$outhead,$tmp,$outmask
 640         vmr             $outhead,$tmp
 641         stvx            $inout,0,$out
 642         addi            $out,$out,16
 643         bge             Lcbc_dec
 644
 645 Lcbc_done:
 646         addi            $out,$out,-1
 647         lvx             $inout,0,$out           # redundant in aligned case
 648         vsel            $inout,$outhead,$inout,$outmask
 649         stvx            $inout,0,$out
 650
 651         neg             $enc,$ivp               # write [unaligned] iv
 652         li              $idx,15                 # 15 is not typo
 653         vxor            $rndkey0,$rndkey0,$rndkey0
 654         vspltisb        $outmask,-1
 655         le?vspltisb     $tmp,0x0f
 656         ?lvsl           $outperm,0,$enc
 657         ?vperm          $outmask,$rndkey0,$outmask,$outperm
 658         le?vxor         $outperm,$outperm,$tmp
 659         lvx             $outhead,0,$ivp
 660         vperm           $ivec,$ivec,$ivec,$outperm
 661         vsel            $inout,$outhead,$ivec,$outmask
 662         lvx             $inptail,$idx,$ivp
 663         stvx            $inout,0,$ivp
 664         vsel            $inout,$ivec,$inptail,$outmask
 665         stvx            $inout,$idx,$ivp
 666
 667         mtspr           256,$vrsave
 668         blr
 669         .long           0
 670         .byte           0,12,0x14,0,0,0,6,0
 671         .long           0
 672 ___
 673 #########################################################################
 674 {{      # Optimized CBC decrypt procedure                               #
 675 my $key_="r11";
 676 my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
 677     $x00=0 if ($flavour =~ /osx/);
 678 my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13));
 679 my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(14..21));
 680 my $rndkey0="v23";      # v24-v25 rotating buffer for first found keys
 681                         # v26-v31 last 6 round keys
 682 my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment
 683
 684 $code.=<<___;
 685 .align  5
 686 _aesp8_cbc_decrypt8x:
 687         $STU            $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
 688         li              r10,`$FRAME+8*16+15`
 689         li              r11,`$FRAME+8*16+31`
 690         stvx            v20,r10,$sp             # ABI says so
 691         addi            r10,r10,32
 692         stvx            v21,r11,$sp
 693         addi            r11,r11,32
 694         stvx            v22,r10,$sp
 695         addi            r10,r10,32
 696         stvx            v23,r11,$sp
 697         addi            r11,r11,32
 698         stvx            v24,r10,$sp
 699         addi            r10,r10,32
 700         stvx            v25,r11,$sp
 701         addi            r11,r11,32
 702         stvx            v26,r10,$sp
 703         addi            r10,r10,32
 704         stvx            v27,r11,$sp
 705         addi            r11,r11,32
 706         stvx            v28,r10,$sp
 707         addi            r10,r10,32
 708         stvx            v29,r11,$sp
 709         addi            r11,r11,32
 710         stvx            v30,r10,$sp
 711         stvx            v31,r11,$sp
 712         li              r0,-1
 713         stw             $vrsave,`$FRAME+21*16-4`($sp)   # save vrsave
 714         li              $x10,0x10
 715         $PUSH           r26,`$FRAME+21*16+0*$SIZE_T`($sp)
 716         li              $x20,0x20
 717         $PUSH           r27,`$FRAME+21*16+1*$SIZE_T`($sp)
 718         li              $x30,0x30
 719         $PUSH           r28,`$FRAME+21*16+2*$SIZE_T`($sp)
 720         li              $x40,0x40
 721         $PUSH           r29,`$FRAME+21*16+3*$SIZE_T`($sp)
 722         li              $x50,0x50
 723         $PUSH           r30,`$FRAME+21*16+4*$SIZE_T`($sp)
 724         li              $x60,0x60
 725         $PUSH           r31,`$FRAME+21*16+5*$SIZE_T`($sp)
 726         li              $x70,0x70
 727         mtspr           256,r0
 728
 729         subi            $rounds,$rounds,3       # -4 in total
 730         subi            $len,$len,128           # bias
 731
 732         lvx             $rndkey0,$x00,$key      # load key schedule
 733         lvx             v30,$x10,$key
 734         addi            $key,$key,0x20
 735         lvx             v31,$x00,$key
 736         ?vperm          $rndkey0,$rndkey0,v30,$keyperm
 737         addi            $key_,$sp,$FRAME+15
 738         mtctr           $rounds
 739
 740 Load_cbc_dec_key:
 741         ?vperm          v24,v30,v31,$keyperm
 742         lvx             v30,$x10,$key
 743         addi            $key,$key,0x20
 744         stvx            v24,$x00,$key_          # off-load round[1]
 745         ?vperm          v25,v31,v30,$keyperm
 746         lvx             v31,$x00,$key
 747         stvx            v25,$x10,$key_          # off-load round[2]
 748         addi            $key_,$key_,0x20
 749         bdnz            Load_cbc_dec_key
 750
 751         lvx             v26,$x10,$key
 752         ?vperm          v24,v30,v31,$keyperm
 753         lvx             v27,$x20,$key
 754         stvx            v24,$x00,$key_          # off-load round[3]
 755         ?vperm          v25,v31,v26,$keyperm
 756         lvx             v28,$x30,$key
 757         stvx            v25,$x10,$key_          # off-load round[4]
 758         addi            $key_,$sp,$FRAME+15     # rewind $key_
 759         ?vperm          v26,v26,v27,$keyperm
 760         lvx             v29,$x40,$key
 761         ?vperm          v27,v27,v28,$keyperm
 762         lvx             v30,$x50,$key
 763         ?vperm          v28,v28,v29,$keyperm
 764         lvx             v31,$x60,$key
 765         ?vperm          v29,v29,v30,$keyperm
 766         lvx             $out0,$x70,$key         # borrow $out0
 767         ?vperm          v30,v30,v31,$keyperm
 768         lvx             v24,$x00,$key_          # pre-load round[1]
 769         ?vperm          v31,v31,$out0,$keyperm
 770         lvx             v25,$x10,$key_          # pre-load round[2]
 771
 772         #lvx            $inptail,0,$inp         # "caller" already did this
 773         #addi           $inp,$inp,15            # 15 is not typo
 774         subi            $inp,$inp,15            # undo "caller"
 775
 776          le?li          $idx,8
 777         lvx_u           $in0,$x00,$inp          # load first 8 "words"
 778          le?lvsl        $inpperm,0,$idx
 779          le?vspltisb    $tmp,0x0f
 780         lvx_u           $in1,$x10,$inp
 781          le?vxor        $inpperm,$inpperm,$tmp  # transform for lvx_u/stvx_u
 782         lvx_u           $in2,$x20,$inp
 783          le?vperm       $in0,$in0,$in0,$inpperm
 784         lvx_u           $in3,$x30,$inp
 785          le?vperm       $in1,$in1,$in1,$inpperm
 786         lvx_u           $in4,$x40,$inp
 787          le?vperm       $in2,$in2,$in2,$inpperm
 788         vxor            $out0,$in0,$rndkey0
 789         lvx_u           $in5,$x50,$inp
 790          le?vperm       $in3,$in3,$in3,$inpperm
 791         vxor            $out1,$in1,$rndkey0
 792         lvx_u           $in6,$x60,$inp
 793          le?vperm       $in4,$in4,$in4,$inpperm
 794         vxor            $out2,$in2,$rndkey0
 795         lvx_u           $in7,$x70,$inp
 796         addi            $inp,$inp,0x80
 797          le?vperm       $in5,$in5,$in5,$inpperm
 798         vxor            $out3,$in3,$rndkey0
 799          le?vperm       $in6,$in6,$in6,$inpperm
 800         vxor            $out4,$in4,$rndkey0
 801          le?vperm       $in7,$in7,$in7,$inpperm
 802         vxor            $out5,$in5,$rndkey0
 803         vxor            $out6,$in6,$rndkey0
 804         vxor            $out7,$in7,$rndkey0
 805
 806         mtctr           $rounds
 807         b               Loop_cbc_dec8x
 808 .align  5
 809 Loop_cbc_dec8x:
 810         vncipher        $out0,$out0,v24
 811         vncipher        $out1,$out1,v24
 812         vncipher        $out2,$out2,v24
 813         vncipher        $out3,$out3,v24
 814         vncipher        $out4,$out4,v24
 815         vncipher        $out5,$out5,v24
 816         vncipher        $out6,$out6,v24
 817         vncipher        $out7,$out7,v24
 818         lvx             v24,$x20,$key_          # round[3]
 819         addi            $key_,$key_,0x20
 820
 821         vncipher        $out0,$out0,v25
 822         vncipher        $out1,$out1,v25
 823         vncipher        $out2,$out2,v25
 824         vncipher        $out3,$out3,v25
 825         vncipher        $out4,$out4,v25
 826         vncipher        $out5,$out5,v25
 827         vncipher        $out6,$out6,v25
 828         vncipher        $out7,$out7,v25
 829         lvx             v25,$x10,$key_          # round[4]
 830         bdnz            Loop_cbc_dec8x
 831
 832         subic           $len,$len,128           # $len-=128
 833         vncipher        $out0,$out0,v24
 834         vncipher        $out1,$out1,v24
 835         vncipher        $out2,$out2,v24
 836         vncipher        $out3,$out3,v24
 837         vncipher        $out4,$out4,v24
 838         vncipher        $out5,$out5,v24
 839         vncipher        $out6,$out6,v24
 840         vncipher        $out7,$out7,v24
 841
 842         subfe.          r0,r0,r0                # borrow?-1:0
 843         vncipher        $out0,$out0,v25
 844         vncipher        $out1,$out1,v25
 845         vncipher        $out2,$out2,v25
 846         vncipher        $out3,$out3,v25
 847         vncipher        $out4,$out4,v25
 848         vncipher        $out5,$out5,v25
 849         vncipher        $out6,$out6,v25
 850         vncipher        $out7,$out7,v25
 851
 852         and             r0,r0,$len
 853         vncipher        $out0,$out0,v26
 854         vncipher        $out1,$out1,v26
 855         vncipher        $out2,$out2,v26
 856         vncipher        $out3,$out3,v26
 857         vncipher        $out4,$out4,v26
 858         vncipher        $out5,$out5,v26
 859         vncipher        $out6,$out6,v26
 860         vncipher        $out7,$out7,v26
 861
 862         add             $inp,$inp,r0            # $inp is adjusted in such
 863                                                 # way that at exit from the
 864                                                 # loop inX-in7 are loaded
 865                                                 # with last "words"
 866         vncipher        $out0,$out0,v27
 867         vncipher        $out1,$out1,v27
 868         vncipher        $out2,$out2,v27
 869         vncipher        $out3,$out3,v27
 870         vncipher        $out4,$out4,v27
 871         vncipher        $out5,$out5,v27
 872         vncipher        $out6,$out6,v27
 873         vncipher        $out7,$out7,v27
 874
 875         addi            $key_,$sp,$FRAME+15     # rewind $key_
 876         vncipher        $out0,$out0,v28
 877         vncipher        $out1,$out1,v28
 878         vncipher        $out2,$out2,v28
 879         vncipher        $out3,$out3,v28
 880         vncipher        $out4,$out4,v28
 881         vncipher        $out5,$out5,v28
 882         vncipher        $out6,$out6,v28
 883         vncipher        $out7,$out7,v28
 884         lvx             v24,$x00,$key_          # re-pre-load round[1]
 885
 886         vncipher        $out0,$out0,v29
 887         vncipher        $out1,$out1,v29
 888         vncipher        $out2,$out2,v29
 889         vncipher        $out3,$out3,v29
 890         vncipher        $out4,$out4,v29
 891         vncipher        $out5,$out5,v29
 892         vncipher        $out6,$out6,v29
 893         vncipher        $out7,$out7,v29
 894         lvx             v25,$x10,$key_          # re-pre-load round[2]
 895
 896         vncipher        $out0,$out0,v30
 897          vxor           $ivec,$ivec,v31         # xor with last round key
 898         vncipher        $out1,$out1,v30
 899          vxor           $in0,$in0,v31
 900         vncipher        $out2,$out2,v30
 901          vxor           $in1,$in1,v31
 902         vncipher        $out3,$out3,v30
 903          vxor           $in2,$in2,v31
 904         vncipher        $out4,$out4,v30
 905          vxor           $in3,$in3,v31
 906         vncipher        $out5,$out5,v30
 907          vxor           $in4,$in4,v31
 908         vncipher        $out6,$out6,v30
 909          vxor           $in5,$in5,v31
 910         vncipher        $out7,$out7,v30
 911          vxor           $in6,$in6,v31
 912
 913         vncipherlast    $out0,$out0,$ivec
 914         vncipherlast    $out1,$out1,$in0
 915          lvx_u          $in0,$x00,$inp          # load next input block
 916         vncipherlast    $out2,$out2,$in1
 917          lvx_u          $in1,$x10,$inp
 918         vncipherlast    $out3,$out3,$in2
 919          le?vperm       $in0,$in0,$in0,$inpperm
 920          lvx_u          $in2,$x20,$inp
 921         vncipherlast    $out4,$out4,$in3
 922          le?vperm       $in1,$in1,$in1,$inpperm
 923          lvx_u          $in3,$x30,$inp
 924         vncipherlast    $out5,$out5,$in4
 925          le?vperm       $in2,$in2,$in2,$inpperm
 926          lvx_u          $in4,$x40,$inp
 927         vncipherlast    $out6,$out6,$in5
 928          le?vperm       $in3,$in3,$in3,$inpperm
 929          lvx_u          $in5,$x50,$inp
 930         vncipherlast    $out7,$out7,$in6
 931          le?vperm       $in4,$in4,$in4,$inpperm
 932          lvx_u          $in6,$x60,$inp
 933         vmr             $ivec,$in7
 934          le?vperm       $in5,$in5,$in5,$inpperm
 935          lvx_u          $in7,$x70,$inp
 936          addi           $inp,$inp,0x80
 937
 938         le?vperm        $out0,$out0,$out0,$inpperm
 939         le?vperm        $out1,$out1,$out1,$inpperm
 940         stvx_u          $out0,$x00,$out
 941          le?vperm       $in6,$in6,$in6,$inpperm
 942          vxor           $out0,$in0,$rndkey0
 943         le?vperm        $out2,$out2,$out2,$inpperm
 944         stvx_u          $out1,$x10,$out
 945          le?vperm       $in7,$in7,$in7,$inpperm
 946          vxor           $out1,$in1,$rndkey0
 947         le?vperm        $out3,$out3,$out3,$inpperm
 948         stvx_u          $out2,$x20,$out
 949          vxor           $out2,$in2,$rndkey0
 950         le?vperm        $out4,$out4,$out4,$inpperm
 951         stvx_u          $out3,$x30,$out
 952          vxor           $out3,$in3,$rndkey0
 953         le?vperm        $out5,$out5,$out5,$inpperm
 954         stvx_u          $out4,$x40,$out
 955          vxor           $out4,$in4,$rndkey0
 956         le?vperm        $out6,$out6,$out6,$inpperm
 957         stvx_u          $out5,$x50,$out
 958          vxor           $out5,$in5,$rndkey0
 959         le?vperm        $out7,$out7,$out7,$inpperm
 960         stvx_u          $out6,$x60,$out
 961          vxor           $out6,$in6,$rndkey0
 962         stvx_u          $out7,$x70,$out
 963         addi            $out,$out,0x80
 964          vxor           $out7,$in7,$rndkey0
 965
 966         mtctr           $rounds
 967         beq             Loop_cbc_dec8x          # did $len-=128 borrow?
 968
 969         addic.          $len,$len,128
 970         beq             Lcbc_dec8x_done
 971         nop
 972         nop
 973
 974 Loop_cbc_dec8x_tail:                            # up to 7 "words" tail...
 975         vncipher        $out1,$out1,v24
 976         vncipher        $out2,$out2,v24
 977         vncipher        $out3,$out3,v24
 978         vncipher        $out4,$out4,v24
 979         vncipher        $out5,$out5,v24
 980         vncipher        $out6,$out6,v24
 981         vncipher        $out7,$out7,v24
 982         lvx             v24,$x20,$key_          # round[3]
 983         addi            $key_,$key_,0x20
 984
 985         vncipher        $out1,$out1,v25
 986         vncipher        $out2,$out2,v25
 987         vncipher        $out3,$out3,v25
 988         vncipher        $out4,$out4,v25
 989         vncipher        $out5,$out5,v25
 990         vncipher        $out6,$out6,v25
 991         vncipher        $out7,$out7,v25
 992         lvx             v25,$x10,$key_          # round[4]
 993         bdnz            Loop_cbc_dec8x_tail
 994
 995         vncipher        $out1,$out1,v24
 996         vncipher        $out2,$out2,v24
 997         vncipher        $out3,$out3,v24
 998         vncipher        $out4,$out4,v24
 999         vncipher        $out5,$out5,v24
1000         vncipher        $out6,$out6,v24
1001         vncipher        $out7,$out7,v24
1002
1003         vncipher        $out1,$out1,v25
1004         vncipher        $out2,$out2,v25
1005         vncipher        $out3,$out3,v25
1006         vncipher        $out4,$out4,v25
1007         vncipher        $out5,$out5,v25
1008         vncipher        $out6,$out6,v25
1009         vncipher        $out7,$out7,v25
1010
1011         vncipher        $out1,$out1,v26
1012         vncipher        $out2,$out2,v26
1013         vncipher        $out3,$out3,v26
1014         vncipher        $out4,$out4,v26
1015         vncipher        $out5,$out5,v26
1016         vncipher        $out6,$out6,v26
1017         vncipher        $out7,$out7,v26
1018
1019         vncipher        $out1,$out1,v27
1020         vncipher        $out2,$out2,v27
1021         vncipher        $out3,$out3,v27
1022         vncipher        $out4,$out4,v27
1023         vncipher        $out5,$out5,v27
1024         vncipher        $out6,$out6,v27
1025         vncipher        $out7,$out7,v27
1026
1027         vncipher        $out1,$out1,v28
1028         vncipher        $out2,$out2,v28
1029         vncipher        $out3,$out3,v28
1030         vncipher        $out4,$out4,v28
1031         vncipher        $out5,$out5,v28
1032         vncipher        $out6,$out6,v28
1033         vncipher        $out7,$out7,v28
1034
1035         vncipher        $out1,$out1,v29
1036         vncipher        $out2,$out2,v29
1037         vncipher        $out3,$out3,v29
1038         vncipher        $out4,$out4,v29
1039         vncipher        $out5,$out5,v29
1040         vncipher        $out6,$out6,v29
1041         vncipher        $out7,$out7,v29
1042
1043         vncipher        $out1,$out1,v30
1044          vxor           $ivec,$ivec,v31         # last round key
1045         vncipher        $out2,$out2,v30
1046          vxor           $in1,$in1,v31
1047         vncipher        $out3,$out3,v30
1048          vxor           $in2,$in2,v31
1049         vncipher        $out4,$out4,v30
1050          vxor           $in3,$in3,v31
1051         vncipher        $out5,$out5,v30
1052          vxor           $in4,$in4,v31
1053         vncipher        $out6,$out6,v30
1054          vxor           $in5,$in5,v31
1055         vncipher        $out7,$out7,v30
1056          vxor           $in6,$in6,v31
1057
1058         cmplwi          $len,32                 # switch($len)
1059         blt             Lcbc_dec8x_one
1060         nop
1061         beq             Lcbc_dec8x_two
1062         cmplwi          $len,64
1063         blt             Lcbc_dec8x_three
1064         nop
1065         beq             Lcbc_dec8x_four
1066         cmplwi          $len,96
1067         blt             Lcbc_dec8x_five
1068         nop
1069         beq             Lcbc_dec8x_six
1070
1071 Lcbc_dec8x_seven:
1072         vncipherlast    $out1,$out1,$ivec
1073         vncipherlast    $out2,$out2,$in1
1074         vncipherlast    $out3,$out3,$in2
1075         vncipherlast    $out4,$out4,$in3
1076         vncipherlast    $out5,$out5,$in4
1077         vncipherlast    $out6,$out6,$in5
1078         vncipherlast    $out7,$out7,$in6
1079         vmr             $ivec,$in7
1080
1081         le?vperm        $out1,$out1,$out1,$inpperm
1082         le?vperm        $out2,$out2,$out2,$inpperm
1083         stvx_u          $out1,$x00,$out
1084         le?vperm        $out3,$out3,$out3,$inpperm
1085         stvx_u          $out2,$x10,$out
1086         le?vperm        $out4,$out4,$out4,$inpperm
1087         stvx_u          $out3,$x20,$out
1088         le?vperm        $out5,$out5,$out5,$inpperm
1089         stvx_u          $out4,$x30,$out
1090         le?vperm        $out6,$out6,$out6,$inpperm
1091         stvx_u          $out5,$x40,$out
1092         le?vperm        $out7,$out7,$out7,$inpperm
1093         stvx_u          $out6,$x50,$out
1094         stvx_u          $out7,$x60,$out
1095         addi            $out,$out,0x70
1096         b               Lcbc_dec8x_done
1097
1098 .align  5
1099 Lcbc_dec8x_six:
1100         vncipherlast    $out2,$out2,$ivec
1101         vncipherlast    $out3,$out3,$in2
1102         vncipherlast    $out4,$out4,$in3
1103         vncipherlast    $out5,$out5,$in4
1104         vncipherlast    $out6,$out6,$in5
1105         vncipherlast    $out7,$out7,$in6
1106         vmr             $ivec,$in7
1107
1108         le?vperm        $out2,$out2,$out2,$inpperm
1109         le?vperm        $out3,$out3,$out3,$inpperm
1110         stvx_u          $out2,$x00,$out
1111         le?vperm        $out4,$out4,$out4,$inpperm
1112         stvx_u          $out3,$x10,$out
1113         le?vperm        $out5,$out5,$out5,$inpperm
1114         stvx_u          $out4,$x20,$out
1115         le?vperm        $out6,$out6,$out6,$inpperm
1116         stvx_u          $out5,$x30,$out
1117         le?vperm        $out7,$out7,$out7,$inpperm
1118         stvx_u          $out6,$x40,$out
1119         stvx_u          $out7,$x50,$out
1120         addi            $out,$out,0x60
1121         b               Lcbc_dec8x_done
1122
1123 .align  5
1124 Lcbc_dec8x_five:
1125         vncipherlast    $out3,$out3,$ivec
1126         vncipherlast    $out4,$out4,$in3
1127         vncipherlast    $out5,$out5,$in4
1128         vncipherlast    $out6,$out6,$in5
1129         vncipherlast    $out7,$out7,$in6
1130         vmr             $ivec,$in7
1131
1132         le?vperm        $out3,$out3,$out3,$inpperm
1133         le?vperm        $out4,$out4,$out4,$inpperm
1134         stvx_u          $out3,$x00,$out
1135         le?vperm        $out5,$out5,$out5,$inpperm
1136         stvx_u          $out4,$x10,$out
1137         le?vperm        $out6,$out6,$out6,$inpperm
1138         stvx_u          $out5,$x20,$out
1139         le?vperm        $out7,$out7,$out7,$inpperm
1140         stvx_u          $out6,$x30,$out
1141         stvx_u          $out7,$x40,$out
1142         addi            $out,$out,0x50
1143         b               Lcbc_dec8x_done
1144
1145 .align  5
1146 Lcbc_dec8x_four:
1147         vncipherlast    $out4,$out4,$ivec
1148         vncipherlast    $out5,$out5,$in4
1149         vncipherlast    $out6,$out6,$in5
1150         vncipherlast    $out7,$out7,$in6
1151         vmr             $ivec,$in7
1152
1153         le?vperm        $out4,$out4,$out4,$inpperm
1154         le?vperm        $out5,$out5,$out5,$inpperm
1155         stvx_u          $out4,$x00,$out
1156         le?vperm        $out6,$out6,$out6,$inpperm
1157         stvx_u          $out5,$x10,$out
1158         le?vperm        $out7,$out7,$out7,$inpperm
1159         stvx_u          $out6,$x20,$out
1160         stvx_u          $out7,$x30,$out
1161         addi            $out,$out,0x40
1162         b               Lcbc_dec8x_done
1163
1164 .align  5
1165 Lcbc_dec8x_three:
1166         vncipherlast    $out5,$out5,$ivec
1167         vncipherlast    $out6,$out6,$in5
1168         vncipherlast    $out7,$out7,$in6
1169         vmr             $ivec,$in7
1170
1171         le?vperm        $out5,$out5,$out5,$inpperm
1172         le?vperm        $out6,$out6,$out6,$inpperm
1173         stvx_u          $out5,$x00,$out
1174         le?vperm        $out7,$out7,$out7,$inpperm
1175         stvx_u          $out6,$x10,$out
1176         stvx_u          $out7,$x20,$out
1177         addi            $out,$out,0x30
1178         b               Lcbc_dec8x_done
1179
1180 .align  5
1181 Lcbc_dec8x_two:
1182         vncipherlast    $out6,$out6,$ivec
1183         vncipherlast    $out7,$out7,$in6
1184         vmr             $ivec,$in7
1185
1186         le?vperm        $out6,$out6,$out6,$inpperm
1187         le?vperm        $out7,$out7,$out7,$inpperm
1188         stvx_u          $out6,$x00,$out
1189         stvx_u          $out7,$x10,$out
1190         addi            $out,$out,0x20
1191         b               Lcbc_dec8x_done
1192
1193 .align  5
1194 Lcbc_dec8x_one:
1195         vncipherlast    $out7,$out7,$ivec
1196         vmr             $ivec,$in7
1197
1198         le?vperm        $out7,$out7,$out7,$inpperm
1199         stvx_u          $out7,0,$out
1200         addi            $out,$out,0x10
1201
1202 Lcbc_dec8x_done:
1203         le?vperm        $ivec,$ivec,$ivec,$inpperm
1204         stvx_u          $ivec,0,$ivp            # write [unaligned] iv
1205
1206         li              r10,`$FRAME+15`
1207         li              r11,`$FRAME+31`
1208         stvx            $inpperm,r10,$sp        # wipe copies of round keys
1209         addi            r10,r10,32
1210         stvx            $inpperm,r11,$sp
1211         addi            r11,r11,32
1212         stvx            $inpperm,r10,$sp
1213         addi            r10,r10,32
1214         stvx            $inpperm,r11,$sp
1215         addi            r11,r11,32
1216         stvx            $inpperm,r10,$sp
1217         addi            r10,r10,32
1218         stvx            $inpperm,r11,$sp
1219         addi            r11,r11,32
1220         stvx            $inpperm,r10,$sp
1221         addi            r10,r10,32
1222         stvx            $inpperm,r11,$sp
1223         addi            r11,r11,32
1224
1225         mtspr           256,$vrsave
1226         lvx             v20,r10,$sp             # ABI says so
1227         addi            r10,r10,32
1228         lvx             v21,r11,$sp
1229         addi            r11,r11,32
1230         lvx             v22,r10,$sp
1231         addi            r10,r10,32
1232         lvx             v23,r11,$sp
1233         addi            r11,r11,32
1234         lvx             v24,r10,$sp
1235         addi            r10,r10,32
1236         lvx             v25,r11,$sp
1237         addi            r11,r11,32
1238         lvx             v26,r10,$sp
1239         addi            r10,r10,32
1240         lvx             v27,r11,$sp
1241         addi            r11,r11,32
1242         lvx             v28,r10,$sp
1243         addi            r10,r10,32
1244         lvx             v29,r11,$sp
1245         addi            r11,r11,32
1246         lvx             v30,r10,$sp
1247         lvx             v31,r11,$sp
1248         $POP            r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1249         $POP            r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1250         $POP            r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1251         $POP            r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1252         $POP            r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1253         $POP            r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1254         addi            $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
1255         blr
1256         .long           0
1257         .byte           0,12,0x04,0,0x80,6,6,0
1258         .long           0
1259 .size   .${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt
1260 ___
1261 }}      }}}
1262
1263 #########################################################################
1264 {{{     # CTR procedure[s]                                              #
1265 my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10));
1266 my ($rndkey0,$rndkey1,$inout,$tmp)=             map("v$_",(0..3));
1267 my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)=
1268                                                 map("v$_",(4..11));
1269 my $dat=$tmp;
1270
1271 $code.=<<___;
1272 .globl  .${prefix}_ctr32_encrypt_blocks
1273 .align  5
1274 .${prefix}_ctr32_encrypt_blocks:
1275         ${UCMP}i        $len,1
1276         bltlr-
1277
1278         lis             r0,0xfff0
1279         mfspr           $vrsave,256
1280         mtspr           256,r0
1281
1282         li              $idx,15
1283         vxor            $rndkey0,$rndkey0,$rndkey0
1284         le?vspltisb     $tmp,0x0f
1285
1286         lvx             $ivec,0,$ivp            # load [unaligned] iv
1287         lvsl            $inpperm,0,$ivp
1288         lvx             $inptail,$idx,$ivp
1289          vspltisb       $one,1
1290         le?vxor         $inpperm,$inpperm,$tmp
1291         vperm           $ivec,$ivec,$inptail,$inpperm
1292          vsldoi         $one,$rndkey0,$one,1
1293
1294         neg             r11,$inp
1295         ?lvsl           $keyperm,0,$key         # prepare for unaligned key
1296         lwz             $rounds,240($key)
1297
1298         lvsr            $inpperm,0,r11          # prepare for unaligned load
1299         lvx             $inptail,0,$inp
1300         addi            $inp,$inp,15            # 15 is not typo
1301         le?vxor         $inpperm,$inpperm,$tmp
1302
1303         srwi            $rounds,$rounds,1
1304         li              $idx,16
1305         subi            $rounds,$rounds,1
1306
1307         ${UCMP}i        $len,8
1308         bge             _aesp8_ctr32_encrypt8x
1309
1310         ?lvsr           $outperm,0,$out         # prepare for unaligned store
1311         vspltisb        $outmask,-1
1312         lvx             $outhead,0,$out
1313         ?vperm          $outmask,$rndkey0,$outmask,$outperm
1314         le?vxor         $outperm,$outperm,$tmp
1315
1316         lvx             $rndkey0,0,$key
1317         mtctr           $rounds
1318         lvx             $rndkey1,$idx,$key
1319         addi            $idx,$idx,16
1320         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
1321         vxor            $inout,$ivec,$rndkey0
1322         lvx             $rndkey0,$idx,$key
1323         addi            $idx,$idx,16
1324         b               Loop_ctr32_enc
1325
1326 .align  5
1327 Loop_ctr32_enc:
1328         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
1329         vcipher         $inout,$inout,$rndkey1
1330         lvx             $rndkey1,$idx,$key
1331         addi            $idx,$idx,16
1332         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
1333         vcipher         $inout,$inout,$rndkey0
1334         lvx             $rndkey0,$idx,$key
1335         addi            $idx,$idx,16
1336         bdnz            Loop_ctr32_enc
1337
1338         vadduwm         $ivec,$ivec,$one
1339          vmr            $dat,$inptail
1340          lvx            $inptail,0,$inp
1341          addi           $inp,$inp,16
1342          subic.         $len,$len,1             # blocks--
1343
1344         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
1345         vcipher         $inout,$inout,$rndkey1
1346         lvx             $rndkey1,$idx,$key
1347          vperm          $dat,$dat,$inptail,$inpperm
1348          li             $idx,16
1349         ?vperm          $rndkey1,$rndkey0,$rndkey1,$keyperm
1350          lvx            $rndkey0,0,$key
1351         vxor            $dat,$dat,$rndkey1      # last round key
1352         vcipherlast     $inout,$inout,$dat
1353
1354          lvx            $rndkey1,$idx,$key
1355          addi           $idx,$idx,16
1356         vperm           $inout,$inout,$inout,$outperm
1357         vsel            $dat,$outhead,$inout,$outmask
1358          mtctr          $rounds
1359          ?vperm         $rndkey0,$rndkey0,$rndkey1,$keyperm
1360         vmr             $outhead,$inout
1361          vxor           $inout,$ivec,$rndkey0
1362          lvx            $rndkey0,$idx,$key
1363          addi           $idx,$idx,16
1364         stvx            $dat,0,$out
1365         addi            $out,$out,16
1366         bne             Loop_ctr32_enc
1367
1368         addi            $out,$out,-1
1369         lvx             $inout,0,$out           # redundant in aligned case
1370         vsel            $inout,$outhead,$inout,$outmask
1371         stvx            $inout,0,$out
1372
1373         mtspr           256,$vrsave
1374         blr
1375         .long           0
1376         .byte           0,12,0x14,0,0,0,6,0
1377         .long           0
1378 ___
1379 #########################################################################
1380 {{      # Optimized CTR procedure                                       #
1381 my $key_="r11";
1382 my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
1383     $x00=0 if ($flavour =~ /osx/);
1384 my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14));
1385 my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(15..22));
1386 my $rndkey0="v23";      # v24-v25 rotating buffer for first found keys
1387                         # v26-v31 last 6 round keys
1388 my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment
1389 my ($two,$three,$four)=($outhead,$outperm,$outmask);
1390
1391 $code.=<<___;
1392 .align  5
1393 _aesp8_ctr32_encrypt8x:
1394         $STU            $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
1395         li              r10,`$FRAME+8*16+15`
1396         li              r11,`$FRAME+8*16+31`
1397         stvx            v20,r10,$sp             # ABI says so
1398         addi            r10,r10,32
1399         stvx            v21,r11,$sp
1400         addi            r11,r11,32
1401         stvx            v22,r10,$sp
1402         addi            r10,r10,32
1403         stvx            v23,r11,$sp
1404         addi            r11,r11,32
1405         stvx            v24,r10,$sp
1406         addi            r10,r10,32
1407         stvx            v25,r11,$sp
1408         addi            r11,r11,32
1409         stvx            v26,r10,$sp
1410         addi            r10,r10,32
1411         stvx            v27,r11,$sp
1412         addi            r11,r11,32
1413         stvx            v28,r10,$sp
1414         addi            r10,r10,32
1415         stvx            v29,r11,$sp
1416         addi            r11,r11,32
1417         stvx            v30,r10,$sp
1418         stvx            v31,r11,$sp
1419         li              r0,-1
1420         stw             $vrsave,`$FRAME+21*16-4`($sp)   # save vrsave
1421         li              $x10,0x10
1422         $PUSH           r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1423         li              $x20,0x20
1424         $PUSH           r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1425         li              $x30,0x30
1426         $PUSH           r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1427         li              $x40,0x40
1428         $PUSH           r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1429         li              $x50,0x50
1430         $PUSH           r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1431         li              $x60,0x60
1432         $PUSH           r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1433         li              $x70,0x70
1434         mtspr           256,r0
1435
1436         subi            $rounds,$rounds,3       # -4 in total
1437
1438         lvx             $rndkey0,$x00,$key      # load key schedule
1439         lvx             v30,$x10,$key
1440         addi            $key,$key,0x20
1441         lvx             v31,$x00,$key
1442         ?vperm          $rndkey0,$rndkey0,v30,$keyperm
1443         addi            $key_,$sp,$FRAME+15
1444         mtctr           $rounds
1445
1446 Load_ctr32_enc_key:
1447         ?vperm          v24,v30,v31,$keyperm
1448         lvx             v30,$x10,$key
1449         addi            $key,$key,0x20
1450         stvx            v24,$x00,$key_          # off-load round[1]
1451         ?vperm          v25,v31,v30,$keyperm
1452         lvx             v31,$x00,$key
1453         stvx            v25,$x10,$key_          # off-load round[2]
1454         addi            $key_,$key_,0x20
1455         bdnz            Load_ctr32_enc_key
1456
1457         lvx             v26,$x10,$key
1458         ?vperm          v24,v30,v31,$keyperm
1459         lvx             v27,$x20,$key
1460         stvx            v24,$x00,$key_          # off-load round[3]
1461         ?vperm          v25,v31,v26,$keyperm
1462         lvx             v28,$x30,$key
1463         stvx            v25,$x10,$key_          # off-load round[4]
1464         addi            $key_,$sp,$FRAME+15     # rewind $key_
1465         ?vperm          v26,v26,v27,$keyperm
1466         lvx             v29,$x40,$key
1467         ?vperm          v27,v27,v28,$keyperm
1468         lvx             v30,$x50,$key
1469         ?vperm          v28,v28,v29,$keyperm
1470         lvx             v31,$x60,$key
1471         ?vperm          v29,v29,v30,$keyperm
1472         lvx             $out0,$x70,$key         # borrow $out0
1473         ?vperm          v30,v30,v31,$keyperm
1474         lvx             v24,$x00,$key_          # pre-load round[1]
1475         ?vperm          v31,v31,$out0,$keyperm
1476         lvx             v25,$x10,$key_          # pre-load round[2]
1477
1478         vadduwm         $two,$one,$one
1479         subi            $inp,$inp,15            # undo "caller"
1480         $SHL            $len,$len,4
1481
1482         vadduwm         $out1,$ivec,$one        # counter values ...
1483         vadduwm         $out2,$ivec,$two
1484         vxor            $out0,$ivec,$rndkey0    # ... xored with rndkey[0]
1485          le?li          $idx,8
1486         vadduwm         $out3,$out1,$two
1487         vxor            $out1,$out1,$rndkey0
1488          le?lvsl        $inpperm,0,$idx
1489         vadduwm         $out4,$out2,$two
1490         vxor            $out2,$out2,$rndkey0
1491          le?vspltisb    $tmp,0x0f
1492         vadduwm         $out5,$out3,$two
1493         vxor            $out3,$out3,$rndkey0
1494          le?vxor        $inpperm,$inpperm,$tmp  # transform for lvx_u/stvx_u
1495         vadduwm         $out6,$out4,$two
1496         vxor            $out4,$out4,$rndkey0
1497         vadduwm         $out7,$out5,$two
1498         vxor            $out5,$out5,$rndkey0
1499         vadduwm         $ivec,$out6,$two        # next counter value
1500         vxor            $out6,$out6,$rndkey0
1501         vxor            $out7,$out7,$rndkey0
1502
1503         mtctr           $rounds
1504         b               Loop_ctr32_enc8x
1505 .align  5
1506 Loop_ctr32_enc8x:
1507         vcipher         $out0,$out0,v24
1508         vcipher         $out1,$out1,v24
1509         vcipher         $out2,$out2,v24
1510         vcipher         $out3,$out3,v24
1511         vcipher         $out4,$out4,v24
1512         vcipher         $out5,$out5,v24
1513         vcipher         $out6,$out6,v24
1514         vcipher         $out7,$out7,v24
1515 Loop_ctr32_enc8x_middle:
1516         lvx             v24,$x20,$key_          # round[3]
1517         addi            $key_,$key_,0x20
1518
1519         vcipher         $out0,$out0,v25
1520         vcipher         $out1,$out1,v25
1521         vcipher         $out2,$out2,v25
1522         vcipher         $out3,$out3,v25
1523         vcipher         $out4,$out4,v25
1524         vcipher         $out5,$out5,v25
1525         vcipher         $out6,$out6,v25
1526         vcipher         $out7,$out7,v25
1527         lvx             v25,$x10,$key_          # round[4]
1528         bdnz            Loop_ctr32_enc8x
1529
1530         subic           r11,$len,256            # $len-256, borrow $key_
1531         vcipher         $out0,$out0,v24
1532         vcipher         $out1,$out1,v24
1533         vcipher         $out2,$out2,v24
1534         vcipher         $out3,$out3,v24
1535         vcipher         $out4,$out4,v24
1536         vcipher         $out5,$out5,v24
1537         vcipher         $out6,$out6,v24
1538         vcipher         $out7,$out7,v24
1539
1540         subfe           r0,r0,r0                # borrow?-1:0
1541         vcipher         $out0,$out0,v25
1542         vcipher         $out1,$out1,v25
1543         vcipher         $out2,$out2,v25
1544         vcipher         $out3,$out3,v25
1545         vcipher         $out4,$out4,v25
1546         vcipher         $out5,$out5,v25
1547         vcipher         $out6,$out6,v25
1548         vcipher         $out7,$out7,v25
1549
1550         and             r0,r0,r11
1551         addi            $key_,$sp,$FRAME+15     # rewind $key_
1552         vcipher         $out0,$out0,v26
1553         vcipher         $out1,$out1,v26
1554         vcipher         $out2,$out2,v26
1555         vcipher         $out3,$out3,v26
1556         vcipher         $out4,$out4,v26
1557         vcipher         $out5,$out5,v26
1558         vcipher         $out6,$out6,v26
1559         vcipher         $out7,$out7,v26
1560         lvx             v24,$x00,$key_          # re-pre-load round[1]
1561
1562         subic           $len,$len,129           # $len-=129
1563         vcipher         $out0,$out0,v27
1564         addi            $len,$len,1             # $len-=128 really
1565         vcipher         $out1,$out1,v27
1566         vcipher         $out2,$out2,v27
1567         vcipher         $out3,$out3,v27
1568         vcipher         $out4,$out4,v27
1569         vcipher         $out5,$out5,v27
1570         vcipher         $out6,$out6,v27
1571         vcipher         $out7,$out7,v27
1572         lvx             v25,$x10,$key_          # re-pre-load round[2]
1573
1574         vcipher         $out0,$out0,v28
1575          lvx_u          $in0,$x00,$inp          # load input
1576         vcipher         $out1,$out1,v28
1577          lvx_u          $in1,$x10,$inp
1578         vcipher         $out2,$out2,v28
1579          lvx_u          $in2,$x20,$inp
1580         vcipher         $out3,$out3,v28
1581          lvx_u          $in3,$x30,$inp
1582         vcipher         $out4,$out4,v28
1583          lvx_u          $in4,$x40,$inp
1584         vcipher         $out5,$out5,v28
1585          lvx_u          $in5,$x50,$inp
1586         vcipher         $out6,$out6,v28
1587          lvx_u          $in6,$x60,$inp
1588         vcipher         $out7,$out7,v28
1589          lvx_u          $in7,$x70,$inp
1590          addi           $inp,$inp,0x80
1591
1592         vcipher         $out0,$out0,v29
1593          le?vperm       $in0,$in0,$in0,$inpperm
1594         vcipher         $out1,$out1,v29
1595          le?vperm       $in1,$in1,$in1,$inpperm
1596         vcipher         $out2,$out2,v29
1597          le?vperm       $in2,$in2,$in2,$inpperm
1598         vcipher         $out3,$out3,v29
1599          le?vperm       $in3,$in3,$in3,$inpperm
1600         vcipher         $out4,$out4,v29
1601          le?vperm       $in4,$in4,$in4,$inpperm
1602         vcipher         $out5,$out5,v29
1603          le?vperm       $in5,$in5,$in5,$inpperm
1604         vcipher         $out6,$out6,v29
1605          le?vperm       $in6,$in6,$in6,$inpperm
1606         vcipher         $out7,$out7,v29
1607          le?vperm       $in7,$in7,$in7,$inpperm
1608
1609         add             $inp,$inp,r0            # $inp is adjusted in such
1610                                                 # way that at exit from the
1611                                                 # loop inX-in7 are loaded
1612                                                 # with last "words"
1613         subfe.          r0,r0,r0                # borrow?-1:0
1614         vcipher         $out0,$out0,v30
1615          vxor           $in0,$in0,v31           # xor with last round key
1616         vcipher         $out1,$out1,v30
1617          vxor           $in1,$in1,v31
1618         vcipher         $out2,$out2,v30
1619          vxor           $in2,$in2,v31
1620         vcipher         $out3,$out3,v30
1621          vxor           $in3,$in3,v31
1622         vcipher         $out4,$out4,v30
1623          vxor           $in4,$in4,v31
1624         vcipher         $out5,$out5,v30
1625          vxor           $in5,$in5,v31
1626         vcipher         $out6,$out6,v30
1627          vxor           $in6,$in6,v31
1628         vcipher         $out7,$out7,v30
1629          vxor           $in7,$in7,v31
1630
1631         bne             Lctr32_enc8x_break      # did $len-129 borrow?
1632
1633         vcipherlast     $in0,$out0,$in0
1634         vcipherlast     $in1,$out1,$in1
1635          vadduwm        $out1,$ivec,$one        # counter values ...
1636         vcipherlast     $in2,$out2,$in2
1637          vadduwm        $out2,$ivec,$two
1638          vxor           $out0,$ivec,$rndkey0    # ... xored with rndkey[0]
1639         vcipherlast     $in3,$out3,$in3
1640          vadduwm        $out3,$out1,$two
1641          vxor           $out1,$out1,$rndkey0
1642         vcipherlast     $in4,$out4,$in4
1643          vadduwm        $out4,$out2,$two
1644          vxor           $out2,$out2,$rndkey0
1645         vcipherlast     $in5,$out5,$in5
1646          vadduwm        $out5,$out3,$two
1647          vxor           $out3,$out3,$rndkey0
1648         vcipherlast     $in6,$out6,$in6
1649          vadduwm        $out6,$out4,$two
1650          vxor           $out4,$out4,$rndkey0
1651         vcipherlast     $in7,$out7,$in7
1652          vadduwm        $out7,$out5,$two
1653          vxor           $out5,$out5,$rndkey0
1654         le?vperm        $in0,$in0,$in0,$inpperm
1655          vadduwm        $ivec,$out6,$two        # next counter value
1656          vxor           $out6,$out6,$rndkey0
1657         le?vperm        $in1,$in1,$in1,$inpperm
1658          vxor           $out7,$out7,$rndkey0
1659         mtctr           $rounds
1660
1661          vcipher        $out0,$out0,v24
1662         stvx_u          $in0,$x00,$out
1663         le?vperm        $in2,$in2,$in2,$inpperm
1664          vcipher        $out1,$out1,v24
1665         stvx_u          $in1,$x10,$out
1666         le?vperm        $in3,$in3,$in3,$inpperm
1667          vcipher        $out2,$out2,v24
1668         stvx_u          $in2,$x20,$out
1669         le?vperm        $in4,$in4,$in4,$inpperm
1670          vcipher        $out3,$out3,v24
1671         stvx_u          $in3,$x30,$out
1672         le?vperm        $in5,$in5,$in5,$inpperm
1673          vcipher        $out4,$out4,v24
1674         stvx_u          $in4,$x40,$out
1675         le?vperm        $in6,$in6,$in6,$inpperm
1676          vcipher        $out5,$out5,v24
1677         stvx_u          $in5,$x50,$out
1678         le?vperm        $in7,$in7,$in7,$inpperm
1679          vcipher        $out6,$out6,v24
1680         stvx_u          $in6,$x60,$out
1681          vcipher        $out7,$out7,v24
1682         stvx_u          $in7,$x70,$out
1683         addi            $out,$out,0x80
1684
1685         b               Loop_ctr32_enc8x_middle
1686
1687 .align  5
1688 Lctr32_enc8x_break:
1689         cmpwi           $len,-0x60
1690         blt             Lctr32_enc8x_one
1691         nop
1692         beq             Lctr32_enc8x_two
1693         cmpwi           $len,-0x40
1694         blt             Lctr32_enc8x_three
1695         nop
1696         beq             Lctr32_enc8x_four
1697         cmpwi           $len,-0x20
1698         blt             Lctr32_enc8x_five
1699         nop
1700         beq             Lctr32_enc8x_six
1701         cmpwi           $len,0x00
1702         blt             Lctr32_enc8x_seven
1703
1704 Lctr32_enc8x_eight:
1705         vcipherlast     $out0,$out0,$in0
1706         vcipherlast     $out1,$out1,$in1
1707         vcipherlast     $out2,$out2,$in2
1708         vcipherlast     $out3,$out3,$in3
1709         vcipherlast     $out4,$out4,$in4
1710         vcipherlast     $out5,$out5,$in5
1711         vcipherlast     $out6,$out6,$in6
1712         vcipherlast     $out7,$out7,$in7
1713
1714         le?vperm        $out0,$out0,$out0,$inpperm
1715         le?vperm        $out1,$out1,$out1,$inpperm
1716         stvx_u          $out0,$x00,$out
1717         le?vperm        $out2,$out2,$out2,$inpperm
1718         stvx_u          $out1,$x10,$out
1719         le?vperm        $out3,$out3,$out3,$inpperm
1720         stvx_u          $out2,$x20,$out
1721         le?vperm        $out4,$out4,$out4,$inpperm
1722         stvx_u          $out3,$x30,$out
1723         le?vperm        $out5,$out5,$out5,$inpperm
1724         stvx_u          $out4,$x40,$out
1725         le?vperm        $out6,$out6,$out6,$inpperm
1726         stvx_u          $out5,$x50,$out
1727         le?vperm        $out7,$out7,$out7,$inpperm
1728         stvx_u          $out6,$x60,$out
1729         stvx_u          $out7,$x70,$out
1730         addi            $out,$out,0x80
1731         b               Lctr32_enc8x_done
1732
1733 .align  5
1734 Lctr32_enc8x_seven:
1735         vcipherlast     $out0,$out0,$in1
1736         vcipherlast     $out1,$out1,$in2
1737         vcipherlast     $out2,$out2,$in3
1738         vcipherlast     $out3,$out3,$in4
1739         vcipherlast     $out4,$out4,$in5
1740         vcipherlast     $out5,$out5,$in6
1741         vcipherlast     $out6,$out6,$in7
1742
1743         le?vperm        $out0,$out0,$out0,$inpperm
1744         le?vperm        $out1,$out1,$out1,$inpperm
1745         stvx_u          $out0,$x00,$out
1746         le?vperm        $out2,$out2,$out2,$inpperm
1747         stvx_u          $out1,$x10,$out
1748         le?vperm        $out3,$out3,$out3,$inpperm
1749         stvx_u          $out2,$x20,$out
1750         le?vperm        $out4,$out4,$out4,$inpperm
1751         stvx_u          $out3,$x30,$out
1752         le?vperm        $out5,$out5,$out5,$inpperm
1753         stvx_u          $out4,$x40,$out
1754         le?vperm        $out6,$out6,$out6,$inpperm
1755         stvx_u          $out5,$x50,$out
1756         stvx_u          $out6,$x60,$out
1757         addi            $out,$out,0x70
1758         b               Lctr32_enc8x_done
1759
1760 .align  5
1761 Lctr32_enc8x_six:
1762         vcipherlast     $out0,$out0,$in2
1763         vcipherlast     $out1,$out1,$in3
1764         vcipherlast     $out2,$out2,$in4
1765         vcipherlast     $out3,$out3,$in5
1766         vcipherlast     $out4,$out4,$in6
1767         vcipherlast     $out5,$out5,$in7
1768
1769         le?vperm        $out0,$out0,$out0,$inpperm
1770         le?vperm        $out1,$out1,$out1,$inpperm
1771         stvx_u          $out0,$x00,$out
1772         le?vperm        $out2,$out2,$out2,$inpperm
1773         stvx_u          $out1,$x10,$out
1774         le?vperm        $out3,$out3,$out3,$inpperm
1775         stvx_u          $out2,$x20,$out
1776         le?vperm        $out4,$out4,$out4,$inpperm
1777         stvx_u          $out3,$x30,$out
1778         le?vperm        $out5,$out5,$out5,$inpperm
1779         stvx_u          $out4,$x40,$out
1780         stvx_u          $out5,$x50,$out
1781         addi            $out,$out,0x60
1782         b               Lctr32_enc8x_done
1783
1784 .align  5
1785 Lctr32_enc8x_five:
1786         vcipherlast     $out0,$out0,$in3
1787         vcipherlast     $out1,$out1,$in4
1788         vcipherlast     $out2,$out2,$in5
1789         vcipherlast     $out3,$out3,$in6
1790         vcipherlast     $out4,$out4,$in7
1791
1792         le?vperm        $out0,$out0,$out0,$inpperm
1793         le?vperm        $out1,$out1,$out1,$inpperm
1794         stvx_u          $out0,$x00,$out
1795         le?vperm        $out2,$out2,$out2,$inpperm
1796         stvx_u          $out1,$x10,$out
1797         le?vperm        $out3,$out3,$out3,$inpperm
1798         stvx_u          $out2,$x20,$out
1799         le?vperm        $out4,$out4,$out4,$inpperm
1800         stvx_u          $out3,$x30,$out
1801         stvx_u          $out4,$x40,$out
1802         addi            $out,$out,0x50
1803         b               Lctr32_enc8x_done
1804
1805 .align  5
1806 Lctr32_enc8x_four:
1807         vcipherlast     $out0,$out0,$in4
1808         vcipherlast     $out1,$out1,$in5
1809         vcipherlast     $out2,$out2,$in6
1810         vcipherlast     $out3,$out3,$in7
1811
1812         le?vperm        $out0,$out0,$out0,$inpperm
1813         le?vperm        $out1,$out1,$out1,$inpperm
1814         stvx_u          $out0,$x00,$out
1815         le?vperm        $out2,$out2,$out2,$inpperm
1816         stvx_u          $out1,$x10,$out
1817         le?vperm        $out3,$out3,$out3,$inpperm
1818         stvx_u          $out2,$x20,$out
1819         stvx_u          $out3,$x30,$out
1820         addi            $out,$out,0x40
1821         b               Lctr32_enc8x_done
1822
1823 .align  5
1824 Lctr32_enc8x_three:
1825         vcipherlast     $out0,$out0,$in5
1826         vcipherlast     $out1,$out1,$in6
1827         vcipherlast     $out2,$out2,$in7
1828
1829         le?vperm        $out0,$out0,$out0,$inpperm
1830         le?vperm        $out1,$out1,$out1,$inpperm
1831         stvx_u          $out0,$x00,$out
1832         le?vperm        $out2,$out2,$out2,$inpperm
1833         stvx_u          $out1,$x10,$out
1834         stvx_u          $out2,$x20,$out
1835         addi            $out,$out,0x30
1836         b               Lctr32_enc8x_done
1837
1838 .align  5
1839 Lctr32_enc8x_two:
1840         vcipherlast     $out0,$out0,$in6
1841         vcipherlast     $out1,$out1,$in7
1842
1843         le?vperm        $out0,$out0,$out0,$inpperm
1844         le?vperm        $out1,$out1,$out1,$inpperm
1845         stvx_u          $out0,$x00,$out
1846         stvx_u          $out1,$x10,$out
1847         addi            $out,$out,0x20
1848         b               Lctr32_enc8x_done
1849
1850 .align  5
1851 Lctr32_enc8x_one:
1852         vcipherlast     $out0,$out0,$in7
1853
1854         le?vperm        $out0,$out0,$out0,$inpperm
1855         stvx_u          $out0,0,$out
1856         addi            $out,$out,0x10
1857
1858 Lctr32_enc8x_done:
1859         li              r10,`$FRAME+15`
1860         li              r11,`$FRAME+31`
1861         stvx            $inpperm,r10,$sp        # wipe copies of round keys
1862         addi            r10,r10,32
1863         stvx            $inpperm,r11,$sp
1864         addi            r11,r11,32
1865         stvx            $inpperm,r10,$sp
1866         addi            r10,r10,32
1867         stvx            $inpperm,r11,$sp
1868         addi            r11,r11,32
1869         stvx            $inpperm,r10,$sp
1870         addi            r10,r10,32
1871         stvx            $inpperm,r11,$sp
1872         addi            r11,r11,32
1873         stvx            $inpperm,r10,$sp
1874         addi            r10,r10,32
1875         stvx            $inpperm,r11,$sp
1876         addi            r11,r11,32
1877
1878         mtspr           256,$vrsave
1879         lvx             v20,r10,$sp             # ABI says so
1880         addi            r10,r10,32
1881         lvx             v21,r11,$sp
1882         addi            r11,r11,32
1883         lvx             v22,r10,$sp
1884         addi            r10,r10,32
1885         lvx             v23,r11,$sp
1886         addi            r11,r11,32
1887         lvx             v24,r10,$sp
1888         addi            r10,r10,32
1889         lvx             v25,r11,$sp
1890         addi            r11,r11,32
1891         lvx             v26,r10,$sp
1892         addi            r10,r10,32
1893         lvx             v27,r11,$sp
1894         addi            r11,r11,32
1895         lvx             v28,r10,$sp
1896         addi            r10,r10,32
1897         lvx             v29,r11,$sp
1898         addi            r11,r11,32
1899         lvx             v30,r10,$sp
1900         lvx             v31,r11,$sp
1901         $POP            r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1902         $POP            r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1903         $POP            r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1904         $POP            r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1905         $POP            r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1906         $POP            r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1907         addi            $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
1908         blr
1909         .long           0
1910         .byte           0,12,0x04,0,0x80,6,6,0
1911         .long           0
1912 .size   .${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks
1913 ___
1914 }}      }}}
1915
1916 #########################################################################
1917 {{{     # XTS procedures                                                #
1918 # int aes_p8_xts_[en|de]crypt(const char *inp, char *out, size_t len,   #
1919 #                             const AES_KEY *key1, const AES_KEY *key2, #
1920 #                             [const] unsigned char iv[16]);            #
1921 # If $key2 is NULL, then a "tweak chaining" mode is engaged, in which   #
1922 # input tweak value is assumed to be encrypted already, and last tweak  #
1923 # value, one suitable for consecutive call on same chunk of data, is    #
1924 # written back to original buffer. In addition, in "tweak chaining"     #
1925 # mode only complete input blocks are processed.                        #
1926
1927 my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) =     map("r$_",(3..10));
1928 my ($rndkey0,$rndkey1,$inout) =                         map("v$_",(0..2));
1929 my ($output,$inptail,$inpperm,$leperm,$keyperm) =       map("v$_",(3..7));
1930 my ($tweak,$seven,$eighty7,$tmp,$tweak1) =              map("v$_",(8..12));
1931 my $taillen = $key2;
1932
1933    ($inp,$idx) = ($idx,$inp);                           # reassign
1934
1935 $code.=<<___;
1936 .globl  .${prefix}_xts_encrypt
1937 .align  5
1938 .${prefix}_xts_encrypt:
1939         mr              $inp,r3                         # reassign
1940         li              r3,-1
1941         ${UCMP}i        $len,16
1942         bltlr-
1943
1944         lis             r0,0xfff0
1945         mfspr           r12,256                         # save vrsave
1946         li              r11,0
1947         mtspr           256,r0
1948
1949         vspltisb        $seven,0x07                     # 0x070707..07
1950         le?lvsl         $leperm,r11,r11
1951         le?vspltisb     $tmp,0x0f
1952         le?vxor         $leperm,$leperm,$seven
1953
1954         li              $idx,15
1955         lvx             $tweak,0,$ivp                   # load [unaligned] iv
1956         lvsl            $inpperm,0,$ivp
1957         lvx             $inptail,$idx,$ivp
1958         le?vxor         $inpperm,$inpperm,$tmp
1959         vperm           $tweak,$tweak,$inptail,$inpperm
1960
1961         neg             r11,$inp
1962         lvsr            $inpperm,0,r11                  # prepare for unaligned load
1963         lvx             $inout,0,$inp
1964         addi            $inp,$inp,15                    # 15 is not typo
1965         le?vxor         $inpperm,$inpperm,$tmp
1966
1967         ${UCMP}i        $key2,0                         # key2==NULL?
1968         beq             Lxts_enc_no_key2
1969
1970         ?lvsl           $keyperm,0,$key2                # prepare for unaligned key
1971         lwz             $rounds,240($key2)
1972         srwi            $rounds,$rounds,1
1973         subi            $rounds,$rounds,1
1974         li              $idx,16
1975
1976         lvx             $rndkey0,0,$key2
1977         lvx             $rndkey1,$idx,$key2
1978         addi            $idx,$idx,16
1979         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
1980         vxor            $tweak,$tweak,$rndkey0
1981         lvx             $rndkey0,$idx,$key2
1982         addi            $idx,$idx,16
1983         mtctr           $rounds
1984
1985 Ltweak_xts_enc:
1986         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
1987         vcipher         $tweak,$tweak,$rndkey1
1988         lvx             $rndkey1,$idx,$key2
1989         addi            $idx,$idx,16
1990         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
1991         vcipher         $tweak,$tweak,$rndkey0
1992         lvx             $rndkey0,$idx,$key2
1993         addi            $idx,$idx,16
1994         bdnz            Ltweak_xts_enc
1995
1996         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
1997         vcipher         $tweak,$tweak,$rndkey1
1998         lvx             $rndkey1,$idx,$key2
1999         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
2000         vcipherlast     $tweak,$tweak,$rndkey0
2001
2002         li              $ivp,0                          # don't chain the tweak
2003         b               Lxts_enc
2004
2005 Lxts_enc_no_key2:
2006         li              $idx,-16
2007         and             $len,$len,$idx                  # in "tweak chaining"
2008                                                         # mode only complete
2009                                                         # blocks are processed
2010 Lxts_enc:
2011         lvx             $inptail,0,$inp
2012         addi            $inp,$inp,16
2013
2014         ?lvsl           $keyperm,0,$key1                # prepare for unaligned key
2015         lwz             $rounds,240($key1)
2016         srwi            $rounds,$rounds,1
2017         subi            $rounds,$rounds,1
2018         li              $idx,16
2019
2020         vslb            $eighty7,$seven,$seven          # 0x808080..80
2021         vor             $eighty7,$eighty7,$seven        # 0x878787..87
2022         vspltisb        $tmp,1                          # 0x010101..01
2023         vsldoi          $eighty7,$eighty7,$tmp,15       # 0x870101..01
2024
2025         ${UCMP}i        $len,96
2026         bge             _aesp8_xts_encrypt6x
2027
2028         andi.           $taillen,$len,15
2029         subic           r0,$len,32
2030         subi            $taillen,$taillen,16
2031         subfe           r0,r0,r0
2032         and             r0,r0,$taillen
2033         add             $inp,$inp,r0
2034
2035         lvx             $rndkey0,0,$key1
2036         lvx             $rndkey1,$idx,$key1
2037         addi            $idx,$idx,16
2038         vperm           $inout,$inout,$inptail,$inpperm
2039         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
2040         vxor            $inout,$inout,$tweak
2041         vxor            $inout,$inout,$rndkey0
2042         lvx             $rndkey0,$idx,$key1
2043         addi            $idx,$idx,16
2044         mtctr           $rounds
2045         b               Loop_xts_enc
2046
2047 .align  5
2048 Loop_xts_enc:
2049         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
2050         vcipher         $inout,$inout,$rndkey1
2051         lvx             $rndkey1,$idx,$key1
2052         addi            $idx,$idx,16
2053         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
2054         vcipher         $inout,$inout,$rndkey0
2055         lvx             $rndkey0,$idx,$key1
2056         addi            $idx,$idx,16
2057         bdnz            Loop_xts_enc
2058
2059         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
2060         vcipher         $inout,$inout,$rndkey1
2061         lvx             $rndkey1,$idx,$key1
2062         li              $idx,16
2063         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
2064         vxor            $rndkey0,$rndkey0,$tweak
2065         vcipherlast     $output,$inout,$rndkey0
2066
2067         le?vperm        $tmp,$output,$output,$leperm
2068         be?nop
2069         le?stvx_u       $tmp,0,$out
2070         be?stvx_u       $output,0,$out
2071         addi            $out,$out,16
2072
2073         subic.          $len,$len,16
2074         beq             Lxts_enc_done
2075
2076         vmr             $inout,$inptail
2077         lvx             $inptail,0,$inp
2078         addi            $inp,$inp,16
2079         lvx             $rndkey0,0,$key1
2080         lvx             $rndkey1,$idx,$key1
2081         addi            $idx,$idx,16
2082
2083         subic           r0,$len,32
2084         subfe           r0,r0,r0
2085         and             r0,r0,$taillen
2086         add             $inp,$inp,r0
2087
2088         vsrab           $tmp,$tweak,$seven              # next tweak value
2089         vaddubm         $tweak,$tweak,$tweak
2090         vsldoi          $tmp,$tmp,$tmp,15
2091         vand            $tmp,$tmp,$eighty7
2092         vxor            $tweak,$tweak,$tmp
2093
2094         vperm           $inout,$inout,$inptail,$inpperm
2095         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
2096         vxor            $inout,$inout,$tweak
2097         vxor            $output,$output,$rndkey0        # just in case $len<16
2098         vxor            $inout,$inout,$rndkey0
2099         lvx             $rndkey0,$idx,$key1
2100         addi            $idx,$idx,16
2101
2102         mtctr           $rounds
2103         ${UCMP}i        $len,16
2104         bge             Loop_xts_enc
2105
2106         vxor            $output,$output,$tweak
2107         lvsr            $inpperm,0,$len                 # $inpperm is no longer needed
2108         vxor            $inptail,$inptail,$inptail      # $inptail is no longer needed
2109         vspltisb        $tmp,-1
2110         vperm           $inptail,$inptail,$tmp,$inpperm
2111         vsel            $inout,$inout,$output,$inptail
2112
2113         subi            r11,$out,17
2114         subi            $out,$out,16
2115         mtctr           $len
2116         li              $len,16
2117 Loop_xts_enc_steal:
2118         lbzu            r0,1(r11)
2119         stb             r0,16(r11)
2120         bdnz            Loop_xts_enc_steal
2121
2122         mtctr           $rounds
2123         b               Loop_xts_enc                    # one more time...
2124
2125 Lxts_enc_done:
2126         ${UCMP}i        $ivp,0
2127         beq             Lxts_enc_ret
2128
2129         vsrab           $tmp,$tweak,$seven              # next tweak value
2130         vaddubm         $tweak,$tweak,$tweak
2131         vsldoi          $tmp,$tmp,$tmp,15
2132         vand            $tmp,$tmp,$eighty7
2133         vxor            $tweak,$tweak,$tmp
2134
2135         le?vperm        $tweak,$tweak,$tweak,$leperm
2136         stvx_u          $tweak,0,$ivp
2137
2138 Lxts_enc_ret:
2139         mtspr           256,r12                         # restore vrsave
2140         li              r3,0
2141         blr
2142         .long           0
2143         .byte           0,12,0x04,0,0x80,6,6,0
2144         .long           0
2145 .size   .${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt
2146
2147 .globl  .${prefix}_xts_decrypt
2148 .align  5
2149 .${prefix}_xts_decrypt:
2150         mr              $inp,r3                         # reassign
2151         li              r3,-1
2152         ${UCMP}i        $len,16
2153         bltlr-
2154
2155         lis             r0,0xfff8
2156         mfspr           r12,256                         # save vrsave
2157         li              r11,0
2158         mtspr           256,r0
2159
2160         andi.           r0,$len,15
2161         neg             r0,r0
2162         andi.           r0,r0,16
2163         sub             $len,$len,r0
2164
2165         vspltisb        $seven,0x07                     # 0x070707..07
2166         le?lvsl         $leperm,r11,r11
2167         le?vspltisb     $tmp,0x0f
2168         le?vxor         $leperm,$leperm,$seven
2169
2170         li              $idx,15
2171         lvx             $tweak,0,$ivp                   # load [unaligned] iv
2172         lvsl            $inpperm,0,$ivp
2173         lvx             $inptail,$idx,$ivp
2174         le?vxor         $inpperm,$inpperm,$tmp
2175         vperm           $tweak,$tweak,$inptail,$inpperm
2176
2177         neg             r11,$inp
2178         lvsr            $inpperm,0,r11                  # prepare for unaligned load
2179         lvx             $inout,0,$inp
2180         addi            $inp,$inp,15                    # 15 is not typo
2181         le?vxor         $inpperm,$inpperm,$tmp
2182
2183         ${UCMP}i        $key2,0                         # key2==NULL?
2184         beq             Lxts_dec_no_key2
2185
2186         ?lvsl           $keyperm,0,$key2                # prepare for unaligned key
2187         lwz             $rounds,240($key2)
2188         srwi            $rounds,$rounds,1
2189         subi            $rounds,$rounds,1
2190         li              $idx,16
2191
2192         lvx             $rndkey0,0,$key2
2193         lvx             $rndkey1,$idx,$key2
2194         addi            $idx,$idx,16
2195         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
2196         vxor            $tweak,$tweak,$rndkey0
2197         lvx             $rndkey0,$idx,$key2
2198         addi            $idx,$idx,16
2199         mtctr           $rounds
2200
2201 Ltweak_xts_dec:
2202         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
2203         vcipher         $tweak,$tweak,$rndkey1
2204         lvx             $rndkey1,$idx,$key2
2205         addi            $idx,$idx,16
2206         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
2207         vcipher         $tweak,$tweak,$rndkey0
2208         lvx             $rndkey0,$idx,$key2
2209         addi            $idx,$idx,16
2210         bdnz            Ltweak_xts_dec
2211
2212         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
2213         vcipher         $tweak,$tweak,$rndkey1
2214         lvx             $rndkey1,$idx,$key2
2215         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
2216         vcipherlast     $tweak,$tweak,$rndkey0
2217
2218         li              $ivp,0                          # don't chain the tweak
2219         b               Lxts_dec
2220
2221 Lxts_dec_no_key2:
2222         neg             $idx,$len
2223         andi.           $idx,$idx,15
2224         add             $len,$len,$idx                  # in "tweak chaining"
2225                                                         # mode only complete
2226                                                         # blocks are processed
2227 Lxts_dec:
2228         lvx             $inptail,0,$inp
2229         addi            $inp,$inp,16
2230
2231         ?lvsl           $keyperm,0,$key1                # prepare for unaligned key
2232         lwz             $rounds,240($key1)
2233         srwi            $rounds,$rounds,1
2234         subi            $rounds,$rounds,1
2235         li              $idx,16
2236
2237         vslb            $eighty7,$seven,$seven          # 0x808080..80
2238         vor             $eighty7,$eighty7,$seven        # 0x878787..87
2239         vspltisb        $tmp,1                          # 0x010101..01
2240         vsldoi          $eighty7,$eighty7,$tmp,15       # 0x870101..01
2241
2242         ${UCMP}i        $len,96
2243         bge             _aesp8_xts_decrypt6x
2244
2245         lvx             $rndkey0,0,$key1
2246         lvx             $rndkey1,$idx,$key1
2247         addi            $idx,$idx,16
2248         vperm           $inout,$inout,$inptail,$inpperm
2249         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
2250         vxor            $inout,$inout,$tweak
2251         vxor            $inout,$inout,$rndkey0
2252         lvx             $rndkey0,$idx,$key1
2253         addi            $idx,$idx,16
2254         mtctr           $rounds
2255
2256         ${UCMP}i        $len,16
2257         blt             Ltail_xts_dec
2258         be?b            Loop_xts_dec
2259
2260 .align  5
2261 Loop_xts_dec:
2262         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
2263         vncipher        $inout,$inout,$rndkey1
2264         lvx             $rndkey1,$idx,$key1
2265         addi            $idx,$idx,16
2266         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
2267         vncipher        $inout,$inout,$rndkey0
2268         lvx             $rndkey0,$idx,$key1
2269         addi            $idx,$idx,16
2270         bdnz            Loop_xts_dec
2271
2272         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
2273         vncipher        $inout,$inout,$rndkey1
2274         lvx             $rndkey1,$idx,$key1
2275         li              $idx,16
2276         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
2277         vxor            $rndkey0,$rndkey0,$tweak
2278         vncipherlast    $output,$inout,$rndkey0
2279
2280         le?vperm        $tmp,$output,$output,$leperm
2281         be?nop
2282         le?stvx_u       $tmp,0,$out
2283         be?stvx_u       $output,0,$out
2284         addi            $out,$out,16
2285
2286         subic.          $len,$len,16
2287         beq             Lxts_dec_done
2288
2289         vmr             $inout,$inptail
2290         lvx             $inptail,0,$inp
2291         addi            $inp,$inp,16
2292         lvx             $rndkey0,0,$key1
2293         lvx             $rndkey1,$idx,$key1
2294         addi            $idx,$idx,16
2295
2296         vsrab           $tmp,$tweak,$seven              # next tweak value
2297         vaddubm         $tweak,$tweak,$tweak
2298         vsldoi          $tmp,$tmp,$tmp,15
2299         vand            $tmp,$tmp,$eighty7
2300         vxor            $tweak,$tweak,$tmp
2301
2302         vperm           $inout,$inout,$inptail,$inpperm
2303         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
2304         vxor            $inout,$inout,$tweak
2305         vxor            $inout,$inout,$rndkey0
2306         lvx             $rndkey0,$idx,$key1
2307         addi            $idx,$idx,16
2308
2309         mtctr           $rounds
2310         ${UCMP}i        $len,16
2311         bge             Loop_xts_dec
2312
2313 Ltail_xts_dec:
2314         vsrab           $tmp,$tweak,$seven              # next tweak value
2315         vaddubm         $tweak1,$tweak,$tweak
2316         vsldoi          $tmp,$tmp,$tmp,15
2317         vand            $tmp,$tmp,$eighty7
2318         vxor            $tweak1,$tweak1,$tmp
2319
2320         subi            $inp,$inp,16
2321         add             $inp,$inp,$len
2322
2323         vxor            $inout,$inout,$tweak            # :-(
2324         vxor            $inout,$inout,$tweak1           # :-)
2325
2326 Loop_xts_dec_short:
2327         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
2328         vncipher        $inout,$inout,$rndkey1
2329         lvx             $rndkey1,$idx,$key1
2330         addi            $idx,$idx,16
2331         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
2332         vncipher        $inout,$inout,$rndkey0
2333         lvx             $rndkey0,$idx,$key1
2334         addi            $idx,$idx,16
2335         bdnz            Loop_xts_dec_short
2336
2337         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
2338         vncipher        $inout,$inout,$rndkey1
2339         lvx             $rndkey1,$idx,$key1
2340         li              $idx,16
2341         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
2342         vxor            $rndkey0,$rndkey0,$tweak1
2343         vncipherlast    $output,$inout,$rndkey0
2344
2345         le?vperm        $tmp,$output,$output,$leperm
2346         be?nop
2347         le?stvx_u       $tmp,0,$out
2348         be?stvx_u       $output,0,$out
2349
2350         vmr             $inout,$inptail
2351         lvx             $inptail,0,$inp
2352         #addi           $inp,$inp,16
2353         lvx             $rndkey0,0,$key1
2354         lvx             $rndkey1,$idx,$key1
2355         addi            $idx,$idx,16
2356         vperm           $inout,$inout,$inptail,$inpperm
2357         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
2358
2359         lvsr            $inpperm,0,$len                 # $inpperm is no longer needed
2360         vxor            $inptail,$inptail,$inptail      # $inptail is no longer needed
2361         vspltisb        $tmp,-1
2362         vperm           $inptail,$inptail,$tmp,$inpperm
2363         vsel            $inout,$inout,$output,$inptail
2364
2365         vxor            $rndkey0,$rndkey0,$tweak
2366         vxor            $inout,$inout,$rndkey0
2367         lvx             $rndkey0,$idx,$key1
2368         addi            $idx,$idx,16
2369
2370         subi            r11,$out,1
2371         mtctr           $len
2372         li              $len,16
2373 Loop_xts_dec_steal:
2374         lbzu            r0,1(r11)
2375         stb             r0,16(r11)
2376         bdnz            Loop_xts_dec_steal
2377
2378         mtctr           $rounds
2379         b               Loop_xts_dec                    # one more time...
2380
2381 Lxts_dec_done:
2382         ${UCMP}i        $ivp,0
2383         beq             Lxts_dec_ret
2384
2385         vsrab           $tmp,$tweak,$seven              # next tweak value
2386         vaddubm         $tweak,$tweak,$tweak
2387         vsldoi          $tmp,$tmp,$tmp,15
2388         vand            $tmp,$tmp,$eighty7
2389         vxor            $tweak,$tweak,$tmp
2390
2391         le?vperm        $tweak,$tweak,$tweak,$leperm
2392         stvx_u          $tweak,0,$ivp
2393
2394 Lxts_dec_ret:
2395         mtspr           256,r12                         # restore vrsave
2396         li              r3,0
2397         blr
2398         .long           0
2399         .byte           0,12,0x04,0,0x80,6,6,0
2400         .long           0
2401 .size   .${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt
2402 ___
2403 #########################################################################
2404 {{      # Optimized XTS procedures                                      #
2405 my $key_=$key2;
2406 my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31));
2407     $x00=0 if ($flavour =~ /osx/);
2408 my ($in0,  $in1,  $in2,  $in3,  $in4,  $in5 )=map("v$_",(0..5));
2409 my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16));
2410 my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22));
2411 my $rndkey0="v23";      # v24-v25 rotating buffer for first found keys
2412                         # v26-v31 last 6 round keys
2413 my ($keyperm)=($out0);  # aliases with "caller", redundant assignment
2414 my $taillen=$x70;
2415
2416 $code.=<<___;
2417 .align  5
2418 _aesp8_xts_encrypt6x:
2419         $STU            $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
2420         mflr            r11
2421         li              r7,`$FRAME+8*16+15`
2422         li              r3,`$FRAME+8*16+31`
2423         $PUSH           r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
2424         stvx            v20,r7,$sp              # ABI says so
2425         addi            r7,r7,32
2426         stvx            v21,r3,$sp
2427         addi            r3,r3,32
2428         stvx            v22,r7,$sp
2429         addi            r7,r7,32
2430         stvx            v23,r3,$sp
2431         addi            r3,r3,32
2432         stvx            v24,r7,$sp
2433         addi            r7,r7,32
2434         stvx            v25,r3,$sp
2435         addi            r3,r3,32
2436         stvx            v26,r7,$sp
2437         addi            r7,r7,32
2438         stvx            v27,r3,$sp
2439         addi            r3,r3,32
2440         stvx            v28,r7,$sp
2441         addi            r7,r7,32
2442         stvx            v29,r3,$sp
2443         addi            r3,r3,32
2444         stvx            v30,r7,$sp
2445         stvx            v31,r3,$sp
2446         li              r0,-1
2447         stw             $vrsave,`$FRAME+21*16-4`($sp)   # save vrsave
2448         li              $x10,0x10
2449         $PUSH           r26,`$FRAME+21*16+0*$SIZE_T`($sp)
2450         li              $x20,0x20
2451         $PUSH           r27,`$FRAME+21*16+1*$SIZE_T`($sp)
2452         li              $x30,0x30
2453         $PUSH           r28,`$FRAME+21*16+2*$SIZE_T`($sp)
2454         li              $x40,0x40
2455         $PUSH           r29,`$FRAME+21*16+3*$SIZE_T`($sp)
2456         li              $x50,0x50
2457         $PUSH           r30,`$FRAME+21*16+4*$SIZE_T`($sp)
2458         li              $x60,0x60
2459         $PUSH           r31,`$FRAME+21*16+5*$SIZE_T`($sp)
2460         li              $x70,0x70
2461         mtspr           256,r0
2462
2463         subi            $rounds,$rounds,3       # -4 in total
2464
2465         lvx             $rndkey0,$x00,$key1     # load key schedule
2466         lvx             v30,$x10,$key1
2467         addi            $key1,$key1,0x20
2468         lvx             v31,$x00,$key1
2469         ?vperm          $rndkey0,$rndkey0,v30,$keyperm
2470         addi            $key_,$sp,$FRAME+15
2471         mtctr           $rounds
2472
2473 Load_xts_enc_key:
2474         ?vperm          v24,v30,v31,$keyperm
2475         lvx             v30,$x10,$key1
2476         addi            $key1,$key1,0x20
2477         stvx            v24,$x00,$key_          # off-load round[1]
2478         ?vperm          v25,v31,v30,$keyperm
2479         lvx             v31,$x00,$key1
2480         stvx            v25,$x10,$key_          # off-load round[2]
2481         addi            $key_,$key_,0x20
2482         bdnz            Load_xts_enc_key
2483
2484         lvx             v26,$x10,$key1
2485         ?vperm          v24,v30,v31,$keyperm
2486         lvx             v27,$x20,$key1
2487         stvx            v24,$x00,$key_          # off-load round[3]
2488         ?vperm          v25,v31,v26,$keyperm
2489         lvx             v28,$x30,$key1
2490         stvx            v25,$x10,$key_          # off-load round[4]
2491         addi            $key_,$sp,$FRAME+15     # rewind $key_
2492         ?vperm          v26,v26,v27,$keyperm
2493         lvx             v29,$x40,$key1
2494         ?vperm          v27,v27,v28,$keyperm
2495         lvx             v30,$x50,$key1
2496         ?vperm          v28,v28,v29,$keyperm
2497         lvx             v31,$x60,$key1
2498         ?vperm          v29,v29,v30,$keyperm
2499         lvx             $twk5,$x70,$key1        # borrow $twk5
2500         ?vperm          v30,v30,v31,$keyperm
2501         lvx             v24,$x00,$key_          # pre-load round[1]
2502         ?vperm          v31,v31,$twk5,$keyperm
2503         lvx             v25,$x10,$key_          # pre-load round[2]
2504
2505          vperm          $in0,$inout,$inptail,$inpperm
2506          subi           $inp,$inp,31            # undo "caller"
2507         vxor            $twk0,$tweak,$rndkey0
2508         vsrab           $tmp,$tweak,$seven      # next tweak value
2509         vaddubm         $tweak,$tweak,$tweak
2510         vsldoi          $tmp,$tmp,$tmp,15
2511         vand            $tmp,$tmp,$eighty7
2512          vxor           $out0,$in0,$twk0
2513         vxor            $tweak,$tweak,$tmp
2514
2515          lvx_u          $in1,$x10,$inp
2516         vxor            $twk1,$tweak,$rndkey0
2517         vsrab           $tmp,$tweak,$seven      # next tweak value
2518         vaddubm         $tweak,$tweak,$tweak
2519         vsldoi          $tmp,$tmp,$tmp,15
2520          le?vperm       $in1,$in1,$in1,$leperm
2521         vand            $tmp,$tmp,$eighty7
2522          vxor           $out1,$in1,$twk1
2523         vxor            $tweak,$tweak,$tmp
2524
2525          lvx_u          $in2,$x20,$inp
2526          andi.          $taillen,$len,15
2527         vxor            $twk2,$tweak,$rndkey0
2528         vsrab           $tmp,$tweak,$seven      # next tweak value
2529         vaddubm         $tweak,$tweak,$tweak
2530         vsldoi          $tmp,$tmp,$tmp,15
2531          le?vperm       $in2,$in2,$in2,$leperm
2532         vand            $tmp,$tmp,$eighty7
2533          vxor           $out2,$in2,$twk2
2534         vxor            $tweak,$tweak,$tmp
2535
2536          lvx_u          $in3,$x30,$inp
2537          sub            $len,$len,$taillen
2538         vxor            $twk3,$tweak,$rndkey0
2539         vsrab           $tmp,$tweak,$seven      # next tweak value
2540         vaddubm         $tweak,$tweak,$tweak
2541         vsldoi          $tmp,$tmp,$tmp,15
2542          le?vperm       $in3,$in3,$in3,$leperm
2543         vand            $tmp,$tmp,$eighty7
2544          vxor           $out3,$in3,$twk3
2545         vxor            $tweak,$tweak,$tmp
2546
2547          lvx_u          $in4,$x40,$inp
2548          subi           $len,$len,0x60
2549         vxor            $twk4,$tweak,$rndkey0
2550         vsrab           $tmp,$tweak,$seven      # next tweak value
2551         vaddubm         $tweak,$tweak,$tweak
2552         vsldoi          $tmp,$tmp,$tmp,15
2553          le?vperm       $in4,$in4,$in4,$leperm
2554         vand            $tmp,$tmp,$eighty7
2555          vxor           $out4,$in4,$twk4
2556         vxor            $tweak,$tweak,$tmp
2557
2558          lvx_u          $in5,$x50,$inp
2559          addi           $inp,$inp,0x60
2560         vxor            $twk5,$tweak,$rndkey0
2561         vsrab           $tmp,$tweak,$seven      # next tweak value
2562         vaddubm         $tweak,$tweak,$tweak
2563         vsldoi          $tmp,$tmp,$tmp,15
2564          le?vperm       $in5,$in5,$in5,$leperm
2565         vand            $tmp,$tmp,$eighty7
2566          vxor           $out5,$in5,$twk5
2567         vxor            $tweak,$tweak,$tmp
2568
2569         vxor            v31,v31,$rndkey0
2570         mtctr           $rounds
2571         b               Loop_xts_enc6x
2572
2573 .align  5
2574 Loop_xts_enc6x:
2575         vcipher         $out0,$out0,v24
2576         vcipher         $out1,$out1,v24
2577         vcipher         $out2,$out2,v24
2578         vcipher         $out3,$out3,v24
2579         vcipher         $out4,$out4,v24
2580         vcipher         $out5,$out5,v24
2581         lvx             v24,$x20,$key_          # round[3]
2582         addi            $key_,$key_,0x20
2583
2584         vcipher         $out0,$out0,v25
2585         vcipher         $out1,$out1,v25
2586         vcipher         $out2,$out2,v25
2587         vcipher         $out3,$out3,v25
2588         vcipher         $out4,$out4,v25
2589         vcipher         $out5,$out5,v25
2590         lvx             v25,$x10,$key_          # round[4]
2591         bdnz            Loop_xts_enc6x
2592
2593         subic           $len,$len,96            # $len-=96
2594          vxor           $in0,$twk0,v31          # xor with last round key
2595         vcipher         $out0,$out0,v24
2596         vcipher         $out1,$out1,v24
2597          vsrab          $tmp,$tweak,$seven      # next tweak value
2598          vxor           $twk0,$tweak,$rndkey0
2599          vaddubm        $tweak,$tweak,$tweak
2600         vcipher         $out2,$out2,v24
2601         vcipher         $out3,$out3,v24
2602          vsldoi         $tmp,$tmp,$tmp,15
2603         vcipher         $out4,$out4,v24
2604         vcipher         $out5,$out5,v24
2605
2606         subfe.          r0,r0,r0                # borrow?-1:0
2607          vand           $tmp,$tmp,$eighty7
2608         vcipher         $out0,$out0,v25
2609         vcipher         $out1,$out1,v25
2610          vxor           $tweak,$tweak,$tmp
2611         vcipher         $out2,$out2,v25
2612         vcipher         $out3,$out3,v25
2613          vxor           $in1,$twk1,v31
2614          vsrab          $tmp,$tweak,$seven      # next tweak value
2615          vxor           $twk1,$tweak,$rndkey0
2616         vcipher         $out4,$out4,v25
2617         vcipher         $out5,$out5,v25
2618
2619         and             r0,r0,$len
2620          vaddubm        $tweak,$tweak,$tweak
2621          vsldoi         $tmp,$tmp,$tmp,15
2622         vcipher         $out0,$out0,v26
2623         vcipher         $out1,$out1,v26
2624          vand           $tmp,$tmp,$eighty7
2625         vcipher         $out2,$out2,v26
2626         vcipher         $out3,$out3,v26
2627          vxor           $tweak,$tweak,$tmp
2628         vcipher         $out4,$out4,v26
2629         vcipher         $out5,$out5,v26
2630
2631         add             $inp,$inp,r0            # $inp is adjusted in such
2632                                                 # way that at exit from the
2633                                                 # loop inX-in5 are loaded
2634                                                 # with last "words"
2635          vxor           $in2,$twk2,v31
2636          vsrab          $tmp,$tweak,$seven      # next tweak value
2637          vxor           $twk2,$tweak,$rndkey0
2638          vaddubm        $tweak,$tweak,$tweak
2639         vcipher         $out0,$out0,v27
2640         vcipher         $out1,$out1,v27
2641          vsldoi         $tmp,$tmp,$tmp,15
2642         vcipher         $out2,$out2,v27
2643         vcipher         $out3,$out3,v27
2644          vand           $tmp,$tmp,$eighty7
2645         vcipher         $out4,$out4,v27
2646         vcipher         $out5,$out5,v27
2647
2648         addi            $key_,$sp,$FRAME+15     # rewind $key_
2649          vxor           $tweak,$tweak,$tmp
2650         vcipher         $out0,$out0,v28
2651         vcipher         $out1,$out1,v28
2652          vxor           $in3,$twk3,v31
2653          vsrab          $tmp,$tweak,$seven      # next tweak value
2654          vxor           $twk3,$tweak,$rndkey0
2655         vcipher         $out2,$out2,v28
2656         vcipher         $out3,$out3,v28
2657          vaddubm        $tweak,$tweak,$tweak
2658          vsldoi         $tmp,$tmp,$tmp,15
2659         vcipher         $out4,$out4,v28
2660         vcipher         $out5,$out5,v28
2661         lvx             v24,$x00,$key_          # re-pre-load round[1]
2662          vand           $tmp,$tmp,$eighty7
2663
2664         vcipher         $out0,$out0,v29
2665         vcipher         $out1,$out1,v29
2666          vxor           $tweak,$tweak,$tmp
2667         vcipher         $out2,$out2,v29
2668         vcipher         $out3,$out3,v29
2669          vxor           $in4,$twk4,v31
2670          vsrab          $tmp,$tweak,$seven      # next tweak value
2671          vxor           $twk4,$tweak,$rndkey0
2672         vcipher         $out4,$out4,v29
2673         vcipher         $out5,$out5,v29
2674         lvx             v25,$x10,$key_          # re-pre-load round[2]
2675          vaddubm        $tweak,$tweak,$tweak
2676          vsldoi         $tmp,$tmp,$tmp,15
2677
2678         vcipher         $out0,$out0,v30
2679         vcipher         $out1,$out1,v30
2680          vand           $tmp,$tmp,$eighty7
2681         vcipher         $out2,$out2,v30
2682         vcipher         $out3,$out3,v30
2683          vxor           $tweak,$tweak,$tmp
2684         vcipher         $out4,$out4,v30
2685         vcipher         $out5,$out5,v30
2686          vxor           $in5,$twk5,v31
2687          vsrab          $tmp,$tweak,$seven      # next tweak value
2688          vxor           $twk5,$tweak,$rndkey0
2689
2690         vcipherlast     $out0,$out0,$in0
2691          lvx_u          $in0,$x00,$inp          # load next input block
2692          vaddubm        $tweak,$tweak,$tweak
2693          vsldoi         $tmp,$tmp,$tmp,15
2694         vcipherlast     $out1,$out1,$in1
2695          lvx_u          $in1,$x10,$inp
2696         vcipherlast     $out2,$out2,$in2
2697          le?vperm       $in0,$in0,$in0,$leperm
2698          lvx_u          $in2,$x20,$inp
2699          vand           $tmp,$tmp,$eighty7
2700         vcipherlast     $out3,$out3,$in3
2701          le?vperm       $in1,$in1,$in1,$leperm
2702          lvx_u          $in3,$x30,$inp
2703         vcipherlast     $out4,$out4,$in4
2704          le?vperm       $in2,$in2,$in2,$leperm
2705          lvx_u          $in4,$x40,$inp
2706          vxor           $tweak,$tweak,$tmp
2707         vcipherlast     $tmp,$out5,$in5         # last block might be needed
2708                                                 # in stealing mode
2709          le?vperm       $in3,$in3,$in3,$leperm
2710          lvx_u          $in5,$x50,$inp
2711          addi           $inp,$inp,0x60
2712          le?vperm       $in4,$in4,$in4,$leperm
2713          le?vperm       $in5,$in5,$in5,$leperm
2714
2715         le?vperm        $out0,$out0,$out0,$leperm
2716         le?vperm        $out1,$out1,$out1,$leperm
2717         stvx_u          $out0,$x00,$out         # store output
2718          vxor           $out0,$in0,$twk0
2719         le?vperm        $out2,$out2,$out2,$leperm
2720         stvx_u          $out1,$x10,$out
2721          vxor           $out1,$in1,$twk1
2722         le?vperm        $out3,$out3,$out3,$leperm
2723         stvx_u          $out2,$x20,$out
2724          vxor           $out2,$in2,$twk2
2725         le?vperm        $out4,$out4,$out4,$leperm
2726         stvx_u          $out3,$x30,$out
2727          vxor           $out3,$in3,$twk3
2728         le?vperm        $out5,$tmp,$tmp,$leperm
2729         stvx_u          $out4,$x40,$out
2730          vxor           $out4,$in4,$twk4
2731         le?stvx_u       $out5,$x50,$out
2732         be?stvx_u       $tmp, $x50,$out
2733          vxor           $out5,$in5,$twk5
2734         addi            $out,$out,0x60
2735
2736         mtctr           $rounds
2737         beq             Loop_xts_enc6x          # did $len-=96 borrow?
2738
2739         addic.          $len,$len,0x60
2740         beq             Lxts_enc6x_zero
2741         cmpwi           $len,0x20
2742         blt             Lxts_enc6x_one
2743         nop
2744         beq             Lxts_enc6x_two
2745         cmpwi           $len,0x40
2746         blt             Lxts_enc6x_three
2747         nop
2748         beq             Lxts_enc6x_four
2749
2750 Lxts_enc6x_five:
2751         vxor            $out0,$in1,$twk0
2752         vxor            $out1,$in2,$twk1
2753         vxor            $out2,$in3,$twk2
2754         vxor            $out3,$in4,$twk3
2755         vxor            $out4,$in5,$twk4
2756
2757         bl              _aesp8_xts_enc5x
2758
2759         le?vperm        $out0,$out0,$out0,$leperm
2760         vmr             $twk0,$twk5             # unused tweak
2761         le?vperm        $out1,$out1,$out1,$leperm
2762         stvx_u          $out0,$x00,$out         # store output
2763         le?vperm        $out2,$out2,$out2,$leperm
2764         stvx_u          $out1,$x10,$out
2765         le?vperm        $out3,$out3,$out3,$leperm
2766         stvx_u          $out2,$x20,$out
2767         vxor            $tmp,$out4,$twk5        # last block prep for stealing
2768         le?vperm        $out4,$out4,$out4,$leperm
2769         stvx_u          $out3,$x30,$out
2770         stvx_u          $out4,$x40,$out
2771         addi            $out,$out,0x50
2772         bne             Lxts_enc6x_steal
2773         b               Lxts_enc6x_done
2774
2775 .align  4
2776 Lxts_enc6x_four:
2777         vxor            $out0,$in2,$twk0
2778         vxor            $out1,$in3,$twk1
2779         vxor            $out2,$in4,$twk2
2780         vxor            $out3,$in5,$twk3
2781         vxor            $out4,$out4,$out4
2782
2783         bl              _aesp8_xts_enc5x
2784
2785         le?vperm        $out0,$out0,$out0,$leperm
2786         vmr             $twk0,$twk4             # unused tweak
2787         le?vperm        $out1,$out1,$out1,$leperm
2788         stvx_u          $out0,$x00,$out         # store output
2789         le?vperm        $out2,$out2,$out2,$leperm
2790         stvx_u          $out1,$x10,$out
2791         vxor            $tmp,$out3,$twk4        # last block prep for stealing
2792         le?vperm        $out3,$out3,$out3,$leperm
2793         stvx_u          $out2,$x20,$out
2794         stvx_u          $out3,$x30,$out
2795         addi            $out,$out,0x40
2796         bne             Lxts_enc6x_steal
2797         b               Lxts_enc6x_done
2798
2799 .align  4
2800 Lxts_enc6x_three:
2801         vxor            $out0,$in3,$twk0
2802         vxor            $out1,$in4,$twk1
2803         vxor            $out2,$in5,$twk2
2804         vxor            $out3,$out3,$out3
2805         vxor            $out4,$out4,$out4
2806
2807         bl              _aesp8_xts_enc5x
2808
2809         le?vperm        $out0,$out0,$out0,$leperm
2810         vmr             $twk0,$twk3             # unused tweak
2811         le?vperm        $out1,$out1,$out1,$leperm
2812         stvx_u          $out0,$x00,$out         # store output
2813         vxor            $tmp,$out2,$twk3        # last block prep for stealing
2814         le?vperm        $out2,$out2,$out2,$leperm
2815         stvx_u          $out1,$x10,$out
2816         stvx_u          $out2,$x20,$out
2817         addi            $out,$out,0x30
2818         bne             Lxts_enc6x_steal
2819         b               Lxts_enc6x_done
2820
2821 .align  4
2822 Lxts_enc6x_two:
2823         vxor            $out0,$in4,$twk0
2824         vxor            $out1,$in5,$twk1
2825         vxor            $out2,$out2,$out2
2826         vxor            $out3,$out3,$out3
2827         vxor            $out4,$out4,$out4
2828
2829         bl              _aesp8_xts_enc5x
2830
2831         le?vperm        $out0,$out0,$out0,$leperm
2832         vmr             $twk0,$twk2             # unused tweak
2833         vxor            $tmp,$out1,$twk2        # last block prep for stealing
2834         le?vperm        $out1,$out1,$out1,$leperm
2835         stvx_u          $out0,$x00,$out         # store output
2836         stvx_u          $out1,$x10,$out
2837         addi            $out,$out,0x20
2838         bne             Lxts_enc6x_steal
2839         b               Lxts_enc6x_done
2840
2841 .align  4
2842 Lxts_enc6x_one:
2843         vxor            $out0,$in5,$twk0
2844         nop
2845 Loop_xts_enc1x:
2846         vcipher         $out0,$out0,v24
2847         lvx             v24,$x20,$key_          # round[3]
2848         addi            $key_,$key_,0x20
2849
2850         vcipher         $out0,$out0,v25
2851         lvx             v25,$x10,$key_          # round[4]
2852         bdnz            Loop_xts_enc1x
2853
2854         add             $inp,$inp,$taillen
2855         cmpwi           $taillen,0
2856         vcipher         $out0,$out0,v24
2857
2858         subi            $inp,$inp,16
2859         vcipher         $out0,$out0,v25
2860
2861         lvsr            $inpperm,0,$taillen
2862         vcipher         $out0,$out0,v26
2863
2864         lvx_u           $in0,0,$inp
2865         vcipher         $out0,$out0,v27
2866
2867         addi            $key_,$sp,$FRAME+15     # rewind $key_
2868         vcipher         $out0,$out0,v28
2869         lvx             v24,$x00,$key_          # re-pre-load round[1]
2870
2871         vcipher         $out0,$out0,v29
2872         lvx             v25,$x10,$key_          # re-pre-load round[2]
2873          vxor           $twk0,$twk0,v31
2874
2875         le?vperm        $in0,$in0,$in0,$leperm
2876         vcipher         $out0,$out0,v30
2877
2878         vperm           $in0,$in0,$in0,$inpperm
2879         vcipherlast     $out0,$out0,$twk0
2880
2881         vmr             $twk0,$twk1             # unused tweak
2882         vxor            $tmp,$out0,$twk1        # last block prep for stealing
2883         le?vperm        $out0,$out0,$out0,$leperm
2884         stvx_u          $out0,$x00,$out         # store output
2885         addi            $out,$out,0x10
2886         bne             Lxts_enc6x_steal
2887         b               Lxts_enc6x_done
2888
2889 .align  4
2890 Lxts_enc6x_zero:
2891         cmpwi           $taillen,0
2892         beq             Lxts_enc6x_done
2893
2894         add             $inp,$inp,$taillen
2895         subi            $inp,$inp,16
2896         lvx_u           $in0,0,$inp
2897         lvsr            $inpperm,0,$taillen     # $in5 is no more
2898         le?vperm        $in0,$in0,$in0,$leperm
2899         vperm           $in0,$in0,$in0,$inpperm
2900         vxor            $tmp,$tmp,$twk0
2901 Lxts_enc6x_steal:
2902         vxor            $in0,$in0,$twk0
2903         vxor            $out0,$out0,$out0
2904         vspltisb        $out1,-1
2905         vperm           $out0,$out0,$out1,$inpperm
2906         vsel            $out0,$in0,$tmp,$out0   # $tmp is last block, remember?
2907
2908         subi            r30,$out,17
2909         subi            $out,$out,16
2910         mtctr           $taillen
2911 Loop_xts_enc6x_steal:
2912         lbzu            r0,1(r30)
2913         stb             r0,16(r30)
2914         bdnz            Loop_xts_enc6x_steal
2915
2916         li              $taillen,0
2917         mtctr           $rounds
2918         b               Loop_xts_enc1x          # one more time...
2919
2920 .align  4
2921 Lxts_enc6x_done:
2922         ${UCMP}i        $ivp,0
2923         beq             Lxts_enc6x_ret
2924
2925         vxor            $tweak,$twk0,$rndkey0
2926         le?vperm        $tweak,$tweak,$tweak,$leperm
2927         stvx_u          $tweak,0,$ivp
2928
2929 Lxts_enc6x_ret:
2930         mtlr            r11
2931         li              r10,`$FRAME+15`
2932         li              r11,`$FRAME+31`
2933         stvx            $seven,r10,$sp          # wipe copies of round keys
2934         addi            r10,r10,32
2935         stvx            $seven,r11,$sp
2936         addi            r11,r11,32
2937         stvx            $seven,r10,$sp
2938         addi            r10,r10,32
2939         stvx            $seven,r11,$sp
2940         addi            r11,r11,32
2941         stvx            $seven,r10,$sp
2942         addi            r10,r10,32
2943         stvx            $seven,r11,$sp
2944         addi            r11,r11,32
2945         stvx            $seven,r10,$sp
2946         addi            r10,r10,32
2947         stvx            $seven,r11,$sp
2948         addi            r11,r11,32
2949
2950         mtspr           256,$vrsave
2951         lvx             v20,r10,$sp             # ABI says so
2952         addi            r10,r10,32
2953         lvx             v21,r11,$sp
2954         addi            r11,r11,32
2955         lvx             v22,r10,$sp
2956         addi            r10,r10,32
2957         lvx             v23,r11,$sp
2958         addi            r11,r11,32
2959         lvx             v24,r10,$sp
2960         addi            r10,r10,32
2961         lvx             v25,r11,$sp
2962         addi            r11,r11,32
2963         lvx             v26,r10,$sp
2964         addi            r10,r10,32
2965         lvx             v27,r11,$sp
2966         addi            r11,r11,32
2967         lvx             v28,r10,$sp
2968         addi            r10,r10,32
2969         lvx             v29,r11,$sp
2970         addi            r11,r11,32
2971         lvx             v30,r10,$sp
2972         lvx             v31,r11,$sp
2973         $POP            r26,`$FRAME+21*16+0*$SIZE_T`($sp)
2974         $POP            r27,`$FRAME+21*16+1*$SIZE_T`($sp)
2975         $POP            r28,`$FRAME+21*16+2*$SIZE_T`($sp)
2976         $POP            r29,`$FRAME+21*16+3*$SIZE_T`($sp)
2977         $POP            r30,`$FRAME+21*16+4*$SIZE_T`($sp)
2978         $POP            r31,`$FRAME+21*16+5*$SIZE_T`($sp)
2979         addi            $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
2980         blr
2981         .long           0
2982         .byte           0,12,0x04,1,0x80,6,6,0
2983         .long           0
2984
2985 .align  5
2986 _aesp8_xts_enc5x:
2987         vcipher         $out0,$out0,v24
2988         vcipher         $out1,$out1,v24
2989         vcipher         $out2,$out2,v24
2990         vcipher         $out3,$out3,v24
2991         vcipher         $out4,$out4,v24
2992         lvx             v24,$x20,$key_          # round[3]
2993         addi            $key_,$key_,0x20
2994
2995         vcipher         $out0,$out0,v25
2996         vcipher         $out1,$out1,v25
2997         vcipher         $out2,$out2,v25
2998         vcipher         $out3,$out3,v25
2999         vcipher         $out4,$out4,v25
3000         lvx             v25,$x10,$key_          # round[4]
3001         bdnz            _aesp8_xts_enc5x
3002
3003         add             $inp,$inp,$taillen
3004         cmpwi           $taillen,0
3005         vcipher         $out0,$out0,v24
3006         vcipher         $out1,$out1,v24
3007         vcipher         $out2,$out2,v24
3008         vcipher         $out3,$out3,v24
3009         vcipher         $out4,$out4,v24
3010
3011         subi            $inp,$inp,16
3012         vcipher         $out0,$out0,v25
3013         vcipher         $out1,$out1,v25
3014         vcipher         $out2,$out2,v25
3015         vcipher         $out3,$out3,v25
3016         vcipher         $out4,$out4,v25
3017          vxor           $twk0,$twk0,v31
3018
3019         vcipher         $out0,$out0,v26
3020         lvsr            $inpperm,0,$taillen     # $in5 is no more
3021         vcipher         $out1,$out1,v26
3022         vcipher         $out2,$out2,v26
3023         vcipher         $out3,$out3,v26
3024         vcipher         $out4,$out4,v26
3025          vxor           $in1,$twk1,v31
3026
3027         vcipher         $out0,$out0,v27
3028         lvx_u           $in0,0,$inp
3029         vcipher         $out1,$out1,v27
3030         vcipher         $out2,$out2,v27
3031         vcipher         $out3,$out3,v27
3032         vcipher         $out4,$out4,v27
3033          vxor           $in2,$twk2,v31
3034
3035         addi            $key_,$sp,$FRAME+15     # rewind $key_
3036         vcipher         $out0,$out0,v28
3037         vcipher         $out1,$out1,v28
3038         vcipher         $out2,$out2,v28
3039         vcipher         $out3,$out3,v28
3040         vcipher         $out4,$out4,v28
3041         lvx             v24,$x00,$key_          # re-pre-load round[1]
3042          vxor           $in3,$twk3,v31
3043
3044         vcipher         $out0,$out0,v29
3045         le?vperm        $in0,$in0,$in0,$leperm
3046         vcipher         $out1,$out1,v29
3047         vcipher         $out2,$out2,v29
3048         vcipher         $out3,$out3,v29
3049         vcipher         $out4,$out4,v29
3050         lvx             v25,$x10,$key_          # re-pre-load round[2]
3051          vxor           $in4,$twk4,v31
3052
3053         vcipher         $out0,$out0,v30
3054         vperm           $in0,$in0,$in0,$inpperm
3055         vcipher         $out1,$out1,v30
3056         vcipher         $out2,$out2,v30
3057         vcipher         $out3,$out3,v30
3058         vcipher         $out4,$out4,v30
3059
3060         vcipherlast     $out0,$out0,$twk0
3061         vcipherlast     $out1,$out1,$in1
3062         vcipherlast     $out2,$out2,$in2
3063         vcipherlast     $out3,$out3,$in3
3064         vcipherlast     $out4,$out4,$in4
3065         blr
3066         .long           0
3067         .byte           0,12,0x14,0,0,0,0,0
3068
3069 .align  5
3070 _aesp8_xts_decrypt6x:
3071         $STU            $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
3072         mflr            r11
3073         li              r7,`$FRAME+8*16+15`
3074         li              r3,`$FRAME+8*16+31`
3075         $PUSH           r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
3076         stvx            v20,r7,$sp              # ABI says so
3077         addi            r7,r7,32
3078         stvx            v21,r3,$sp
3079         addi            r3,r3,32
3080         stvx            v22,r7,$sp
3081         addi            r7,r7,32
3082         stvx            v23,r3,$sp
3083         addi            r3,r3,32
3084         stvx            v24,r7,$sp
3085         addi            r7,r7,32
3086         stvx            v25,r3,$sp
3087         addi            r3,r3,32
3088         stvx            v26,r7,$sp
3089         addi            r7,r7,32
3090         stvx            v27,r3,$sp
3091         addi            r3,r3,32
3092         stvx            v28,r7,$sp
3093         addi            r7,r7,32
3094         stvx            v29,r3,$sp
3095         addi            r3,r3,32
3096         stvx            v30,r7,$sp
3097         stvx            v31,r3,$sp
3098         li              r0,-1
3099         stw             $vrsave,`$FRAME+21*16-4`($sp)   # save vrsave
3100         li              $x10,0x10
3101         $PUSH           r26,`$FRAME+21*16+0*$SIZE_T`($sp)
3102         li              $x20,0x20
3103         $PUSH           r27,`$FRAME+21*16+1*$SIZE_T`($sp)
3104         li              $x30,0x30
3105         $PUSH           r28,`$FRAME+21*16+2*$SIZE_T`($sp)
3106         li              $x40,0x40
3107         $PUSH           r29,`$FRAME+21*16+3*$SIZE_T`($sp)
3108         li              $x50,0x50
3109         $PUSH           r30,`$FRAME+21*16+4*$SIZE_T`($sp)
3110         li              $x60,0x60
3111         $PUSH           r31,`$FRAME+21*16+5*$SIZE_T`($sp)
3112         li              $x70,0x70
3113         mtspr           256,r0
3114
3115         subi            $rounds,$rounds,3       # -4 in total
3116
3117         lvx             $rndkey0,$x00,$key1     # load key schedule
3118         lvx             v30,$x10,$key1
3119         addi            $key1,$key1,0x20
3120         lvx             v31,$x00,$key1
3121         ?vperm          $rndkey0,$rndkey0,v30,$keyperm
3122         addi            $key_,$sp,$FRAME+15
3123         mtctr           $rounds
3124
3125 Load_xts_dec_key:
3126         ?vperm          v24,v30,v31,$keyperm
3127         lvx             v30,$x10,$key1
3128         addi            $key1,$key1,0x20
3129         stvx            v24,$x00,$key_          # off-load round[1]
3130         ?vperm          v25,v31,v30,$keyperm
3131         lvx             v31,$x00,$key1
3132         stvx            v25,$x10,$key_          # off-load round[2]
3133         addi            $key_,$key_,0x20
3134         bdnz            Load_xts_dec_key
3135
3136         lvx             v26,$x10,$key1
3137         ?vperm          v24,v30,v31,$keyperm
3138         lvx             v27,$x20,$key1
3139         stvx            v24,$x00,$key_          # off-load round[3]
3140         ?vperm          v25,v31,v26,$keyperm
3141         lvx             v28,$x30,$key1
3142         stvx            v25,$x10,$key_          # off-load round[4]
3143         addi            $key_,$sp,$FRAME+15     # rewind $key_
3144         ?vperm          v26,v26,v27,$keyperm
3145         lvx             v29,$x40,$key1
3146         ?vperm          v27,v27,v28,$keyperm
3147         lvx             v30,$x50,$key1
3148         ?vperm          v28,v28,v29,$keyperm
3149         lvx             v31,$x60,$key1
3150         ?vperm          v29,v29,v30,$keyperm
3151         lvx             $twk5,$x70,$key1        # borrow $twk5
3152         ?vperm          v30,v30,v31,$keyperm
3153         lvx             v24,$x00,$key_          # pre-load round[1]
3154         ?vperm          v31,v31,$twk5,$keyperm
3155         lvx             v25,$x10,$key_          # pre-load round[2]
3156
3157          vperm          $in0,$inout,$inptail,$inpperm
3158          subi           $inp,$inp,31            # undo "caller"
3159         vxor            $twk0,$tweak,$rndkey0
3160         vsrab           $tmp,$tweak,$seven      # next tweak value
3161         vaddubm         $tweak,$tweak,$tweak
3162         vsldoi          $tmp,$tmp,$tmp,15
3163         vand            $tmp,$tmp,$eighty7
3164          vxor           $out0,$in0,$twk0
3165         vxor            $tweak,$tweak,$tmp
3166
3167          lvx_u          $in1,$x10,$inp
3168         vxor            $twk1,$tweak,$rndkey0
3169         vsrab           $tmp,$tweak,$seven      # next tweak value
3170         vaddubm         $tweak,$tweak,$tweak
3171         vsldoi          $tmp,$tmp,$tmp,15
3172          le?vperm       $in1,$in1,$in1,$leperm
3173         vand            $tmp,$tmp,$eighty7
3174          vxor           $out1,$in1,$twk1
3175         vxor            $tweak,$tweak,$tmp
3176
3177          lvx_u          $in2,$x20,$inp
3178          andi.          $taillen,$len,15
3179         vxor            $twk2,$tweak,$rndkey0
3180         vsrab           $tmp,$tweak,$seven      # next tweak value
3181         vaddubm         $tweak,$tweak,$tweak
3182         vsldoi          $tmp,$tmp,$tmp,15
3183          le?vperm       $in2,$in2,$in2,$leperm
3184         vand            $tmp,$tmp,$eighty7
3185          vxor           $out2,$in2,$twk2
3186         vxor            $tweak,$tweak,$tmp
3187
3188          lvx_u          $in3,$x30,$inp
3189          sub            $len,$len,$taillen
3190         vxor            $twk3,$tweak,$rndkey0
3191         vsrab           $tmp,$tweak,$seven      # next tweak value
3192         vaddubm         $tweak,$tweak,$tweak
3193         vsldoi          $tmp,$tmp,$tmp,15
3194          le?vperm       $in3,$in3,$in3,$leperm
3195         vand            $tmp,$tmp,$eighty7
3196          vxor           $out3,$in3,$twk3
3197         vxor            $tweak,$tweak,$tmp
3198
3199          lvx_u          $in4,$x40,$inp
3200          subi           $len,$len,0x60
3201         vxor            $twk4,$tweak,$rndkey0
3202         vsrab           $tmp,$tweak,$seven      # next tweak value
3203         vaddubm         $tweak,$tweak,$tweak
3204         vsldoi          $tmp,$tmp,$tmp,15
3205          le?vperm       $in4,$in4,$in4,$leperm
3206         vand            $tmp,$tmp,$eighty7
3207          vxor           $out4,$in4,$twk4
3208         vxor            $tweak,$tweak,$tmp
3209
3210          lvx_u          $in5,$x50,$inp
3211          addi           $inp,$inp,0x60
3212         vxor            $twk5,$tweak,$rndkey0
3213         vsrab           $tmp,$tweak,$seven      # next tweak value
3214         vaddubm         $tweak,$tweak,$tweak
3215         vsldoi          $tmp,$tmp,$tmp,15
3216          le?vperm       $in5,$in5,$in5,$leperm
3217         vand            $tmp,$tmp,$eighty7
3218          vxor           $out5,$in5,$twk5
3219         vxor            $tweak,$tweak,$tmp
3220
3221         vxor            v31,v31,$rndkey0
3222         mtctr           $rounds
3223         b               Loop_xts_dec6x
3224
3225 .align  5
3226 Loop_xts_dec6x:
3227         vncipher        $out0,$out0,v24
3228         vncipher        $out1,$out1,v24
3229         vncipher        $out2,$out2,v24
3230         vncipher        $out3,$out3,v24
3231         vncipher        $out4,$out4,v24
3232         vncipher        $out5,$out5,v24
3233         lvx             v24,$x20,$key_          # round[3]
3234         addi            $key_,$key_,0x20
3235
3236         vncipher        $out0,$out0,v25
3237         vncipher        $out1,$out1,v25
3238         vncipher        $out2,$out2,v25
3239         vncipher        $out3,$out3,v25
3240         vncipher        $out4,$out4,v25
3241         vncipher        $out5,$out5,v25
3242         lvx             v25,$x10,$key_          # round[4]
3243         bdnz            Loop_xts_dec6x
3244
3245         subic           $len,$len,96            # $len-=96
3246          vxor           $in0,$twk0,v31          # xor with last round key
3247         vncipher        $out0,$out0,v24
3248         vncipher        $out1,$out1,v24
3249          vsrab          $tmp,$tweak,$seven      # next tweak value
3250          vxor           $twk0,$tweak,$rndkey0
3251          vaddubm        $tweak,$tweak,$tweak
3252         vncipher        $out2,$out2,v24
3253         vncipher        $out3,$out3,v24
3254          vsldoi         $tmp,$tmp,$tmp,15
3255         vncipher        $out4,$out4,v24
3256         vncipher        $out5,$out5,v24
3257
3258         subfe.          r0,r0,r0                # borrow?-1:0
3259          vand           $tmp,$tmp,$eighty7
3260         vncipher        $out0,$out0,v25
3261         vncipher        $out1,$out1,v25
3262          vxor           $tweak,$tweak,$tmp
3263         vncipher        $out2,$out2,v25
3264         vncipher        $out3,$out3,v25
3265          vxor           $in1,$twk1,v31
3266          vsrab          $tmp,$tweak,$seven      # next tweak value
3267          vxor           $twk1,$tweak,$rndkey0
3268         vncipher        $out4,$out4,v25
3269         vncipher        $out5,$out5,v25
3270
3271         and             r0,r0,$len
3272          vaddubm        $tweak,$tweak,$tweak
3273          vsldoi         $tmp,$tmp,$tmp,15
3274         vncipher        $out0,$out0,v26
3275         vncipher        $out1,$out1,v26
3276          vand           $tmp,$tmp,$eighty7
3277         vncipher        $out2,$out2,v26
3278         vncipher        $out3,$out3,v26
3279          vxor           $tweak,$tweak,$tmp
3280         vncipher        $out4,$out4,v26
3281         vncipher        $out5,$out5,v26
3282
3283         add             $inp,$inp,r0            # $inp is adjusted in such
3284                                                 # way that at exit from the
3285                                                 # loop inX-in5 are loaded
3286                                                 # with last "words"
3287          vxor           $in2,$twk2,v31
3288          vsrab          $tmp,$tweak,$seven      # next tweak value
3289          vxor           $twk2,$tweak,$rndkey0
3290          vaddubm        $tweak,$tweak,$tweak
3291         vncipher        $out0,$out0,v27
3292         vncipher        $out1,$out1,v27
3293          vsldoi         $tmp,$tmp,$tmp,15
3294         vncipher        $out2,$out2,v27
3295         vncipher        $out3,$out3,v27
3296          vand           $tmp,$tmp,$eighty7
3297         vncipher        $out4,$out4,v27
3298         vncipher        $out5,$out5,v27
3299
3300         addi            $key_,$sp,$FRAME+15     # rewind $key_
3301          vxor           $tweak,$tweak,$tmp
3302         vncipher        $out0,$out0,v28
3303         vncipher        $out1,$out1,v28
3304          vxor           $in3,$twk3,v31
3305          vsrab          $tmp,$tweak,$seven      # next tweak value
3306          vxor           $twk3,$tweak,$rndkey0
3307         vncipher        $out2,$out2,v28
3308         vncipher        $out3,$out3,v28
3309          vaddubm        $tweak,$tweak,$tweak
3310          vsldoi         $tmp,$tmp,$tmp,15
3311         vncipher        $out4,$out4,v28
3312         vncipher        $out5,$out5,v28
3313         lvx             v24,$x00,$key_          # re-pre-load round[1]
3314          vand           $tmp,$tmp,$eighty7
3315
3316         vncipher        $out0,$out0,v29
3317         vncipher        $out1,$out1,v29
3318          vxor           $tweak,$tweak,$tmp
3319         vncipher        $out2,$out2,v29
3320         vncipher        $out3,$out3,v29
3321          vxor           $in4,$twk4,v31
3322          vsrab          $tmp,$tweak,$seven      # next tweak value
3323          vxor           $twk4,$tweak,$rndkey0
3324         vncipher        $out4,$out4,v29
3325         vncipher        $out5,$out5,v29
3326         lvx             v25,$x10,$key_          # re-pre-load round[2]
3327          vaddubm        $tweak,$tweak,$tweak
3328          vsldoi         $tmp,$tmp,$tmp,15
3329
3330         vncipher        $out0,$out0,v30
3331         vncipher        $out1,$out1,v30
3332          vand           $tmp,$tmp,$eighty7
3333         vncipher        $out2,$out2,v30
3334         vncipher        $out3,$out3,v30
3335          vxor           $tweak,$tweak,$tmp
3336         vncipher        $out4,$out4,v30
3337         vncipher        $out5,$out5,v30
3338          vxor           $in5,$twk5,v31
3339          vsrab          $tmp,$tweak,$seven      # next tweak value
3340          vxor           $twk5,$tweak,$rndkey0
3341
3342         vncipherlast    $out0,$out0,$in0
3343          lvx_u          $in0,$x00,$inp          # load next input block
3344          vaddubm        $tweak,$tweak,$tweak
3345          vsldoi         $tmp,$tmp,$tmp,15
3346         vncipherlast    $out1,$out1,$in1
3347          lvx_u          $in1,$x10,$inp
3348         vncipherlast    $out2,$out2,$in2
3349          le?vperm       $in0,$in0,$in0,$leperm
3350          lvx_u          $in2,$x20,$inp
3351          vand           $tmp,$tmp,$eighty7
3352         vncipherlast    $out3,$out3,$in3
3353          le?vperm       $in1,$in1,$in1,$leperm
3354          lvx_u          $in3,$x30,$inp
3355         vncipherlast    $out4,$out4,$in4
3356          le?vperm       $in2,$in2,$in2,$leperm
3357          lvx_u          $in4,$x40,$inp
3358          vxor           $tweak,$tweak,$tmp
3359         vncipherlast    $out5,$out5,$in5
3360          le?vperm       $in3,$in3,$in3,$leperm
3361          lvx_u          $in5,$x50,$inp
3362          addi           $inp,$inp,0x60
3363          le?vperm       $in4,$in4,$in4,$leperm
3364          le?vperm       $in5,$in5,$in5,$leperm
3365
3366         le?vperm        $out0,$out0,$out0,$leperm
3367         le?vperm        $out1,$out1,$out1,$leperm
3368         stvx_u          $out0,$x00,$out         # store output
3369          vxor           $out0,$in0,$twk0
3370         le?vperm        $out2,$out2,$out2,$leperm
3371         stvx_u          $out1,$x10,$out
3372          vxor           $out1,$in1,$twk1
3373         le?vperm        $out3,$out3,$out3,$leperm
3374         stvx_u          $out2,$x20,$out
3375          vxor           $out2,$in2,$twk2
3376         le?vperm        $out4,$out4,$out4,$leperm
3377         stvx_u          $out3,$x30,$out
3378          vxor           $out3,$in3,$twk3
3379         le?vperm        $out5,$out5,$out5,$leperm
3380         stvx_u          $out4,$x40,$out
3381          vxor           $out4,$in4,$twk4
3382         stvx_u          $out5,$x50,$out
3383          vxor           $out5,$in5,$twk5
3384         addi            $out,$out,0x60
3385
3386         mtctr           $rounds
3387         beq             Loop_xts_dec6x          # did $len-=96 borrow?
3388
3389         addic.          $len,$len,0x60
3390         beq             Lxts_dec6x_zero
3391         cmpwi           $len,0x20
3392         blt             Lxts_dec6x_one
3393         nop
3394         beq             Lxts_dec6x_two
3395         cmpwi           $len,0x40
3396         blt             Lxts_dec6x_three
3397         nop
3398         beq             Lxts_dec6x_four
3399
3400 Lxts_dec6x_five:
3401         vxor            $out0,$in1,$twk0
3402         vxor            $out1,$in2,$twk1
3403         vxor            $out2,$in3,$twk2
3404         vxor            $out3,$in4,$twk3
3405         vxor            $out4,$in5,$twk4
3406
3407         bl              _aesp8_xts_dec5x
3408
3409         le?vperm        $out0,$out0,$out0,$leperm
3410         vmr             $twk0,$twk5             # unused tweak
3411         vxor            $twk1,$tweak,$rndkey0
3412         le?vperm        $out1,$out1,$out1,$leperm
3413         stvx_u          $out0,$x00,$out         # store output
3414         vxor            $out0,$in0,$twk1
3415         le?vperm        $out2,$out2,$out2,$leperm
3416         stvx_u          $out1,$x10,$out
3417         le?vperm        $out3,$out3,$out3,$leperm
3418         stvx_u          $out2,$x20,$out
3419         le?vperm        $out4,$out4,$out4,$leperm
3420         stvx_u          $out3,$x30,$out
3421         stvx_u          $out4,$x40,$out
3422         addi            $out,$out,0x50
3423         bne             Lxts_dec6x_steal
3424         b               Lxts_dec6x_done
3425
3426 .align  4
3427 Lxts_dec6x_four:
3428         vxor            $out0,$in2,$twk0
3429         vxor            $out1,$in3,$twk1
3430         vxor            $out2,$in4,$twk2
3431         vxor            $out3,$in5,$twk3
3432         vxor            $out4,$out4,$out4
3433
3434         bl              _aesp8_xts_dec5x
3435
3436         le?vperm        $out0,$out0,$out0,$leperm
3437         vmr             $twk0,$twk4             # unused tweak
3438         vmr             $twk1,$twk5
3439         le?vperm        $out1,$out1,$out1,$leperm
3440         stvx_u          $out0,$x00,$out         # store output
3441         vxor            $out0,$in0,$twk5
3442         le?vperm        $out2,$out2,$out2,$leperm
3443         stvx_u          $out1,$x10,$out
3444         le?vperm        $out3,$out3,$out3,$leperm
3445         stvx_u          $out2,$x20,$out
3446         stvx_u          $out3,$x30,$out
3447         addi            $out,$out,0x40
3448         bne             Lxts_dec6x_steal
3449         b               Lxts_dec6x_done
3450
3451 .align  4
3452 Lxts_dec6x_three:
3453         vxor            $out0,$in3,$twk0
3454         vxor            $out1,$in4,$twk1
3455         vxor            $out2,$in5,$twk2
3456         vxor            $out3,$out3,$out3
3457         vxor            $out4,$out4,$out4
3458
3459         bl              _aesp8_xts_dec5x
3460
3461         le?vperm        $out0,$out0,$out0,$leperm
3462         vmr             $twk0,$twk3             # unused tweak
3463         vmr             $twk1,$twk4
3464         le?vperm        $out1,$out1,$out1,$leperm
3465         stvx_u          $out0,$x00,$out         # store output
3466         vxor            $out0,$in0,$twk4
3467         le?vperm        $out2,$out2,$out2,$leperm
3468         stvx_u          $out1,$x10,$out
3469         stvx_u          $out2,$x20,$out
3470         addi            $out,$out,0x30
3471         bne             Lxts_dec6x_steal
3472         b               Lxts_dec6x_done
3473
3474 .align  4
3475 Lxts_dec6x_two:
3476         vxor            $out0,$in4,$twk0
3477         vxor            $out1,$in5,$twk1
3478         vxor            $out2,$out2,$out2
3479         vxor            $out3,$out3,$out3
3480         vxor            $out4,$out4,$out4
3481
3482         bl              _aesp8_xts_dec5x
3483
3484         le?vperm        $out0,$out0,$out0,$leperm
3485         vmr             $twk0,$twk2             # unused tweak
3486         vmr             $twk1,$twk3
3487         le?vperm        $out1,$out1,$out1,$leperm
3488         stvx_u          $out0,$x00,$out         # store output
3489         vxor            $out0,$in0,$twk3
3490         stvx_u          $out1,$x10,$out
3491         addi            $out,$out,0x20
3492         bne             Lxts_dec6x_steal
3493         b               Lxts_dec6x_done
3494
3495 .align  4
3496 Lxts_dec6x_one:
3497         vxor            $out0,$in5,$twk0
3498         nop
3499 Loop_xts_dec1x:
3500         vncipher        $out0,$out0,v24
3501         lvx             v24,$x20,$key_          # round[3]
3502         addi            $key_,$key_,0x20
3503
3504         vncipher        $out0,$out0,v25
3505         lvx             v25,$x10,$key_          # round[4]
3506         bdnz            Loop_xts_dec1x
3507
3508         subi            r0,$taillen,1
3509         vncipher        $out0,$out0,v24
3510
3511         andi.           r0,r0,16
3512         cmpwi           $taillen,0
3513         vncipher        $out0,$out0,v25
3514
3515         sub             $inp,$inp,r0
3516         vncipher        $out0,$out0,v26
3517
3518         lvx_u           $in0,0,$inp
3519         vncipher        $out0,$out0,v27
3520
3521         addi            $key_,$sp,$FRAME+15     # rewind $key_
3522         vncipher        $out0,$out0,v28
3523         lvx             v24,$x00,$key_          # re-pre-load round[1]
3524
3525         vncipher        $out0,$out0,v29
3526         lvx             v25,$x10,$key_          # re-pre-load round[2]
3527          vxor           $twk0,$twk0,v31
3528
3529         le?vperm        $in0,$in0,$in0,$leperm
3530         vncipher        $out0,$out0,v30
3531
3532         mtctr           $rounds
3533         vncipherlast    $out0,$out0,$twk0
3534
3535         vmr             $twk0,$twk1             # unused tweak
3536         vmr             $twk1,$twk2
3537         le?vperm        $out0,$out0,$out0,$leperm
3538         stvx_u          $out0,$x00,$out         # store output
3539         addi            $out,$out,0x10
3540         vxor            $out0,$in0,$twk2
3541         bne             Lxts_dec6x_steal
3542         b               Lxts_dec6x_done
3543
3544 .align  4
3545 Lxts_dec6x_zero:
3546         cmpwi           $taillen,0
3547         beq             Lxts_dec6x_done
3548
3549         lvx_u           $in0,0,$inp
3550         le?vperm        $in0,$in0,$in0,$leperm
3551         vxor            $out0,$in0,$twk1
3552 Lxts_dec6x_steal:
3553         vncipher        $out0,$out0,v24
3554         lvx             v24,$x20,$key_          # round[3]
3555         addi            $key_,$key_,0x20
3556
3557         vncipher        $out0,$out0,v25
3558         lvx             v25,$x10,$key_          # round[4]
3559         bdnz            Lxts_dec6x_steal
3560
3561         add             $inp,$inp,$taillen
3562         vncipher        $out0,$out0,v24
3563
3564         cmpwi           $taillen,0
3565         vncipher        $out0,$out0,v25
3566
3567         lvx_u           $in0,0,$inp
3568         vncipher        $out0,$out0,v26
3569
3570         lvsr            $inpperm,0,$taillen     # $in5 is no more
3571         vncipher        $out0,$out0,v27
3572
3573         addi            $key_,$sp,$FRAME+15     # rewind $key_
3574         vncipher        $out0,$out0,v28
3575         lvx             v24,$x00,$key_          # re-pre-load round[1]
3576
3577         vncipher        $out0,$out0,v29
3578         lvx             v25,$x10,$key_          # re-pre-load round[2]
3579          vxor           $twk1,$twk1,v31
3580
3581         le?vperm        $in0,$in0,$in0,$leperm
3582         vncipher        $out0,$out0,v30
3583
3584         vperm           $in0,$in0,$in0,$inpperm
3585         vncipherlast    $tmp,$out0,$twk1
3586
3587         le?vperm        $out0,$tmp,$tmp,$leperm
3588         le?stvx_u       $out0,0,$out
3589         be?stvx_u       $tmp,0,$out
3590
3591         vxor            $out0,$out0,$out0
3592         vspltisb        $out1,-1
3593         vperm           $out0,$out0,$out1,$inpperm
3594         vsel            $out0,$in0,$tmp,$out0
3595         vxor            $out0,$out0,$twk0
3596
3597         subi            r30,$out,1
3598         mtctr           $taillen
3599 Loop_xts_dec6x_steal:
3600         lbzu            r0,1(r30)
3601         stb             r0,16(r30)
3602         bdnz            Loop_xts_dec6x_steal
3603
3604         li              $taillen,0
3605         mtctr           $rounds
3606         b               Loop_xts_dec1x          # one more time...
3607
3608 .align  4
3609 Lxts_dec6x_done:
3610         ${UCMP}i        $ivp,0
3611         beq             Lxts_dec6x_ret
3612
3613         vxor            $tweak,$twk0,$rndkey0
3614         le?vperm        $tweak,$tweak,$tweak,$leperm
3615         stvx_u          $tweak,0,$ivp
3616
3617 Lxts_dec6x_ret:
3618         mtlr            r11
3619         li              r10,`$FRAME+15`
3620         li              r11,`$FRAME+31`
3621         stvx            $seven,r10,$sp          # wipe copies of round keys
3622         addi            r10,r10,32
3623         stvx            $seven,r11,$sp
3624         addi            r11,r11,32
3625         stvx            $seven,r10,$sp
3626         addi            r10,r10,32
3627         stvx            $seven,r11,$sp
3628         addi            r11,r11,32
3629         stvx            $seven,r10,$sp
3630         addi            r10,r10,32
3631         stvx            $seven,r11,$sp
3632         addi            r11,r11,32
3633         stvx            $seven,r10,$sp
3634         addi            r10,r10,32
3635         stvx            $seven,r11,$sp
3636         addi            r11,r11,32
3637
3638         mtspr           256,$vrsave
3639         lvx             v20,r10,$sp             # ABI says so
3640         addi            r10,r10,32
3641         lvx             v21,r11,$sp
3642         addi            r11,r11,32
3643         lvx             v22,r10,$sp
3644         addi            r10,r10,32
3645         lvx             v23,r11,$sp
3646         addi            r11,r11,32
3647         lvx             v24,r10,$sp
3648         addi            r10,r10,32
3649         lvx             v25,r11,$sp
3650         addi            r11,r11,32
3651         lvx             v26,r10,$sp
3652         addi            r10,r10,32
3653         lvx             v27,r11,$sp
3654         addi            r11,r11,32
3655         lvx             v28,r10,$sp
3656         addi            r10,r10,32
3657         lvx             v29,r11,$sp
3658         addi            r11,r11,32
3659         lvx             v30,r10,$sp
3660         lvx             v31,r11,$sp
3661         $POP            r26,`$FRAME+21*16+0*$SIZE_T`($sp)
3662         $POP            r27,`$FRAME+21*16+1*$SIZE_T`($sp)
3663         $POP            r28,`$FRAME+21*16+2*$SIZE_T`($sp)
3664         $POP            r29,`$FRAME+21*16+3*$SIZE_T`($sp)
3665         $POP            r30,`$FRAME+21*16+4*$SIZE_T`($sp)
3666         $POP            r31,`$FRAME+21*16+5*$SIZE_T`($sp)
3667         addi            $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
3668         blr
3669         .long           0
3670         .byte           0,12,0x04,1,0x80,6,6,0
3671         .long           0
3672
3673 .align  5
3674 _aesp8_xts_dec5x:
3675         vncipher        $out0,$out0,v24
3676         vncipher        $out1,$out1,v24
3677         vncipher        $out2,$out2,v24
3678         vncipher        $out3,$out3,v24
3679         vncipher        $out4,$out4,v24
3680         lvx             v24,$x20,$key_          # round[3]
3681         addi            $key_,$key_,0x20
3682
3683         vncipher        $out0,$out0,v25
3684         vncipher        $out1,$out1,v25
3685         vncipher        $out2,$out2,v25
3686         vncipher        $out3,$out3,v25
3687         vncipher        $out4,$out4,v25
3688         lvx             v25,$x10,$key_          # round[4]
3689         bdnz            _aesp8_xts_dec5x
3690
3691         subi            r0,$taillen,1
3692         vncipher        $out0,$out0,v24
3693         vncipher        $out1,$out1,v24
3694         vncipher        $out2,$out2,v24
3695         vncipher        $out3,$out3,v24
3696         vncipher        $out4,$out4,v24
3697
3698         andi.           r0,r0,16
3699         cmpwi           $taillen,0
3700         vncipher        $out0,$out0,v25
3701         vncipher        $out1,$out1,v25
3702         vncipher        $out2,$out2,v25
3703         vncipher        $out3,$out3,v25
3704         vncipher        $out4,$out4,v25
3705          vxor           $twk0,$twk0,v31
3706
3707         sub             $inp,$inp,r0
3708         vncipher        $out0,$out0,v26
3709         vncipher        $out1,$out1,v26
3710         vncipher        $out2,$out2,v26
3711         vncipher        $out3,$out3,v26
3712         vncipher        $out4,$out4,v26
3713          vxor           $in1,$twk1,v31
3714
3715         vncipher        $out0,$out0,v27
3716         lvx_u           $in0,0,$inp
3717         vncipher        $out1,$out1,v27
3718         vncipher        $out2,$out2,v27
3719         vncipher        $out3,$out3,v27
3720         vncipher        $out4,$out4,v27
3721          vxor           $in2,$twk2,v31
3722
3723         addi            $key_,$sp,$FRAME+15     # rewind $key_
3724         vncipher        $out0,$out0,v28
3725         vncipher        $out1,$out1,v28
3726         vncipher        $out2,$out2,v28
3727         vncipher        $out3,$out3,v28
3728         vncipher        $out4,$out4,v28
3729         lvx             v24,$x00,$key_          # re-pre-load round[1]
3730          vxor           $in3,$twk3,v31
3731
3732         vncipher        $out0,$out0,v29
3733         le?vperm        $in0,$in0,$in0,$leperm
3734         vncipher        $out1,$out1,v29
3735         vncipher        $out2,$out2,v29
3736         vncipher        $out3,$out3,v29
3737         vncipher        $out4,$out4,v29
3738         lvx             v25,$x10,$key_          # re-pre-load round[2]
3739          vxor           $in4,$twk4,v31
3740
3741         vncipher        $out0,$out0,v30
3742         vncipher        $out1,$out1,v30
3743         vncipher        $out2,$out2,v30
3744         vncipher        $out3,$out3,v30
3745         vncipher        $out4,$out4,v30
3746
3747         vncipherlast    $out0,$out0,$twk0
3748         vncipherlast    $out1,$out1,$in1
3749         vncipherlast    $out2,$out2,$in2
3750         vncipherlast    $out3,$out3,$in3
3751         vncipherlast    $out4,$out4,$in4
3752         mtctr           $rounds
3753         blr
3754         .long           0
3755         .byte           0,12,0x14,0,0,0,0,0
3756 ___
3757 }}      }}}
3758
3759 my $consts=1;
3760 foreach(split("\n",$code)) {
3761         s/\`([^\`]*)\`/eval($1)/geo;
3762
3763         # constants table endian-specific conversion
3764         if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) {
3765             my $conv=$3;
3766             my @bytes=();
3767
3768             # convert to endian-agnostic format
3769             if ($1 eq "long") {
3770               foreach (split(/,\s*/,$2)) {
3771                 my $l = /^0/?oct:int;
3772                 push @bytes,($l>>24)&0xff,($l>>16)&0xff,($l>>8)&0xff,$l&0xff;
3773               }
3774             } else {
3775                 @bytes = map(/^0/?oct:int,split(/,\s*/,$2));
3776             }
3777
3778             # little-endian conversion
3779             if ($flavour =~ /le$/o) {
3780                 SWITCH: for($conv)  {
3781                     /\?inv/ && do   { @bytes=map($_^0xf,@bytes); last; };
3782                     /\?rev/ && do   { @bytes=reverse(@bytes);    last; };
3783                 }
3784             }
3785
3786             #emit
3787             print ".byte\t",join(',',map (sprintf("0x%02x",$_),@bytes)),"\n";
3788             next;
3789         }
3790         $consts=0 if (m/Lconsts:/o);    # end of table
3791
3792         # instructions prefixed with '?' are endian-specific and need
3793         # to be adjusted accordingly...
3794         if ($flavour =~ /le$/o) {       # little-endian
3795             s/le\?//o           or
3796             s/be\?/#be#/o       or
3797             s/\?lvsr/lvsl/o     or
3798             s/\?lvsl/lvsr/o     or
3799             s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or
3800             s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or
3801             s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o;
3802         } else {                        # big-endian
3803             s/le\?/#le#/o       or
3804             s/be\?//o           or
3805             s/\?([a-z]+)/$1/o;
3806         }
3807
3808         print $_,"\n";
3809 }
3810
3811 close STDOUT or die "error closing STDOUT: $!";