poly1305/asm/poly1305-ppc.pl: add vector base 2^26 implementation.
authorAndy Polyakov <appro@openssl.org>
Wed, 23 Jan 2019 13:56:19 +0000 (14:56 +0100)
committerRichard Levitte <levitte@openssl.org>
Fri, 1 Feb 2019 08:23:25 +0000 (09:23 +0100)
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/8120)

crypto/poly1305/asm/poly1305-ppc.pl
crypto/ppccap.c

index e9118ba..9f15c0d 100755 (executable)
@@ -8,10 +8,10 @@
 
 #
 # ====================================================================
-# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
-# project. The module is, however, dual licensed under OpenSSL and
-# CRYPTOGAMS licenses depending on where you obtain it. For further
-# details see http://www.openssl.org/~appro/cryptogams/.
+# Written by Andy Polyakov, @dot-asm, initially for use in the OpenSSL
+# project. The module is dual licensed under OpenSSL and CRYPTOGAMS
+# licenses depending on where you obtain it. For further details see
+# https://github.com/dot-asm/cryptogams/.
 # ====================================================================
 #
 # This module implements Poly1305 hash for PowerPC.
 #
 # On side note, Power ISA 2.07 enables vector base 2^26 implementation,
 # and POWER8 might have capacity to break 1.0 cycle per byte barrier...
+#
+# January 2019
+#
+# ... Unfortunately not:-( Estimate was a projection of ARM result,
+# but ARM has vector multiply-n-add instruction, while PowerISA does
+# not, not one usable in the context. Improvement is ~40% over -m64
+# result above and is ~1.43 on little-endian systems.
 
 $flavour = shift;
 
@@ -99,6 +106,7 @@ $code.=<<___;
        std     r0,0($ctx)              # zero hash value
        std     r0,8($ctx)
        std     r0,16($ctx)
+       stw     r0,24($ctx)             # clear is_base2_26
 
        $UCMP   $inp,r0
        beq-    Lno_key
@@ -140,6 +148,7 @@ Lno_key:
 .globl .poly1305_blocks
 .align 4
 .poly1305_blocks:
+Lpoly1305_blocks:
        srdi.   $len,$len,4
        beq-    Labort
 
@@ -238,60 +247,120 @@ Labort:
        .long   0
        .byte   0,12,4,1,0x80,5,4,0
 .size  .poly1305_blocks,.-.poly1305_blocks
+___
+{
+my ($h0,$h1,$h2,$h3,$h4,$t0) = map("r$_",(7..12));
 
+$code.=<<___;
 .globl .poly1305_emit
-.align 4
+.align 5
 .poly1305_emit:
-       ld      $h0,0($ctx)             # load hash
-       ld      $h1,8($ctx)
-       ld      $h2,16($ctx)
-       ld      $padbit,0($nonce)       # load nonce
-       ld      $nonce,8($nonce)
-
-       addic   $d0,$h0,5               # compare to modulus
-       addze   $d1,$h1
-       addze   $d2,$h2
-
-       srdi    $mask,$d2,2             # did it carry/borrow?
-       neg     $mask,$mask
+       lwz     $h0,0($ctx)     # load hash value base 2^26
+       lwz     $h1,4($ctx)
+       lwz     $h2,8($ctx)
+       lwz     $h3,12($ctx)
+       lwz     $h4,16($ctx)
+       lwz     r0,24($ctx)     # is_base2_26
+
+       sldi    $h1,$h1,26      # base 2^26 -> base 2^64
+       sldi    $t0,$h2,52
+       srdi    $h2,$h2,12
+       sldi    $h3,$h3,14
+       add     $h0,$h0,$h1
+       addc    $h0,$h0,$t0
+       sldi    $t0,$h4,40
+       srdi    $h4,$h4,24
+       adde    $h1,$h2,$h3
+       addc    $h1,$h1,$t0
+       addze   $h2,$h4
+
+       ld      $h3,0($ctx)     # load hash value base 2^64
+       ld      $h4,8($ctx)
+       ld      $t0,16($ctx)
+
+       neg     r0,r0
+       xor     $h0,$h0,$h3     # choose between radixes
+       xor     $h1,$h1,$h4
+       xor     $h2,$h2,$t0
+       and     $h0,$h0,r0
+       and     $h1,$h1,r0
+       and     $h2,$h2,r0
+       xor     $h0,$h0,$h3
+       xor     $h1,$h1,$h4
+       xor     $h2,$h2,$t0
+
+       addic   $h3,$h0,5       # compare to modulus
+       addze   $h4,$h1
+       addze   $t0,$h2
+
+       srdi    $t0,$t0,2       # see if it carried/borrowed
+       neg     $t0,$t0
+
+       andc    $h0,$h0,$t0
+       and     $h3,$h3,$t0
+       andc    $h1,$h1,$t0
+       and     $h4,$h4,$t0
+       or      $h0,$h0,$h3
+       or      $h1,$h1,$h4
+
+       lwz     $t0,4($nonce)
+       lwz     $h2,12($nonce)
+       lwz     $h3,0($nonce)
+       lwz     $h4,8($nonce)
+
+       insrdi  $h3,$t0,32,0
+       insrdi  $h4,$h2,32,0
+
+       addc    $h0,$h0,$h3     # accumulate nonce
+       adde    $h1,$h1,$h4
+
+       addi    $ctx,$mac,-1
+       addi    $mac,$mac,7
+
+       stbu    $h0,1($ctx)     # write [little-endian] result
+       srdi    $h0,$h0,8
+       stbu    $h1,1($mac)
+       srdi    $h1,$h1,8
+
+       stbu    $h0,1($ctx)
+       srdi    $h0,$h0,8
+       stbu    $h1,1($mac)
+       srdi    $h1,$h1,8
+
+       stbu    $h0,1($ctx)
+       srdi    $h0,$h0,8
+       stbu    $h1,1($mac)
+       srdi    $h1,$h1,8
+
+       stbu    $h0,1($ctx)
+       srdi    $h0,$h0,8
+       stbu    $h1,1($mac)
+       srdi    $h1,$h1,8
+
+       stbu    $h0,1($ctx)
+       srdi    $h0,$h0,8
+       stbu    $h1,1($mac)
+       srdi    $h1,$h1,8
+
+       stbu    $h0,1($ctx)
+       srdi    $h0,$h0,8
+       stbu    $h1,1($mac)
+       srdi    $h1,$h1,8
+
+       stbu    $h0,1($ctx)
+       srdi    $h0,$h0,8
+       stbu    $h1,1($mac)
+       srdi    $h1,$h1,8
+
+       stbu    $h0,1($ctx)
+       stbu    $h1,1($mac)
 
-       andc    $h0,$h0,$mask
-       and     $d0,$d0,$mask
-       andc    $h1,$h1,$mask
-       and     $d1,$d1,$mask
-       or      $h0,$h0,$d0
-       or      $h1,$h1,$d1
-___
-$code.=<<___   if (!$LITTLE_ENDIAN);
-       rotldi  $padbit,$padbit,32      # flip nonce words
-       rotldi  $nonce,$nonce,32
-___
-$code.=<<___;
-       addc    $h0,$h0,$padbit         # accumulate nonce
-       adde    $h1,$h1,$nonce
-___
-$code.=<<___   if ($LITTLE_ENDIAN);
-       std     $h0,0($mac)             # write result
-       std     $h1,8($mac)
-___
-$code.=<<___   if (!$LITTLE_ENDIAN);
-       extrdi  r0,$h0,32,0
-       li      $d0,4
-       stwbrx  $h0,0,$mac              # write result
-       extrdi  $h0,$h1,32,0
-       li      $d1,8
-       stwbrx  r0,$d0,$mac
-       li      $d2,12
-       stwbrx  $h1,$d1,$mac
-       stwbrx  $h0,$d2,$mac
-___
-$code.=<<___;
        blr
        .long   0
        .byte   0,12,0x14,0,0,0,3,0
 .size  .poly1305_emit,.-.poly1305_emit
 ___
-                                                       } else {
+}                                                      } else {
 ###############################################################################
 # base 2^32 implementation
 
@@ -309,6 +378,7 @@ $code.=<<___;
        stw     r0,8($ctx)
        stw     r0,12($ctx)
        stw     r0,16($ctx)
+       stw     r0,24($ctx)             # clear is_base2_26
 
        $UCMP   $inp,r0
        beq-    Lno_key
@@ -353,6 +423,7 @@ Lno_key:
 .globl .poly1305_blocks
 .align 4
 .poly1305_blocks:
+Lpoly1305_blocks:
        srwi.   $len,$len,4
        beq-    Labort
 
@@ -560,17 +631,389 @@ Labort:
        .long   0
        .byte   0,12,4,1,0x80,18,4,0
 .size  .poly1305_blocks,.-.poly1305_blocks
+___
+{
+my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(6..12));
 
+$code.=<<___;
 .globl .poly1305_emit
-.align 4
+.align 5
 .poly1305_emit:
-       $STU    $sp,-$FRAME($sp)
+       lwz     r0,24($ctx)     # is_base2_26
+       lwz     $h0,0($ctx)     # load hash value
+       lwz     $h1,4($ctx)
+       lwz     $h2,8($ctx)
+       lwz     $h3,12($ctx)
+       lwz     $h4,16($ctx)
+       cmplwi  r0,0
+       beq     Lemit_base2_32
+
+       slwi    $t0,$h1,26      # base 2^26 -> base 2^32
+       srwi    $h1,$h1,6
+       slwi    $t1,$h2,20
+       srwi    $h2,$h2,12
+       addc    $h0,$h0,$t0
+       slwi    $t0,$h3,14
+       srwi    $h3,$h3,18
+       adde    $h1,$h1,$t1
+       slwi    $t1,$h4,8
+       srwi    $h4,$h4,24
+       adde    $h2,$h2,$t0
+       adde    $h3,$h3,$t1
+       addze   $h4,$h4
+
+Lemit_base2_32:
+       addic   r0,$h0,5        # compare to modulus
+       addze   r0,$h1
+       addze   r0,$h2
+       addze   r0,$h3
+       addze   r0,$h4
+
+       srwi    r0,r0,2         # see if it carried/borrowed
+       neg     r0,r0
+       andi.   r0,r0,5
+
+       addc    $h0,$h0,r0
+       lwz     r0,0($nonce)
+       addze   $h1,$h1
+       lwz     $t0,4($nonce)
+       addze   $h2,$h2
+       lwz     $t1,8($nonce)
+       addze   $h3,$h3
+       lwz     $h4,12($nonce)
+
+       addc    $h0,$h0,r0      # accumulate nonce
+       adde    $h1,$h1,$t0
+       adde    $h2,$h2,$t1
+       adde    $h3,$h3,$h4
+
+       addi    $ctx,$mac,-1
+       addi    $mac,$mac,7
+
+       stbu    $h0,1($ctx)     # write [little-endian] result
+       srwi    $h0,$h0,8
+       stbu    $h2,1($mac)
+       srwi    $h2,$h2,8
+
+       stbu    $h0,1($ctx)
+       srwi    $h0,$h0,8
+       stbu    $h2,1($mac)
+       srwi    $h2,$h2,8
+
+       stbu    $h0,1($ctx)
+       srwi    $h0,$h0,8
+       stbu    $h2,1($mac)
+       srwi    $h2,$h2,8
+
+       stbu    $h0,1($ctx)
+       stbu    $h2,1($mac)
+
+       stbu    $h1,1($ctx)
+       srwi    $h1,$h1,8
+       stbu    $h3,1($mac)
+       srwi    $h3,$h3,8
+
+       stbu    $h1,1($ctx)
+       srwi    $h1,$h1,8
+       stbu    $h3,1($mac)
+       srwi    $h3,$h3,8
+
+       stbu    $h1,1($ctx)
+       srwi    $h1,$h1,8
+       stbu    $h3,1($mac)
+       srwi    $h3,$h3,8
+
+       stbu    $h1,1($ctx)
+       stbu    $h3,1($mac)
+
+       blr
+       .long   0
+       .byte   0,12,0x14,0,0,0,3,0
+.size  .poly1305_emit,.-.poly1305_emit
+___
+}                                                      }
+{{{
+########################################################################
+# PowerISA 2.07/VSX section                                            #
+########################################################################
+
+my $LOCALS= 6*$SIZE_T;
+my $VSXFRAME = $LOCALS + 6*$SIZE_T;
+   $VSXFRAME += 128;   # local variables
+   $VSXFRAME += 13*16; # v20-v31 offload
+
+my $BIG_ENDIAN = ($flavour !~ /le/) ? 4 : 0;
+
+########################################################################
+# Layout of opaque area is following:
+#
+#      unsigned __int32 h[5];          # current hash value base 2^26
+#      unsigned __int32 pad;
+#      unsigned __int32 is_base2_26, pad;
+#      unsigned __int64 r[2];          # key value base 2^64
+#      struct { unsigned __int32 r^2, r^4, r^1, r^3; } r[9];
+#
+# where r^n are base 2^26 digits of powers of multiplier key. There are
+# 5 digits, but last four are interleaved with multiples of 5, totalling
+# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4. Order of
+# powers is as they appear in register, not memory.
+
+my ($H0, $H1, $H2, $H3, $H4) = map("v$_",(0..4));
+my ($I0, $I1, $I2, $I3, $I4) = map("v$_",(5..9));
+my ($R0, $R1, $S1, $R2, $S2) = map("v$_",(10..14));
+my      ($R3, $S3, $R4, $S4) = ($R1, $S1, $R2, $S2);
+my ($ACC0, $ACC1, $ACC2, $ACC3, $ACC4) = map("v$_",(15..19));
+my ($T0, $T1, $T2, $T3, $T4) = map("v$_",(20..24));
+my ($_26,$_4,$_40,$_14,$mask26,$padbits,$I2perm) = map("v$_",(25..31));
+my ($x00,$x60,$x70,$x10,$x20,$x30,$x40,$x50) = (0, map("r$_",(7,8,27..31)));
+my ($ctx_,$_ctx,$const) = map("r$_",(10..12));
+
+                                                       if ($flavour =~ /64/) {
+###############################################################################
+# setup phase of poly1305_blocks_vsx is different on 32- and 64-bit platforms,
+# but the base 2^26 computational part is same...
+
+my ($h0,$h1,$h2,$d0,$d1,$d2, $r0,$r1,$s1, $t0,$t1) = map("r$_",(6..11,27..31));
+my $mask = "r0";
+
+$code.=<<___;
+.globl .poly1305_blocks_vsx
+.align 5
+.poly1305_blocks_vsx:
+       lwz     r7,24($ctx)             # is_base2_26
+       cmpldi  $len,128
+       bge     __poly1305_blocks_vsx
+
+       neg     r0,r7                   # is_base2_26 as mask
+       lwz     r7,0($ctx)              # load hash base 2^26
+       lwz     r8,4($ctx)
+       lwz     r9,8($ctx)
+       lwz     r10,12($ctx)
+       lwz     r11,16($ctx)
+
+       sldi    r8,r8,26                # base 2^26 -> base 2^64
+       sldi    r12,r9,52
+       add     r7,r7,r8
+       srdi    r9,r9,12
+       sldi    r10,r10,14
+       addc    r7,r7,r12
+       sldi    r8,r11,40
+       adde    r9,r9,r10
+       srdi    r11,r11,24
+       addc    r9,r9,r8
+       addze   r11,r11
+
+       ld      r8,0($ctx)              # load hash base 2^64
+       ld      r10,8($ctx)
+       ld      r12,16($ctx)
+
+       xor     r7,r7,r8                # select between radixes
+       xor     r9,r9,r10
+       xor     r11,r11,r12
+       and     r7,r7,r0
+       and     r9,r9,r0
+       and     r11,r11,r0
+       xor     r7,r7,r8
+       xor     r9,r9,r10
+       xor     r11,r11,r12
+
+       li      r0,0
+       std     r7,0($ctx)              # store hash base 2^64
+       std     r9,8($ctx)
+       std     r11,16($ctx)
+       stw     r0,24($ctx)             # clear is_base2_26
+
+       b       Lpoly1305_blocks
+       .long   0
+       .byte   0,12,0x14,0,0,0,4,0
+.size  .poly1305_blocks_vsx,.-.poly1305_blocks_vsx
+
+.align 5
+__poly1305_mul:
+       mulld   $d0,$h0,$r0             # h0*r0
+       mulhdu  $d1,$h0,$r0
+
+       mulld   $t0,$h1,$s1             # h1*5*r1
+       mulhdu  $t1,$h1,$s1
+       addc    $d0,$d0,$t0
+       adde    $d1,$d1,$t1
+
+       mulld   $t0,$h0,$r1             # h0*r1
+       mulhdu  $d2,$h0,$r1
+       addc    $d1,$d1,$t0
+       addze   $d2,$d2
+
+       mulld   $t0,$h1,$r0             # h1*r0
+       mulhdu  $t1,$h1,$r0
+       addc    $d1,$d1,$t0
+       adde    $d2,$d2,$t1
+
+       mulld   $t0,$h2,$s1             # h2*5*r1
+       mulld   $t1,$h2,$r0             # h2*r0
+       addc    $d1,$d1,$t0
+       adde    $d2,$d2,$t1
+
+       andc    $t0,$d2,$mask           # final reduction step
+       and     $h2,$d2,$mask
+       srdi    $t1,$t0,2
+       add     $t0,$t0,$t1
+       addc    $h0,$d0,$t0
+       addze   $h1,$d1
+       addze   $h2,$h2
+
+       blr
+       .long   0
+       .byte   0,12,0x14,0,0,0,0,0
+.size  __poly1305_mul,.-__poly1305_mul
+
+.align 5
+__poly1305_splat:
+       extrdi  $d0,$h0,26,38
+       extrdi  $d1,$h0,26,12
+       stw     $d0,0x00($t1)
+
+       extrdi  $d2,$h0,12,0
+       slwi    $d0,$d1,2
+       stw     $d1,0x10($t1)
+       add     $d0,$d0,$d1             # * 5
+       stw     $d0,0x20($t1)
+
+       insrdi  $d2,$h1,14,38
+       slwi    $d0,$d2,2
+       stw     $d2,0x30($t1)
+       add     $d0,$d0,$d2             # * 5
+       stw     $d0,0x40($t1)
+
+       extrdi  $d1,$h1,26,24
+       extrdi  $d2,$h1,24,0
+       slwi    $d0,$d1,2
+       stw     $d1,0x50($t1)
+       add     $d0,$d0,$d1             # * 5
+       stw     $d0,0x60($t1)
+
+       insrdi  $d2,$h2,3,37
+       slwi    $d0,$d2,2
+       stw     $d2,0x70($t1)
+       add     $d0,$d0,$d2             # * 5
+       stw     $d0,0x80($t1)
+
+       blr
+       .long   0
+       .byte   0,12,0x14,0,0,0,0,0
+.size  __poly1305_splat,.-__poly1305_splat
+
+.align 5
+__poly1305_blocks_vsx:
+       $STU    $sp,-$VSXFRAME($sp)
        mflr    r0
-       $PUSH   r28,`$FRAME-$SIZE_T*4`($sp)
-       $PUSH   r29,`$FRAME-$SIZE_T*3`($sp)
-       $PUSH   r30,`$FRAME-$SIZE_T*2`($sp)
-       $PUSH   r31,`$FRAME-$SIZE_T*1`($sp)
-       $PUSH   r0,`$FRAME+$LRSAVE`($sp)
+       li      r10,`15+$LOCALS+128`
+       li      r11,`31+$LOCALS+128`
+       mfspr   r12,256
+       stvx    v20,r10,$sp
+       addi    r10,r10,32
+       stvx    v21,r11,$sp
+       addi    r11,r11,32
+       stvx    v22,r10,$sp
+       addi    r10,r10,32
+       stvx    v23,r10,$sp
+       addi    r10,r10,32
+       stvx    v24,r11,$sp
+       addi    r11,r11,32
+       stvx    v25,r10,$sp
+       addi    r10,r10,32
+       stvx    v26,r10,$sp
+       addi    r10,r10,32
+       stvx    v27,r11,$sp
+       addi    r11,r11,32
+       stvx    v28,r10,$sp
+       addi    r10,r10,32
+       stvx    v29,r11,$sp
+       addi    r11,r11,32
+       stvx    v30,r10,$sp
+       stvx    v31,r11,$sp
+       stw     r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
+       li      r12,-1
+       mtspr   256,r12                 # preserve all AltiVec registers
+       $PUSH   r27,`$VSXFRAME-$SIZE_T*5`($sp)
+       $PUSH   r28,`$VSXFRAME-$SIZE_T*4`($sp)
+       $PUSH   r29,`$VSXFRAME-$SIZE_T*3`($sp)
+       $PUSH   r30,`$VSXFRAME-$SIZE_T*2`($sp)
+       $PUSH   r31,`$VSXFRAME-$SIZE_T*1`($sp)
+       $PUSH   r0,`$VSXFRAME+$LRSAVE`($sp)
+
+       bl      LPICmeup
+
+       li      $x10,0x10
+       li      $x20,0x20
+       li      $x30,0x30
+       li      $x40,0x40
+       li      $x50,0x50
+       lvx_u   $mask26,$x00,$const
+       lvx_u   $_26,$x10,$const
+       lvx_u   $_40,$x20,$const
+       lvx_u   $I2perm,$x30,$const
+       lvx_u   $padbits,$x40,$const
+
+       cmplwi  r7,0                    # is_base2_26?
+       bne     Lskip_init_vsx
+
+       ld      $r0,32($ctx)            # load key base 2^64
+       ld      $r1,40($ctx)
+       srdi    $s1,$r1,2
+       li      $mask,3
+       add     $s1,$s1,$r1             # s1 = r1 + r1>>2
+
+       mr      $h0,$r0                 # "calculate" r^1
+       mr      $h1,$r1
+       li      $h2,0
+       addi    $t1,$ctx,`48+(12^$BIG_ENDIAN)`
+       bl      __poly1305_splat
+
+       bl      __poly1305_mul          # caclulate r^2
+       addi    $t1,$ctx,`48+(4^$BIG_ENDIAN)`
+       bl      __poly1305_splat
+
+       bl      __poly1305_mul          # caclulate r^3
+       addi    $t1,$ctx,`48+(8^$BIG_ENDIAN)`
+       bl      __poly1305_splat
+
+       bl      __poly1305_mul          # caclulate r^4
+       addi    $t1,$ctx,`48+(0^$BIG_ENDIAN)`
+       bl      __poly1305_splat
+
+       ld      $h0,0($ctx)             # load hash
+       ld      $h1,8($ctx)
+       ld      $h2,16($ctx)
+
+       extrdi  $d0,$h0,26,38           # base 2^64 -> base 2^26
+       extrdi  $d1,$h0,26,12
+       extrdi  $d2,$h0,12,0
+       mtvrwz  $H0,$d0
+       insrdi  $d2,$h1,14,38
+       mtvrwz  $H1,$d1
+       extrdi  $d1,$h1,26,24
+       mtvrwz  $H2,$d2
+       extrdi  $d2,$h1,24,0
+       mtvrwz  $H3,$d1
+       insrdi  $d2,$h2,3,37
+       mtvrwz  $H4,$d2
+___
+                                                       } else {
+###############################################################################
+# 32-bit initialization
+
+my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(7..11,0,12));
+my ($R3,$S3,$R4,$S4)=($I1,$I2,$I3,$I4);
+
+$code.=<<___;
+.globl .poly1305_blocks_vsx
+.align 5
+.poly1305_blocks_vsx:
+       lwz     r7,24($ctx)             # is_base2_26
+       cmplwi  $len,128
+       bge     __poly1305_blocks_vsx
+       cmplwi  r7,0
+       beq     Lpoly1305_blocks
 
        lwz     $h0,0($ctx)             # load hash
        lwz     $h1,4($ctx)
@@ -578,68 +1021,957 @@ Labort:
        lwz     $h3,12($ctx)
        lwz     $h4,16($ctx)
 
-       addic   $d0,$h0,5               # compare to modulus
-       addze   $d1,$h1
-       addze   $d2,$h2
-       addze   $d3,$h3
-       addze   $mask,$h4
+       slwi    $t0,$h1,26              # base 2^26 -> base 2^32
+       srwi    $h1,$h1,6
+       slwi    $t1,$h2,20
+       srwi    $h2,$h2,12
+       addc    $h0,$h0,$t0
+       slwi    $t0,$h3,14
+       srwi    $h3,$h3,18
+       adde    $h1,$h1,$t1
+       slwi    $t1,$h4,8
+       srwi    $h4,$h4,24
+       adde    $h2,$h2,$t0
+       li      $t0,0
+       adde    $h3,$h3,$t1
+       addze   $h4,$h4
 
-       srwi    $mask,$mask,2           # did it carry/borrow?
-       neg     $mask,$mask
+       stw     $h0,0($ctx)             # store hash base 2^32
+       stw     $h1,4($ctx)
+       stw     $h2,8($ctx)
+       stw     $h3,12($ctx)
+       stw     $h4,16($ctx)
+       stw     $t0,24($ctx)            # clear is_base2_26
 
-       andc    $h0,$h0,$mask
-       and     $d0,$d0,$mask
-       andc    $h1,$h1,$mask
-       and     $d1,$d1,$mask
-       or      $h0,$h0,$d0
-       lwz     $d0,0($nonce)           # load nonce
-       andc    $h2,$h2,$mask
-       and     $d2,$d2,$mask
-       or      $h1,$h1,$d1
-       lwz     $d1,4($nonce)
-       andc    $h3,$h3,$mask
-       and     $d3,$d3,$mask
-       or      $h2,$h2,$d2
-       lwz     $d2,8($nonce)
-       or      $h3,$h3,$d3
-       lwz     $d3,12($nonce)
-
-       addc    $h0,$h0,$d0             # accumulate nonce
-       adde    $h1,$h1,$d1
-       adde    $h2,$h2,$d2
-       adde    $h3,$h3,$d3
-___
-$code.=<<___   if ($LITTLE_ENDIAN);
-       stw     $h0,0($mac)             # write result
-       stw     $h1,4($mac)
-       stw     $h2,8($mac)
-       stw     $h3,12($mac)
-___
-$code.=<<___   if (!$LITTLE_ENDIAN);
-       li      $d1,4
-       stwbrx  $h0,0,$mac              # write result
-       li      $d2,8
-       stwbrx  $h1,$d1,$mac
-       li      $d3,12
-       stwbrx  $h2,$d2,$mac
-       stwbrx  $h3,$d3,$mac
+       b       Lpoly1305_blocks
+       .long   0
+       .byte   0,12,0x14,0,0,0,4,0
+.size  .poly1305_blocks_vsx,.-.poly1305_blocks_vsx
+
+.align 5
+__poly1305_mul:
+       vmulouw         $ACC0,$H0,$R0
+       vmulouw         $ACC1,$H1,$R0
+       vmulouw         $ACC2,$H2,$R0
+       vmulouw         $ACC3,$H3,$R0
+       vmulouw         $ACC4,$H4,$R0
+
+       vmulouw         $T0,$H4,$S1
+       vaddudm         $ACC0,$ACC0,$T0
+       vmulouw         $T0,$H0,$R1
+       vaddudm         $ACC1,$ACC1,$T0
+       vmulouw         $T0,$H1,$R1
+       vaddudm         $ACC2,$ACC2,$T0
+       vmulouw         $T0,$H2,$R1
+       vaddudm         $ACC3,$ACC3,$T0
+       vmulouw         $T0,$H3,$R1
+       vaddudm         $ACC4,$ACC4,$T0
+
+       vmulouw         $T0,$H3,$S2
+       vaddudm         $ACC0,$ACC0,$T0
+       vmulouw         $T0,$H4,$S2
+       vaddudm         $ACC1,$ACC1,$T0
+       vmulouw         $T0,$H0,$R2
+       vaddudm         $ACC2,$ACC2,$T0
+       vmulouw         $T0,$H1,$R2
+       vaddudm         $ACC3,$ACC3,$T0
+       vmulouw         $T0,$H2,$R2
+       vaddudm         $ACC4,$ACC4,$T0
+
+       vmulouw         $T0,$H2,$S3
+       vaddudm         $ACC0,$ACC0,$T0
+       vmulouw         $T0,$H3,$S3
+       vaddudm         $ACC1,$ACC1,$T0
+       vmulouw         $T0,$H4,$S3
+       vaddudm         $ACC2,$ACC2,$T0
+       vmulouw         $T0,$H0,$R3
+       vaddudm         $ACC3,$ACC3,$T0
+       vmulouw         $T0,$H1,$R3
+       vaddudm         $ACC4,$ACC4,$T0
+
+       vmulouw         $T0,$H1,$S4
+       vaddudm         $ACC0,$ACC0,$T0
+       vmulouw         $T0,$H2,$S4
+       vaddudm         $ACC1,$ACC1,$T0
+       vmulouw         $T0,$H3,$S4
+       vaddudm         $ACC2,$ACC2,$T0
+       vmulouw         $T0,$H4,$S4
+       vaddudm         $ACC3,$ACC3,$T0
+       vmulouw         $T0,$H0,$R4
+       vaddudm         $ACC4,$ACC4,$T0
+
+       ################################################################
+       # lazy reduction
+
+       vspltisb        $T0,2
+       vsrd            $H4,$ACC3,$_26
+       vsrd            $H1,$ACC0,$_26
+       vand            $H3,$ACC3,$mask26
+       vand            $H0,$ACC0,$mask26
+       vaddudm         $H4,$H4,$ACC4           # h3 -> h4
+       vaddudm         $H1,$H1,$ACC1           # h0 -> h1
+
+       vsrd            $ACC4,$H4,$_26
+       vsrd            $ACC1,$H1,$_26
+       vand            $H4,$H4,$mask26
+       vand            $H1,$H1,$mask26
+       vaddudm         $H0,$H0,$ACC4
+       vaddudm         $H2,$ACC2,$ACC1         # h1 -> h2
+
+       vsld            $ACC4,$ACC4,$T0         # <<2
+       vsrd            $ACC2,$H2,$_26
+       vand            $H2,$H2,$mask26
+       vaddudm         $H0,$H0,$ACC4           # h4 -> h0
+       vaddudm         $H3,$H3,$ACC2           # h2 -> h3
+
+       vsrd            $ACC0,$H0,$_26
+       vsrd            $ACC3,$H3,$_26
+       vand            $H0,$H0,$mask26
+       vand            $H3,$H3,$mask26
+       vaddudm         $H1,$H1,$ACC0           # h0 -> h1
+       vaddudm         $H4,$H4,$ACC3           # h3 -> h4
+
+       blr
+       .long   0
+       .byte   0,12,0x14,0,0,0,0,0
+.size  __poly1305_mul,.-__poly1305_mul
+
+.align 5
+__poly1305_blocks_vsx:
+       $STU    $sp,-$VSXFRAME($sp)
+       mflr    r0
+       li      r10,`15+$LOCALS+128`
+       li      r11,`31+$LOCALS+128`
+       mfspr   r12,256
+       stvx    v20,r10,$sp
+       addi    r10,r10,32
+       stvx    v21,r11,$sp
+       addi    r11,r11,32
+       stvx    v22,r10,$sp
+       addi    r10,r10,32
+       stvx    v23,r10,$sp
+       addi    r10,r10,32
+       stvx    v24,r11,$sp
+       addi    r11,r11,32
+       stvx    v25,r10,$sp
+       addi    r10,r10,32
+       stvx    v26,r10,$sp
+       addi    r10,r10,32
+       stvx    v27,r11,$sp
+       addi    r11,r11,32
+       stvx    v28,r10,$sp
+       addi    r10,r10,32
+       stvx    v29,r11,$sp
+       addi    r11,r11,32
+       stvx    v30,r10,$sp
+       stvx    v31,r11,$sp
+       stw     r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
+       li      r12,-1
+       mtspr   256,r12                 # preserve all AltiVec registers
+       $PUSH   r27,`$VSXFRAME-$SIZE_T*5`($sp)
+       $PUSH   r28,`$VSXFRAME-$SIZE_T*4`($sp)
+       $PUSH   r29,`$VSXFRAME-$SIZE_T*3`($sp)
+       $PUSH   r30,`$VSXFRAME-$SIZE_T*2`($sp)
+       $PUSH   r31,`$VSXFRAME-$SIZE_T*1`($sp)
+       $PUSH   r0,`$VSXFRAME+$LRSAVE`($sp)
+
+       bl      LPICmeup
+
+       li      $x10,0x10
+       li      $x20,0x20
+       li      $x30,0x30
+       li      $x40,0x40
+       li      $x50,0x50
+       lvx_u   $mask26,$x00,$const
+       lvx_u   $_26,$x10,$const
+       lvx_u   $_40,$x20,$const
+       lvx_u   $I2perm,$x30,$const
+       lvx_u   $padbits,$x40,$const
+
+       cmplwi  r7,0                    # is_base2_26?
+       bne     Lskip_init_vsx
+
+       lwz     $h1,32($ctx)            # load key base 2^32
+       lwz     $h2,36($ctx)
+       lwz     $h3,40($ctx)
+       lwz     $h4,44($ctx)
+
+       extrwi  $h0,$h1,26,6            # base 2^32 -> base 2^26
+       extrwi  $h1,$h1,6,0
+       insrwi  $h1,$h2,20,6
+       extrwi  $h2,$h2,12,0
+       insrwi  $h2,$h3,14,6
+       extrwi  $h3,$h3,18,0
+       insrwi  $h3,$h4,8,6
+       extrwi  $h4,$h4,24,0
+
+       mtvrwz  $R0,$h0
+       slwi    $h0,$h1,2
+       mtvrwz  $R1,$h1
+       add     $h1,$h1,$h0
+       mtvrwz  $S1,$h1
+       slwi    $h1,$h2,2
+       mtvrwz  $R2,$h2
+       add     $h2,$h2,$h1
+       mtvrwz  $S2,$h2
+       slwi    $h2,$h3,2
+       mtvrwz  $R3,$h3
+       add     $h3,$h3,$h2
+       mtvrwz  $S3,$h3
+       slwi    $h3,$h4,2
+       mtvrwz  $R4,$h4
+       add     $h4,$h4,$h3
+       mtvrwz  $S4,$h4
+
+       vmr     $H0,$R0
+       vmr     $H1,$R1
+       vmr     $H2,$R2
+       vmr     $H3,$R3
+       vmr     $H4,$R4
+
+       bl      __poly1305_mul          # r^1:- * r^1:-
+
+       vpermdi $R0,$H0,$R0,0b00
+       vpermdi $R1,$H1,$R1,0b00
+       vpermdi $R2,$H2,$R2,0b00
+       vpermdi $R3,$H3,$R3,0b00
+       vpermdi $R4,$H4,$R4,0b00
+       vpermdi $H0,$H0,$H0,0b00
+       vpermdi $H1,$H1,$H1,0b00
+       vpermdi $H2,$H2,$H2,0b00
+       vpermdi $H3,$H3,$H3,0b00
+       vpermdi $H4,$H4,$H4,0b00
+       vsld    $S1,$R1,$T0             # <<2
+       vsld    $S2,$R2,$T0
+       vsld    $S3,$R3,$T0
+       vsld    $S4,$R4,$T0
+       vaddudm $S1,$S1,$R1
+       vaddudm $S2,$S2,$R2
+       vaddudm $S3,$S3,$R3
+       vaddudm $S4,$S4,$R4
+
+       bl      __poly1305_mul          # r^2:r^2 * r^2:r^1
+
+       addi    $h0,$ctx,0x60
+       lwz     $h1,0($ctx)             # load hash
+       lwz     $h2,4($ctx)
+       lwz     $h3,8($ctx)
+       lwz     $h4,12($ctx)
+       lwz     $t0,16($ctx)
+
+       vmrgow  $R0,$R0,$H0             # r^2:r^4:r^1:r^3
+       vmrgow  $R1,$R1,$H1
+       vmrgow  $R2,$R2,$H2
+       vmrgow  $R3,$R3,$H3
+       vmrgow  $R4,$R4,$H4
+       vslw    $S1,$R1,$T0             # <<2
+       vslw    $S2,$R2,$T0
+       vslw    $S3,$R3,$T0
+       vslw    $S4,$R4,$T0
+       vadduwm $S1,$S1,$R1
+       vadduwm $S2,$S2,$R2
+       vadduwm $S3,$S3,$R3
+       vadduwm $S4,$S4,$R4
+
+       stvx_u  $R0,$x30,$ctx
+       stvx_u  $R1,$x40,$ctx
+       stvx_u  $S1,$x50,$ctx
+       stvx_u  $R2,$x00,$h0
+       stvx_u  $S2,$x10,$h0
+       stvx_u  $R3,$x20,$h0
+       stvx_u  $S3,$x30,$h0
+       stvx_u  $R4,$x40,$h0
+       stvx_u  $S4,$x50,$h0
+
+       extrwi  $h0,$h1,26,6            # base 2^32 -> base 2^26
+       extrwi  $h1,$h1,6,0
+       mtvrwz  $H0,$h0
+       insrwi  $h1,$h2,20,6
+       extrwi  $h2,$h2,12,0
+       mtvrwz  $H1,$h1
+       insrwi  $h2,$h3,14,6
+       extrwi  $h3,$h3,18,0
+       mtvrwz  $H2,$h2
+       insrwi  $h3,$h4,8,6
+       extrwi  $h4,$h4,24,0
+       mtvrwz  $H3,$h3
+       insrwi  $h4,$t0,3,5
+       mtvrwz  $H4,$h4
 ___
+                                                       }
 $code.=<<___;
-       $POP    r28,`$FRAME-$SIZE_T*4`($sp)
-       $POP    r29,`$FRAME-$SIZE_T*3`($sp)
-       $POP    r30,`$FRAME-$SIZE_T*2`($sp)
-       $POP    r31,`$FRAME-$SIZE_T*1`($sp)
-       addi    $sp,$sp,$FRAME
+       li      r0,1
+       stw     r0,24($ctx)             # set is_base2_26
+       b       Loaded_vsx
+
+.align 4
+Lskip_init_vsx:
+       li              $x10,4
+       li              $x20,8
+       li              $x30,12
+       li              $x40,16
+       lvwzx_u         $H0,$x00,$ctx
+       lvwzx_u         $H1,$x10,$ctx
+       lvwzx_u         $H2,$x20,$ctx
+       lvwzx_u         $H3,$x30,$ctx
+       lvwzx_u         $H4,$x40,$ctx
+
+Loaded_vsx:
+       li              $x10,0x10
+       li              $x20,0x20
+       li              $x30,0x30
+       li              $x40,0x40
+       li              $x50,0x50
+       li              $x60,0x60
+       li              $x70,0x70
+       addi            $ctx_,$ctx,64           # &ctx->r[1]
+       addi            $_ctx,$sp,`$LOCALS+15`  # &ctx->r[1], r^2:r^4 shadow
+
+       vxor            $T0,$T0,$T0             # ensure second half is zero
+       vpermdi         $H0,$H0,$T0,0b00
+       vpermdi         $H1,$H1,$T0,0b00
+       vpermdi         $H2,$H2,$T0,0b00
+       vpermdi         $H3,$H3,$T0,0b00
+       vpermdi         $H4,$H4,$T0,0b00
+
+       be?lvx_u        $_4,$x50,$const         # byte swap mask
+       lvx_u           $T1,$x00,$inp           # load first input block
+       lvx_u           $T2,$x10,$inp
+       lvx_u           $T3,$x20,$inp
+       lvx_u           $T4,$x30,$inp
+       be?vperm        $T1,$T1,$T1,$_4
+       be?vperm        $T2,$T2,$T2,$_4
+       be?vperm        $T3,$T3,$T3,$_4
+       be?vperm        $T4,$T4,$T4,$_4
+
+       vpermdi         $I0,$T1,$T2,0b00        # smash input to base 2^26
+       vspltisb        $_4,4
+       vperm           $I2,$T1,$T2,$I2perm     # 0x...0e0f0001...1e1f1011
+       vspltisb        $_14,14
+       vpermdi         $I3,$T1,$T2,0b11
+
+       vsrd            $I1,$I0,$_26
+       vsrd            $I2,$I2,$_4
+       vsrd            $I4,$I3,$_40
+       vsrd            $I3,$I3,$_14
+       vand            $I0,$I0,$mask26
+       vand            $I1,$I1,$mask26
+       vand            $I2,$I2,$mask26
+       vand            $I3,$I3,$mask26
+
+       vpermdi         $T1,$T3,$T4,0b00
+       vperm           $T2,$T3,$T4,$I2perm     # 0x...0e0f0001...1e1f1011
+       vpermdi         $T3,$T3,$T4,0b11
+
+       vsrd            $T0,$T1,$_26
+       vsrd            $T2,$T2,$_4
+       vsrd            $T4,$T3,$_40
+       vsrd            $T3,$T3,$_14
+       vand            $T1,$T1,$mask26
+       vand            $T0,$T0,$mask26
+       vand            $T2,$T2,$mask26
+       vand            $T3,$T3,$mask26
+
+       # inp[2]:inp[0]:inp[3]:inp[1]
+       vmrgow          $I4,$T4,$I4
+       vmrgow          $I0,$T1,$I0
+       vmrgow          $I1,$T0,$I1
+       vmrgow          $I2,$T2,$I2
+       vmrgow          $I3,$T3,$I3
+       vor             $I4,$I4,$padbits
+
+       lvx_splt        $R0,$x30,$ctx           # taking lvx_vsplt out of loop
+       lvx_splt        $R1,$x00,$ctx_          # gives ~8% improvement
+       lvx_splt        $S1,$x10,$ctx_
+       lvx_splt        $R2,$x20,$ctx_
+       lvx_splt        $S2,$x30,$ctx_
+       lvx_splt        $T1,$x40,$ctx_
+       lvx_splt        $T2,$x50,$ctx_
+       lvx_splt        $T3,$x60,$ctx_
+       lvx_splt        $T4,$x70,$ctx_
+       stvx            $R1,$x00,$_ctx
+       stvx            $S1,$x10,$_ctx
+       stvx            $R2,$x20,$_ctx
+       stvx            $S2,$x30,$_ctx
+       stvx            $T1,$x40,$_ctx
+       stvx            $T2,$x50,$_ctx
+       stvx            $T3,$x60,$_ctx
+       stvx            $T4,$x70,$_ctx
+
+       addi            $inp,$inp,0x40
+       addi            $const,$const,0x50
+       addi            r0,$len,-64
+       srdi            r0,r0,6
+       mtctr           r0
+       b               Loop_vsx
+
+.align 4
+Loop_vsx:
+       ################################################################
+       ## ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
+       ## ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
+       ##   \___________________/
+       ##
+       ## Note that we start with inp[2:3]*r^2. This is because it
+       ## doesn't depend on reduction in previous iteration.
+       ################################################################
+       ## d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4
+       ## d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4
+       ## d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4
+       ## d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4
+       ## d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
+
+       vmuleuw         $ACC0,$I0,$R0
+       vmuleuw         $ACC1,$I0,$R1
+       vmuleuw         $ACC2,$I0,$R2
+       vmuleuw         $ACC3,$I1,$R2
+
+       vmuleuw         $T0,$I1,$R0
+       vaddudm         $ACC1,$ACC1,$T0
+       vmuleuw         $T0,$I1,$R1
+       vaddudm         $ACC2,$ACC2,$T0
+        vmuleuw        $ACC4,$I2,$R2
+       vmuleuw         $T0,$I4,$S1
+       vaddudm         $ACC0,$ACC0,$T0
+       vmuleuw         $T0,$I2,$R1
+       vaddudm         $ACC3,$ACC3,$T0
+       lvx             $S3,$x50,$_ctx
+       vmuleuw         $T0,$I3,$R1
+       vaddudm         $ACC4,$ACC4,$T0
+       lvx             $R3,$x40,$_ctx
+
+        vaddudm        $H2,$H2,$I2
+        vaddudm        $H0,$H0,$I0
+        vaddudm        $H3,$H3,$I3
+        vaddudm        $H1,$H1,$I1
+        vaddudm        $H4,$H4,$I4
+
+       vmuleuw         $T0,$I3,$S2
+       vaddudm         $ACC0,$ACC0,$T0
+       vmuleuw         $T0,$I4,$S2
+       vaddudm         $ACC1,$ACC1,$T0
+       vmuleuw         $T0,$I2,$R0
+       vaddudm         $ACC2,$ACC2,$T0
+       vmuleuw         $T0,$I3,$R0
+       vaddudm         $ACC3,$ACC3,$T0
+       lvx             $S4,$x70,$_ctx
+       vmuleuw         $T0,$I4,$R0
+       vaddudm         $ACC4,$ACC4,$T0
+       lvx             $R4,$x60,$_ctx
+
+       vmuleuw         $T0,$I2,$S3
+       vaddudm         $ACC0,$ACC0,$T0
+       vmuleuw         $T0,$I3,$S3
+       vaddudm         $ACC1,$ACC1,$T0
+       vmuleuw         $T0,$I4,$S3
+       vaddudm         $ACC2,$ACC2,$T0
+       vmuleuw         $T0,$I0,$R3
+       vaddudm         $ACC3,$ACC3,$T0
+       vmuleuw         $T0,$I1,$R3
+       vaddudm         $ACC4,$ACC4,$T0
+
+        be?lvx_u       $_4,$x00,$const         # byte swap mask
+        lvx_u          $T1,$x00,$inp           # load next input block
+        lvx_u          $T2,$x10,$inp
+        lvx_u          $T3,$x20,$inp
+        lvx_u          $T4,$x30,$inp
+        be?vperm       $T1,$T1,$T1,$_4
+        be?vperm       $T2,$T2,$T2,$_4
+        be?vperm       $T3,$T3,$T3,$_4
+        be?vperm       $T4,$T4,$T4,$_4
+
+       vmuleuw         $T0,$I1,$S4
+       vaddudm         $ACC0,$ACC0,$T0
+       vmuleuw         $T0,$I2,$S4
+       vaddudm         $ACC1,$ACC1,$T0
+       vmuleuw         $T0,$I3,$S4
+       vaddudm         $ACC2,$ACC2,$T0
+       vmuleuw         $T0,$I4,$S4
+       vaddudm         $ACC3,$ACC3,$T0
+       vmuleuw         $T0,$I0,$R4
+       vaddudm         $ACC4,$ACC4,$T0
+
+        vpermdi        $I0,$T1,$T2,0b00        # smash input to base 2^26
+        vspltisb       $_4,4
+        vperm          $I2,$T1,$T2,$I2perm     # 0x...0e0f0001...1e1f1011
+        vpermdi        $I3,$T1,$T2,0b11
+
+       # (hash + inp[0:1]) * r^4
+       vmulouw         $T0,$H0,$R0
+       vaddudm         $ACC0,$ACC0,$T0
+       vmulouw         $T0,$H1,$R0
+       vaddudm         $ACC1,$ACC1,$T0
+       vmulouw         $T0,$H2,$R0
+       vaddudm         $ACC2,$ACC2,$T0
+       vmulouw         $T0,$H3,$R0
+       vaddudm         $ACC3,$ACC3,$T0
+       vmulouw         $T0,$H4,$R0
+       vaddudm         $ACC4,$ACC4,$T0
+
+        vpermdi        $T1,$T3,$T4,0b00
+        vperm          $T2,$T3,$T4,$I2perm     # 0x...0e0f0001...1e1f1011
+        vpermdi        $T3,$T3,$T4,0b11
+
+       vmulouw         $T0,$H2,$S3
+       vaddudm         $ACC0,$ACC0,$T0
+       vmulouw         $T0,$H3,$S3
+       vaddudm         $ACC1,$ACC1,$T0
+       vmulouw         $T0,$H4,$S3
+       vaddudm         $ACC2,$ACC2,$T0
+       vmulouw         $T0,$H0,$R3
+       vaddudm         $ACC3,$ACC3,$T0
+       lvx             $S1,$x10,$_ctx
+       vmulouw         $T0,$H1,$R3
+       vaddudm         $ACC4,$ACC4,$T0
+       lvx             $R1,$x00,$_ctx
+
+        vsrd           $I1,$I0,$_26
+        vsrd           $I2,$I2,$_4
+        vsrd           $I4,$I3,$_40
+        vsrd           $I3,$I3,$_14
+
+       vmulouw         $T0,$H1,$S4
+       vaddudm         $ACC0,$ACC0,$T0
+       vmulouw         $T0,$H2,$S4
+       vaddudm         $ACC1,$ACC1,$T0
+       vmulouw         $T0,$H3,$S4
+       vaddudm         $ACC2,$ACC2,$T0
+       vmulouw         $T0,$H4,$S4
+       vaddudm         $ACC3,$ACC3,$T0
+       lvx             $S2,$x30,$_ctx
+       vmulouw         $T0,$H0,$R4
+       vaddudm         $ACC4,$ACC4,$T0
+       lvx             $R2,$x20,$_ctx
+
+        vand           $I0,$I0,$mask26
+        vand           $I1,$I1,$mask26
+        vand           $I2,$I2,$mask26
+        vand           $I3,$I3,$mask26
+
+       vmulouw         $T0,$H4,$S1
+       vaddudm         $ACC0,$ACC0,$T0
+       vmulouw         $T0,$H0,$R1
+       vaddudm         $ACC1,$ACC1,$T0
+       vmulouw         $T0,$H1,$R1
+       vaddudm         $ACC2,$ACC2,$T0
+       vmulouw         $T0,$H2,$R1
+       vaddudm         $ACC3,$ACC3,$T0
+       vmulouw         $T0,$H3,$R1
+       vaddudm         $ACC4,$ACC4,$T0
+
+        vsrd           $T2,$T2,$_4
+        vsrd           $_4,$T1,$_26
+        vsrd           $T4,$T3,$_40
+        vsrd           $T3,$T3,$_14
+
+       vmulouw         $T0,$H3,$S2
+       vaddudm         $ACC0,$ACC0,$T0
+       vmulouw         $T0,$H4,$S2
+       vaddudm         $ACC1,$ACC1,$T0
+       vmulouw         $T0,$H0,$R2
+       vaddudm         $ACC2,$ACC2,$T0
+       vmulouw         $T0,$H1,$R2
+       vaddudm         $ACC3,$ACC3,$T0
+       vmulouw         $T0,$H2,$R2
+       vaddudm         $ACC4,$ACC4,$T0
+
+        vand           $T1,$T1,$mask26
+        vand           $_4,$_4,$mask26
+        vand           $T2,$T2,$mask26
+        vand           $T3,$T3,$mask26
+
+       ################################################################
+       # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
+       # and P. Schwabe
+
+       vspltisb        $T0,2
+       vsrd            $H4,$ACC3,$_26
+       vsrd            $H1,$ACC0,$_26
+       vand            $H3,$ACC3,$mask26
+       vand            $H0,$ACC0,$mask26
+       vaddudm         $H4,$H4,$ACC4           # h3 -> h4
+       vaddudm         $H1,$H1,$ACC1           # h0 -> h1
+
+        vmrgow         $I4,$T4,$I4
+        vmrgow         $I0,$T1,$I0
+        vmrgow         $I1,$_4,$I1
+        vmrgow         $I2,$T2,$I2
+        vmrgow         $I3,$T3,$I3
+        vor            $I4,$I4,$padbits
+
+       vsrd            $ACC4,$H4,$_26
+       vsrd            $ACC1,$H1,$_26
+       vand            $H4,$H4,$mask26
+       vand            $H1,$H1,$mask26
+       vaddudm         $H0,$H0,$ACC4
+       vaddudm         $H2,$ACC2,$ACC1         # h1 -> h2
+
+       vsld            $ACC4,$ACC4,$T0         # <<2
+       vsrd            $ACC2,$H2,$_26
+       vand            $H2,$H2,$mask26
+       vaddudm         $H0,$H0,$ACC4           # h4 -> h0
+       vaddudm         $H3,$H3,$ACC2           # h2 -> h3
+
+       vsrd            $ACC0,$H0,$_26
+       vsrd            $ACC3,$H3,$_26
+       vand            $H0,$H0,$mask26
+       vand            $H3,$H3,$mask26
+       vaddudm         $H1,$H1,$ACC0           # h0 -> h1
+       vaddudm         $H4,$H4,$ACC3           # h3 -> h4
+
+       addi            $inp,$inp,0x40
+       bdnz            Loop_vsx
+
+       neg             $len,$len
+       andi.           $len,$len,0x30
+       sub             $inp,$inp,$len
+
+       lvx_u           $R0,$x30,$ctx           # load all powers
+       lvx_u           $R1,$x00,$ctx_
+       lvx_u           $S1,$x10,$ctx_
+       lvx_u           $R2,$x20,$ctx_
+       lvx_u           $S2,$x30,$ctx_
+
+Last_vsx:
+       vmuleuw         $ACC0,$I0,$R0
+       vmuleuw         $ACC1,$I1,$R0
+       vmuleuw         $ACC2,$I2,$R0
+       vmuleuw         $ACC3,$I3,$R0
+       vmuleuw         $ACC4,$I4,$R0
+
+       vmuleuw         $T0,$I4,$S1
+       vaddudm         $ACC0,$ACC0,$T0
+       vmuleuw         $T0,$I0,$R1
+       vaddudm         $ACC1,$ACC1,$T0
+       vmuleuw         $T0,$I1,$R1
+       vaddudm         $ACC2,$ACC2,$T0
+       vmuleuw         $T0,$I2,$R1
+       vaddudm         $ACC3,$ACC3,$T0
+       lvx_u           $S3,$x50,$ctx_
+       vmuleuw         $T0,$I3,$R1
+       vaddudm         $ACC4,$ACC4,$T0
+       lvx_u           $R3,$x40,$ctx_
+
+        vaddudm        $H2,$H2,$I2
+        vaddudm        $H0,$H0,$I0
+        vaddudm        $H3,$H3,$I3
+        vaddudm        $H1,$H1,$I1
+        vaddudm        $H4,$H4,$I4
+
+       vmuleuw         $T0,$I3,$S2
+       vaddudm         $ACC0,$ACC0,$T0
+       vmuleuw         $T0,$I4,$S2
+       vaddudm         $ACC1,$ACC1,$T0
+       vmuleuw         $T0,$I0,$R2
+       vaddudm         $ACC2,$ACC2,$T0
+       vmuleuw         $T0,$I1,$R2
+       vaddudm         $ACC3,$ACC3,$T0
+       lvx_u           $S4,$x70,$ctx_
+       vmuleuw         $T0,$I2,$R2
+       vaddudm         $ACC4,$ACC4,$T0
+       lvx_u           $R4,$x60,$ctx_
+
+       vmuleuw         $T0,$I2,$S3
+       vaddudm         $ACC0,$ACC0,$T0
+       vmuleuw         $T0,$I3,$S3
+       vaddudm         $ACC1,$ACC1,$T0
+       vmuleuw         $T0,$I4,$S3
+       vaddudm         $ACC2,$ACC2,$T0
+       vmuleuw         $T0,$I0,$R3
+       vaddudm         $ACC3,$ACC3,$T0
+       vmuleuw         $T0,$I1,$R3
+       vaddudm         $ACC4,$ACC4,$T0
+
+       vmuleuw         $T0,$I1,$S4
+       vaddudm         $ACC0,$ACC0,$T0
+       vmuleuw         $T0,$I2,$S4
+       vaddudm         $ACC1,$ACC1,$T0
+       vmuleuw         $T0,$I3,$S4
+       vaddudm         $ACC2,$ACC2,$T0
+       vmuleuw         $T0,$I4,$S4
+       vaddudm         $ACC3,$ACC3,$T0
+       vmuleuw         $T0,$I0,$R4
+       vaddudm         $ACC4,$ACC4,$T0
+
+       # (hash + inp[0:1]) * r^4
+       vmulouw         $T0,$H0,$R0
+       vaddudm         $ACC0,$ACC0,$T0
+       vmulouw         $T0,$H1,$R0
+       vaddudm         $ACC1,$ACC1,$T0
+       vmulouw         $T0,$H2,$R0
+       vaddudm         $ACC2,$ACC2,$T0
+       vmulouw         $T0,$H3,$R0
+       vaddudm         $ACC3,$ACC3,$T0
+       vmulouw         $T0,$H4,$R0
+       vaddudm         $ACC4,$ACC4,$T0
+
+       vmulouw         $T0,$H2,$S3
+       vaddudm         $ACC0,$ACC0,$T0
+       vmulouw         $T0,$H3,$S3
+       vaddudm         $ACC1,$ACC1,$T0
+       vmulouw         $T0,$H4,$S3
+       vaddudm         $ACC2,$ACC2,$T0
+       vmulouw         $T0,$H0,$R3
+       vaddudm         $ACC3,$ACC3,$T0
+       lvx_u           $S1,$x10,$ctx_
+       vmulouw         $T0,$H1,$R3
+       vaddudm         $ACC4,$ACC4,$T0
+       lvx_u           $R1,$x00,$ctx_
+
+       vmulouw         $T0,$H1,$S4
+       vaddudm         $ACC0,$ACC0,$T0
+       vmulouw         $T0,$H2,$S4
+       vaddudm         $ACC1,$ACC1,$T0
+       vmulouw         $T0,$H3,$S4
+       vaddudm         $ACC2,$ACC2,$T0
+       vmulouw         $T0,$H4,$S4
+       vaddudm         $ACC3,$ACC3,$T0
+       lvx_u           $S2,$x30,$ctx_
+       vmulouw         $T0,$H0,$R4
+       vaddudm         $ACC4,$ACC4,$T0
+       lvx_u           $R2,$x20,$ctx_
+
+       vmulouw         $T0,$H4,$S1
+       vaddudm         $ACC0,$ACC0,$T0
+       vmulouw         $T0,$H0,$R1
+       vaddudm         $ACC1,$ACC1,$T0
+       vmulouw         $T0,$H1,$R1
+       vaddudm         $ACC2,$ACC2,$T0
+       vmulouw         $T0,$H2,$R1
+       vaddudm         $ACC3,$ACC3,$T0
+       vmulouw         $T0,$H3,$R1
+       vaddudm         $ACC4,$ACC4,$T0
+
+       vmulouw         $T0,$H3,$S2
+       vaddudm         $ACC0,$ACC0,$T0
+       vmulouw         $T0,$H4,$S2
+       vaddudm         $ACC1,$ACC1,$T0
+       vmulouw         $T0,$H0,$R2
+       vaddudm         $ACC2,$ACC2,$T0
+       vmulouw         $T0,$H1,$R2
+       vaddudm         $ACC3,$ACC3,$T0
+       vmulouw         $T0,$H2,$R2
+       vaddudm         $ACC4,$ACC4,$T0
+
+       ################################################################
+       # horizontal addition
+
+       vpermdi         $H0,$ACC0,$ACC0,0b10
+       vpermdi         $H1,$ACC1,$ACC1,0b10
+       vpermdi         $H2,$ACC2,$ACC2,0b10
+       vpermdi         $H3,$ACC3,$ACC3,0b10
+       vpermdi         $H4,$ACC4,$ACC4,0b10
+       vaddudm         $ACC0,$ACC0,$H0
+       vaddudm         $ACC1,$ACC1,$H1
+       vaddudm         $ACC2,$ACC2,$H2
+       vaddudm         $ACC3,$ACC3,$H3
+       vaddudm         $ACC4,$ACC4,$H4
+
+       ################################################################
+       # lazy reduction
+
+       vspltisb        $T0,2
+       vsrd            $H4,$ACC3,$_26
+       vsrd            $H1,$ACC0,$_26
+       vand            $H3,$ACC3,$mask26
+       vand            $H0,$ACC0,$mask26
+       vaddudm         $H4,$H4,$ACC4           # h3 -> h4
+       vaddudm         $H1,$H1,$ACC1           # h0 -> h1
+
+       vsrd            $ACC4,$H4,$_26
+       vsrd            $ACC1,$H1,$_26
+       vand            $H4,$H4,$mask26
+       vand            $H1,$H1,$mask26
+       vaddudm         $H0,$H0,$ACC4
+       vaddudm         $H2,$ACC2,$ACC1         # h1 -> h2
+
+       vsld            $ACC4,$ACC4,$T0         # <<2
+       vsrd            $ACC2,$H2,$_26
+       vand            $H2,$H2,$mask26
+       vaddudm         $H0,$H0,$ACC4           # h4 -> h0
+       vaddudm         $H3,$H3,$ACC2           # h2 -> h3
+
+       vsrd            $ACC0,$H0,$_26
+       vsrd            $ACC3,$H3,$_26
+       vand            $H0,$H0,$mask26
+       vand            $H3,$H3,$mask26
+       vaddudm         $H1,$H1,$ACC0           # h0 -> h1
+       vaddudm         $H4,$H4,$ACC3           # h3 -> h4
+
+       beq             Ldone_vsx
+
+       add             r6,$const,$len
+
+       be?lvx_u        $_4,$x00,$const         # byte swap mask
+       lvx_u           $T1,$x00,$inp           # load last partial input block
+       lvx_u           $T2,$x10,$inp
+       lvx_u           $T3,$x20,$inp
+       lvx_u           $T4,$x30,$inp
+       be?vperm        $T1,$T1,$T1,$_4
+       be?vperm        $T2,$T2,$T2,$_4
+       be?vperm        $T3,$T3,$T3,$_4
+       be?vperm        $T4,$T4,$T4,$_4
+
+       vpermdi         $I0,$T1,$T2,0b00        # smash input to base 2^26
+       vspltisb        $_4,4
+       vperm           $I2,$T1,$T2,$I2perm     # 0x...0e0f0001...1e1f1011
+       vpermdi         $I3,$T1,$T2,0b11
+
+       vsrd            $I1,$I0,$_26
+       vsrd            $I2,$I2,$_4
+       vsrd            $I4,$I3,$_40
+       vsrd            $I3,$I3,$_14
+       vand            $I0,$I0,$mask26
+       vand            $I1,$I1,$mask26
+       vand            $I2,$I2,$mask26
+       vand            $I3,$I3,$mask26
+
+       vpermdi         $T0,$T3,$T4,0b00
+       vperm           $T1,$T3,$T4,$I2perm     # 0x...0e0f0001...1e1f1011
+       vpermdi         $T2,$T3,$T4,0b11
+
+       lvx_u           $ACC0,$x00,r6
+       lvx_u           $ACC1,$x30,r6
+
+       vsrd            $T3,$T0,$_26
+       vsrd            $T1,$T1,$_4
+       vsrd            $T4,$T2,$_40
+       vsrd            $T2,$T2,$_14
+       vand            $T0,$T0,$mask26
+       vand            $T3,$T3,$mask26
+       vand            $T1,$T1,$mask26
+       vand            $T2,$T2,$mask26
+
+       # inp[2]:inp[0]:inp[3]:inp[1]
+       vmrgow          $I4,$T4,$I4
+       vmrgow          $I0,$T0,$I0
+       vmrgow          $I1,$T3,$I1
+       vmrgow          $I2,$T1,$I2
+       vmrgow          $I3,$T2,$I3
+       vor             $I4,$I4,$padbits
+
+       vperm           $H0,$H0,$H0,$ACC0       # move hash to right lane
+       vand            $I0,$I0,    $ACC1       # mask redundant input lane[s]
+       vperm           $H1,$H1,$H1,$ACC0
+       vand            $I1,$I1,    $ACC1
+       vperm           $H2,$H2,$H2,$ACC0
+       vand            $I2,$I2,    $ACC1
+       vperm           $H3,$H3,$H3,$ACC0
+       vand            $I3,$I3,    $ACC1
+       vperm           $H4,$H4,$H4,$ACC0
+       vand            $I4,$I4,    $ACC1
+
+       vaddudm         $I0,$I0,$H0             # accumulate hash
+       vxor            $H0,$H0,$H0             # wipe hash value
+       vaddudm         $I1,$I1,$H1
+       vxor            $H1,$H1,$H1
+       vaddudm         $I2,$I2,$H2
+       vxor            $H2,$H2,$H2
+       vaddudm         $I3,$I3,$H3
+       vxor            $H3,$H3,$H3
+       vaddudm         $I4,$I4,$H4
+       vxor            $H4,$H4,$H4
+
+       xor.            $len,$len,$len
+       b               Last_vsx
+
+.align 4
+Ldone_vsx:
+       $POP    r0,`$VSXFRAME+$LRSAVE`($sp)
+       li      $x10,4
+       li      $x20,8
+       li      $x30,12
+       li      $x40,16
+       stvwx_u $H0,$x00,$ctx                   # store hash
+       stvwx_u $H1,$x10,$ctx
+       stvwx_u $H2,$x20,$ctx
+       stvwx_u $H3,$x30,$ctx
+       stvwx_u $H4,$x40,$ctx
+
+       lwz     r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# pull vrsave
+       mtlr    r0
+       li      r10,`15+$LOCALS+128`
+       li      r11,`31+$LOCALS+128`
+       mtspr   256,r12                         # restore vrsave
+       lvx     v20,r10,$sp
+       addi    r10,r10,32
+       lvx     v21,r10,$sp
+       addi    r10,r10,32
+       lvx     v22,r11,$sp
+       addi    r11,r11,32
+       lvx     v23,r10,$sp
+       addi    r10,r10,32
+       lvx     v24,r11,$sp
+       addi    r11,r11,32
+       lvx     v25,r10,$sp
+       addi    r10,r10,32
+       lvx     v26,r11,$sp
+       addi    r11,r11,32
+       lvx     v27,r10,$sp
+       addi    r10,r10,32
+       lvx     v28,r11,$sp
+       addi    r11,r11,32
+       lvx     v29,r10,$sp
+       addi    r10,r10,32
+       lvx     v30,r11,$sp
+       lvx     v31,r10,$sp
+       $POP    r27,`$VSXFRAME-$SIZE_T*5`($sp)
+       $POP    r28,`$VSXFRAME-$SIZE_T*4`($sp)
+       $POP    r29,`$VSXFRAME-$SIZE_T*3`($sp)
+       $POP    r30,`$VSXFRAME-$SIZE_T*2`($sp)
+       $POP    r31,`$VSXFRAME-$SIZE_T*1`($sp)
+       addi    $sp,$sp,$VSXFRAME
        blr
        .long   0
-       .byte   0,12,4,1,0x80,4,3,0
-.size  .poly1305_emit,.-.poly1305_emit
+       .byte   0,12,0x04,1,0x80,5,4,0
+       .long   0
+.size  __poly1305_blocks_vsx,.-__poly1305_blocks_vsx
+
+.align 6
+LPICmeup:
+       mflr    r0
+       bcl     20,31,\$+4
+       mflr    $const      # vvvvvv "distance" between . and 1st data entry
+       addi    $const,$const,`64-8`
+       mtlr    r0
+       blr
+       .long   0
+       .byte   0,12,0x14,0,0,0,0,0
+       .space  `64-9*4`
+
+.quad  0x0000000003ffffff,0x0000000003ffffff   # mask26
+.quad  0x000000000000001a,0x000000000000001a   # _26
+.quad  0x0000000000000028,0x0000000000000028   # _40
+.quad  0x000000000e0f0001,0x000000001e1f1011   # I2perm
+.quad  0x0100000001000000,0x0100000001000000   # padbits
+.quad  0x0706050403020100,0x0f0e0d0c0b0a0908   # byte swap for big-endian
+
+.quad  0x0000000000000000,0x0000000004050607   # magic tail masks
+.quad  0x0405060700000000,0x0000000000000000
+.quad  0x0000000000000000,0x0405060700000000
+
+.quad  0xffffffff00000000,0xffffffffffffffff
+.quad  0xffffffff00000000,0xffffffff00000000
+.quad  0x0000000000000000,0xffffffff00000000
 ___
-                                                       }
+}}}
 $code.=<<___;
-.asciz "Poly1305 for PPC, CRYPTOGAMS by <appro\@openssl.org>"
+.asciz "Poly1305 for PPC, CRYPTOGAMS by \@dot-asm"
 ___
 
-$code =~ s/\`([^\`]*)\`/eval $1/gem;
-print $code;
+foreach (split("\n",$code)) {
+       s/\`([^\`]*)\`/eval($1)/ge;
+
+       # instructions prefixed with '?' are endian-specific and need
+       # to be adjusted accordingly...
+       if ($flavour !~ /le$/) {        # big-endian
+           s/be\?//            or
+           s/le\?/#le#/
+       } else {                        # little-endian
+           s/le\?//            or
+           s/be\?/#be#/
+       }
+
+       print $_,"\n";
+}
 close STDOUT;
index 70829e4..9601674 100644 (file)
@@ -116,10 +116,19 @@ void poly1305_blocks_fpu(void *ctx, const unsigned char *inp, size_t len,
                          unsigned int padbit);
 void poly1305_emit_fpu(void *ctx, unsigned char mac[16],
                        const unsigned int nonce[4]);
+void poly1305_init_vsx(void *ctx, const unsigned char key[16]);
+void poly1305_blocks_vsx(void *ctx, const unsigned char *inp, size_t len,
+                         unsigned int padbit);
+void poly1305_emit_vsx(void *ctx, unsigned char mac[16],
+                       const unsigned int nonce[4]);
 int poly1305_init(void *ctx, const unsigned char key[16], void *func[2]);
 int poly1305_init(void *ctx, const unsigned char key[16], void *func[2])
 {
-    if (sizeof(size_t) == 4 && (OPENSSL_ppccap_P & PPC_FPU)) {
+    if (OPENSSL_ppccap_P & PPC_CRYPTO207) {
+        poly1305_init_int(ctx, key);
+        func[0] = (void*)(uintptr_t)poly1305_blocks_vsx;
+        func[1] = (void*)(uintptr_t)poly1305_emit;
+    } else if (sizeof(size_t) == 4 && (OPENSSL_ppccap_P & PPC_FPU)) {
         poly1305_init_fpu(ctx, key);
         func[0] = (void*)(uintptr_t)poly1305_blocks_fpu;
         func[1] = (void*)(uintptr_t)poly1305_emit_fpu;