Add ecp_nistz256-ppc64 module.
authorAndy Polyakov <appro@openssl.org>
Sun, 14 Aug 2016 17:19:11 +0000 (19:19 +0200)
committerAndy Polyakov <appro@openssl.org>
Mon, 29 Aug 2016 21:12:32 +0000 (23:12 +0200)
Reviewed-by: Rich Salz <rsalz@openssl.org>
Configurations/00-base-templates.conf
crypto/ec/asm/ecp_nistz256-ppc64.pl [new file with mode: 0755]
crypto/ec/build.info
crypto/ppccap.c

index 566b89b..1c63a50 100644 (file)
        inherit_from    => [ "parisc11_asm" ],
        perlasm_scheme  => "64",
     },
-    ppc64_asm => {
+    ppc32_asm => {
        template        => 1,
        cpuid_asm_src   => "ppccpuid.s ppccap.c",
        bn_asm_src      => "bn-ppc.s ppc-mont.s ppc64-mont.s",
        chacha_asm_src  => "chacha-ppc.s",
        poly1305_asm_src=> "poly1305-ppc.s poly1305-ppcfp.s",
     },
-    ppc32_asm => {
-       inherit_from    => [ "ppc64_asm" ],
-       template        => 1
+    ppc64_asm => {
+       inherit_from    => [ "ppc32_asm" ],
+       template        => 1,
+       ec_asm_src      => "ecp_nistz256.c ecp_nistz256-ppc64.s",
+
     },
 );
diff --git a/crypto/ec/asm/ecp_nistz256-ppc64.pl b/crypto/ec/asm/ecp_nistz256-ppc64.pl
new file mode 100755 (executable)
index 0000000..fadb9f7
--- /dev/null
@@ -0,0 +1,2375 @@
+#! /usr/bin/env perl
+#
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see http://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+#
+# ECP_NISTZ256 module for PPC64.
+#
+# August 2016.
+#
+# Original ECP_NISTZ256 submission targeting x86_64 is detailed in
+# http://eprint.iacr.org/2013/816.
+#
+#                      with/without -DECP_NISTZ256_ASM
+# POWER7               +260-530%
+# POWER8               +220-340%
+
+$flavour = shift;
+while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
+
+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
+die "can't locate ppc-xlate.pl";
+
+open OUT,"| \"$^X\" $xlate $flavour $output";
+*STDOUT=*OUT;
+
+my $sp="r1";
+
+{
+my ($rp,$ap,$bp,$bi,$acc0,$acc1,$acc2,$acc3,$poly1,$poly3,
+    $acc4,$acc5,$a0,$a1,$a2,$a3,$t0,$t1,$t2,$t3) =
+    map("r$_",(3..12,22..31));
+
+my ($acc6,$acc7)=($bp,$bi);    # used in __ecp_nistz256_sqr_mont
+
+$code.=<<___;
+.machine       "any"
+.text
+___
+########################################################################
+# Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7
+#
+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+open TABLE,"<ecp_nistz256_table.c"             or
+open TABLE,"<${dir}../ecp_nistz256_table.c"    or
+die "failed to open ecp_nistz256_table.c:",$!;
+
+use integer;
+
+foreach(<TABLE>) {
+       s/TOBN\(\s*(0x[0-9a-f]+),\s*(0x[0-9a-f]+)\s*\)/push @arr,hex($2),hex($1)/geo;
+}
+close TABLE;
+
+# See ecp_nistz256_table.c for explanation for why it's 64*16*37.
+# 64*16*37-1 is because $#arr returns last valid index or @arr, not
+# amount of elements.
+die "insane number of elements" if ($#arr != 64*16*37-1);
+
+$code.=<<___;
+.type  ecp_nistz256_precomputed,\@object
+.globl ecp_nistz256_precomputed
+.align 12
+ecp_nistz256_precomputed:
+___
+########################################################################
+# this conversion smashes P256_POINT_AFFINE by individual bytes with
+# 64 byte interval, similar to
+#      1111222233334444
+#      1234123412341234
+for(1..37) {
+       @tbl = splice(@arr,0,64*16);
+       for($i=0;$i<64;$i++) {
+               undef @line;
+               for($j=0;$j<64;$j++) {
+                       push @line,(@tbl[$j*16+$i/4]>>(($i%4)*8))&0xff;
+               }
+               $code.=".byte\t";
+               $code.=join(',',map { sprintf "0x%02x",$_} @line);
+               $code.="\n";
+       }
+}
+
+$code.=<<___;
+.size  ecp_nistz256_precomputed,.-ecp_nistz256_precomputed
+.asciz "ECP_NISTZ256 for PPC64, CRYPTOGAMS by <appro\@openssl.org>"
+
+# void ecp_nistz256_mul_mont(BN_ULONG x0[4],const BN_ULONG x1[4],
+#                                           const BN_ULONG x2[4]);
+.globl ecp_nistz256_mul_mont
+.align 5
+ecp_nistz256_mul_mont:
+       stdu    $sp,-128($sp)
+       mflr    r0
+       std     r22,48($sp)
+       std     r23,56($sp)
+       std     r24,64($sp)
+       std     r25,72($sp)
+       std     r26,80($sp)
+       std     r27,88($sp)
+       std     r28,96($sp)
+       std     r29,104($sp)
+       std     r30,112($sp)
+       std     r31,120($sp)
+
+       ld      $a0,0($ap)
+       ld      $bi,0($bp)
+       ld      $a1,8($ap)
+       ld      $a2,16($ap)
+       ld      $a3,24($ap)
+
+       li      $poly1,-1
+       srdi    $poly1,$poly1,32        # 0x00000000ffffffff
+       li      $poly3,1
+       orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
+
+       bl      __ecp_nistz256_mul_mont
+
+       mtlr    r0
+       ld      r22,48($sp)
+       ld      r23,56($sp)
+       ld      r24,64($sp)
+       ld      r25,72($sp)
+       ld      r26,80($sp)
+       ld      r27,88($sp)
+       ld      r28,96($sp)
+       ld      r29,104($sp)
+       ld      r30,112($sp)
+       ld      r31,120($sp)
+       addi    $sp,$sp,128
+       blr
+       .long   0
+       .byte   0,12,4,0,0x80,10,3,0
+       .long   0
+.size  ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont
+
+# void ecp_nistz256_sqr_mont(BN_ULONG x0[4],const BN_ULONG x1[4]);
+.globl ecp_nistz256_sqr_mont
+.align 4
+ecp_nistz256_sqr_mont:
+       stdu    $sp,-128($sp)
+       mflr    r0
+       std     r22,48($sp)
+       std     r23,56($sp)
+       std     r24,64($sp)
+       std     r25,72($sp)
+       std     r26,80($sp)
+       std     r27,88($sp)
+       std     r28,96($sp)
+       std     r29,104($sp)
+       std     r30,112($sp)
+       std     r31,120($sp)
+
+       ld      $a0,0($ap)
+       ld      $a1,8($ap)
+       ld      $a2,16($ap)
+       ld      $a3,24($ap)
+
+       li      $poly1,-1
+       srdi    $poly1,$poly1,32        # 0x00000000ffffffff
+       li      $poly3,1
+       orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
+
+       bl      __ecp_nistz256_sqr_mont
+
+       mtlr    r0
+       ld      r22,48($sp)
+       ld      r23,56($sp)
+       ld      r24,64($sp)
+       ld      r25,72($sp)
+       ld      r26,80($sp)
+       ld      r27,88($sp)
+       ld      r28,96($sp)
+       ld      r29,104($sp)
+       ld      r30,112($sp)
+       ld      r31,120($sp)
+       addi    $sp,$sp,128
+       blr
+       .long   0
+       .byte   0,12,4,0,0x80,10,2,0
+       .long   0
+.size  ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont
+
+# void ecp_nistz256_add(BN_ULONG x0[4],const BN_ULONG x1[4],
+#                                      const BN_ULONG x2[4]);
+.globl ecp_nistz256_add
+.align 4
+ecp_nistz256_add:
+       stdu    $sp,-128($sp)
+       mflr    r0
+       std     r28,96($sp)
+       std     r29,104($sp)
+       std     r30,112($sp)
+       std     r31,120($sp)
+
+       ld      $acc0,0($ap)
+       ld      $t0,  0($bp)
+       ld      $acc1,8($ap)
+       ld      $t1,  8($bp)
+       ld      $acc2,16($ap)
+       ld      $t2,  16($bp)
+       ld      $acc3,24($ap)
+       ld      $t3,  24($bp)
+
+       li      $poly1,-1
+       srdi    $poly1,$poly1,32        # 0x00000000ffffffff
+       li      $poly3,1
+       orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
+
+       bl      __ecp_nistz256_add
+
+       mtlr    r0
+       ld      r28,96($sp)
+       ld      r29,104($sp)
+       ld      r30,112($sp)
+       ld      r31,120($sp)
+       addi    $sp,$sp,128
+       blr
+       .long   0
+       .byte   0,12,4,0,0x80,4,3,0
+       .long   0
+.size  ecp_nistz256_add,.-ecp_nistz256_add
+
+# void ecp_nistz256_div_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]);
+.globl ecp_nistz256_div_by_2
+.align 4
+ecp_nistz256_div_by_2:
+       stdu    $sp,-128($sp)
+       mflr    r0
+       std     r28,96($sp)
+       std     r29,104($sp)
+       std     r30,112($sp)
+       std     r31,120($sp)
+
+       ld      $acc0,0($ap)
+       ld      $acc1,8($ap)
+       ld      $acc2,16($ap)
+       ld      $acc3,24($ap)
+
+       li      $poly1,-1
+       srdi    $poly1,$poly1,32        # 0x00000000ffffffff
+       li      $poly3,1
+       orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
+
+       bl      __ecp_nistz256_div_by_2
+
+       mtlr    r0
+       ld      r28,96($sp)
+       ld      r29,104($sp)
+       ld      r30,112($sp)
+       ld      r31,120($sp)
+       addi    $sp,$sp,128
+       blr
+       .long   0
+       .byte   0,12,4,0,0x80,4,2,0
+       .long   0
+.size  ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2
+
+# void ecp_nistz256_mul_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]);
+.globl ecp_nistz256_mul_by_2
+.align 4
+ecp_nistz256_mul_by_2:
+       stdu    $sp,-128($sp)
+       mflr    r0
+       std     r28,96($sp)
+       std     r29,104($sp)
+       std     r30,112($sp)
+       std     r31,120($sp)
+
+       ld      $acc0,0($ap)
+       ld      $acc1,8($ap)
+       ld      $acc2,16($ap)
+       ld      $acc3,24($ap)
+
+       mr      $t0,$acc0
+       mr      $t1,$acc1
+       mr      $t2,$acc2
+       mr      $t3,$acc3
+
+       li      $poly1,-1
+       srdi    $poly1,$poly1,32        # 0x00000000ffffffff
+       li      $poly3,1
+       orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
+
+       bl      __ecp_nistz256_add      # ret = a+a     // 2*a
+
+       mtlr    r0
+       ld      r28,96($sp)
+       ld      r29,104($sp)
+       ld      r30,112($sp)
+       ld      r31,120($sp)
+       addi    $sp,$sp,128
+       blr
+       .long   0
+       .byte   0,12,4,0,0x80,4,3,0
+       .long   0
+.size  ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2
+
+# void ecp_nistz256_mul_by_3(BN_ULONG x0[4],const BN_ULONG x1[4]);
+.globl ecp_nistz256_mul_by_3
+.align 4
+ecp_nistz256_mul_by_3:
+       stdu    $sp,-128($sp)
+       mflr    r0
+       std     r28,96($sp)
+       std     r29,104($sp)
+       std     r30,112($sp)
+       std     r31,120($sp)
+
+       ld      $acc0,0($ap)
+       ld      $acc1,8($ap)
+       ld      $acc2,16($ap)
+       ld      $acc3,24($ap)
+
+       mr      $t0,$acc0
+       std     $acc0,64($sp)
+       mr      $t1,$acc1
+       std     $acc1,72($sp)
+       mr      $t2,$acc2
+       std     $acc2,80($sp)
+       mr      $t3,$acc3
+       std     $acc3,88($sp)
+
+       li      $poly1,-1
+       srdi    $poly1,$poly1,32        # 0x00000000ffffffff
+       li      $poly3,1
+       orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
+
+       bl      __ecp_nistz256_add      # ret = a+a     // 2*a
+
+       ld      $t0,64($sp)
+       ld      $t1,72($sp)
+       ld      $t2,80($sp)
+       ld      $t3,88($sp)
+
+       bl      __ecp_nistz256_add      # ret += a      // 2*a+a=3*a
+
+       mtlr    r0
+       ld      r28,96($sp)
+       ld      r29,104($sp)
+       ld      r30,112($sp)
+       ld      r31,120($sp)
+       addi    $sp,$sp,128
+       blr
+       .long   0
+       .byte   0,12,4,0,0x80,4,2,0
+       .long   0
+.size  ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3
+
+# void ecp_nistz256_sub(BN_ULONG x0[4],const BN_ULONG x1[4],
+#                                      const BN_ULONG x2[4]);
+.globl ecp_nistz256_sub
+.align 4
+ecp_nistz256_sub:
+       stdu    $sp,-128($sp)
+       mflr    r0
+       std     r28,96($sp)
+       std     r29,104($sp)
+       std     r30,112($sp)
+       std     r31,120($sp)
+
+       ld      $acc0,0($ap)
+       ld      $acc1,8($ap)
+       ld      $acc2,16($ap)
+       ld      $acc3,24($ap)
+
+       li      $poly1,-1
+       srdi    $poly1,$poly1,32        # 0x00000000ffffffff
+       li      $poly3,1
+       orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
+
+       bl      __ecp_nistz256_sub_from
+
+       mtlr    r0
+       ld      r28,96($sp)
+       ld      r29,104($sp)
+       ld      r30,112($sp)
+       ld      r31,120($sp)
+       addi    $sp,$sp,128
+       blr
+       .long   0
+       .byte   0,12,4,0,0x80,4,3,0
+       .long   0
+.size  ecp_nistz256_sub,.-ecp_nistz256_sub
+
+# void ecp_nistz256_neg(BN_ULONG x0[4],const BN_ULONG x1[4]);
+.globl ecp_nistz256_neg
+.align 4
+ecp_nistz256_neg:
+       stdu    $sp,-128($sp)
+       mflr    r0
+       std     r28,96($sp)
+       std     r29,104($sp)
+       std     r30,112($sp)
+       std     r31,120($sp)
+
+       mr      $bp,$ap
+       li      $acc0,0
+       li      $acc1,0
+       li      $acc2,0
+       li      $acc3,0
+
+       li      $poly1,-1
+       srdi    $poly1,$poly1,32        # 0x00000000ffffffff
+       li      $poly3,1
+       orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
+
+       bl      __ecp_nistz256_sub_from
+
+       mtlr    r0
+       ld      r28,96($sp)
+       ld      r29,104($sp)
+       ld      r30,112($sp)
+       ld      r31,120($sp)
+       addi    $sp,$sp,128
+       blr
+       .long   0
+       .byte   0,12,4,0,0x80,4,2,0
+       .long   0
+.size  ecp_nistz256_neg,.-ecp_nistz256_neg
+
+# note that __ecp_nistz256_mul_mont expects a[0-3] input pre-loaded
+# to $a0-$a3 and b[0] - to $bi
+.type  __ecp_nistz256_mul_mont,\@function
+.align 4
+__ecp_nistz256_mul_mont:
+       mulld   $acc0,$a0,$bi           # a[0]*b[0]
+       mulhdu  $t0,$a0,$bi
+
+       mulld   $acc1,$a1,$bi           # a[1]*b[0]
+       mulhdu  $t1,$a1,$bi
+
+       mulld   $acc2,$a2,$bi           # a[2]*b[0]
+       mulhdu  $t2,$a2,$bi
+
+       mulld   $acc3,$a3,$bi           # a[3]*b[0]
+       mulhdu  $t3,$a3,$bi
+       ld      $bi,8($bp)              # b[1]
+
+       addc    $acc1,$acc1,$t0         # accumulate high parts of multiplication
+        sldi   $t0,$acc0,32
+       adde    $acc2,$acc2,$t1
+        srdi   $t1,$acc0,32
+       adde    $acc3,$acc3,$t2
+       addze   $acc4,$t3
+       li      $acc5,0
+___
+for($i=1;$i<4;$i++) {
+       ################################################################
+       # Reduction iteration is normally performed by accumulating
+       # result of multiplication of modulus by "magic" digit [and
+       # omitting least significant word, which is guaranteed to
+       # be 0], but thanks to special form of modulus and "magic"
+       # digit being equal to least significant word, it can be
+       # performed with additions and subtractions alone. Indeed:
+       #
+       #            ffff0001.00000000.0000ffff.ffffffff
+       # *                                     abcdefgh
+       # + xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.abcdefgh
+       #
+       # Now observing that ff..ff*x = (2^n-1)*x = 2^n*x-x, we
+       # rewrite above as:
+       #
+       #   xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.abcdefgh
+       # + abcdefgh.abcdefgh.0000abcd.efgh0000.00000000
+       # - 0000abcd.efgh0000.00000000.00000000.abcdefgh
+       #
+       # or marking redundant operations:
+       #
+       #   xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.--------
+       # + abcdefgh.abcdefgh.0000abcd.efgh0000.--------
+       # - 0000abcd.efgh0000.--------.--------.--------
+
+$code.=<<___;
+       subfc   $t2,$t0,$acc0           # "*0xffff0001"
+       subfe   $t3,$t1,$acc0
+       addc    $acc0,$acc1,$t0         # +=acc[0]<<96 and omit acc[0]
+        mulld  $t0,$a0,$bi             # lo(a[0]*b[i])
+       adde    $acc1,$acc2,$t1
+        mulld  $t1,$a1,$bi             # lo(a[1]*b[i])
+       adde    $acc2,$acc3,$t2         # +=acc[0]*0xffff0001
+        mulld  $t2,$a2,$bi             # lo(a[2]*b[i])
+       adde    $acc3,$acc4,$t3
+        mulld  $t3,$a3,$bi             # lo(a[3]*b[i])
+       addze   $acc4,$acc5
+
+       addc    $acc0,$acc0,$t0         # accumulate low parts of multiplication
+        mulhdu $t0,$a0,$bi             # hi(a[0]*b[i])
+       adde    $acc1,$acc1,$t1
+        mulhdu $t1,$a1,$bi             # hi(a[1]*b[i])
+       adde    $acc2,$acc2,$t2
+        mulhdu $t2,$a2,$bi             # hi(a[2]*b[i])
+       adde    $acc3,$acc3,$t3
+        mulhdu $t3,$a3,$bi             # hi(a[3]*b[i])
+       addze   $acc4,$acc4
+___
+$code.=<<___   if ($i<3);
+       ld      $bi,8*($i+1)($bp)       # b[$i+1]
+___
+$code.=<<___;
+       addc    $acc1,$acc1,$t0         # accumulate high parts of multiplication
+        sldi   $t0,$acc0,32
+       adde    $acc2,$acc2,$t1
+        srdi   $t1,$acc0,32
+       adde    $acc3,$acc3,$t2
+        li     $acc5,0
+       adde    $acc4,$acc4,$t3
+       addze   $acc5,$acc5
+___
+}
+$code.=<<___;
+       # last reduction
+       subfc   $t2,$t0,$acc0           # "*0xffff0001"
+       subfe   $t3,$t1,$acc0
+       addc    $acc0,$acc1,$t0         # +=acc[0]<<96 and omit acc[0]
+       adde    $acc1,$acc2,$t1
+       adde    $acc2,$acc3,$t2         # +=acc[0]*0xffff0001
+       adde    $acc3,$acc4,$t3
+       addze   $acc4,$acc5
+
+       li      $t2,0
+       addic   $acc0,$acc0,1           # ret -= modulus
+       subfe   $acc1,$poly1,$acc1
+       subfe   $acc2,$t2,$acc2
+       subfe   $acc3,$poly3,$acc3
+       subfe   $acc4,$t2,$acc4
+
+       addc    $acc0,$acc0,$acc4       # ret += modulus if borrow
+       and     $t1,$poly1,$acc4
+       and     $t3,$poly3,$acc4
+       adde    $acc1,$acc1,$t1
+       addze   $acc2,$acc2
+       adde    $acc3,$acc3,$t3
+
+       std     $acc0,0($rp)
+       std     $acc1,8($rp)
+       std     $acc2,16($rp)
+       std     $acc3,24($rp)
+
+       blr
+       .long   0
+       .byte   0,12,0x14,0,0,0,1,0
+       .long   0
+.size  __ecp_nistz256_mul_mont,.-__ecp_nistz256_mul_mont
+
+# note that __ecp_nistz256_sqr_mont expects a[0-3] input pre-loaded
+# to $a0-$a3
+.type  __ecp_nistz256_sqr_mont,\@function
+.align 4
+__ecp_nistz256_sqr_mont:
+       ################################################################
+       #  |  |  |  |  |  |a1*a0|  |
+       #  |  |  |  |  |a2*a0|  |  |
+       #  |  |a3*a2|a3*a0|  |  |  |
+       #  |  |  |  |a2*a1|  |  |  |
+       #  |  |  |a3*a1|  |  |  |  |
+       # *|  |  |  |  |  |  |  | 2|
+       # +|a3*a3|a2*a2|a1*a1|a0*a0|
+       #  |--+--+--+--+--+--+--+--|
+       #  |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is $accx, i.e. follow $accx
+       #
+       #  "can't overflow" below mark carrying into high part of
+       #  multiplication result, which can't overflow, because it
+       #  can never be all ones.
+
+       mulld   $acc1,$a1,$a0           # a[1]*a[0]
+       mulhdu  $t1,$a1,$a0
+       mulld   $acc2,$a2,$a0           # a[2]*a[0]
+       mulhdu  $t2,$a2,$a0
+       mulld   $acc3,$a3,$a0           # a[3]*a[0]
+       mulhdu  $acc4,$a3,$a0
+
+       addc    $acc2,$acc2,$t1         # accumulate high parts of multiplication
+        mulld  $t0,$a2,$a1             # a[2]*a[1]
+        mulhdu $t1,$a2,$a1
+       adde    $acc3,$acc3,$t2
+        mulld  $t2,$a3,$a1             # a[3]*a[1]
+        mulhdu $t3,$a3,$a1
+       addze   $acc4,$acc4             # can't overflow
+
+       mulld   $acc5,$a3,$a2           # a[3]*a[2]
+       mulhdu  $acc6,$a3,$a2
+
+       addc    $t1,$t1,$t2             # accumulate high parts of multiplication
+        mulld  $acc0,$a0,$a0           # a[0]*a[0]
+       addze   $t2,$t3                 # can't overflow
+
+       addc    $acc3,$acc3,$t0         # accumulate low parts of multiplication
+        mulhdu $a0,$a0,$a0
+       adde    $acc4,$acc4,$t1
+        mulld  $t1,$a1,$a1             # a[1]*a[1]
+       adde    $acc5,$acc5,$t2
+        mulhdu $a1,$a1,$a1
+       addze   $acc6,$acc6             # can't overflow
+
+       addc    $acc1,$acc1,$acc1       # acc[1-6]*=2
+        mulld  $t2,$a2,$a2             # a[2]*a[2]
+       adde    $acc2,$acc2,$acc2
+        mulhdu $a2,$a2,$a2
+       adde    $acc3,$acc3,$acc3
+        mulld  $t3,$a3,$a3             # a[3]*a[3]
+       adde    $acc4,$acc4,$acc4
+        mulhdu $a3,$a3,$a3
+       adde    $acc5,$acc5,$acc5
+       adde    $acc6,$acc6,$acc6
+       li      $acc7,0
+       addze   $acc7,$acc7
+
+       addc    $acc1,$acc1,$a0         # +a[i]*a[i]
+       adde    $acc2,$acc2,$t1
+       adde    $acc3,$acc3,$a1
+       adde    $acc4,$acc4,$t2
+       adde    $acc5,$acc5,$a2
+        sldi   $t0,$acc0,32
+       adde    $acc6,$acc6,$t3
+        srdi   $t1,$acc0,32
+       adde    $acc7,$acc7,$a3
+___
+for($i=0;$i<3;$i++) {                  # reductions, see commentary in
+                                       # multiplication for details
+$code.=<<___;
+       subfc   $t2,$t0,$acc0           # "*0xffff0001"
+       subfe   $t3,$t1,$acc0
+       addc    $acc0,$acc1,$t0         # +=acc[0]<<96 and omit acc[0]
+       adde    $acc1,$acc2,$t1
+        sldi   $t0,$acc0,32
+       adde    $acc2,$acc3,$t2         # +=acc[0]*0xffff0001
+        srdi   $t1,$acc0,32
+       addze   $acc3,$t3               # can't overflow
+___
+}
+$code.=<<___;
+       subfc   $t2,$t0,$acc0           # "*0xffff0001"
+       subfe   $t3,$t1,$acc0
+       addc    $acc0,$acc1,$t0         # +=acc[0]<<96 and omit acc[0]
+       adde    $acc1,$acc2,$t1
+       adde    $acc2,$acc3,$t2         # +=acc[0]*0xffff0001
+       li      $t2,0
+       addze   $acc3,$t3               # can't overflow
+
+       addc    $acc0,$acc0,$acc4       # accumulate upper half
+       adde    $acc1,$acc1,$acc5
+       adde    $acc2,$acc2,$acc6
+       adde    $acc3,$acc3,$acc7
+       addze   $acc4,$t2
+
+       addic   $acc0,$acc0,1           # ret -= modulus
+       subfe   $acc1,$poly1,$acc1
+       subfe   $acc2,$t2,$acc2
+       subfe   $acc3,$poly3,$acc3
+       subfe   $acc4,$t2,$acc4
+
+       addc    $acc0,$acc0,$acc4       # ret += modulus if borrow
+       and     $t1,$poly1,$acc4
+       and     $t3,$poly3,$acc4
+       adde    $acc1,$acc1,$t1
+       addze   $acc2,$acc2
+       adde    $acc3,$acc3,$t3
+
+       std     $acc0,0($rp)
+       std     $acc1,8($rp)
+       std     $acc2,16($rp)
+       std     $acc3,24($rp)
+
+       blr
+       .long   0
+       .byte   0,12,0x14,0,0,0,1,0
+       .long   0
+.size  __ecp_nistz256_sqr_mont,.-__ecp_nistz256_sqr_mont
+
+# Note that __ecp_nistz256_add expects both input vectors pre-loaded to
+# $a0-$a3 and $t0-$t3. This is done because it's used in multiple
+# contexts, e.g. in multiplication by 2 and 3...
+.type  __ecp_nistz256_add,\@function
+.align 4
+__ecp_nistz256_add:
+       addc    $acc0,$acc0,$t0         # ret = a+b
+       adde    $acc1,$acc1,$t1
+       adde    $acc2,$acc2,$t2
+       li      $t2,0
+       adde    $acc3,$acc3,$t3
+       addze   $t0,$t2
+
+       # if a+b >= modulus, subtract modulus
+       #
+       # But since comparison implies subtraction, we subtract
+       # modulus and then add it back if subraction borrowed.
+
+       subic   $acc0,$acc0,-1
+       subfe   $acc1,$poly1,$acc1
+       subfe   $acc2,$t2,$acc2
+       subfe   $acc3,$poly3,$acc3
+       subfe   $t0,$t2,$t0
+
+       addc    $acc0,$acc0,$t0
+       and     $t1,$poly1,$t0
+       and     $t3,$poly3,$t0
+       adde    $acc1,$acc1,$t1
+       addze   $acc2,$acc2
+       adde    $acc3,$acc3,$t3
+
+       std     $acc0,0($rp)
+       std     $acc1,8($rp)
+       std     $acc2,16($rp)
+       std     $acc3,24($rp)
+
+       blr
+       .long   0
+       .byte   0,12,0x14,0,0,0,3,0
+       .long   0
+.size  __ecp_nistz256_add,.-__ecp_nistz256_add
+
+.type  __ecp_nistz256_sub_from,\@function
+.align 4
+__ecp_nistz256_sub_from:
+       ld      $t0,0($bp)
+       ld      $t1,8($bp)
+       ld      $t2,16($bp)
+       ld      $t3,24($bp)
+       subfc   $acc0,$t0,$acc0         # ret = a-b
+       subfe   $acc1,$t1,$acc1
+       subfe   $acc2,$t2,$acc2
+       subfe   $acc3,$t3,$acc3
+       subfe   $t0,$t0,$t0             # t0 = borrow ? -1 : 0
+
+       # if a-b borrowed, add modulus
+
+       addc    $acc0,$acc0,$t0         # ret -= modulus & t0
+       and     $t1,$poly1,$t0
+       and     $t3,$poly3,$t0
+       adde    $acc1,$acc1,$t1
+       addze   $acc2,$acc2
+       adde    $acc3,$acc3,$t3
+
+       std     $acc0,0($rp)
+       std     $acc1,8($rp)
+       std     $acc2,16($rp)
+       std     $acc3,24($rp)
+
+       blr
+       .long   0
+       .byte   0,12,0x14,0,0,0,3,0
+       .long   0
+.size  __ecp_nistz256_sub_from,.-__ecp_nistz256_sub_from
+
+.type  __ecp_nistz256_sub_morf,\@function
+.align 4
+__ecp_nistz256_sub_morf:
+       ld      $t0,0($bp)
+       ld      $t1,8($bp)
+       ld      $t2,16($bp)
+       ld      $t3,24($bp)
+       subfc   $acc0,$acc0,$t0         # ret = b-a
+       subfe   $acc1,$acc1,$t1
+       subfe   $acc2,$acc2,$t2
+       subfe   $acc3,$acc3,$t3
+       subfe   $t0,$t0,$t0             # t0 = borrow ? -1 : 0
+
+       # if b-a borrowed, add modulus
+
+       addc    $acc0,$acc0,$t0         # ret -= modulus & t0
+       and     $t1,$poly1,$t0
+       and     $t3,$poly3,$t0
+       adde    $acc1,$acc1,$t1
+       addze   $acc2,$acc2
+       adde    $acc3,$acc3,$t3
+
+       std     $acc0,0($rp)
+       std     $acc1,8($rp)
+       std     $acc2,16($rp)
+       std     $acc3,24($rp)
+
+       blr
+       .long   0
+       .byte   0,12,0x14,0,0,0,3,0
+       .long   0
+.size  __ecp_nistz256_sub_morf,.-__ecp_nistz256_sub_morf
+
+.type  __ecp_nistz256_div_by_2,\@function
+.align 4
+__ecp_nistz256_div_by_2:
+       andi.   $t0,$acc0,1
+       addic   $acc0,$acc0,-1          # a += modulus
+        neg    $t0,$t0
+       adde    $acc1,$acc1,$poly1
+        not    $t0,$t0
+       addze   $acc2,$acc2
+        li     $t2,0
+       adde    $acc3,$acc3,$poly3
+        and    $t1,$poly1,$t0
+       addze   $ap,$t2                 # ap = carry
+        and    $t3,$poly3,$t0
+
+       subfc   $acc0,$t0,$acc0         # a -= modulus if a was even
+       subfe   $acc1,$t1,$acc1
+       subfe   $acc2,$t2,$acc2
+       subfe   $acc3,$t3,$acc3
+       subfe   $ap,  $t2,$ap
+
+       srdi    $acc0,$acc0,1
+       sldi    $t0,$acc1,63
+       srdi    $acc1,$acc1,1
+       sldi    $t1,$acc2,63
+       srdi    $acc2,$acc2,1
+       sldi    $t2,$acc3,63
+       srdi    $acc3,$acc3,1
+       sldi    $t3,$ap,63
+       or      $acc0,$acc0,$t0
+       or      $acc1,$acc1,$t1
+       or      $acc2,$acc2,$t2
+       or      $acc3,$acc3,$t3
+
+       std     $acc0,0($rp)
+       std     $acc1,8($rp)
+       std     $acc2,16($rp)
+       std     $acc3,24($rp)
+
+       blr
+       .long   0
+       .byte   0,12,0x14,0,0,0,1,0
+       .long   0
+.size  __ecp_nistz256_div_by_2,.-__ecp_nistz256_div_by_2
+___
+########################################################################
+# following subroutines are "literal" implementation of those found in
+# ecp_nistz256.c
+#
+########################################################################
+# void ecp_nistz256_point_double(P256_POINT *out,const P256_POINT *inp);
+#
+if (1) {
+my $FRAME=64+32*4+12*8;
+my ($S,$M,$Zsqr,$tmp0)=map(64+32*$_,(0..3));
+# above map() describes stack layout with 4 temporary
+# 256-bit vectors on top.
+my ($rp_real,$ap_real) = map("r$_",(20,21));
+
+$code.=<<___;
+.globl ecp_nistz256_point_double
+.align 5
+ecp_nistz256_point_double:
+       stdu    $sp,-$FRAME($sp)
+       mflr    r0
+       std     r20,$FRAME-8*12($sp)
+       std     r21,$FRAME-8*11($sp)
+       std     r22,$FRAME-8*10($sp)
+       std     r23,$FRAME-8*9($sp)
+       std     r24,$FRAME-8*8($sp)
+       std     r25,$FRAME-8*7($sp)
+       std     r26,$FRAME-8*6($sp)
+       std     r27,$FRAME-8*5($sp)
+       std     r28,$FRAME-8*4($sp)
+       std     r29,$FRAME-8*3($sp)
+       std     r30,$FRAME-8*2($sp)
+       std     r31,$FRAME-8*1($sp)
+
+       li      $poly1,-1
+       srdi    $poly1,$poly1,32        # 0x00000000ffffffff
+       li      $poly3,1
+       orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
+.Ldouble_shortcut:
+       ld      $acc0,32($ap)
+       ld      $acc1,40($ap)
+       ld      $acc2,48($ap)
+       ld      $acc3,56($ap)
+       mr      $t0,$acc0
+       mr      $t1,$acc1
+       mr      $t2,$acc2
+       mr      $t3,$acc3
+        ld     $a0,64($ap)             # forward load for p256_sqr_mont
+        ld     $a1,72($ap)
+        ld     $a2,80($ap)
+        ld     $a3,88($ap)
+        mr     $rp_real,$rp
+        mr     $ap_real,$ap
+       addi    $rp,$sp,$S
+       bl      __ecp_nistz256_add      # p256_mul_by_2(S, in_y);
+
+       addi    $rp,$sp,$Zsqr
+       bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Zsqr, in_z);
+
+       ld      $t0,0($ap_real)
+       ld      $t1,8($ap_real)
+       ld      $t2,16($ap_real)
+       ld      $t3,24($ap_real)
+       mr      $a0,$acc0               # put Zsqr aside for p256_sub
+       mr      $a1,$acc1
+       mr      $a2,$acc2
+       mr      $a3,$acc3
+       addi    $rp,$sp,$M
+       bl      __ecp_nistz256_add      # p256_add(M, Zsqr, in_x);
+
+       addi    $bp,$ap_real,0
+       mr      $acc0,$a0               # restore Zsqr
+       mr      $acc1,$a1
+       mr      $acc2,$a2
+       mr      $acc3,$a3
+        ld     $a0,$S+0($sp)           # forward load for p256_sqr_mont
+        ld     $a1,$S+8($sp)
+        ld     $a2,$S+16($sp)
+        ld     $a3,$S+24($sp)
+       addi    $rp,$sp,$Zsqr
+       bl      __ecp_nistz256_sub_morf # p256_sub(Zsqr, in_x, Zsqr);
+
+       addi    $rp,$sp,$S
+       bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(S, S);
+
+       ld      $bi,32($ap_real)
+       ld      $a0,64($ap_real)
+       ld      $a1,72($ap_real)
+       ld      $a2,80($ap_real)
+       ld      $a3,88($ap_real)
+       addi    $bp,$ap_real,32
+       addi    $rp,$sp,$tmp0
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(tmp0, in_z, in_y);
+
+       mr      $t0,$acc0
+       mr      $t1,$acc1
+       mr      $t2,$acc2
+       mr      $t3,$acc3
+        ld     $a0,$S+0($sp)           # forward load for p256_sqr_mont
+        ld     $a1,$S+8($sp)
+        ld     $a2,$S+16($sp)
+        ld     $a3,$S+24($sp)
+       addi    $rp,$rp_real,64
+       bl      __ecp_nistz256_add      # p256_mul_by_2(res_z, tmp0);
+
+       addi    $rp,$sp,$tmp0
+       bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(tmp0, S);
+
+        ld     $bi,$Zsqr($sp)          # forward load for p256_mul_mont
+        ld     $a0,$M+0($sp)
+        ld     $a1,$M+8($sp)
+        ld     $a2,$M+16($sp)
+        ld     $a3,$M+24($sp)
+       addi    $rp,$rp_real,32
+       bl      __ecp_nistz256_div_by_2 # p256_div_by_2(res_y, tmp0);
+
+       addi    $bp,$sp,$Zsqr
+       addi    $rp,$sp,$M
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(M, M, Zsqr);
+
+       mr      $t0,$acc0               # duplicate M
+       mr      $t1,$acc1
+       mr      $t2,$acc2
+       mr      $t3,$acc3
+       mr      $a0,$acc0               # put M aside
+       mr      $a1,$acc1
+       mr      $a2,$acc2
+       mr      $a3,$acc3
+       addi    $rp,$sp,$M
+       bl      __ecp_nistz256_add
+       mr      $t0,$a0                 # restore M
+       mr      $t1,$a1
+       mr      $t2,$a2
+       mr      $t3,$a3
+        ld     $bi,0($ap_real)         # forward load for p256_mul_mont
+        ld     $a0,$S+0($sp)
+        ld     $a1,$S+8($sp)
+        ld     $a2,$S+16($sp)
+        ld     $a3,$S+24($sp)
+       bl      __ecp_nistz256_add      # p256_mul_by_3(M, M);
+
+       addi    $bp,$ap_real,0
+       addi    $rp,$sp,$S
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(S, S, in_x);
+
+       mr      $t0,$acc0
+       mr      $t1,$acc1
+       mr      $t2,$acc2
+       mr      $t3,$acc3
+        ld     $a0,$M+0($sp)           # forward load for p256_sqr_mont
+        ld     $a1,$M+8($sp)
+        ld     $a2,$M+16($sp)
+        ld     $a3,$M+24($sp)
+       addi    $rp,$sp,$tmp0
+       bl      __ecp_nistz256_add      # p256_mul_by_2(tmp0, S);
+
+       addi    $rp,$rp_real,0
+       bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(res_x, M);
+
+       addi    $bp,$sp,$tmp0
+       bl      __ecp_nistz256_sub_from # p256_sub(res_x, res_x, tmp0);
+
+       addi    $bp,$sp,$S
+       addi    $rp,$sp,$S
+       bl      __ecp_nistz256_sub_morf # p256_sub(S, S, res_x);
+
+       ld      $bi,$M($sp)
+       mr      $a0,$acc0               # copy S
+       mr      $a1,$acc1
+       mr      $a2,$acc2
+       mr      $a3,$acc3
+       addi    $bp,$sp,$M
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(S, S, M);
+
+       addi    $bp,$rp_real,32
+       addi    $rp,$rp_real,32
+       bl      __ecp_nistz256_sub_from # p256_sub(res_y, S, res_y);
+
+       mtlr    r0
+       ld      r20,$FRAME-8*12($sp)
+       ld      r21,$FRAME-8*11($sp)
+       ld      r22,$FRAME-8*10($sp)
+       ld      r23,$FRAME-8*9($sp)
+       ld      r24,$FRAME-8*8($sp)
+       ld      r25,$FRAME-8*7($sp)
+       ld      r26,$FRAME-8*6($sp)
+       ld      r27,$FRAME-8*5($sp)
+       ld      r28,$FRAME-8*4($sp)
+       ld      r29,$FRAME-8*3($sp)
+       ld      r30,$FRAME-8*2($sp)
+       ld      r31,$FRAME-8*1($sp)
+       addi    $sp,$sp,$FRAME
+       blr
+       .long   0
+       .byte   0,12,4,0,0x80,12,2,0
+       .long   0
+.size  ecp_nistz256_point_double,.-ecp_nistz256_point_double
+___
+}
+
+########################################################################
+# void ecp_nistz256_point_add(P256_POINT *out,const P256_POINT *in1,
+#                            const P256_POINT *in2);
+if (1) {
+my $FRAME = 64 + 32*12 + 16*8;
+my ($res_x,$res_y,$res_z,
+    $H,$Hsqr,$R,$Rsqr,$Hcub,
+    $U1,$U2,$S1,$S2)=map(64+32*$_,(0..11));
+my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
+# above map() describes stack layout with 12 temporary
+# 256-bit vectors on top.
+my ($rp_real,$ap_real,$bp_real,$in1infty,$in2infty,$temp)=map("r$_",(16..21));
+
+$code.=<<___;
+.globl ecp_nistz256_point_add
+.align 5
+ecp_nistz256_point_add:
+       stdu    $sp,-$FRAME($sp)
+       mflr    r0
+       std     r16,$FRAME-8*16($sp)
+       std     r17,$FRAME-8*15($sp)
+       std     r18,$FRAME-8*14($sp)
+       std     r19,$FRAME-8*13($sp)
+       std     r20,$FRAME-8*12($sp)
+       std     r21,$FRAME-8*11($sp)
+       std     r22,$FRAME-8*10($sp)
+       std     r23,$FRAME-8*9($sp)
+       std     r24,$FRAME-8*8($sp)
+       std     r25,$FRAME-8*7($sp)
+       std     r26,$FRAME-8*6($sp)
+       std     r27,$FRAME-8*5($sp)
+       std     r28,$FRAME-8*4($sp)
+       std     r29,$FRAME-8*3($sp)
+       std     r30,$FRAME-8*2($sp)
+       std     r31,$FRAME-8*1($sp)
+
+       li      $poly1,-1
+       srdi    $poly1,$poly1,32        # 0x00000000ffffffff
+       li      $poly3,1
+       orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
+
+       ld      $a0,64($bp)             # in2_z
+       ld      $a1,72($bp)
+       ld      $a2,80($bp)
+       ld      $a3,88($bp)
+        mr     $rp_real,$rp
+        mr     $ap_real,$ap
+        mr     $bp_real,$bp
+       or      $t0,$a0,$a1
+       or      $t2,$a2,$a3
+       or      $in2infty,$t0,$t2
+       neg     $t0,$in2infty
+       or      $in2infty,$in2infty,$t0
+       sradi   $in2infty,$in2infty,63  # !in2infty
+       addi    $rp,$sp,$Z2sqr
+       bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Z2sqr, in2_z);
+
+       ld      $a0,64($ap_real)        # in1_z
+       ld      $a1,72($ap_real)
+       ld      $a2,80($ap_real)
+       ld      $a3,88($ap_real)
+       or      $t0,$a0,$a1
+       or      $t2,$a2,$a3
+       or      $in1infty,$t0,$t2
+       neg     $t0,$in1infty
+       or      $in1infty,$in1infty,$t0
+       sradi   $in1infty,$in1infty,63  # !in1infty
+       addi    $rp,$sp,$Z1sqr
+       bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Z1sqr, in1_z);
+
+       ld      $bi,64($bp_real)
+       ld      $a0,$Z2sqr+0($sp)
+       ld      $a1,$Z2sqr+8($sp)
+       ld      $a2,$Z2sqr+16($sp)
+       ld      $a3,$Z2sqr+24($sp)
+       addi    $bp,$bp_real,64
+       addi    $rp,$sp,$S1
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(S1, Z2sqr, in2_z);
+
+       ld      $bi,64($ap_real)
+       ld      $a0,$Z1sqr+0($sp)
+       ld      $a1,$Z1sqr+8($sp)
+       ld      $a2,$Z1sqr+16($sp)
+       ld      $a3,$Z1sqr+24($sp)
+       addi    $bp,$ap_real,64
+       addi    $rp,$sp,$S2
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(S2, Z1sqr, in1_z);
+
+       ld      $bi,32($ap_real)
+       ld      $a0,$S1+0($sp)
+       ld      $a1,$S1+8($sp)
+       ld      $a2,$S1+16($sp)
+       ld      $a3,$S1+24($sp)
+       addi    $bp,$ap_real,32
+       addi    $rp,$sp,$S1
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(S1, S1, in1_y);
+
+       ld      $bi,32($bp_real)
+       ld      $a0,$S2+0($sp)
+       ld      $a1,$S2+8($sp)
+       ld      $a2,$S2+16($sp)
+       ld      $a3,$S2+24($sp)
+       addi    $bp,$bp_real,32
+       addi    $rp,$sp,$S2
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(S2, S2, in2_y);
+
+       addi    $bp,$sp,$S1
+        ld     $bi,$Z2sqr($sp)         # forward load for p256_mul_mont
+        ld     $a0,0($ap_real)
+        ld     $a1,8($ap_real)
+        ld     $a2,16($ap_real)
+        ld     $a3,24($ap_real)
+       addi    $rp,$sp,$R
+       bl      __ecp_nistz256_sub_from # p256_sub(R, S2, S1);
+
+       or      $acc0,$acc0,$acc1       # see if result is zero
+       or      $acc2,$acc2,$acc3
+       or      $temp,$acc0,$acc2
+
+       addi    $bp,$sp,$Z2sqr
+       addi    $rp,$sp,$U1
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(U1, in1_x, Z2sqr);
+
+       ld      $bi,$Z1sqr($sp)
+       ld      $a0,0($bp_real)
+       ld      $a1,8($bp_real)
+       ld      $a2,16($bp_real)
+       ld      $a3,24($bp_real)
+       addi    $bp,$sp,$Z1sqr
+       addi    $rp,$sp,$U2
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(U2, in2_x, Z1sqr);
+
+       addi    $bp,$sp,$U1
+        ld     $a0,$R+0($sp)           # forward load for p256_sqr_mont
+        ld     $a1,$R+8($sp)
+        ld     $a2,$R+16($sp)
+        ld     $a3,$R+24($sp)
+       addi    $rp,$sp,$H
+       bl      __ecp_nistz256_sub_from # p256_sub(H, U2, U1);
+
+       or      $acc0,$acc0,$acc1       # see if result is zero
+       or      $acc2,$acc2,$acc3
+       or.     $acc0,$acc0,$acc2
+       bne     .Ladd_proceed           # is_equal(U1,U2)?
+
+       and.    $t0,$in1infty,$in2infty
+       beq     .Ladd_proceed           # (in1infty || in2infty)?
+
+       cmpldi  $temp,0
+       beq     .Ladd_double            # is_equal(S1,S2)?
+
+       xor     $a0,$a0,$a0
+       std     $a0,0($rp_real)
+       std     $a0,8($rp_real)
+       std     $a0,16($rp_real)
+       std     $a0,24($rp_real)
+       std     $a0,32($rp_real)
+       std     $a0,40($rp_real)
+       std     $a0,48($rp_real)
+       std     $a0,56($rp_real)
+       std     $a0,64($rp_real)
+       std     $a0,72($rp_real)
+       std     $a0,80($rp_real)
+       std     $a0,88($rp_real)
+       b       .Ladd_done
+
+.align 4
+.Ladd_double:
+       ld      $bp,0($sp)              # back-link
+       mr      $ap,$ap_real
+       mr      $rp,$rp_real
+       ld      r16,$FRAME-8*16($sp)
+       ld      r17,$FRAME-8*15($sp)
+       ld      r18,$FRAME-8*14($sp)
+       ld      r19,$FRAME-8*13($sp)
+       stdu    $bp,$FRAME-288($sp)     # difference in stack frame sizes
+       b       .Ldouble_shortcut
+
+.align 4
+.Ladd_proceed:
+       addi    $rp,$sp,$Rsqr
+       bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Rsqr, R);
+
+       ld      $bi,64($ap_real)
+       ld      $a0,$H+0($sp)
+       ld      $a1,$H+8($sp)
+       ld      $a2,$H+16($sp)
+       ld      $a3,$H+24($sp)
+       addi    $bp,$ap_real,64
+       addi    $rp,$sp,$res_z
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(res_z, H, in1_z);
+
+       ld      $a0,$H+0($sp)
+       ld      $a1,$H+8($sp)
+       ld      $a2,$H+16($sp)
+       ld      $a3,$H+24($sp)
+       addi    $rp,$sp,$Hsqr
+       bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Hsqr, H);
+
+       ld      $bi,64($bp_real)
+       ld      $a0,$res_z+0($sp)
+       ld      $a1,$res_z+8($sp)
+       ld      $a2,$res_z+16($sp)
+       ld      $a3,$res_z+24($sp)
+       addi    $bp,$bp_real,64
+       addi    $rp,$sp,$res_z
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(res_z, res_z, in2_z);
+
+       ld      $bi,$H($sp)
+       ld      $a0,$Hsqr+0($sp)
+       ld      $a1,$Hsqr+8($sp)
+       ld      $a2,$Hsqr+16($sp)
+       ld      $a3,$Hsqr+24($sp)
+       addi    $bp,$sp,$H
+       addi    $rp,$sp,$Hcub
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(Hcub, Hsqr, H);
+
+       ld      $bi,$Hsqr($sp)
+       ld      $a0,$U1+0($sp)
+       ld      $a1,$U1+8($sp)
+       ld      $a2,$U1+16($sp)
+       ld      $a3,$U1+24($sp)
+       addi    $bp,$sp,$Hsqr
+       addi    $rp,$sp,$U2
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(U2, U1, Hsqr);
+
+       mr      $t0,$acc0
+       mr      $t1,$acc1
+       mr      $t2,$acc2
+       mr      $t3,$acc3
+       addi    $rp,$sp,$Hsqr
+       bl      __ecp_nistz256_add      # p256_mul_by_2(Hsqr, U2);
+
+       addi    $bp,$sp,$Rsqr
+       addi    $rp,$sp,$res_x
+       bl      __ecp_nistz256_sub_morf # p256_sub(res_x, Rsqr, Hsqr);
+
+       addi    $bp,$sp,$Hcub
+       bl      __ecp_nistz256_sub_from # p256_sub(res_x, res_x, Hcub);
+
+       addi    $bp,$sp,$U2
+        ld     $bi,$Hcub($sp)          # forward load for p256_mul_mont
+        ld     $a0,$S1+0($sp)
+        ld     $a1,$S1+8($sp)
+        ld     $a2,$S1+16($sp)
+        ld     $a3,$S1+24($sp)
+       addi    $rp,$sp,$res_y
+       bl      __ecp_nistz256_sub_morf # p256_sub(res_y, U2, res_x);
+
+       addi    $bp,$sp,$Hcub
+       addi    $rp,$sp,$S2
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(S2, S1, Hcub);
+
+       ld      $bi,$R($sp)
+       ld      $a0,$res_y+0($sp)
+       ld      $a1,$res_y+8($sp)
+       ld      $a2,$res_y+16($sp)
+       ld      $a3,$res_y+24($sp)
+       addi    $bp,$sp,$R
+       addi    $rp,$sp,$res_y
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(res_y, res_y, R);
+
+       addi    $bp,$sp,$S2
+       bl      __ecp_nistz256_sub_from # p256_sub(res_y, res_y, S2);
+
+       ld      $t0,0($bp_real)         # in2
+       ld      $t1,8($bp_real)
+       ld      $t2,16($bp_real)
+       ld      $t3,24($bp_real)
+       ld      $a0,$res_x+0($sp)       # res
+       ld      $a1,$res_x+8($sp)
+       ld      $a2,$res_x+16($sp)
+       ld      $a3,$res_x+24($sp)
+___
+for($i=0;$i<64;$i+=32) {               # conditional moves
+$code.=<<___;
+       ld      $acc0,$i+0($ap_real)    # in1
+       ld      $acc1,$i+8($ap_real)
+       ld      $acc2,$i+16($ap_real)
+       ld      $acc3,$i+24($ap_real)
+       andc    $t0,$t0,$in1infty
+       andc    $t1,$t1,$in1infty
+       andc    $t2,$t2,$in1infty
+       andc    $t3,$t3,$in1infty
+       and     $a0,$a0,$in1infty
+       and     $a1,$a1,$in1infty
+       and     $a2,$a2,$in1infty
+       and     $a3,$a3,$in1infty
+       or      $t0,$t0,$a0
+       or      $t1,$t1,$a1
+       or      $t2,$t2,$a2
+       or      $t3,$t3,$a3
+       andc    $acc0,$acc0,$in2infty
+       andc    $acc1,$acc1,$in2infty
+       andc    $acc2,$acc2,$in2infty
+       andc    $acc3,$acc3,$in2infty
+       and     $t0,$t0,$in2infty
+       and     $t1,$t1,$in2infty
+       and     $t2,$t2,$in2infty
+       and     $t3,$t3,$in2infty
+       or      $acc0,$acc0,$t0
+       or      $acc1,$acc1,$t1
+       or      $acc2,$acc2,$t2
+       or      $acc3,$acc3,$t3
+
+       ld      $t0,$i+32($bp_real)     # in2
+       ld      $t1,$i+40($bp_real)
+       ld      $t2,$i+48($bp_real)
+       ld      $t3,$i+56($bp_real)
+       ld      $a0,$res_x+$i+32($sp)
+       ld      $a1,$res_x+$i+40($sp)
+       ld      $a2,$res_x+$i+48($sp)
+       ld      $a3,$res_x+$i+56($sp)
+       std     $acc0,$i+0($rp_real)
+       std     $acc1,$i+8($rp_real)
+       std     $acc2,$i+16($rp_real)
+       std     $acc3,$i+24($rp_real)
+___
+}
+$code.=<<___;
+       ld      $acc0,$i+0($ap_real)    # in1
+       ld      $acc1,$i+8($ap_real)
+       ld      $acc2,$i+16($ap_real)
+       ld      $acc3,$i+24($ap_real)
+       andc    $t0,$t0,$in1infty
+       andc    $t1,$t1,$in1infty
+       andc    $t2,$t2,$in1infty
+       andc    $t3,$t3,$in1infty
+       and     $a0,$a0,$in1infty
+       and     $a1,$a1,$in1infty
+       and     $a2,$a2,$in1infty
+       and     $a3,$a3,$in1infty
+       or      $t0,$t0,$a0
+       or      $t1,$t1,$a1
+       or      $t2,$t2,$a2
+       or      $t3,$t3,$a3
+       andc    $acc0,$acc0,$in2infty
+       andc    $acc1,$acc1,$in2infty
+       andc    $acc2,$acc2,$in2infty
+       andc    $acc3,$acc3,$in2infty
+       and     $t0,$t0,$in2infty
+       and     $t1,$t1,$in2infty
+       and     $t2,$t2,$in2infty
+       and     $t3,$t3,$in2infty
+       or      $acc0,$acc0,$t0
+       or      $acc1,$acc1,$t1
+       or      $acc2,$acc2,$t2
+       or      $acc3,$acc3,$t3
+       std     $acc0,$i+0($rp_real)
+       std     $acc1,$i+8($rp_real)
+       std     $acc2,$i+16($rp_real)
+       std     $acc3,$i+24($rp_real)
+
+.Ladd_done:
+       mtlr    r0
+       ld      r16,$FRAME-8*16($sp)
+       ld      r17,$FRAME-8*15($sp)
+       ld      r18,$FRAME-8*14($sp)
+       ld      r19,$FRAME-8*13($sp)
+       ld      r20,$FRAME-8*12($sp)
+       ld      r21,$FRAME-8*11($sp)
+       ld      r22,$FRAME-8*10($sp)
+       ld      r23,$FRAME-8*9($sp)
+       ld      r24,$FRAME-8*8($sp)
+       ld      r25,$FRAME-8*7($sp)
+       ld      r26,$FRAME-8*6($sp)
+       ld      r27,$FRAME-8*5($sp)
+       ld      r28,$FRAME-8*4($sp)
+       ld      r29,$FRAME-8*3($sp)
+       ld      r30,$FRAME-8*2($sp)
+       ld      r31,$FRAME-8*1($sp)
+       addi    $sp,$sp,$FRAME
+       blr
+       .long   0
+       .byte   0,12,4,0,0x80,16,3,0
+       .long   0
+.size  ecp_nistz256_point_add,.-ecp_nistz256_point_add
+___
+}
+
+########################################################################
+# void ecp_nistz256_point_add_affine(P256_POINT *out,const P256_POINT *in1,
+#                                   const P256_POINT_AFFINE *in2);
+if (1) {
+my $FRAME = 64 + 32*10 + 16*8;
+my ($res_x,$res_y,$res_z,
+    $U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr)=map(64+32*$_,(0..9));
+my $Z1sqr = $S2;
+# above map() describes stack layout with 10 temporary
+# 256-bit vectors on top.
+my ($rp_real,$ap_real,$bp_real,$in1infty,$in2infty,$temp)=map("r$_",(16..21));
+
+$code.=<<___;
+.globl ecp_nistz256_point_add_affine
+.align 5
+ecp_nistz256_point_add_affine:
+       stdu    $sp,-$FRAME($sp)
+       mflr    r0
+       std     r16,$FRAME-8*16($sp)
+       std     r17,$FRAME-8*15($sp)
+       std     r18,$FRAME-8*14($sp)
+       std     r19,$FRAME-8*13($sp)
+       std     r20,$FRAME-8*12($sp)
+       std     r21,$FRAME-8*11($sp)
+       std     r22,$FRAME-8*10($sp)
+       std     r23,$FRAME-8*9($sp)
+       std     r24,$FRAME-8*8($sp)
+       std     r25,$FRAME-8*7($sp)
+       std     r26,$FRAME-8*6($sp)
+       std     r27,$FRAME-8*5($sp)
+       std     r28,$FRAME-8*4($sp)
+       std     r29,$FRAME-8*3($sp)
+       std     r30,$FRAME-8*2($sp)
+       std     r31,$FRAME-8*1($sp)
+
+       li      $poly1,-1
+       srdi    $poly1,$poly1,32        # 0x00000000ffffffff
+       li      $poly3,1
+       orc     $poly3,$poly3,$poly1    # 0xffffffff00000001
+
+       mr      $rp_real,$rp
+       mr      $ap_real,$ap
+       mr      $bp_real,$bp
+
+       ld      $a0,64($ap)             # in1_z
+       ld      $a1,72($ap)
+       ld      $a2,80($ap)
+       ld      $a3,88($ap)
+       or      $t0,$a0,$a1
+       or      $t2,$a2,$a3
+       or      $in1infty,$t0,$t2
+       neg     $t0,$in1infty
+       or      $in1infty,$in1infty,$t0
+       sradi   $in1infty,$in1infty,63  # !in1infty
+
+       ld      $acc0,0($bp)            # in2_x
+       ld      $acc1,8($bp)
+       ld      $acc2,16($bp)
+       ld      $acc3,24($bp)
+       ld      $t0,32($bp)             # in2_y
+       ld      $t1,40($bp)
+       ld      $t2,48($bp)
+       ld      $t3,56($bp)
+       or      $acc0,$acc0,$acc1
+       or      $acc2,$acc2,$acc3
+       or      $acc0,$acc0,$acc2
+       or      $t0,$t0,$t1
+       or      $t2,$t2,$t3
+       or      $t0,$t0,$t2
+       or      $in2infty,$acc0,$t0
+       neg     $t0,$in2infty
+       or      $in2infty,$in2infty,$t0
+       sradi   $in2infty,$in2infty,63  # !in2infty
+
+       addi    $rp,$sp,$Z1sqr
+       bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Z1sqr, in1_z);
+
+       mr      $a0,$acc0
+       mr      $a1,$acc1
+       mr      $a2,$acc2
+       mr      $a3,$acc3
+       ld      $bi,0($bp_real)
+       addi    $bp,$bp_real,0
+       addi    $rp,$sp,$U2
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(U2, Z1sqr, in2_x);
+
+       addi    $bp,$ap_real,0
+        ld     $bi,64($ap_real)        # forward load for p256_mul_mont
+        ld     $a0,$Z1sqr+0($sp)
+        ld     $a1,$Z1sqr+8($sp)
+        ld     $a2,$Z1sqr+16($sp)
+        ld     $a3,$Z1sqr+24($sp)
+       addi    $rp,$sp,$H
+       bl      __ecp_nistz256_sub_from # p256_sub(H, U2, in1_x);
+
+       addi    $bp,$ap_real,64
+       addi    $rp,$sp,$S2
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(S2, Z1sqr, in1_z);
+
+       ld      $bi,64($ap_real)
+       ld      $a0,$H+0($sp)
+       ld      $a1,$H+8($sp)
+       ld      $a2,$H+16($sp)
+       ld      $a3,$H+24($sp)
+       addi    $bp,$ap_real,64
+       addi    $rp,$sp,$res_z
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(res_z, H, in1_z);
+
+       ld      $bi,32($bp_real)
+       ld      $a0,$S2+0($sp)
+       ld      $a1,$S2+8($sp)
+       ld      $a2,$S2+16($sp)
+       ld      $a3,$S2+24($sp)
+       addi    $bp,$bp_real,32
+       addi    $rp,$sp,$S2
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(S2, S2, in2_y);
+
+       addi    $bp,$ap_real,32
+        ld     $a0,$H+0($sp)           # forward load for p256_sqr_mont
+        ld     $a1,$H+8($sp)
+        ld     $a2,$H+16($sp)
+        ld     $a3,$H+24($sp)
+       addi    $rp,$sp,$R
+       bl      __ecp_nistz256_sub_from # p256_sub(R, S2, in1_y);
+
+       addi    $rp,$sp,$Hsqr
+       bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Hsqr, H);
+
+       ld      $a0,$R+0($sp)
+       ld      $a1,$R+8($sp)
+       ld      $a2,$R+16($sp)
+       ld      $a3,$R+24($sp)
+       addi    $rp,$sp,$Rsqr
+       bl      __ecp_nistz256_sqr_mont # p256_sqr_mont(Rsqr, R);
+
+       ld      $bi,$H($sp)
+       ld      $a0,$Hsqr+0($sp)
+       ld      $a1,$Hsqr+8($sp)
+       ld      $a2,$Hsqr+16($sp)
+       ld      $a3,$Hsqr+24($sp)
+       addi    $bp,$sp,$H
+       addi    $rp,$sp,$Hcub
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(Hcub, Hsqr, H);
+
+       ld      $bi,0($ap_real)
+       ld      $a0,$Hsqr+0($sp)
+       ld      $a1,$Hsqr+8($sp)
+       ld      $a2,$Hsqr+16($sp)
+       ld      $a3,$Hsqr+24($sp)
+       addi    $bp,$ap_real,0
+       addi    $rp,$sp,$U2
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(U2, in1_x, Hsqr);
+
+       mr      $t0,$acc0
+       mr      $t1,$acc1
+       mr      $t2,$acc2
+       mr      $t3,$acc3
+       addi    $rp,$sp,$Hsqr
+       bl      __ecp_nistz256_add      # p256_mul_by_2(Hsqr, U2);
+
+       addi    $bp,$sp,$Rsqr
+       addi    $rp,$sp,$res_x
+       bl      __ecp_nistz256_sub_morf # p256_sub(res_x, Rsqr, Hsqr);
+
+       addi    $bp,$sp,$Hcub
+       bl      __ecp_nistz256_sub_from #  p256_sub(res_x, res_x, Hcub);
+
+       addi    $bp,$sp,$U2
+        ld     $bi,32($ap_real)        # forward load for p256_mul_mont
+        ld     $a0,$Hcub+0($sp)
+        ld     $a1,$Hcub+8($sp)
+        ld     $a2,$Hcub+16($sp)
+        ld     $a3,$Hcub+24($sp)
+       addi    $rp,$sp,$res_y
+       bl      __ecp_nistz256_sub_morf # p256_sub(res_y, U2, res_x);
+
+       addi    $bp,$ap_real,32
+       addi    $rp,$sp,$S2
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(S2, in1_y, Hcub);
+
+       ld      $bi,$R($sp)
+       ld      $a0,$res_y+0($sp)
+       ld      $a1,$res_y+8($sp)
+       ld      $a2,$res_y+16($sp)
+       ld      $a3,$res_y+24($sp)
+       addi    $bp,$sp,$R
+       addi    $rp,$sp,$res_y
+       bl      __ecp_nistz256_mul_mont # p256_mul_mont(res_y, res_y, R);
+
+       addi    $bp,$sp,$S2
+       bl      __ecp_nistz256_sub_from # p256_sub(res_y, res_y, S2);
+
+       ld      $t0,0($bp_real)         # in2
+       ld      $t1,8($bp_real)
+       ld      $t2,16($bp_real)
+       ld      $t3,24($bp_real)
+       ld      $a0,$res_x+0($sp)       # res
+       ld      $a1,$res_x+8($sp)
+       ld      $a2,$res_x+16($sp)
+       ld      $a3,$res_x+24($sp)
+___
+for($i=0;$i<64;$i+=32) {               # conditional moves
+$code.=<<___;
+       ld      $acc0,$i+0($ap_real)    # in1
+       ld      $acc1,$i+8($ap_real)
+       ld      $acc2,$i+16($ap_real)
+       ld      $acc3,$i+24($ap_real)
+       andc    $t0,$t0,$in1infty
+       andc    $t1,$t1,$in1infty
+       andc    $t2,$t2,$in1infty
+       andc    $t3,$t3,$in1infty
+       and     $a0,$a0,$in1infty
+       and     $a1,$a1,$in1infty
+       and     $a2,$a2,$in1infty
+       and     $a3,$a3,$in1infty
+       or      $t0,$t0,$a0
+       or      $t1,$t1,$a1
+       or      $t2,$t2,$a2
+       or      $t3,$t3,$a3
+       andc    $acc0,$acc0,$in2infty
+       andc    $acc1,$acc1,$in2infty
+       andc    $acc2,$acc2,$in2infty
+       andc    $acc3,$acc3,$in2infty
+       and     $t0,$t0,$in2infty
+       and     $t1,$t1,$in2infty
+       and     $t2,$t2,$in2infty
+       and     $t3,$t3,$in2infty
+       or      $acc0,$acc0,$t0
+       or      $acc1,$acc1,$t1
+       or      $acc2,$acc2,$t2
+       or      $acc3,$acc3,$t3
+___
+$code.=<<___   if ($i==0);
+       ld      $t0,32($bp_real)        # in2
+       ld      $t1,40($bp_real)
+       ld      $t2,48($bp_real)
+       ld      $t3,56($bp_real)
+___
+$code.=<<___   if ($i==32);
+       li      $t0,1                   # Lone_mont
+       not     $t1,$poly1
+       li      $t2,-1
+       not     $t3,$poly3
+___
+$code.=<<___;
+       ld      $a0,$res_x+$i+32($sp)
+       ld      $a1,$res_x+$i+40($sp)
+       ld      $a2,$res_x+$i+48($sp)
+       ld      $a3,$res_x+$i+56($sp)
+       std     $acc0,$i+0($rp_real)
+       std     $acc1,$i+8($rp_real)
+       std     $acc2,$i+16($rp_real)
+       std     $acc3,$i+24($rp_real)
+___
+}
+$code.=<<___;
+       ld      $acc0,$i+0($ap_real)    # in1
+       ld      $acc1,$i+8($ap_real)
+       ld      $acc2,$i+16($ap_real)
+       ld      $acc3,$i+24($ap_real)
+       andc    $t0,$t0,$in1infty
+       andc    $t1,$t1,$in1infty
+       andc    $t2,$t2,$in1infty
+       andc    $t3,$t3,$in1infty
+       and     $a0,$a0,$in1infty
+       and     $a1,$a1,$in1infty
+       and     $a2,$a2,$in1infty
+       and     $a3,$a3,$in1infty
+       or      $t0,$t0,$a0
+       or      $t1,$t1,$a1
+       or      $t2,$t2,$a2
+       or      $t3,$t3,$a3
+       andc    $acc0,$acc0,$in2infty
+       andc    $acc1,$acc1,$in2infty
+       andc    $acc2,$acc2,$in2infty
+       andc    $acc3,$acc3,$in2infty
+       and     $t0,$t0,$in2infty
+       and     $t1,$t1,$in2infty
+       and     $t2,$t2,$in2infty
+       and     $t3,$t3,$in2infty
+       or      $acc0,$acc0,$t0
+       or      $acc1,$acc1,$t1
+       or      $acc2,$acc2,$t2
+       or      $acc3,$acc3,$t3
+       std     $acc0,$i+0($rp_real)
+       std     $acc1,$i+8($rp_real)
+       std     $acc2,$i+16($rp_real)
+       std     $acc3,$i+24($rp_real)
+
+       mtlr    r0
+       ld      r16,$FRAME-8*16($sp)
+       ld      r17,$FRAME-8*15($sp)
+       ld      r18,$FRAME-8*14($sp)
+       ld      r19,$FRAME-8*13($sp)
+       ld      r20,$FRAME-8*12($sp)
+       ld      r21,$FRAME-8*11($sp)
+       ld      r22,$FRAME-8*10($sp)
+       ld      r23,$FRAME-8*9($sp)
+       ld      r24,$FRAME-8*8($sp)
+       ld      r25,$FRAME-8*7($sp)
+       ld      r26,$FRAME-8*6($sp)
+       ld      r27,$FRAME-8*5($sp)
+       ld      r28,$FRAME-8*4($sp)
+       ld      r29,$FRAME-8*3($sp)
+       ld      r30,$FRAME-8*2($sp)
+       ld      r31,$FRAME-8*1($sp)
+       addi    $sp,$sp,$FRAME
+       blr
+       .long   0
+       .byte   0,12,4,0,0x80,16,3,0
+       .long   0
+.size  ecp_nistz256_point_add_affine,.-ecp_nistz256_point_add_affine
+___
+}
+if (1) {
+my ($ordk,$ord0,$ord1,$t4) = map("r$_",(18..21));
+my ($ord2,$ord3,$zr) = ($poly1,$poly3,"r0");
+
+$code.=<<___;
+########################################################################
+# void ecp_nistz256_ord_mul_mont(uint64_t res[4], uint64_t a[4],
+#                                uint64_t b[4]);
+.globl ecp_nistz256_ord_mul_mont
+.align 5
+ecp_nistz256_ord_mul_mont:
+       stdu    $sp,-160($sp)
+       std     r18,48($sp)
+       std     r19,56($sp)
+       std     r20,64($sp)
+       std     r21,72($sp)
+       std     r22,80($sp)
+       std     r23,88($sp)
+       std     r24,96($sp)
+       std     r25,104($sp)
+       std     r26,112($sp)
+       std     r27,120($sp)
+       std     r28,128($sp)
+       std     r29,136($sp)
+       std     r30,144($sp)
+       std     r31,152($sp)
+
+       ld      $a0,0($ap)
+       ld      $bi,0($bp)
+       ld      $a1,8($ap)
+       ld      $a2,16($ap)
+       ld      $a3,24($ap)
+
+       lis     $ordk,0xccd1
+       lis     $ord0,0xf3b9
+       lis     $ord1,0xbce6
+       ori     $ordk,$ordk,0xc8aa
+       ori     $ord0,$ord0,0xcac2
+       ori     $ord1,$ord1,0xfaad
+       sldi    $ordk,$ordk,32
+       sldi    $ord0,$ord0,32
+       sldi    $ord1,$ord1,32
+       oris    $ordk,$ordk,0xee00
+       oris    $ord0,$ord0,0xfc63
+       oris    $ord1,$ord1,0xa717
+       ori     $ordk,$ordk,0xbc4f      # 0xccd1c8aaee00bc4f
+       ori     $ord0,$ord0,0x2551      # 0xf3b9cac2fc632551
+       ori     $ord1,$ord1,0x9e84      # 0xbce6faada7179e84
+       li      $ord2,-1                # 0xffffffffffffffff
+       sldi    $ord3,$ord2,32          # 0xffffffff00000000
+       li      $zr,0
+
+       mulld   $acc0,$a0,$bi           # a[0]*b[0]
+       mulhdu  $t0,$a0,$bi
+
+       mulld   $acc1,$a1,$bi           # a[1]*b[0]
+       mulhdu  $t1,$a1,$bi
+
+       mulld   $acc2,$a2,$bi           # a[2]*b[0]
+       mulhdu  $t2,$a2,$bi
+
+       mulld   $acc3,$a3,$bi           # a[3]*b[0]
+       mulhdu  $acc4,$a3,$bi
+
+       mulld   $t4,$acc0,$ordk
+
+       addc    $acc1,$acc1,$t0         # accumulate high parts of multiplication
+       adde    $acc2,$acc2,$t1
+       adde    $acc3,$acc3,$t2
+       addze   $acc4,$acc4
+       li      $acc5,0
+___
+for ($i=1;$i<4;$i++) {
+       ################################################################
+       #            ffff0000.ffffffff.yyyyyyyy.zzzzzzzz
+       # *                                     abcdefgh
+       # + xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx
+       #
+       # Now observing that ff..ff*x = (2^n-1)*x = 2^n*x-x, we
+       # rewrite above as:
+       #
+       #   xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx
+       # - 0000abcd.efgh0000.abcdefgh.00000000.00000000
+       # + abcdefgh.abcdefgh.yzayzbyz.cyzdyzey.zfyzgyzh
+$code.=<<___;
+       ld      $bi,8*$i($bp)           # b[i]
+
+       sldi    $t0,$t4,32
+       subfc   $acc2,$t4,$acc2
+       srdi    $t1,$t4,32
+       subfe   $acc3,$t0,$acc3
+       subfe   $acc4,$t1,$acc4
+       subfe   $acc5,$zr,$acc5
+
+       addic   $t0,$acc0,-1            # discarded
+       mulhdu  $t1,$ord0,$t4
+       mulld   $t2,$ord1,$t4
+       mulhdu  $t3,$ord1,$t4
+
+       adde    $t2,$t2,$t1
+        mulld  $t0,$a0,$bi
+       addze   $t3,$t3
+        mulld  $t1,$a1,$bi
+
+       addc    $acc0,$acc1,$t2
+        mulld  $t2,$a2,$bi
+       adde    $acc1,$acc2,$t3
+        mulld  $t3,$a3,$bi
+       adde    $acc2,$acc3,$t4
+       adde    $acc3,$acc4,$t4
+       addze   $acc4,$acc5
+
+       addc    $acc0,$acc0,$t0         # accumulate low parts
+       mulhdu  $t0,$a0,$bi
+       adde    $acc1,$acc1,$t1
+       mulhdu  $t1,$a1,$bi
+       adde    $acc2,$acc2,$t2
+       mulhdu  $t2,$a2,$bi
+       adde    $acc3,$acc3,$t3
+       mulhdu  $t3,$a3,$bi
+       addze   $acc4,$acc4
+       mulld   $t4,$acc0,$ordk
+       addc    $acc1,$acc1,$t0         # accumulate high parts
+       adde    $acc2,$acc2,$t1
+       adde    $acc3,$acc3,$t2
+       adde    $acc4,$acc4,$t3
+       addze   $acc5,$zr
+___
+}
+$code.=<<___;
+       sldi    $t0,$t4,32              # last reduction
+       subfc   $acc2,$t4,$acc2
+       srdi    $t1,$t4,32
+       subfe   $acc3,$t0,$acc3
+       subfe   $acc4,$t1,$acc4
+       subfe   $acc5,$zr,$acc5
+
+       addic   $t0,$acc0,-1            # discarded
+       mulhdu  $t1,$ord0,$t4
+       mulld   $t2,$ord1,$t4
+       mulhdu  $t3,$ord1,$t4
+
+       adde    $t2,$t2,$t1
+       addze   $t3,$t3
+
+       addc    $acc0,$acc1,$t2
+       adde    $acc1,$acc2,$t3
+       adde    $acc2,$acc3,$t4
+       adde    $acc3,$acc4,$t4
+       addze   $acc4,$acc5
+
+       subfc   $acc0,$ord0,$acc0       # ret -= modulus
+       subfe   $acc1,$ord1,$acc1
+       subfe   $acc2,$ord2,$acc2
+       subfe   $acc3,$ord3,$acc3
+       subfe   $acc4,$zr,$acc4
+
+       and     $t0,$ord0,$acc4
+       and     $t1,$ord1,$acc4
+       addc    $acc0,$acc0,$t0         # ret += modulus if borrow
+       and     $t3,$ord3,$acc4
+       adde    $acc1,$acc1,$t1
+       adde    $acc2,$acc2,$acc4
+       adde    $acc3,$acc3,$t3
+
+       std     $acc0,0($rp)
+       std     $acc1,8($rp)
+       std     $acc2,16($rp)
+       std     $acc3,24($rp)
+
+       ld      r18,48($sp)
+       ld      r19,56($sp)
+       ld      r20,64($sp)
+       ld      r21,72($sp)
+       ld      r22,80($sp)
+       ld      r23,88($sp)
+       ld      r24,96($sp)
+       ld      r25,104($sp)
+       ld      r26,112($sp)
+       ld      r27,120($sp)
+       ld      r28,128($sp)
+       ld      r29,136($sp)
+       ld      r30,144($sp)
+       ld      r31,152($sp)
+       addi    $sp,$sp,160
+       blr
+       .long   0
+       .byte   0,12,4,0,0x80,14,3,0
+       .long   0
+.size  ecp_nistz256_ord_mul_mont,.-ecp_nistz256_ord_mul_mont
+
+################################################################################
+# void ecp_nistz256_ord_sqr_mont(uint64_t res[4], uint64_t a[4],
+#                                int rep);
+.globl ecp_nistz256_ord_sqr_mont
+.align 5
+ecp_nistz256_ord_sqr_mont:
+       stdu    $sp,-160($sp)
+       std     r18,48($sp)
+       std     r19,56($sp)
+       std     r20,64($sp)
+       std     r21,72($sp)
+       std     r22,80($sp)
+       std     r23,88($sp)
+       std     r24,96($sp)
+       std     r25,104($sp)
+       std     r26,112($sp)
+       std     r27,120($sp)
+       std     r28,128($sp)
+       std     r29,136($sp)
+       std     r30,144($sp)
+       std     r31,152($sp)
+
+       mtctr   $bp
+
+       ld      $a0,0($ap)
+       ld      $a1,8($ap)
+       ld      $a2,16($ap)
+       ld      $a3,24($ap)
+
+       lis     $ordk,0xccd1
+       lis     $ord0,0xf3b9
+       lis     $ord1,0xbce6
+       ori     $ordk,$ordk,0xc8aa
+       ori     $ord0,$ord0,0xcac2
+       ori     $ord1,$ord1,0xfaad
+       sldi    $ordk,$ordk,32
+       sldi    $ord0,$ord0,32
+       sldi    $ord1,$ord1,32
+       oris    $ordk,$ordk,0xee00
+       oris    $ord0,$ord0,0xfc63
+       oris    $ord1,$ord1,0xa717
+       ori     $ordk,$ordk,0xbc4f      # 0xccd1c8aaee00bc4f
+       ori     $ord0,$ord0,0x2551      # 0xf3b9cac2fc632551
+       ori     $ord1,$ord1,0x9e84      # 0xbce6faada7179e84
+       li      $ord2,-1                # 0xffffffffffffffff
+       sldi    $ord3,$ord2,32          # 0xffffffff00000000
+       li      $zr,0
+       b       .Loop_ord_sqr
+
+.align 5
+.Loop_ord_sqr:
+       ################################################################
+       #  |  |  |  |  |  |a1*a0|  |
+       #  |  |  |  |  |a2*a0|  |  |
+       #  |  |a3*a2|a3*a0|  |  |  |
+       #  |  |  |  |a2*a1|  |  |  |
+       #  |  |  |a3*a1|  |  |  |  |
+       # *|  |  |  |  |  |  |  | 2|
+       # +|a3*a3|a2*a2|a1*a1|a0*a0|
+       #  |--+--+--+--+--+--+--+--|
+       #  |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is $accx, i.e. follow $accx
+       #
+       #  "can't overflow" below mark carrying into high part of
+       #  multiplication result, which can't overflow, because it
+       #  can never be all ones.
+
+       mulld   $acc1,$a1,$a0           # a[1]*a[0]
+       mulhdu  $t1,$a1,$a0
+       mulld   $acc2,$a2,$a0           # a[2]*a[0]
+       mulhdu  $t2,$a2,$a0
+       mulld   $acc3,$a3,$a0           # a[3]*a[0]
+       mulhdu  $acc4,$a3,$a0
+
+       addc    $acc2,$acc2,$t1         # accumulate high parts of multiplication
+        mulld  $t0,$a2,$a1             # a[2]*a[1]
+        mulhdu $t1,$a2,$a1
+       adde    $acc3,$acc3,$t2
+        mulld  $t2,$a3,$a1             # a[3]*a[1]
+        mulhdu $t3,$a3,$a1
+       addze   $acc4,$acc4             # can't overflow
+
+       mulld   $acc5,$a3,$a2           # a[3]*a[2]
+       mulhdu  $acc6,$a3,$a2
+
+       addc    $t1,$t1,$t2             # accumulate high parts of multiplication
+        mulld  $acc0,$a0,$a0           # a[0]*a[0]
+       addze   $t2,$t3                 # can't overflow
+
+       addc    $acc3,$acc3,$t0         # accumulate low parts of multiplication
+        mulhdu $a0,$a0,$a0
+       adde    $acc4,$acc4,$t1
+        mulld  $t1,$a1,$a1             # a[1]*a[1]
+       adde    $acc5,$acc5,$t2
+        mulhdu $a1,$a1,$a1
+       addze   $acc6,$acc6             # can't overflow
+
+       addc    $acc1,$acc1,$acc1       # acc[1-6]*=2
+        mulld  $t2,$a2,$a2             # a[2]*a[2]
+       adde    $acc2,$acc2,$acc2
+        mulhdu $a2,$a2,$a2
+       adde    $acc3,$acc3,$acc3
+        mulld  $t3,$a3,$a3             # a[3]*a[3]
+       adde    $acc4,$acc4,$acc4
+        mulhdu $a3,$a3,$a3
+       adde    $acc5,$acc5,$acc5
+       adde    $acc6,$acc6,$acc6
+       addze   $acc7,$zr
+
+       addc    $acc1,$acc1,$a0         # +a[i]*a[i]
+        mulld  $t4,$acc0,$ordk
+       adde    $acc2,$acc2,$t1
+       adde    $acc3,$acc3,$a1
+       adde    $acc4,$acc4,$t2
+       adde    $acc5,$acc5,$a2
+       adde    $acc6,$acc6,$t3
+       adde    $acc7,$acc7,$a3
+___
+for($i=0; $i<4; $i++) {                        # reductions
+$code.=<<___;
+       addic   $t0,$acc0,-1            # discarded
+       mulhdu  $t1,$ord0,$t4
+       mulld   $t2,$ord1,$t4
+       mulhdu  $t3,$ord1,$t4
+
+       adde    $t2,$t2,$t1
+       addze   $t3,$t3
+
+       addc    $acc0,$acc1,$t2
+       adde    $acc1,$acc2,$t3
+       adde    $acc2,$acc3,$t4
+       adde    $acc3,$zr,$t4           # can't overflow
+___
+$code.=<<___   if ($i<3);
+       mulld   $t3,$acc0,$ordk
+___
+$code.=<<___;
+       sldi    $t0,$t4,32
+       subfc   $acc1,$t4,$acc1
+       srdi    $t1,$t4,32
+       subfe   $acc2,$t0,$acc2
+       subfe   $acc3,$t1,$acc3         # can't borrow
+___
+       ($t3,$t4) = ($t4,$t3);
+}
+$code.=<<___;
+       addc    $acc0,$acc0,$acc4       # accumulate upper half
+       adde    $acc1,$acc1,$acc5
+       adde    $acc2,$acc2,$acc6
+       adde    $acc3,$acc3,$acc7
+       addze   $acc4,$zr
+
+       subfc   $acc0,$ord0,$acc0       # ret -= modulus
+       subfe   $acc1,$ord1,$acc1
+       subfe   $acc2,$ord2,$acc2
+       subfe   $acc3,$ord3,$acc3
+       subfe   $acc4,$zr,$acc4
+
+       and     $t0,$ord0,$acc4
+       and     $t1,$ord1,$acc4
+       addc    $a0,$acc0,$t0           # ret += modulus if borrow
+       and     $t3,$ord3,$acc4
+       adde    $a1,$acc1,$t1
+       adde    $a2,$acc2,$acc4
+       adde    $a3,$acc3,$t3
+
+       bdnz    .Loop_ord_sqr
+
+       std     $a0,0($rp)
+       std     $a1,8($rp)
+       std     $a2,16($rp)
+       std     $a3,24($rp)
+
+       ld      r18,48($sp)
+       ld      r19,56($sp)
+       ld      r20,64($sp)
+       ld      r21,72($sp)
+       ld      r22,80($sp)
+       ld      r23,88($sp)
+       ld      r24,96($sp)
+       ld      r25,104($sp)
+       ld      r26,112($sp)
+       ld      r27,120($sp)
+       ld      r28,128($sp)
+       ld      r29,136($sp)
+       ld      r30,144($sp)
+       ld      r31,152($sp)
+       addi    $sp,$sp,160
+       blr
+       .long   0
+       .byte   0,12,4,0,0x80,14,3,0
+       .long   0
+.size  ecp_nistz256_ord_sqr_mont,.-ecp_nistz256_ord_sqr_mont
+___
+}      }
+
+########################################################################
+# scatter-gather subroutines
+{
+my ($out,$inp,$index,$mask)=map("r$_",(3..7));
+$code.=<<___;
+########################################################################
+# void ecp_nistz256_scatter_w5(void *out, const P256_POINT *inp,
+#                              int index);
+.globl ecp_nistz256_scatter_w5
+.align 4
+ecp_nistz256_scatter_w5:
+       slwi    $index,$index,2
+       add     $out,$out,$index
+
+       ld      r8, 0($inp)             # X
+       ld      r9, 8($inp)
+       ld      r10,16($inp)
+       ld      r11,24($inp)
+
+       stw     r8, 64*0-4($out)
+       srdi    r8, r8, 32
+       stw     r9, 64*1-4($out)
+       srdi    r9, r9, 32
+       stw     r10,64*2-4($out)
+       srdi    r10,r10,32
+       stw     r11,64*3-4($out)
+       srdi    r11,r11,32
+       stw     r8, 64*4-4($out)
+       stw     r9, 64*5-4($out)
+       stw     r10,64*6-4($out)
+       stw     r11,64*7-4($out)
+       addi    $out,$out,64*8
+
+       ld      r8, 32($inp)            # Y
+       ld      r9, 40($inp)
+       ld      r10,48($inp)
+       ld      r11,56($inp)
+
+       stw     r8, 64*0-4($out)
+       srdi    r8, r8, 32
+       stw     r9, 64*1-4($out)
+       srdi    r9, r9, 32
+       stw     r10,64*2-4($out)
+       srdi    r10,r10,32
+       stw     r11,64*3-4($out)
+       srdi    r11,r11,32
+       stw     r8, 64*4-4($out)
+       stw     r9, 64*5-4($out)
+       stw     r10,64*6-4($out)
+       stw     r11,64*7-4($out)
+       addi    $out,$out,64*8
+
+       ld      r8, 64($inp)            # Z
+       ld      r9, 72($inp)
+       ld      r10,80($inp)
+       ld      r11,88($inp)
+
+       stw     r8, 64*0-4($out)
+       srdi    r8, r8, 32
+       stw     r9, 64*1-4($out)
+       srdi    r9, r9, 32
+       stw     r10,64*2-4($out)
+       srdi    r10,r10,32
+       stw     r11,64*3-4($out)
+       srdi    r11,r11,32
+       stw     r8, 64*4-4($out)
+       stw     r9, 64*5-4($out)
+       stw     r10,64*6-4($out)
+       stw     r11,64*7-4($out)
+
+       blr
+       .long   0
+       .byte   0,12,0x14,0,0,0,3,0
+       .long   0
+.size  ecp_nistz256_scatter_w5,.-ecp_nistz256_scatter_w5
+
+########################################################################
+# void ecp_nistz256_gather_w5(P256_POINT *out, const void *inp,
+#                              int index);
+.globl ecp_nistz256_gather_w5
+.align 4
+ecp_nistz256_gather_w5:
+       neg     r0,$index
+       sradi   r0,r0,63
+
+       add     $index,$index,r0
+       slwi    $index,$index,2
+       add     $inp,$inp,$index
+
+       lwz     r5, 64*0($inp)
+       lwz     r6, 64*1($inp)
+       lwz     r7, 64*2($inp)
+       lwz     r8, 64*3($inp)
+       lwz     r9, 64*4($inp)
+       lwz     r10,64*5($inp)
+       lwz     r11,64*6($inp)
+       lwz     r12,64*7($inp)
+       addi    $inp,$inp,64*8
+       sldi    r9, r9, 32
+       sldi    r10,r10,32
+       sldi    r11,r11,32
+       sldi    r12,r12,32
+       or      r5,r5,r9
+       or      r6,r6,r10
+       or      r7,r7,r11
+       or      r8,r8,r12
+       and     r5,r5,r0
+       and     r6,r6,r0
+       and     r7,r7,r0
+       and     r8,r8,r0
+       std     r5,0($out)              # X
+       std     r6,8($out)
+       std     r7,16($out)
+       std     r8,24($out)
+
+       lwz     r5, 64*0($inp)
+       lwz     r6, 64*1($inp)
+       lwz     r7, 64*2($inp)
+       lwz     r8, 64*3($inp)
+       lwz     r9, 64*4($inp)
+       lwz     r10,64*5($inp)
+       lwz     r11,64*6($inp)
+       lwz     r12,64*7($inp)
+       addi    $inp,$inp,64*8
+       sldi    r9, r9, 32
+       sldi    r10,r10,32
+       sldi    r11,r11,32
+       sldi    r12,r12,32
+       or      r5,r5,r9
+       or      r6,r6,r10
+       or      r7,r7,r11
+       or      r8,r8,r12
+       and     r5,r5,r0
+       and     r6,r6,r0
+       and     r7,r7,r0
+       and     r8,r8,r0
+       std     r5,32($out)             # Y
+       std     r6,40($out)
+       std     r7,48($out)
+       std     r8,56($out)
+
+       lwz     r5, 64*0($inp)
+       lwz     r6, 64*1($inp)
+       lwz     r7, 64*2($inp)
+       lwz     r8, 64*3($inp)
+       lwz     r9, 64*4($inp)
+       lwz     r10,64*5($inp)
+       lwz     r11,64*6($inp)
+       lwz     r12,64*7($inp)
+       sldi    r9, r9, 32
+       sldi    r10,r10,32
+       sldi    r11,r11,32
+       sldi    r12,r12,32
+       or      r5,r5,r9
+       or      r6,r6,r10
+       or      r7,r7,r11
+       or      r8,r8,r12
+       and     r5,r5,r0
+       and     r6,r6,r0
+       and     r7,r7,r0
+       and     r8,r8,r0
+       std     r5,64($out)             # Z
+       std     r6,72($out)
+       std     r7,80($out)
+       std     r8,88($out)
+
+       blr
+       .long   0
+       .byte   0,12,0x14,0,0,0,3,0
+       .long   0
+.size  ecp_nistz256_gather_w5,.-ecp_nistz256_gather_w5
+
+########################################################################
+# void ecp_nistz256_scatter_w7(void *out, const P256_POINT_AFFINE *inp,
+#                              int index);
+.globl ecp_nistz256_scatter_w7
+.align 4
+ecp_nistz256_scatter_w7:
+       li      r0,8
+       mtctr   r0
+       add     $out,$out,$index
+       subi    $inp,$inp,8
+
+.Loop_scatter_w7:
+       ldu     r0,8($inp)
+       stb     r0,64*0-1($out)
+       srdi    r0,r0,8
+       stb     r0,64*1-1($out)
+       srdi    r0,r0,8
+       stb     r0,64*2-1($out)
+       srdi    r0,r0,8
+       stb     r0,64*3-1($out)
+       srdi    r0,r0,8
+       stb     r0,64*4-1($out)
+       srdi    r0,r0,8
+       stb     r0,64*5-1($out)
+       srdi    r0,r0,8
+       stb     r0,64*6-1($out)
+       srdi    r0,r0,8
+       stb     r0,64*7-1($out)
+       addi    $out,$out,64*8
+       bdnz    .Loop_scatter_w7
+
+       blr
+       .long   0
+       .byte   0,12,0x14,0,0,0,3,0
+       .long   0
+.size  ecp_nistz256_scatter_w7,.-ecp_nistz256_scatter_w7
+
+########################################################################
+# void ecp_nistz256_gather_w7(P256_POINT_AFFINE *out, const void *inp,
+#                              int index);
+.globl ecp_nistz256_gather_w7
+.align 4
+ecp_nistz256_gather_w7:
+       li      r0,8
+       mtctr   r0
+       neg     r0,$index
+       sradi   r0,r0,63
+
+       add     $index,$index,r0
+       add     $inp,$inp,$index
+       subi    $out,$out,8
+
+.Loop_gather_w7:
+       lbz     r5, 64*0($inp)
+       lbz     r6, 64*1($inp)
+       lbz     r7, 64*2($inp)
+       lbz     r8, 64*3($inp)
+       lbz     r9, 64*4($inp)
+       lbz     r10,64*5($inp)
+       lbz     r11,64*6($inp)
+       lbz     r12,64*7($inp)
+       addi    $inp,$inp,64*8
+
+       sldi    r6, r6, 8
+       sldi    r7, r7, 16
+       sldi    r8, r8, 24
+       sldi    r9, r9, 32
+       sldi    r10,r10,40
+       sldi    r11,r11,48
+       sldi    r12,r12,56
+
+       or      r5,r5,r6
+       or      r7,r7,r8
+       or      r9,r9,r10
+       or      r11,r11,r12
+       or      r5,r5,r7
+       or      r9,r9,r11
+       or      r5,r5,r9
+       and     r5,r5,r0
+       stdu    r5,8($out)
+       bdnz    .Loop_gather_w7
+
+       blr
+       .long   0
+       .byte   0,12,0x14,0,0,0,3,0
+       .long   0
+.size  ecp_nistz256_gather_w7,.-ecp_nistz256_gather_w7
+___
+}
+
+foreach (split("\n",$code)) {
+       s/\`([^\`]*)\`/eval $1/ge;
+
+       print $_,"\n";
+}
+close STDOUT;  # enforce flush
index 970c292..6840887 100644 (file)
@@ -21,6 +21,7 @@ GENERATE[ecp_nistz256-armv4.S]=asm/ecp_nistz256-armv4.pl $(PERLASM_SCHEME)
 INCLUDE[ecp_nistz256-armv4.o]=..
 GENERATE[ecp_nistz256-armv8.S]=asm/ecp_nistz256-armv8.pl $(PERLASM_SCHEME)
 INCLUDE[ecp_nistz256-armv8.o]=..
+GENERATE[ecp_nistz256-ppc64.s]=asm/ecp_nistz256-ppc64.pl $(PERLASM_SCHEME)
 
 BEGINRAW[Makefile]
 {- $builddir -}/ecp_nistz256-%.S:      {- $sourcedir -}/asm/ecp_nistz256-%.pl
index ef38b17..28cfa19 100644 (file)
@@ -131,6 +131,30 @@ int poly1305_init(void *ctx, const unsigned char key[16], void *func[2])
 }
 #endif
 
+#ifdef ECP_NISTZ256_ASM
+void ecp_nistz256_mul_mont(unsigned long res[4], const unsigned long a[4],
+                           const unsigned long b[4]);
+
+void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4]);
+void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4])
+{
+    static const unsigned long RR[] = { 0x0000000000000003U,
+                                        0xfffffffbffffffffU,
+                                        0xfffffffffffffffeU,
+                                        0x00000004fffffffdU };
+
+    ecp_nistz256_mul_mont(res, in, RR);
+}
+
+void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4]);
+void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4])
+{
+    static const unsigned long one[] = { 1, 0, 0, 0 };
+
+    ecp_nistz256_mul_mont(res, in, one);
+}
+#endif
+
 static sigjmp_buf ill_jmp;
 static void ill_handler(int sig)
 {