crypto/ec/asm/ecp_nistz256-sparcv9.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2015-2020 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License 2.0 (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9
  10 # ====================================================================
  11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  12 # project. The module is, however, dual licensed under OpenSSL and
  13 # CRYPTOGAMS licenses depending on where you obtain it. For further
  14 # details see http://www.openssl.org/~appro/cryptogams/.
  15 # ====================================================================
  16 #
  17 # ECP_NISTZ256 module for SPARCv9.
  18 #
  19 # February 2015.
  20 #
  21 # Original ECP_NISTZ256 submission targeting x86_64 is detailed in
  22 # http://eprint.iacr.org/2013/816. In the process of adaptation
  23 # original .c module was made 32-bit savvy in order to make this
  24 # implementation possible.
  25 #
  26 #                       with/without -DECP_NISTZ256_ASM
  27 # UltraSPARC III        +12-18%
  28 # SPARC T4              +99-550% (+66-150% on 32-bit Solaris)
  29 #
  30 # Ranges denote minimum and maximum improvement coefficients depending
  31 # on benchmark. Lower coefficients are for ECDSA sign, server-side
  32 # operation. Keep in mind that +200% means 3x improvement.
  33
  34 $output = pop and open STDOUT,">$output";
  35
  36 $code.=<<___;
  37 #include "sparc_arch.h"
  38
  39 #define LOCALS  (STACK_BIAS+STACK_FRAME)
  40 #ifdef  __arch64__
  41 .register       %g2,#scratch
  42 .register       %g3,#scratch
  43 # define STACK64_FRAME  STACK_FRAME
  44 # define LOCALS64       LOCALS
  45 #else
  46 # define STACK64_FRAME  (2047+192)
  47 # define LOCALS64       STACK64_FRAME
  48 #endif
  49
  50 .section        ".text",#alloc,#execinstr
  51 ___
  52 ########################################################################
  53 # Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7
  54 #
  55 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  56 open TABLE,"<ecp_nistz256_table.c"              or
  57 open TABLE,"<${dir}../ecp_nistz256_table.c"     or
  58 die "failed to open ecp_nistz256_table.c:",$!;
  59
  60 use integer;
  61
  62 foreach(<TABLE>) {
  63         s/TOBN\(\s*(0x[0-9a-f]+),\s*(0x[0-9a-f]+)\s*\)/push @arr,hex($2),hex($1)/geo;
  64 }
  65 close TABLE;
  66
  67 # See ecp_nistz256_table.c for explanation for why it's 64*16*37.
  68 # 64*16*37-1 is because $#arr returns last valid index or @arr, not
  69 # amount of elements.
  70 die "insane number of elements" if ($#arr != 64*16*37-1);
  71
  72 $code.=<<___;
  73 .globl  ecp_nistz256_precomputed
  74 .align  4096
  75 ecp_nistz256_precomputed:
  76 ___
  77 ########################################################################
  78 # this conversion smashes P256_POINT_AFFINE by individual bytes with
  79 # 64 byte interval, similar to
  80 #       1111222233334444
  81 #       1234123412341234
  82 for(1..37) {
  83         @tbl = splice(@arr,0,64*16);
  84         for($i=0;$i<64;$i++) {
  85                 undef @line;
  86                 for($j=0;$j<64;$j++) {
  87                         push @line,(@tbl[$j*16+$i/4]>>(($i%4)*8))&0xff;
  88                 }
  89                 $code.=".byte\t";
  90                 $code.=join(',',map { sprintf "0x%02x",$_} @line);
  91                 $code.="\n";
  92         }
  93 }
  94
  95 {{{
  96 my ($rp,$ap,$bp)=map("%i$_",(0..2));
  97 my @acc=map("%l$_",(0..7));
  98 my ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7)=(map("%o$_",(0..5)),"%g4","%g5");
  99 my ($bi,$a0,$mask,$carry)=(map("%i$_",(3..5)),"%g1");
 100 my ($rp_real,$ap_real)=("%g2","%g3");
 101
 102 $code.=<<___;
 103 .type   ecp_nistz256_precomputed,#object
 104 .size   ecp_nistz256_precomputed,.-ecp_nistz256_precomputed
 105 .align  64
 106 .LRR:   ! 2^512 mod P precomputed for NIST P256 polynomial
 107 .long   0x00000003, 0x00000000, 0xffffffff, 0xfffffffb
 108 .long   0xfffffffe, 0xffffffff, 0xfffffffd, 0x00000004
 109 .Lone:
 110 .long   1,0,0,0,0,0,0,0
 111 .asciz  "ECP_NISTZ256 for SPARCv9, CRYPTOGAMS by <appro\@openssl.org>"
 112
 113 ! void  ecp_nistz256_to_mont(BN_ULONG %i0[8],const BN_ULONG %i1[8]);
 114 .globl  ecp_nistz256_to_mont
 115 .align  64
 116 ecp_nistz256_to_mont:
 117         save    %sp,-STACK_FRAME,%sp
 118         nop
 119 1:      call    .+8
 120         add     %o7,.LRR-1b,$bp
 121         call    __ecp_nistz256_mul_mont
 122         nop
 123         ret
 124         restore
 125 .type   ecp_nistz256_to_mont,#function
 126 .size   ecp_nistz256_to_mont,.-ecp_nistz256_to_mont
 127
 128 ! void  ecp_nistz256_from_mont(BN_ULONG %i0[8],const BN_ULONG %i1[8]);
 129 .globl  ecp_nistz256_from_mont
 130 .align  32
 131 ecp_nistz256_from_mont:
 132         save    %sp,-STACK_FRAME,%sp
 133         nop
 134 1:      call    .+8
 135         add     %o7,.Lone-1b,$bp
 136         call    __ecp_nistz256_mul_mont
 137         nop
 138         ret
 139         restore
 140 .type   ecp_nistz256_from_mont,#function
 141 .size   ecp_nistz256_from_mont,.-ecp_nistz256_from_mont
 142
 143 ! void  ecp_nistz256_mul_mont(BN_ULONG %i0[8],const BN_ULONG %i1[8],
 144 !                                             const BN_ULONG %i2[8]);
 145 .globl  ecp_nistz256_mul_mont
 146 .align  32
 147 ecp_nistz256_mul_mont:
 148         save    %sp,-STACK_FRAME,%sp
 149         nop
 150         call    __ecp_nistz256_mul_mont
 151         nop
 152         ret
 153         restore
 154 .type   ecp_nistz256_mul_mont,#function
 155 .size   ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont
 156
 157 ! void  ecp_nistz256_sqr_mont(BN_ULONG %i0[8],const BN_ULONG %i2[8]);
 158 .globl  ecp_nistz256_sqr_mont
 159 .align  32
 160 ecp_nistz256_sqr_mont:
 161         save    %sp,-STACK_FRAME,%sp
 162         mov     $ap,$bp
 163         call    __ecp_nistz256_mul_mont
 164         nop
 165         ret
 166         restore
 167 .type   ecp_nistz256_sqr_mont,#function
 168 .size   ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont
 169 ___
 170
 171 ########################################################################
 172 # Special thing to keep in mind is that $t0-$t7 hold 64-bit values,
 173 # while all others are meant to keep 32. "Meant to" means that additions
 174 # to @acc[0-7] do "contaminate" upper bits, but they are cleared before
 175 # they can affect outcome (follow 'and' with $mask). Also keep in mind
 176 # that addition with carry is addition with 32-bit carry, even though
 177 # CPU is 64-bit. [Addition with 64-bit carry was introduced in T3, see
 178 # below for VIS3 code paths.]
 179
 180 $code.=<<___;
 181 .align  32
 182 __ecp_nistz256_mul_mont:
 183         ld      [$bp+0],$bi             ! b[0]
 184         mov     -1,$mask
 185         ld      [$ap+0],$a0
 186         srl     $mask,0,$mask           ! 0xffffffff
 187         ld      [$ap+4],$t1
 188         ld      [$ap+8],$t2
 189         ld      [$ap+12],$t3
 190         ld      [$ap+16],$t4
 191         ld      [$ap+20],$t5
 192         ld      [$ap+24],$t6
 193         ld      [$ap+28],$t7
 194         mulx    $a0,$bi,$t0             ! a[0-7]*b[0], 64-bit results
 195         mulx    $t1,$bi,$t1
 196         mulx    $t2,$bi,$t2
 197         mulx    $t3,$bi,$t3
 198         mulx    $t4,$bi,$t4
 199         mulx    $t5,$bi,$t5
 200         mulx    $t6,$bi,$t6
 201         mulx    $t7,$bi,$t7
 202         srlx    $t0,32,@acc[1]          ! extract high parts
 203         srlx    $t1,32,@acc[2]
 204         srlx    $t2,32,@acc[3]
 205         srlx    $t3,32,@acc[4]
 206         srlx    $t4,32,@acc[5]
 207         srlx    $t5,32,@acc[6]
 208         srlx    $t6,32,@acc[7]
 209         srlx    $t7,32,@acc[0]          ! "@acc[8]"
 210         mov     0,$carry
 211 ___
 212 for($i=1;$i<8;$i++) {
 213 $code.=<<___;
 214         addcc   @acc[1],$t1,@acc[1]     ! accumulate high parts
 215         ld      [$bp+4*$i],$bi          ! b[$i]
 216         ld      [$ap+4],$t1             ! re-load a[1-7]
 217         addccc  @acc[2],$t2,@acc[2]
 218         addccc  @acc[3],$t3,@acc[3]
 219         ld      [$ap+8],$t2
 220         ld      [$ap+12],$t3
 221         addccc  @acc[4],$t4,@acc[4]
 222         addccc  @acc[5],$t5,@acc[5]
 223         ld      [$ap+16],$t4
 224         ld      [$ap+20],$t5
 225         addccc  @acc[6],$t6,@acc[6]
 226         addccc  @acc[7],$t7,@acc[7]
 227         ld      [$ap+24],$t6
 228         ld      [$ap+28],$t7
 229         addccc  @acc[0],$carry,@acc[0]  ! "@acc[8]"
 230         addc    %g0,%g0,$carry
 231 ___
 232         # Reduction iteration is normally performed by accumulating
 233         # result of multiplication of modulus by "magic" digit [and
 234         # omitting least significant word, which is guaranteed to
 235         # be 0], but thanks to special form of modulus and "magic"
 236         # digit being equal to least significant word, it can be
 237         # performed with additions and subtractions alone. Indeed:
 238         #
 239         #        ffff.0001.0000.0000.0000.ffff.ffff.ffff
 240         # *                                         abcd
 241         # + xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.abcd
 242         #
 243         # Now observing that ff..ff*x = (2^n-1)*x = 2^n*x-x, we
 244         # rewrite above as:
 245         #
 246         #   xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.abcd
 247         # + abcd.0000.abcd.0000.0000.abcd.0000.0000.0000
 248         # -      abcd.0000.0000.0000.0000.0000.0000.abcd
 249         #
 250         # or marking redundant operations:
 251         #
 252         #   xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.----
 253         # + abcd.0000.abcd.0000.0000.abcd.----.----.----
 254         # -      abcd.----.----.----.----.----.----.----
 255
 256 $code.=<<___;
 257         ! multiplication-less reduction
 258         addcc   @acc[3],$t0,@acc[3]     ! r[3]+=r[0]
 259         addccc  @acc[4],%g0,@acc[4]     ! r[4]+=0
 260          and    @acc[1],$mask,@acc[1]
 261          and    @acc[2],$mask,@acc[2]
 262         addccc  @acc[5],%g0,@acc[5]     ! r[5]+=0
 263         addccc  @acc[6],$t0,@acc[6]     ! r[6]+=r[0]
 264          and    @acc[3],$mask,@acc[3]
 265          and    @acc[4],$mask,@acc[4]
 266         addccc  @acc[7],%g0,@acc[7]     ! r[7]+=0
 267         addccc  @acc[0],$t0,@acc[0]     ! r[8]+=r[0]    "@acc[8]"
 268          and    @acc[5],$mask,@acc[5]
 269          and    @acc[6],$mask,@acc[6]
 270         addc    $carry,%g0,$carry       ! top-most carry
 271         subcc   @acc[7],$t0,@acc[7]     ! r[7]-=r[0]
 272         subccc  @acc[0],%g0,@acc[0]     ! r[8]-=0       "@acc[8]"
 273         subc    $carry,%g0,$carry       ! top-most carry
 274          and    @acc[7],$mask,@acc[7]
 275          and    @acc[0],$mask,@acc[0]   ! "@acc[8]"
 276 ___
 277         push(@acc,shift(@acc));         # rotate registers to "omit" acc[0]
 278 $code.=<<___;
 279         mulx    $a0,$bi,$t0             ! a[0-7]*b[$i], 64-bit results
 280         mulx    $t1,$bi,$t1
 281         mulx    $t2,$bi,$t2
 282         mulx    $t3,$bi,$t3
 283         mulx    $t4,$bi,$t4
 284         mulx    $t5,$bi,$t5
 285         mulx    $t6,$bi,$t6
 286         mulx    $t7,$bi,$t7
 287         add     @acc[0],$t0,$t0         ! accumulate low parts, can't overflow
 288         add     @acc[1],$t1,$t1
 289         srlx    $t0,32,@acc[1]          ! extract high parts
 290         add     @acc[2],$t2,$t2
 291         srlx    $t1,32,@acc[2]
 292         add     @acc[3],$t3,$t3
 293         srlx    $t2,32,@acc[3]
 294         add     @acc[4],$t4,$t4
 295         srlx    $t3,32,@acc[4]
 296         add     @acc[5],$t5,$t5
 297         srlx    $t4,32,@acc[5]
 298         add     @acc[6],$t6,$t6
 299         srlx    $t5,32,@acc[6]
 300         add     @acc[7],$t7,$t7
 301         srlx    $t6,32,@acc[7]
 302         srlx    $t7,32,@acc[0]          ! "@acc[8]"
 303 ___
 304 }
 305 $code.=<<___;
 306         addcc   @acc[1],$t1,@acc[1]     ! accumulate high parts
 307         addccc  @acc[2],$t2,@acc[2]
 308         addccc  @acc[3],$t3,@acc[3]
 309         addccc  @acc[4],$t4,@acc[4]
 310         addccc  @acc[5],$t5,@acc[5]
 311         addccc  @acc[6],$t6,@acc[6]
 312         addccc  @acc[7],$t7,@acc[7]
 313         addccc  @acc[0],$carry,@acc[0]  ! "@acc[8]"
 314         addc    %g0,%g0,$carry
 315
 316         addcc   @acc[3],$t0,@acc[3]     ! multiplication-less reduction
 317         addccc  @acc[4],%g0,@acc[4]
 318         addccc  @acc[5],%g0,@acc[5]
 319         addccc  @acc[6],$t0,@acc[6]
 320         addccc  @acc[7],%g0,@acc[7]
 321         addccc  @acc[0],$t0,@acc[0]     ! "@acc[8]"
 322         addc    $carry,%g0,$carry
 323         subcc   @acc[7],$t0,@acc[7]
 324         subccc  @acc[0],%g0,@acc[0]     ! "@acc[8]"
 325         subc    $carry,%g0,$carry       ! top-most carry
 326 ___
 327         push(@acc,shift(@acc));         # rotate registers to omit acc[0]
 328 $code.=<<___;
 329         ! Final step is "if result > mod, subtract mod", but we do it
 330         ! "other way around", namely subtract modulus from result
 331         ! and if it borrowed, add modulus back.
 332
 333         subcc   @acc[0],-1,@acc[0]      ! subtract modulus
 334         subccc  @acc[1],-1,@acc[1]
 335         subccc  @acc[2],-1,@acc[2]
 336         subccc  @acc[3],0,@acc[3]
 337         subccc  @acc[4],0,@acc[4]
 338         subccc  @acc[5],0,@acc[5]
 339         subccc  @acc[6],1,@acc[6]
 340         subccc  @acc[7],-1,@acc[7]
 341         subc    $carry,0,$carry         ! broadcast borrow bit
 342
 343         ! Note that because mod has special form, i.e. consists of
 344         ! 0xffffffff, 1 and 0s, we can conditionally synthesize it by
 345         ! using value of broadcasted borrow and the borrow bit itself.
 346         ! To minimize dependency chain we first broadcast and then
 347         ! extract the bit by negating (follow $bi).
 348
 349         addcc   @acc[0],$carry,@acc[0]  ! add modulus or zero
 350         addccc  @acc[1],$carry,@acc[1]
 351         neg     $carry,$bi
 352         st      @acc[0],[$rp]
 353         addccc  @acc[2],$carry,@acc[2]
 354         st      @acc[1],[$rp+4]
 355         addccc  @acc[3],0,@acc[3]
 356         st      @acc[2],[$rp+8]
 357         addccc  @acc[4],0,@acc[4]
 358         st      @acc[3],[$rp+12]
 359         addccc  @acc[5],0,@acc[5]
 360         st      @acc[4],[$rp+16]
 361         addccc  @acc[6],$bi,@acc[6]
 362         st      @acc[5],[$rp+20]
 363         addc    @acc[7],$carry,@acc[7]
 364         st      @acc[6],[$rp+24]
 365         retl
 366         st      @acc[7],[$rp+28]
 367 .type   __ecp_nistz256_mul_mont,#function
 368 .size   __ecp_nistz256_mul_mont,.-__ecp_nistz256_mul_mont
 369
 370 ! void  ecp_nistz256_add(BN_ULONG %i0[8],const BN_ULONG %i1[8],
 371 !                                        const BN_ULONG %i2[8]);
 372 .globl  ecp_nistz256_add
 373 .align  32
 374 ecp_nistz256_add:
 375         save    %sp,-STACK_FRAME,%sp
 376         ld      [$ap],@acc[0]
 377         ld      [$ap+4],@acc[1]
 378         ld      [$ap+8],@acc[2]
 379         ld      [$ap+12],@acc[3]
 380         ld      [$ap+16],@acc[4]
 381         ld      [$ap+20],@acc[5]
 382         ld      [$ap+24],@acc[6]
 383         call    __ecp_nistz256_add
 384         ld      [$ap+28],@acc[7]
 385         ret
 386         restore
 387 .type   ecp_nistz256_add,#function
 388 .size   ecp_nistz256_add,.-ecp_nistz256_add
 389
 390 .align  32
 391 __ecp_nistz256_add:
 392         ld      [$bp+0],$t0             ! b[0]
 393         ld      [$bp+4],$t1
 394         ld      [$bp+8],$t2
 395         ld      [$bp+12],$t3
 396         addcc   @acc[0],$t0,@acc[0]
 397         ld      [$bp+16],$t4
 398         ld      [$bp+20],$t5
 399         addccc  @acc[1],$t1,@acc[1]
 400         ld      [$bp+24],$t6
 401         ld      [$bp+28],$t7
 402         addccc  @acc[2],$t2,@acc[2]
 403         addccc  @acc[3],$t3,@acc[3]
 404         addccc  @acc[4],$t4,@acc[4]
 405         addccc  @acc[5],$t5,@acc[5]
 406         addccc  @acc[6],$t6,@acc[6]
 407         addccc  @acc[7],$t7,@acc[7]
 408         addc    %g0,%g0,$carry
 409
 410 .Lreduce_by_sub:
 411
 412         ! if a+b >= modulus, subtract modulus.
 413         !
 414         ! But since comparison implies subtraction, we subtract
 415         ! modulus and then add it back if subtraction borrowed.
 416
 417         subcc   @acc[0],-1,@acc[0]
 418         subccc  @acc[1],-1,@acc[1]
 419         subccc  @acc[2],-1,@acc[2]
 420         subccc  @acc[3], 0,@acc[3]
 421         subccc  @acc[4], 0,@acc[4]
 422         subccc  @acc[5], 0,@acc[5]
 423         subccc  @acc[6], 1,@acc[6]
 424         subccc  @acc[7],-1,@acc[7]
 425         subc    $carry,0,$carry
 426
 427         ! Note that because mod has special form, i.e. consists of
 428         ! 0xffffffff, 1 and 0s, we can conditionally synthesize it by
 429         ! using value of borrow and its negative.
 430
 431         addcc   @acc[0],$carry,@acc[0]  ! add synthesized modulus
 432         addccc  @acc[1],$carry,@acc[1]
 433         neg     $carry,$bi
 434         st      @acc[0],[$rp]
 435         addccc  @acc[2],$carry,@acc[2]
 436         st      @acc[1],[$rp+4]
 437         addccc  @acc[3],0,@acc[3]
 438         st      @acc[2],[$rp+8]
 439         addccc  @acc[4],0,@acc[4]
 440         st      @acc[3],[$rp+12]
 441         addccc  @acc[5],0,@acc[5]
 442         st      @acc[4],[$rp+16]
 443         addccc  @acc[6],$bi,@acc[6]
 444         st      @acc[5],[$rp+20]
 445         addc    @acc[7],$carry,@acc[7]
 446         st      @acc[6],[$rp+24]
 447         retl
 448         st      @acc[7],[$rp+28]
 449 .type   __ecp_nistz256_add,#function
 450 .size   __ecp_nistz256_add,.-__ecp_nistz256_add
 451
 452 ! void  ecp_nistz256_mul_by_2(BN_ULONG %i0[8],const BN_ULONG %i1[8]);
 453 .globl  ecp_nistz256_mul_by_2
 454 .align  32
 455 ecp_nistz256_mul_by_2:
 456         save    %sp,-STACK_FRAME,%sp
 457         ld      [$ap],@acc[0]
 458         ld      [$ap+4],@acc[1]
 459         ld      [$ap+8],@acc[2]
 460         ld      [$ap+12],@acc[3]
 461         ld      [$ap+16],@acc[4]
 462         ld      [$ap+20],@acc[5]
 463         ld      [$ap+24],@acc[6]
 464         call    __ecp_nistz256_mul_by_2
 465         ld      [$ap+28],@acc[7]
 466         ret
 467         restore
 468 .type   ecp_nistz256_mul_by_2,#function
 469 .size   ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2
 470
 471 .align  32
 472 __ecp_nistz256_mul_by_2:
 473         addcc   @acc[0],@acc[0],@acc[0] ! a+a=2*a
 474         addccc  @acc[1],@acc[1],@acc[1]
 475         addccc  @acc[2],@acc[2],@acc[2]
 476         addccc  @acc[3],@acc[3],@acc[3]
 477         addccc  @acc[4],@acc[4],@acc[4]
 478         addccc  @acc[5],@acc[5],@acc[5]
 479         addccc  @acc[6],@acc[6],@acc[6]
 480         addccc  @acc[7],@acc[7],@acc[7]
 481         b       .Lreduce_by_sub
 482         addc    %g0,%g0,$carry
 483 .type   __ecp_nistz256_mul_by_2,#function
 484 .size   __ecp_nistz256_mul_by_2,.-__ecp_nistz256_mul_by_2
 485
 486 ! void  ecp_nistz256_mul_by_3(BN_ULONG %i0[8],const BN_ULONG %i1[8]);
 487 .globl  ecp_nistz256_mul_by_3
 488 .align  32
 489 ecp_nistz256_mul_by_3:
 490         save    %sp,-STACK_FRAME,%sp
 491         ld      [$ap],@acc[0]
 492         ld      [$ap+4],@acc[1]
 493         ld      [$ap+8],@acc[2]
 494         ld      [$ap+12],@acc[3]
 495         ld      [$ap+16],@acc[4]
 496         ld      [$ap+20],@acc[5]
 497         ld      [$ap+24],@acc[6]
 498         call    __ecp_nistz256_mul_by_3
 499         ld      [$ap+28],@acc[7]
 500         ret
 501         restore
 502 .type   ecp_nistz256_mul_by_3,#function
 503 .size   ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3
 504
 505 .align  32
 506 __ecp_nistz256_mul_by_3:
 507         addcc   @acc[0],@acc[0],$t0     ! a+a=2*a
 508         addccc  @acc[1],@acc[1],$t1
 509         addccc  @acc[2],@acc[2],$t2
 510         addccc  @acc[3],@acc[3],$t3
 511         addccc  @acc[4],@acc[4],$t4
 512         addccc  @acc[5],@acc[5],$t5
 513         addccc  @acc[6],@acc[6],$t6
 514         addccc  @acc[7],@acc[7],$t7
 515         addc    %g0,%g0,$carry
 516
 517         subcc   $t0,-1,$t0              ! .Lreduce_by_sub but without stores
 518         subccc  $t1,-1,$t1
 519         subccc  $t2,-1,$t2
 520         subccc  $t3, 0,$t3
 521         subccc  $t4, 0,$t4
 522         subccc  $t5, 0,$t5
 523         subccc  $t6, 1,$t6
 524         subccc  $t7,-1,$t7
 525         subc    $carry,0,$carry
 526
 527         addcc   $t0,$carry,$t0          ! add synthesized modulus
 528         addccc  $t1,$carry,$t1
 529         neg     $carry,$bi
 530         addccc  $t2,$carry,$t2
 531         addccc  $t3,0,$t3
 532         addccc  $t4,0,$t4
 533         addccc  $t5,0,$t5
 534         addccc  $t6,$bi,$t6
 535         addc    $t7,$carry,$t7
 536
 537         addcc   $t0,@acc[0],@acc[0]     ! 2*a+a=3*a
 538         addccc  $t1,@acc[1],@acc[1]
 539         addccc  $t2,@acc[2],@acc[2]
 540         addccc  $t3,@acc[3],@acc[3]
 541         addccc  $t4,@acc[4],@acc[4]
 542         addccc  $t5,@acc[5],@acc[5]
 543         addccc  $t6,@acc[6],@acc[6]
 544         addccc  $t7,@acc[7],@acc[7]
 545         b       .Lreduce_by_sub
 546         addc    %g0,%g0,$carry
 547 .type   __ecp_nistz256_mul_by_3,#function
 548 .size   __ecp_nistz256_mul_by_3,.-__ecp_nistz256_mul_by_3
 549
 550 ! void  ecp_nistz256_sub(BN_ULONG %i0[8],const BN_ULONG %i1[8],
 551 !                                        const BN_ULONG %i2[8]);
 552 .globl  ecp_nistz256_sub
 553 .align  32
 554 ecp_nistz256_sub:
 555         save    %sp,-STACK_FRAME,%sp
 556         ld      [$ap],@acc[0]
 557         ld      [$ap+4],@acc[1]
 558         ld      [$ap+8],@acc[2]
 559         ld      [$ap+12],@acc[3]
 560         ld      [$ap+16],@acc[4]
 561         ld      [$ap+20],@acc[5]
 562         ld      [$ap+24],@acc[6]
 563         call    __ecp_nistz256_sub_from
 564         ld      [$ap+28],@acc[7]
 565         ret
 566         restore
 567 .type   ecp_nistz256_sub,#function
 568 .size   ecp_nistz256_sub,.-ecp_nistz256_sub
 569
 570 ! void  ecp_nistz256_neg(BN_ULONG %i0[8],const BN_ULONG %i1[8]);
 571 .globl  ecp_nistz256_neg
 572 .align  32
 573 ecp_nistz256_neg:
 574         save    %sp,-STACK_FRAME,%sp
 575         mov     $ap,$bp
 576         mov     0,@acc[0]
 577         mov     0,@acc[1]
 578         mov     0,@acc[2]
 579         mov     0,@acc[3]
 580         mov     0,@acc[4]
 581         mov     0,@acc[5]
 582         mov     0,@acc[6]
 583         call    __ecp_nistz256_sub_from
 584         mov     0,@acc[7]
 585         ret
 586         restore
 587 .type   ecp_nistz256_neg,#function
 588 .size   ecp_nistz256_neg,.-ecp_nistz256_neg
 589
 590 .align  32
 591 __ecp_nistz256_sub_from:
 592         ld      [$bp+0],$t0             ! b[0]
 593         ld      [$bp+4],$t1
 594         ld      [$bp+8],$t2
 595         ld      [$bp+12],$t3
 596         subcc   @acc[0],$t0,@acc[0]
 597         ld      [$bp+16],$t4
 598         ld      [$bp+20],$t5
 599         subccc  @acc[1],$t1,@acc[1]
 600         subccc  @acc[2],$t2,@acc[2]
 601         ld      [$bp+24],$t6
 602         ld      [$bp+28],$t7
 603         subccc  @acc[3],$t3,@acc[3]
 604         subccc  @acc[4],$t4,@acc[4]
 605         subccc  @acc[5],$t5,@acc[5]
 606         subccc  @acc[6],$t6,@acc[6]
 607         subccc  @acc[7],$t7,@acc[7]
 608         subc    %g0,%g0,$carry          ! broadcast borrow bit
 609
 610 .Lreduce_by_add:
 611
 612         ! if a-b borrows, add modulus.
 613         !
 614         ! Note that because mod has special form, i.e. consists of
 615         ! 0xffffffff, 1 and 0s, we can conditionally synthesize it by
 616         ! using value of broadcasted borrow and the borrow bit itself.
 617         ! To minimize dependency chain we first broadcast and then
 618         ! extract the bit by negating (follow $bi).
 619
 620         addcc   @acc[0],$carry,@acc[0]  ! add synthesized modulus
 621         addccc  @acc[1],$carry,@acc[1]
 622         neg     $carry,$bi
 623         st      @acc[0],[$rp]
 624         addccc  @acc[2],$carry,@acc[2]
 625         st      @acc[1],[$rp+4]
 626         addccc  @acc[3],0,@acc[3]
 627         st      @acc[2],[$rp+8]
 628         addccc  @acc[4],0,@acc[4]
 629         st      @acc[3],[$rp+12]
 630         addccc  @acc[5],0,@acc[5]
 631         st      @acc[4],[$rp+16]
 632         addccc  @acc[6],$bi,@acc[6]
 633         st      @acc[5],[$rp+20]
 634         addc    @acc[7],$carry,@acc[7]
 635         st      @acc[6],[$rp+24]
 636         retl
 637         st      @acc[7],[$rp+28]
 638 .type   __ecp_nistz256_sub_from,#function
 639 .size   __ecp_nistz256_sub_from,.-__ecp_nistz256_sub_from
 640
 641 .align  32
 642 __ecp_nistz256_sub_morf:
 643         ld      [$bp+0],$t0             ! b[0]
 644         ld      [$bp+4],$t1
 645         ld      [$bp+8],$t2
 646         ld      [$bp+12],$t3
 647         subcc   $t0,@acc[0],@acc[0]
 648         ld      [$bp+16],$t4
 649         ld      [$bp+20],$t5
 650         subccc  $t1,@acc[1],@acc[1]
 651         subccc  $t2,@acc[2],@acc[2]
 652         ld      [$bp+24],$t6
 653         ld      [$bp+28],$t7
 654         subccc  $t3,@acc[3],@acc[3]
 655         subccc  $t4,@acc[4],@acc[4]
 656         subccc  $t5,@acc[5],@acc[5]
 657         subccc  $t6,@acc[6],@acc[6]
 658         subccc  $t7,@acc[7],@acc[7]
 659         b       .Lreduce_by_add
 660         subc    %g0,%g0,$carry          ! broadcast borrow bit
 661 .type   __ecp_nistz256_sub_morf,#function
 662 .size   __ecp_nistz256_sub_morf,.-__ecp_nistz256_sub_morf
 663
 664 ! void  ecp_nistz256_div_by_2(BN_ULONG %i0[8],const BN_ULONG %i1[8]);
 665 .globl  ecp_nistz256_div_by_2
 666 .align  32
 667 ecp_nistz256_div_by_2:
 668         save    %sp,-STACK_FRAME,%sp
 669         ld      [$ap],@acc[0]
 670         ld      [$ap+4],@acc[1]
 671         ld      [$ap+8],@acc[2]
 672         ld      [$ap+12],@acc[3]
 673         ld      [$ap+16],@acc[4]
 674         ld      [$ap+20],@acc[5]
 675         ld      [$ap+24],@acc[6]
 676         call    __ecp_nistz256_div_by_2
 677         ld      [$ap+28],@acc[7]
 678         ret
 679         restore
 680 .type   ecp_nistz256_div_by_2,#function
 681 .size   ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2
 682
 683 .align  32
 684 __ecp_nistz256_div_by_2:
 685         ! ret = (a is odd ? a+mod : a) >> 1
 686
 687         and     @acc[0],1,$bi
 688         neg     $bi,$carry
 689         addcc   @acc[0],$carry,@acc[0]
 690         addccc  @acc[1],$carry,@acc[1]
 691         addccc  @acc[2],$carry,@acc[2]
 692         addccc  @acc[3],0,@acc[3]
 693         addccc  @acc[4],0,@acc[4]
 694         addccc  @acc[5],0,@acc[5]
 695         addccc  @acc[6],$bi,@acc[6]
 696         addccc  @acc[7],$carry,@acc[7]
 697         addc    %g0,%g0,$carry
 698
 699         ! ret >>= 1
 700
 701         srl     @acc[0],1,@acc[0]
 702         sll     @acc[1],31,$t0
 703         srl     @acc[1],1,@acc[1]
 704         or      @acc[0],$t0,@acc[0]
 705         sll     @acc[2],31,$t1
 706         srl     @acc[2],1,@acc[2]
 707         or      @acc[1],$t1,@acc[1]
 708         sll     @acc[3],31,$t2
 709         st      @acc[0],[$rp]
 710         srl     @acc[3],1,@acc[3]
 711         or      @acc[2],$t2,@acc[2]
 712         sll     @acc[4],31,$t3
 713         st      @acc[1],[$rp+4]
 714         srl     @acc[4],1,@acc[4]
 715         or      @acc[3],$t3,@acc[3]
 716         sll     @acc[5],31,$t4
 717         st      @acc[2],[$rp+8]
 718         srl     @acc[5],1,@acc[5]
 719         or      @acc[4],$t4,@acc[4]
 720         sll     @acc[6],31,$t5
 721         st      @acc[3],[$rp+12]
 722         srl     @acc[6],1,@acc[6]
 723         or      @acc[5],$t5,@acc[5]
 724         sll     @acc[7],31,$t6
 725         st      @acc[4],[$rp+16]
 726         srl     @acc[7],1,@acc[7]
 727         or      @acc[6],$t6,@acc[6]
 728         sll     $carry,31,$t7
 729         st      @acc[5],[$rp+20]
 730         or      @acc[7],$t7,@acc[7]
 731         st      @acc[6],[$rp+24]
 732         retl
 733         st      @acc[7],[$rp+28]
 734 .type   __ecp_nistz256_div_by_2,#function
 735 .size   __ecp_nistz256_div_by_2,.-__ecp_nistz256_div_by_2
 736 ___
 737
 738 ########################################################################
 739 # following subroutines are "literal" implementation of those found in
 740 # ecp_nistz256.c
 741 #
 742 ########################################################################
 743 # void ecp_nistz256_point_double(P256_POINT *out,const P256_POINT *inp);
 744 #
 745 {
 746 my ($S,$M,$Zsqr,$tmp0)=map(32*$_,(0..3));
 747 # above map() describes stack layout with 4 temporary
 748 # 256-bit vectors on top.
 749
 750 $code.=<<___;
 751 #ifdef __PIC__
 752 SPARC_PIC_THUNK(%g1)
 753 #endif
 754
 755 .globl  ecp_nistz256_point_double
 756 .align  32
 757 ecp_nistz256_point_double:
 758         SPARC_LOAD_ADDRESS_LEAF(OPENSSL_sparcv9cap_P,%g1,%g5)
 759         ld      [%g1],%g1               ! OPENSSL_sparcv9cap_P[0]
 760         and     %g1,(SPARCV9_VIS3|SPARCV9_64BIT_STACK),%g1
 761         cmp     %g1,(SPARCV9_VIS3|SPARCV9_64BIT_STACK)
 762         be      ecp_nistz256_point_double_vis3
 763         nop
 764
 765         save    %sp,-STACK_FRAME-32*4,%sp
 766
 767         mov     $rp,$rp_real
 768         mov     $ap,$ap_real
 769
 770 .Lpoint_double_shortcut:
 771         ld      [$ap+32],@acc[0]
 772         ld      [$ap+32+4],@acc[1]
 773         ld      [$ap+32+8],@acc[2]
 774         ld      [$ap+32+12],@acc[3]
 775         ld      [$ap+32+16],@acc[4]
 776         ld      [$ap+32+20],@acc[5]
 777         ld      [$ap+32+24],@acc[6]
 778         ld      [$ap+32+28],@acc[7]
 779         call    __ecp_nistz256_mul_by_2 ! p256_mul_by_2(S, in_y);
 780         add     %sp,LOCALS+$S,$rp
 781
 782         add     $ap_real,64,$bp
 783         add     $ap_real,64,$ap
 784         call    __ecp_nistz256_mul_mont ! p256_sqr_mont(Zsqr, in_z);
 785         add     %sp,LOCALS+$Zsqr,$rp
 786
 787         add     $ap_real,0,$bp
 788         call    __ecp_nistz256_add      ! p256_add(M, Zsqr, in_x);
 789         add     %sp,LOCALS+$M,$rp
 790
 791         add     %sp,LOCALS+$S,$bp
 792         add     %sp,LOCALS+$S,$ap
 793         call    __ecp_nistz256_mul_mont ! p256_sqr_mont(S, S);
 794         add     %sp,LOCALS+$S,$rp
 795
 796         ld      [$ap_real],@acc[0]
 797         add     %sp,LOCALS+$Zsqr,$bp
 798         ld      [$ap_real+4],@acc[1]
 799         ld      [$ap_real+8],@acc[2]
 800         ld      [$ap_real+12],@acc[3]
 801         ld      [$ap_real+16],@acc[4]
 802         ld      [$ap_real+20],@acc[5]
 803         ld      [$ap_real+24],@acc[6]
 804         ld      [$ap_real+28],@acc[7]
 805         call    __ecp_nistz256_sub_from ! p256_sub(Zsqr, in_x, Zsqr);
 806         add     %sp,LOCALS+$Zsqr,$rp
 807
 808         add     $ap_real,32,$bp
 809         add     $ap_real,64,$ap
 810         call    __ecp_nistz256_mul_mont ! p256_mul_mont(tmp0, in_z, in_y);
 811         add     %sp,LOCALS+$tmp0,$rp
 812
 813         call    __ecp_nistz256_mul_by_2 ! p256_mul_by_2(res_z, tmp0);
 814         add     $rp_real,64,$rp
 815
 816         add     %sp,LOCALS+$Zsqr,$bp
 817         add     %sp,LOCALS+$M,$ap
 818         call    __ecp_nistz256_mul_mont ! p256_mul_mont(M, M, Zsqr);
 819         add     %sp,LOCALS+$M,$rp
 820
 821         call    __ecp_nistz256_mul_by_3 ! p256_mul_by_3(M, M);
 822         add     %sp,LOCALS+$M,$rp
 823
 824         add     %sp,LOCALS+$S,$bp
 825         add     %sp,LOCALS+$S,$ap
 826         call    __ecp_nistz256_mul_mont ! p256_sqr_mont(tmp0, S);
 827         add     %sp,LOCALS+$tmp0,$rp
 828
 829         call    __ecp_nistz256_div_by_2 ! p256_div_by_2(res_y, tmp0);
 830         add     $rp_real,32,$rp
 831
 832         add     $ap_real,0,$bp
 833         add     %sp,LOCALS+$S,$ap
 834         call    __ecp_nistz256_mul_mont ! p256_mul_mont(S, S, in_x);
 835         add     %sp,LOCALS+$S,$rp
 836
 837         call    __ecp_nistz256_mul_by_2 ! p256_mul_by_2(tmp0, S);
 838         add     %sp,LOCALS+$tmp0,$rp
 839
 840         add     %sp,LOCALS+$M,$bp
 841         add     %sp,LOCALS+$M,$ap
 842         call    __ecp_nistz256_mul_mont ! p256_sqr_mont(res_x, M);
 843         add     $rp_real,0,$rp
 844
 845         add     %sp,LOCALS+$tmp0,$bp
 846         call    __ecp_nistz256_sub_from ! p256_sub(res_x, res_x, tmp0);
 847         add     $rp_real,0,$rp
 848
 849         add     %sp,LOCALS+$S,$bp
 850         call    __ecp_nistz256_sub_morf ! p256_sub(S, S, res_x);
 851         add     %sp,LOCALS+$S,$rp
 852
 853         add     %sp,LOCALS+$M,$bp
 854         add     %sp,LOCALS+$S,$ap
 855         call    __ecp_nistz256_mul_mont ! p256_mul_mont(S, S, M);
 856         add     %sp,LOCALS+$S,$rp
 857
 858         add     $rp_real,32,$bp
 859         call    __ecp_nistz256_sub_from ! p256_sub(res_y, S, res_y);
 860         add     $rp_real,32,$rp
 861
 862         ret
 863         restore
 864 .type   ecp_nistz256_point_double,#function
 865 .size   ecp_nistz256_point_double,.-ecp_nistz256_point_double
 866 ___
 867 }
 868
 869 ########################################################################
 870 # void ecp_nistz256_point_add(P256_POINT *out,const P256_POINT *in1,
 871 #                             const P256_POINT *in2);
 872 {
 873 my ($res_x,$res_y,$res_z,
 874     $H,$Hsqr,$R,$Rsqr,$Hcub,
 875     $U1,$U2,$S1,$S2)=map(32*$_,(0..11));
 876 my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
 877
 878 # above map() describes stack layout with 12 temporary
 879 # 256-bit vectors on top. Then we reserve some space for
 880 # !in1infty, !in2infty, result of check for zero and return pointer.
 881
 882 my $bp_real=$rp_real;
 883
 884 $code.=<<___;
 885 .globl  ecp_nistz256_point_add
 886 .align  32
 887 ecp_nistz256_point_add:
 888         SPARC_LOAD_ADDRESS_LEAF(OPENSSL_sparcv9cap_P,%g1,%g5)
 889         ld      [%g1],%g1               ! OPENSSL_sparcv9cap_P[0]
 890         and     %g1,(SPARCV9_VIS3|SPARCV9_64BIT_STACK),%g1
 891         cmp     %g1,(SPARCV9_VIS3|SPARCV9_64BIT_STACK)
 892         be      ecp_nistz256_point_add_vis3
 893         nop
 894
 895         save    %sp,-STACK_FRAME-32*12-32,%sp
 896
 897         stx     $rp,[%fp+STACK_BIAS-8]  ! off-load $rp
 898         mov     $ap,$ap_real
 899         mov     $bp,$bp_real
 900
 901         ld      [$bp+64],$t0            ! in2_z
 902         ld      [$bp+64+4],$t1
 903         ld      [$bp+64+8],$t2
 904         ld      [$bp+64+12],$t3
 905         ld      [$bp+64+16],$t4
 906         ld      [$bp+64+20],$t5
 907         ld      [$bp+64+24],$t6
 908         ld      [$bp+64+28],$t7
 909         or      $t1,$t0,$t0
 910         or      $t3,$t2,$t2
 911         or      $t5,$t4,$t4
 912         or      $t7,$t6,$t6
 913         or      $t2,$t0,$t0
 914         or      $t6,$t4,$t4
 915         or      $t4,$t0,$t0             ! !in2infty
 916         movrnz  $t0,-1,$t0
 917         st      $t0,[%fp+STACK_BIAS-12]
 918
 919         ld      [$ap+64],$t0            ! in1_z
 920         ld      [$ap+64+4],$t1
 921         ld      [$ap+64+8],$t2
 922         ld      [$ap+64+12],$t3
 923         ld      [$ap+64+16],$t4
 924         ld      [$ap+64+20],$t5
 925         ld      [$ap+64+24],$t6
 926         ld      [$ap+64+28],$t7
 927         or      $t1,$t0,$t0
 928         or      $t3,$t2,$t2
 929         or      $t5,$t4,$t4
 930         or      $t7,$t6,$t6
 931         or      $t2,$t0,$t0
 932         or      $t6,$t4,$t4
 933         or      $t4,$t0,$t0             ! !in1infty
 934         movrnz  $t0,-1,$t0
 935         st      $t0,[%fp+STACK_BIAS-16]
 936
 937         add     $bp_real,64,$bp
 938         add     $bp_real,64,$ap
 939         call    __ecp_nistz256_mul_mont ! p256_sqr_mont(Z2sqr, in2_z);
 940         add     %sp,LOCALS+$Z2sqr,$rp
 941
 942         add     $ap_real,64,$bp
 943         add     $ap_real,64,$ap
 944         call    __ecp_nistz256_mul_mont ! p256_sqr_mont(Z1sqr, in1_z);
 945         add     %sp,LOCALS+$Z1sqr,$rp
 946
 947         add     $bp_real,64,$bp
 948         add     %sp,LOCALS+$Z2sqr,$ap
 949         call    __ecp_nistz256_mul_mont ! p256_mul_mont(S1, Z2sqr, in2_z);
 950         add     %sp,LOCALS+$S1,$rp
 951
 952         add     $ap_real,64,$bp
 953         add     %sp,LOCALS+$Z1sqr,$ap
 954         call    __ecp_nistz256_mul_mont ! p256_mul_mont(S2, Z1sqr, in1_z);
 955         add     %sp,LOCALS+$S2,$rp
 956
 957         add     $ap_real,32,$bp
 958         add     %sp,LOCALS+$S1,$ap
 959         call    __ecp_nistz256_mul_mont ! p256_mul_mont(S1, S1, in1_y);
 960         add     %sp,LOCALS+$S1,$rp
 961
 962         add     $bp_real,32,$bp
 963         add     %sp,LOCALS+$S2,$ap
 964         call    __ecp_nistz256_mul_mont ! p256_mul_mont(S2, S2, in2_y);
 965         add     %sp,LOCALS+$S2,$rp
 966
 967         add     %sp,LOCALS+$S1,$bp
 968         call    __ecp_nistz256_sub_from ! p256_sub(R, S2, S1);
 969         add     %sp,LOCALS+$R,$rp
 970
 971         or      @acc[1],@acc[0],@acc[0] ! see if result is zero
 972         or      @acc[3],@acc[2],@acc[2]
 973         or      @acc[5],@acc[4],@acc[4]
 974         or      @acc[7],@acc[6],@acc[6]
 975         or      @acc[2],@acc[0],@acc[0]
 976         or      @acc[6],@acc[4],@acc[4]
 977         or      @acc[4],@acc[0],@acc[0]
 978         st      @acc[0],[%fp+STACK_BIAS-20]
 979
 980         add     $ap_real,0,$bp
 981         add     %sp,LOCALS+$Z2sqr,$ap
 982         call    __ecp_nistz256_mul_mont ! p256_mul_mont(U1, in1_x, Z2sqr);
 983         add     %sp,LOCALS+$U1,$rp
 984
 985         add     $bp_real,0,$bp
 986         add     %sp,LOCALS+$Z1sqr,$ap
 987         call    __ecp_nistz256_mul_mont ! p256_mul_mont(U2, in2_x, Z1sqr);
 988         add     %sp,LOCALS+$U2,$rp
 989
 990         add     %sp,LOCALS+$U1,$bp
 991         call    __ecp_nistz256_sub_from ! p256_sub(H, U2, U1);
 992         add     %sp,LOCALS+$H,$rp
 993
 994         or      @acc[1],@acc[0],@acc[0] ! see if result is zero
 995         or      @acc[3],@acc[2],@acc[2]
 996         or      @acc[5],@acc[4],@acc[4]
 997         or      @acc[7],@acc[6],@acc[6]
 998         or      @acc[2],@acc[0],@acc[0]
 999         or      @acc[6],@acc[4],@acc[4]
1000         orcc    @acc[4],@acc[0],@acc[0]
1001
1002         bne,pt  %icc,.Ladd_proceed      ! is_equal(U1,U2)?
1003         nop
1004
1005         ld      [%fp+STACK_BIAS-12],$t0
1006         ld      [%fp+STACK_BIAS-16],$t1
1007         ld      [%fp+STACK_BIAS-20],$t2
1008         andcc   $t0,$t1,%g0
1009         be,pt   %icc,.Ladd_proceed      ! (in1infty || in2infty)?
1010         nop
1011         andcc   $t2,$t2,%g0
1012         be,pt   %icc,.Ladd_double       ! is_equal(S1,S2)?
1013         nop
1014
1015         ldx     [%fp+STACK_BIAS-8],$rp
1016         st      %g0,[$rp]
1017         st      %g0,[$rp+4]
1018         st      %g0,[$rp+8]
1019         st      %g0,[$rp+12]
1020         st      %g0,[$rp+16]
1021         st      %g0,[$rp+20]
1022         st      %g0,[$rp+24]
1023         st      %g0,[$rp+28]
1024         st      %g0,[$rp+32]
1025         st      %g0,[$rp+32+4]
1026         st      %g0,[$rp+32+8]
1027         st      %g0,[$rp+32+12]
1028         st      %g0,[$rp+32+16]
1029         st      %g0,[$rp+32+20]
1030         st      %g0,[$rp+32+24]
1031         st      %g0,[$rp+32+28]
1032         st      %g0,[$rp+64]
1033         st      %g0,[$rp+64+4]
1034         st      %g0,[$rp+64+8]
1035         st      %g0,[$rp+64+12]
1036         st      %g0,[$rp+64+16]
1037         st      %g0,[$rp+64+20]
1038         st      %g0,[$rp+64+24]
1039         st      %g0,[$rp+64+28]
1040         b       .Ladd_done
1041         nop
1042
1043 .align  16
1044 .Ladd_double:
1045         ldx     [%fp+STACK_BIAS-8],$rp_real
1046         mov     $ap_real,$ap
1047         b       .Lpoint_double_shortcut
1048         add     %sp,32*(12-4)+32,%sp    ! difference in frame sizes
1049
1050 .align  16
1051 .Ladd_proceed:
1052         add     %sp,LOCALS+$R,$bp
1053         add     %sp,LOCALS+$R,$ap
1054         call    __ecp_nistz256_mul_mont ! p256_sqr_mont(Rsqr, R);
1055         add     %sp,LOCALS+$Rsqr,$rp
1056
1057         add     $ap_real,64,$bp
1058         add     %sp,LOCALS+$H,$ap
1059         call    __ecp_nistz256_mul_mont ! p256_mul_mont(res_z, H, in1_z);
1060         add     %sp,LOCALS+$res_z,$rp
1061
1062         add     %sp,LOCALS+$H,$bp
1063         add     %sp,LOCALS+$H,$ap
1064         call    __ecp_nistz256_mul_mont ! p256_sqr_mont(Hsqr, H);
1065         add     %sp,LOCALS+$Hsqr,$rp
1066
1067         add     $bp_real,64,$bp
1068         add     %sp,LOCALS+$res_z,$ap
1069         call    __ecp_nistz256_mul_mont ! p256_mul_mont(res_z, res_z, in2_z);
1070         add     %sp,LOCALS+$res_z,$rp
1071
1072         add     %sp,LOCALS+$H,$bp
1073         add     %sp,LOCALS+$Hsqr,$ap
1074         call    __ecp_nistz256_mul_mont ! p256_mul_mont(Hcub, Hsqr, H);
1075         add     %sp,LOCALS+$Hcub,$rp
1076
1077         add     %sp,LOCALS+$U1,$bp
1078         add     %sp,LOCALS+$Hsqr,$ap
1079         call    __ecp_nistz256_mul_mont ! p256_mul_mont(U2, U1, Hsqr);
1080         add     %sp,LOCALS+$U2,$rp
1081
1082         call    __ecp_nistz256_mul_by_2 ! p256_mul_by_2(Hsqr, U2);
1083         add     %sp,LOCALS+$Hsqr,$rp
1084
1085         add     %sp,LOCALS+$Rsqr,$bp
1086         call    __ecp_nistz256_sub_morf ! p256_sub(res_x, Rsqr, Hsqr);
1087         add     %sp,LOCALS+$res_x,$rp
1088
1089         add     %sp,LOCALS+$Hcub,$bp
1090         call    __ecp_nistz256_sub_from !  p256_sub(res_x, res_x, Hcub);
1091         add     %sp,LOCALS+$res_x,$rp
1092
1093         add     %sp,LOCALS+$U2,$bp
1094         call    __ecp_nistz256_sub_morf ! p256_sub(res_y, U2, res_x);
1095         add     %sp,LOCALS+$res_y,$rp
1096
1097         add     %sp,LOCALS+$Hcub,$bp
1098         add     %sp,LOCALS+$S1,$ap
1099         call    __ecp_nistz256_mul_mont ! p256_mul_mont(S2, S1, Hcub);
1100         add     %sp,LOCALS+$S2,$rp
1101
1102         add     %sp,LOCALS+$R,$bp
1103         add     %sp,LOCALS+$res_y,$ap
1104         call    __ecp_nistz256_mul_mont ! p256_mul_mont(res_y, res_y, R);
1105         add     %sp,LOCALS+$res_y,$rp
1106
1107         add     %sp,LOCALS+$S2,$bp
1108         call    __ecp_nistz256_sub_from ! p256_sub(res_y, res_y, S2);
1109         add     %sp,LOCALS+$res_y,$rp
1110
1111         ld      [%fp+STACK_BIAS-16],$t1 ! !in1infty
1112         ld      [%fp+STACK_BIAS-12],$t2 ! !in2infty
1113         ldx     [%fp+STACK_BIAS-8],$rp
1114 ___
1115 for($i=0;$i<96;$i+=8) {                 # conditional moves
1116 $code.=<<___;
1117         ld      [%sp+LOCALS+$i],@acc[0]         ! res
1118         ld      [%sp+LOCALS+$i+4],@acc[1]
1119         ld      [$bp_real+$i],@acc[2]           ! in2
1120         ld      [$bp_real+$i+4],@acc[3]
1121         ld      [$ap_real+$i],@acc[4]           ! in1
1122         ld      [$ap_real+$i+4],@acc[5]
1123         movrz   $t1,@acc[2],@acc[0]
1124         movrz   $t1,@acc[3],@acc[1]
1125         movrz   $t2,@acc[4],@acc[0]
1126         movrz   $t2,@acc[5],@acc[1]
1127         st      @acc[0],[$rp+$i]
1128         st      @acc[1],[$rp+$i+4]
1129 ___
1130 }
1131 $code.=<<___;
1132 .Ladd_done:
1133         ret
1134         restore
1135 .type   ecp_nistz256_point_add,#function
1136 .size   ecp_nistz256_point_add,.-ecp_nistz256_point_add
1137 ___
1138 }
1139
1140 ########################################################################
1141 # void ecp_nistz256_point_add_affine(P256_POINT *out,const P256_POINT *in1,
1142 #                                    const P256_POINT_AFFINE *in2);
1143 {
1144 my ($res_x,$res_y,$res_z,
1145     $U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr)=map(32*$_,(0..9));
1146 my $Z1sqr = $S2;
1147 # above map() describes stack layout with 10 temporary
1148 # 256-bit vectors on top. Then we reserve some space for
1149 # !in1infty, !in2infty, result of check for zero and return pointer.
1150
1151 my @ONE_mont=(1,0,0,-1,-1,-1,-2,0);
1152 my $bp_real=$rp_real;
1153
1154 $code.=<<___;
1155 .globl  ecp_nistz256_point_add_affine
1156 .align  32
1157 ecp_nistz256_point_add_affine:
1158         SPARC_LOAD_ADDRESS_LEAF(OPENSSL_sparcv9cap_P,%g1,%g5)
1159         ld      [%g1],%g1               ! OPENSSL_sparcv9cap_P[0]
1160         and     %g1,(SPARCV9_VIS3|SPARCV9_64BIT_STACK),%g1
1161         cmp     %g1,(SPARCV9_VIS3|SPARCV9_64BIT_STACK)
1162         be      ecp_nistz256_point_add_affine_vis3
1163         nop
1164
1165         save    %sp,-STACK_FRAME-32*10-32,%sp
1166
1167         stx     $rp,[%fp+STACK_BIAS-8]  ! off-load $rp
1168         mov     $ap,$ap_real
1169         mov     $bp,$bp_real
1170
1171         ld      [$ap+64],$t0            ! in1_z
1172         ld      [$ap+64+4],$t1
1173         ld      [$ap+64+8],$t2
1174         ld      [$ap+64+12],$t3
1175         ld      [$ap+64+16],$t4
1176         ld      [$ap+64+20],$t5
1177         ld      [$ap+64+24],$t6
1178         ld      [$ap+64+28],$t7
1179         or      $t1,$t0,$t0
1180         or      $t3,$t2,$t2
1181         or      $t5,$t4,$t4
1182         or      $t7,$t6,$t6
1183         or      $t2,$t0,$t0
1184         or      $t6,$t4,$t4
1185         or      $t4,$t0,$t0             ! !in1infty
1186         movrnz  $t0,-1,$t0
1187         st      $t0,[%fp+STACK_BIAS-16]
1188
1189         ld      [$bp],@acc[0]           ! in2_x
1190         ld      [$bp+4],@acc[1]
1191         ld      [$bp+8],@acc[2]
1192         ld      [$bp+12],@acc[3]
1193         ld      [$bp+16],@acc[4]
1194         ld      [$bp+20],@acc[5]
1195         ld      [$bp+24],@acc[6]
1196         ld      [$bp+28],@acc[7]
1197         ld      [$bp+32],$t0            ! in2_y
1198         ld      [$bp+32+4],$t1
1199         ld      [$bp+32+8],$t2
1200         ld      [$bp+32+12],$t3
1201         ld      [$bp+32+16],$t4
1202         ld      [$bp+32+20],$t5
1203         ld      [$bp+32+24],$t6
1204         ld      [$bp+32+28],$t7
1205         or      @acc[1],@acc[0],@acc[0]
1206         or      @acc[3],@acc[2],@acc[2]
1207         or      @acc[5],@acc[4],@acc[4]
1208         or      @acc[7],@acc[6],@acc[6]
1209         or      @acc[2],@acc[0],@acc[0]
1210         or      @acc[6],@acc[4],@acc[4]
1211         or      @acc[4],@acc[0],@acc[0]
1212         or      $t1,$t0,$t0
1213         or      $t3,$t2,$t2
1214         or      $t5,$t4,$t4
1215         or      $t7,$t6,$t6
1216         or      $t2,$t0,$t0
1217         or      $t6,$t4,$t4
1218         or      $t4,$t0,$t0
1219         or      @acc[0],$t0,$t0         ! !in2infty
1220         movrnz  $t0,-1,$t0
1221         st      $t0,[%fp+STACK_BIAS-12]
1222
1223         add     $ap_real,64,$bp
1224         add     $ap_real,64,$ap
1225         call    __ecp_nistz256_mul_mont ! p256_sqr_mont(Z1sqr, in1_z);
1226         add     %sp,LOCALS+$Z1sqr,$rp
1227
1228         add     $bp_real,0,$bp
1229         add     %sp,LOCALS+$Z1sqr,$ap
1230         call    __ecp_nistz256_mul_mont ! p256_mul_mont(U2, Z1sqr, in2_x);
1231         add     %sp,LOCALS+$U2,$rp
1232
1233         add     $ap_real,0,$bp
1234         call    __ecp_nistz256_sub_from ! p256_sub(H, U2, in1_x);
1235         add     %sp,LOCALS+$H,$rp
1236
1237         add     $ap_real,64,$bp
1238         add     %sp,LOCALS+$Z1sqr,$ap
1239         call    __ecp_nistz256_mul_mont ! p256_mul_mont(S2, Z1sqr, in1_z);
1240         add     %sp,LOCALS+$S2,$rp
1241
1242         add     $ap_real,64,$bp
1243         add     %sp,LOCALS+$H,$ap
1244         call    __ecp_nistz256_mul_mont ! p256_mul_mont(res_z, H, in1_z);
1245         add     %sp,LOCALS+$res_z,$rp
1246
1247         add     $bp_real,32,$bp
1248         add     %sp,LOCALS+$S2,$ap
1249         call    __ecp_nistz256_mul_mont ! p256_mul_mont(S2, S2, in2_y);
1250         add     %sp,LOCALS+$S2,$rp
1251
1252         add     $ap_real,32,$bp
1253         call    __ecp_nistz256_sub_from ! p256_sub(R, S2, in1_y);
1254         add     %sp,LOCALS+$R,$rp
1255
1256         add     %sp,LOCALS+$H,$bp
1257         add     %sp,LOCALS+$H,$ap
1258         call    __ecp_nistz256_mul_mont ! p256_sqr_mont(Hsqr, H);
1259         add     %sp,LOCALS+$Hsqr,$rp
1260
1261         add     %sp,LOCALS+$R,$bp
1262         add     %sp,LOCALS+$R,$ap
1263         call    __ecp_nistz256_mul_mont ! p256_sqr_mont(Rsqr, R);
1264         add     %sp,LOCALS+$Rsqr,$rp
1265
1266         add     %sp,LOCALS+$H,$bp
1267         add     %sp,LOCALS+$Hsqr,$ap
1268         call    __ecp_nistz256_mul_mont ! p256_mul_mont(Hcub, Hsqr, H);
1269         add     %sp,LOCALS+$Hcub,$rp
1270
1271         add     $ap_real,0,$bp
1272         add     %sp,LOCALS+$Hsqr,$ap
1273         call    __ecp_nistz256_mul_mont ! p256_mul_mont(U2, in1_x, Hsqr);
1274         add     %sp,LOCALS+$U2,$rp
1275
1276         call    __ecp_nistz256_mul_by_2 ! p256_mul_by_2(Hsqr, U2);
1277         add     %sp,LOCALS+$Hsqr,$rp
1278
1279         add     %sp,LOCALS+$Rsqr,$bp
1280         call    __ecp_nistz256_sub_morf ! p256_sub(res_x, Rsqr, Hsqr);
1281         add     %sp,LOCALS+$res_x,$rp
1282
1283         add     %sp,LOCALS+$Hcub,$bp
1284         call    __ecp_nistz256_sub_from !  p256_sub(res_x, res_x, Hcub);
1285         add     %sp,LOCALS+$res_x,$rp
1286
1287         add     %sp,LOCALS+$U2,$bp
1288         call    __ecp_nistz256_sub_morf ! p256_sub(res_y, U2, res_x);
1289         add     %sp,LOCALS+$res_y,$rp
1290
1291         add     $ap_real,32,$bp
1292         add     %sp,LOCALS+$Hcub,$ap
1293         call    __ecp_nistz256_mul_mont ! p256_mul_mont(S2, in1_y, Hcub);
1294         add     %sp,LOCALS+$S2,$rp
1295
1296         add     %sp,LOCALS+$R,$bp
1297         add     %sp,LOCALS+$res_y,$ap
1298         call    __ecp_nistz256_mul_mont ! p256_mul_mont(res_y, res_y, R);
1299         add     %sp,LOCALS+$res_y,$rp
1300
1301         add     %sp,LOCALS+$S2,$bp
1302         call    __ecp_nistz256_sub_from ! p256_sub(res_y, res_y, S2);
1303         add     %sp,LOCALS+$res_y,$rp
1304
1305         ld      [%fp+STACK_BIAS-16],$t1 ! !in1infty
1306         ld      [%fp+STACK_BIAS-12],$t2 ! !in2infty
1307         ldx     [%fp+STACK_BIAS-8],$rp
1308 ___
1309 for($i=0;$i<64;$i+=8) {                 # conditional moves
1310 $code.=<<___;
1311         ld      [%sp+LOCALS+$i],@acc[0]         ! res
1312         ld      [%sp+LOCALS+$i+4],@acc[1]
1313         ld      [$bp_real+$i],@acc[2]           ! in2
1314         ld      [$bp_real+$i+4],@acc[3]
1315         ld      [$ap_real+$i],@acc[4]           ! in1
1316         ld      [$ap_real+$i+4],@acc[5]
1317         movrz   $t1,@acc[2],@acc[0]
1318         movrz   $t1,@acc[3],@acc[1]
1319         movrz   $t2,@acc[4],@acc[0]
1320         movrz   $t2,@acc[5],@acc[1]
1321         st      @acc[0],[$rp+$i]
1322         st      @acc[1],[$rp+$i+4]
1323 ___
1324 }
1325 for(;$i<96;$i+=8) {
1326 my $j=($i-64)/4;
1327 $code.=<<___;
1328         ld      [%sp+LOCALS+$i],@acc[0]         ! res
1329         ld      [%sp+LOCALS+$i+4],@acc[1]
1330         ld      [$ap_real+$i],@acc[4]           ! in1
1331         ld      [$ap_real+$i+4],@acc[5]
1332         movrz   $t1,@ONE_mont[$j],@acc[0]
1333         movrz   $t1,@ONE_mont[$j+1],@acc[1]
1334         movrz   $t2,@acc[4],@acc[0]
1335         movrz   $t2,@acc[5],@acc[1]
1336         st      @acc[0],[$rp+$i]
1337         st      @acc[1],[$rp+$i+4]
1338 ___
1339 }
1340 $code.=<<___;
1341         ret
1342         restore
1343 .type   ecp_nistz256_point_add_affine,#function
1344 .size   ecp_nistz256_point_add_affine,.-ecp_nistz256_point_add_affine
1345 ___
1346 }                                                               }}}
1347 {{{
1348 my ($out,$inp,$index)=map("%i$_",(0..2));
1349 my $mask="%o0";
1350
1351 $code.=<<___;
1352 ! void  ecp_nistz256_scatter_w5(void *%i0,const P256_POINT *%i1,
1353 !                                         int %i2);
1354 .globl  ecp_nistz256_scatter_w5
1355 .align  32
1356 ecp_nistz256_scatter_w5:
1357         save    %sp,-STACK_FRAME,%sp
1358
1359         sll     $index,2,$index
1360         add     $out,$index,$out
1361
1362         ld      [$inp],%l0              ! X
1363         ld      [$inp+4],%l1
1364         ld      [$inp+8],%l2
1365         ld      [$inp+12],%l3
1366         ld      [$inp+16],%l4
1367         ld      [$inp+20],%l5
1368         ld      [$inp+24],%l6
1369         ld      [$inp+28],%l7
1370         add     $inp,32,$inp
1371         st      %l0,[$out+64*0-4]
1372         st      %l1,[$out+64*1-4]
1373         st      %l2,[$out+64*2-4]
1374         st      %l3,[$out+64*3-4]
1375         st      %l4,[$out+64*4-4]
1376         st      %l5,[$out+64*5-4]
1377         st      %l6,[$out+64*6-4]
1378         st      %l7,[$out+64*7-4]
1379         add     $out,64*8,$out
1380
1381         ld      [$inp],%l0              ! Y
1382         ld      [$inp+4],%l1
1383         ld      [$inp+8],%l2
1384         ld      [$inp+12],%l3
1385         ld      [$inp+16],%l4
1386         ld      [$inp+20],%l5
1387         ld      [$inp+24],%l6
1388         ld      [$inp+28],%l7
1389         add     $inp,32,$inp
1390         st      %l0,[$out+64*0-4]
1391         st      %l1,[$out+64*1-4]
1392         st      %l2,[$out+64*2-4]
1393         st      %l3,[$out+64*3-4]
1394         st      %l4,[$out+64*4-4]
1395         st      %l5,[$out+64*5-4]
1396         st      %l6,[$out+64*6-4]
1397         st      %l7,[$out+64*7-4]
1398         add     $out,64*8,$out
1399
1400         ld      [$inp],%l0              ! Z
1401         ld      [$inp+4],%l1
1402         ld      [$inp+8],%l2
1403         ld      [$inp+12],%l3
1404         ld      [$inp+16],%l4
1405         ld      [$inp+20],%l5
1406         ld      [$inp+24],%l6
1407         ld      [$inp+28],%l7
1408         st      %l0,[$out+64*0-4]
1409         st      %l1,[$out+64*1-4]
1410         st      %l2,[$out+64*2-4]
1411         st      %l3,[$out+64*3-4]
1412         st      %l4,[$out+64*4-4]
1413         st      %l5,[$out+64*5-4]
1414         st      %l6,[$out+64*6-4]
1415         st      %l7,[$out+64*7-4]
1416
1417         ret
1418         restore
1419 .type   ecp_nistz256_scatter_w5,#function
1420 .size   ecp_nistz256_scatter_w5,.-ecp_nistz256_scatter_w5
1421
1422 ! void  ecp_nistz256_gather_w5(P256_POINT *%i0,const void *%i1,
1423 !                                              int %i2);
1424 .globl  ecp_nistz256_gather_w5
1425 .align  32
1426 ecp_nistz256_gather_w5:
1427         save    %sp,-STACK_FRAME,%sp
1428
1429         neg     $index,$mask
1430         srax    $mask,63,$mask
1431
1432         add     $index,$mask,$index
1433         sll     $index,2,$index
1434         add     $inp,$index,$inp
1435
1436         ld      [$inp+64*0],%l0
1437         ld      [$inp+64*1],%l1
1438         ld      [$inp+64*2],%l2
1439         ld      [$inp+64*3],%l3
1440         ld      [$inp+64*4],%l4
1441         ld      [$inp+64*5],%l5
1442         ld      [$inp+64*6],%l6
1443         ld      [$inp+64*7],%l7
1444         add     $inp,64*8,$inp
1445         and     %l0,$mask,%l0
1446         and     %l1,$mask,%l1
1447         st      %l0,[$out]              ! X
1448         and     %l2,$mask,%l2
1449         st      %l1,[$out+4]
1450         and     %l3,$mask,%l3
1451         st      %l2,[$out+8]
1452         and     %l4,$mask,%l4
1453         st      %l3,[$out+12]
1454         and     %l5,$mask,%l5
1455         st      %l4,[$out+16]
1456         and     %l6,$mask,%l6
1457         st      %l5,[$out+20]
1458         and     %l7,$mask,%l7
1459         st      %l6,[$out+24]
1460         st      %l7,[$out+28]
1461         add     $out,32,$out
1462
1463         ld      [$inp+64*0],%l0
1464         ld      [$inp+64*1],%l1
1465         ld      [$inp+64*2],%l2
1466         ld      [$inp+64*3],%l3
1467         ld      [$inp+64*4],%l4
1468         ld      [$inp+64*5],%l5
1469         ld      [$inp+64*6],%l6
1470         ld      [$inp+64*7],%l7
1471         add     $inp,64*8,$inp
1472         and     %l0,$mask,%l0
1473         and     %l1,$mask,%l1
1474         st      %l0,[$out]              ! Y
1475         and     %l2,$mask,%l2
1476         st      %l1,[$out+4]
1477         and     %l3,$mask,%l3
1478         st      %l2,[$out+8]
1479         and     %l4,$mask,%l4
1480         st      %l3,[$out+12]
1481         and     %l5,$mask,%l5
1482         st      %l4,[$out+16]
1483         and     %l6,$mask,%l6
1484         st      %l5,[$out+20]
1485         and     %l7,$mask,%l7
1486         st      %l6,[$out+24]
1487         st      %l7,[$out+28]
1488         add     $out,32,$out
1489
1490         ld      [$inp+64*0],%l0
1491         ld      [$inp+64*1],%l1
1492         ld      [$inp+64*2],%l2
1493         ld      [$inp+64*3],%l3
1494         ld      [$inp+64*4],%l4
1495         ld      [$inp+64*5],%l5
1496         ld      [$inp+64*6],%l6
1497         ld      [$inp+64*7],%l7
1498         and     %l0,$mask,%l0
1499         and     %l1,$mask,%l1
1500         st      %l0,[$out]              ! Z
1501         and     %l2,$mask,%l2
1502         st      %l1,[$out+4]
1503         and     %l3,$mask,%l3
1504         st      %l2,[$out+8]
1505         and     %l4,$mask,%l4
1506         st      %l3,[$out+12]
1507         and     %l5,$mask,%l5
1508         st      %l4,[$out+16]
1509         and     %l6,$mask,%l6
1510         st      %l5,[$out+20]
1511         and     %l7,$mask,%l7
1512         st      %l6,[$out+24]
1513         st      %l7,[$out+28]
1514
1515         ret
1516         restore
1517 .type   ecp_nistz256_gather_w5,#function
1518 .size   ecp_nistz256_gather_w5,.-ecp_nistz256_gather_w5
1519
1520 ! void  ecp_nistz256_scatter_w7(void *%i0,const P256_POINT_AFFINE *%i1,
1521 !                                         int %i2);
1522 .globl  ecp_nistz256_scatter_w7
1523 .align  32
1524 ecp_nistz256_scatter_w7:
1525         save    %sp,-STACK_FRAME,%sp
1526         nop
1527         add     $out,$index,$out
1528         mov     64/4,$index
1529 .Loop_scatter_w7:
1530         ld      [$inp],%l0
1531         add     $inp,4,$inp
1532         subcc   $index,1,$index
1533         stb     %l0,[$out+64*0]
1534         srl     %l0,8,%l1
1535         stb     %l1,[$out+64*1]
1536         srl     %l0,16,%l2
1537         stb     %l2,[$out+64*2]
1538         srl     %l0,24,%l3
1539         stb     %l3,[$out+64*3]
1540         bne     .Loop_scatter_w7
1541         add     $out,64*4,$out
1542
1543         ret
1544         restore
1545 .type   ecp_nistz256_scatter_w7,#function
1546 .size   ecp_nistz256_scatter_w7,.-ecp_nistz256_scatter_w7
1547
1548 ! void  ecp_nistz256_gather_w7(P256_POINT_AFFINE *%i0,const void *%i1,
1549 !                                                     int %i2);
1550 .globl  ecp_nistz256_gather_w7
1551 .align  32
1552 ecp_nistz256_gather_w7:
1553         save    %sp,-STACK_FRAME,%sp
1554
1555         neg     $index,$mask
1556         srax    $mask,63,$mask
1557
1558         add     $index,$mask,$index
1559         add     $inp,$index,$inp
1560         mov     64/4,$index
1561
1562 .Loop_gather_w7:
1563         ldub    [$inp+64*0],%l0
1564         prefetch [$inp+3840+64*0],1
1565         subcc   $index,1,$index
1566         ldub    [$inp+64*1],%l1
1567         prefetch [$inp+3840+64*1],1
1568         ldub    [$inp+64*2],%l2
1569         prefetch [$inp+3840+64*2],1
1570         ldub    [$inp+64*3],%l3
1571         prefetch [$inp+3840+64*3],1
1572         add     $inp,64*4,$inp
1573         sll     %l1,8,%l1
1574         sll     %l2,16,%l2
1575         or      %l0,%l1,%l0
1576         sll     %l3,24,%l3
1577         or      %l0,%l2,%l0
1578         or      %l0,%l3,%l0
1579         and     %l0,$mask,%l0
1580         st      %l0,[$out]
1581         bne     .Loop_gather_w7
1582         add     $out,4,$out
1583
1584         ret
1585         restore
1586 .type   ecp_nistz256_gather_w7,#function
1587 .size   ecp_nistz256_gather_w7,.-ecp_nistz256_gather_w7
1588 ___
1589 }}}
1590 {{{
1591 ########################################################################
1592 # Following subroutines are VIS3 counterparts of those above that
1593 # implement ones found in ecp_nistz256.c. Key difference is that they
1594 # use 128-bit multiplication and addition with 64-bit carry, and in order
1595 # to do that they perform conversion from uin32_t[8] to uint64_t[4] upon
1596 # entry and vice versa on return.
1597 #
1598 my ($rp,$ap,$bp)=map("%i$_",(0..2));
1599 my ($t0,$t1,$t2,$t3,$a0,$a1,$a2,$a3)=map("%l$_",(0..7));
1600 my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5)=map("%o$_",(0..5));
1601 my ($bi,$poly1,$poly3,$minus1)=(map("%i$_",(3..5)),"%g1");
1602 my ($rp_real,$ap_real)=("%g2","%g3");
1603 my ($acc6,$acc7)=($bp,$bi);     # used in squaring
1604
1605 $code.=<<___;
1606 .align  32
1607 __ecp_nistz256_mul_by_2_vis3:
1608         addcc   $acc0,$acc0,$acc0
1609         addxccc $acc1,$acc1,$acc1
1610         addxccc $acc2,$acc2,$acc2
1611         addxccc $acc3,$acc3,$acc3
1612         b       .Lreduce_by_sub_vis3
1613         addxc   %g0,%g0,$acc4           ! did it carry?
1614 .type   __ecp_nistz256_mul_by_2_vis3,#function
1615 .size   __ecp_nistz256_mul_by_2_vis3,.-__ecp_nistz256_mul_by_2_vis3
1616
1617 .align  32
1618 __ecp_nistz256_add_vis3:
1619         ldx     [$bp+0],$t0
1620         ldx     [$bp+8],$t1
1621         ldx     [$bp+16],$t2
1622         ldx     [$bp+24],$t3
1623
1624 __ecp_nistz256_add_noload_vis3:
1625
1626         addcc   $t0,$acc0,$acc0
1627         addxccc $t1,$acc1,$acc1
1628         addxccc $t2,$acc2,$acc2
1629         addxccc $t3,$acc3,$acc3
1630         addxc   %g0,%g0,$acc4           ! did it carry?
1631
1632 .Lreduce_by_sub_vis3:
1633
1634         addcc   $acc0,1,$t0             ! add -modulus, i.e. subtract
1635         addxccc $acc1,$poly1,$t1
1636         addxccc $acc2,$minus1,$t2
1637         addxccc $acc3,$poly3,$t3
1638         addxc   $acc4,$minus1,$acc4
1639
1640         movrz   $acc4,$t0,$acc0         ! ret = borrow ? ret : ret-modulus
1641         movrz   $acc4,$t1,$acc1
1642         stx     $acc0,[$rp]
1643         movrz   $acc4,$t2,$acc2
1644         stx     $acc1,[$rp+8]
1645         movrz   $acc4,$t3,$acc3
1646         stx     $acc2,[$rp+16]
1647         retl
1648         stx     $acc3,[$rp+24]
1649 .type   __ecp_nistz256_add_vis3,#function
1650 .size   __ecp_nistz256_add_vis3,.-__ecp_nistz256_add_vis3
1651
1652 ! Trouble with subtraction is that there is no subtraction with 64-bit
1653 ! borrow, only with 32-bit one. For this reason we "decompose" 64-bit
1654 ! $acc0-$acc3 to 32-bit values and pick b[4] in 32-bit pieces. But
1655 ! recall that SPARC is big-endian, which is why you'll observe that
1656 ! b[4] is accessed as 4-0-12-8-20-16-28-24. And prior reduction we
1657 ! "collect" result back to 64-bit $acc0-$acc3.
1658 .align  32
1659 __ecp_nistz256_sub_from_vis3:
1660         ld      [$bp+4],$t0
1661         ld      [$bp+0],$t1
1662         ld      [$bp+12],$t2
1663         ld      [$bp+8],$t3
1664
1665         srlx    $acc0,32,$acc4
1666         not     $poly1,$poly1
1667         srlx    $acc1,32,$acc5
1668         subcc   $acc0,$t0,$acc0
1669         ld      [$bp+20],$t0
1670         subccc  $acc4,$t1,$acc4
1671         ld      [$bp+16],$t1
1672         subccc  $acc1,$t2,$acc1
1673         ld      [$bp+28],$t2
1674         and     $acc0,$poly1,$acc0
1675         subccc  $acc5,$t3,$acc5
1676         ld      [$bp+24],$t3
1677         sllx    $acc4,32,$acc4
1678         and     $acc1,$poly1,$acc1
1679         sllx    $acc5,32,$acc5
1680         or      $acc0,$acc4,$acc0
1681         srlx    $acc2,32,$acc4
1682         or      $acc1,$acc5,$acc1
1683         srlx    $acc3,32,$acc5
1684         subccc  $acc2,$t0,$acc2
1685         subccc  $acc4,$t1,$acc4
1686         subccc  $acc3,$t2,$acc3
1687         and     $acc2,$poly1,$acc2
1688         subccc  $acc5,$t3,$acc5
1689         sllx    $acc4,32,$acc4
1690         and     $acc3,$poly1,$acc3
1691         sllx    $acc5,32,$acc5
1692         or      $acc2,$acc4,$acc2
1693         subc    %g0,%g0,$acc4           ! did it borrow?
1694         b       .Lreduce_by_add_vis3
1695         or      $acc3,$acc5,$acc3
1696 .type   __ecp_nistz256_sub_from_vis3,#function
1697 .size   __ecp_nistz256_sub_from_vis3,.-__ecp_nistz256_sub_from_vis3
1698
1699 .align  32
1700 __ecp_nistz256_sub_morf_vis3:
1701         ld      [$bp+4],$t0
1702         ld      [$bp+0],$t1
1703         ld      [$bp+12],$t2
1704         ld      [$bp+8],$t3
1705
1706         srlx    $acc0,32,$acc4
1707         not     $poly1,$poly1
1708         srlx    $acc1,32,$acc5
1709         subcc   $t0,$acc0,$acc0
1710         ld      [$bp+20],$t0
1711         subccc  $t1,$acc4,$acc4
1712         ld      [$bp+16],$t1
1713         subccc  $t2,$acc1,$acc1
1714         ld      [$bp+28],$t2
1715         and     $acc0,$poly1,$acc0
1716         subccc  $t3,$acc5,$acc5
1717         ld      [$bp+24],$t3
1718         sllx    $acc4,32,$acc4
1719         and     $acc1,$poly1,$acc1
1720         sllx    $acc5,32,$acc5
1721         or      $acc0,$acc4,$acc0
1722         srlx    $acc2,32,$acc4
1723         or      $acc1,$acc5,$acc1
1724         srlx    $acc3,32,$acc5
1725         subccc  $t0,$acc2,$acc2
1726         subccc  $t1,$acc4,$acc4
1727         subccc  $t2,$acc3,$acc3
1728         and     $acc2,$poly1,$acc2
1729         subccc  $t3,$acc5,$acc5
1730         sllx    $acc4,32,$acc4
1731         and     $acc3,$poly1,$acc3
1732         sllx    $acc5,32,$acc5
1733         or      $acc2,$acc4,$acc2
1734         subc    %g0,%g0,$acc4           ! did it borrow?
1735         or      $acc3,$acc5,$acc3
1736
1737 .Lreduce_by_add_vis3:
1738
1739         addcc   $acc0,-1,$t0            ! add modulus
1740         not     $poly3,$t3
1741         addxccc $acc1,$poly1,$t1
1742         not     $poly1,$poly1           ! restore $poly1
1743         addxccc $acc2,%g0,$t2
1744         addxc   $acc3,$t3,$t3
1745
1746         movrnz  $acc4,$t0,$acc0         ! if a-b borrowed, ret = ret+mod
1747         movrnz  $acc4,$t1,$acc1
1748         stx     $acc0,[$rp]
1749         movrnz  $acc4,$t2,$acc2
1750         stx     $acc1,[$rp+8]
1751         movrnz  $acc4,$t3,$acc3
1752         stx     $acc2,[$rp+16]
1753         retl
1754         stx     $acc3,[$rp+24]
1755 .type   __ecp_nistz256_sub_morf_vis3,#function
1756 .size   __ecp_nistz256_sub_morf_vis3,.-__ecp_nistz256_sub_morf_vis3
1757
1758 .align  32
1759 __ecp_nistz256_div_by_2_vis3:
1760         ! ret = (a is odd ? a+mod : a) >> 1
1761
1762         not     $poly1,$t1
1763         not     $poly3,$t3
1764         and     $acc0,1,$acc5
1765         addcc   $acc0,-1,$t0            ! add modulus
1766         addxccc $acc1,$t1,$t1
1767         addxccc $acc2,%g0,$t2
1768         addxccc $acc3,$t3,$t3
1769         addxc   %g0,%g0,$acc4           ! carry bit
1770
1771         movrnz  $acc5,$t0,$acc0
1772         movrnz  $acc5,$t1,$acc1
1773         movrnz  $acc5,$t2,$acc2
1774         movrnz  $acc5,$t3,$acc3
1775         movrz   $acc5,%g0,$acc4
1776
1777         ! ret >>= 1
1778
1779         srlx    $acc0,1,$acc0
1780         sllx    $acc1,63,$t0
1781         srlx    $acc1,1,$acc1
1782         or      $acc0,$t0,$acc0
1783         sllx    $acc2,63,$t1
1784         srlx    $acc2,1,$acc2
1785         or      $acc1,$t1,$acc1
1786         sllx    $acc3,63,$t2
1787         stx     $acc0,[$rp]
1788         srlx    $acc3,1,$acc3
1789         or      $acc2,$t2,$acc2
1790         sllx    $acc4,63,$t3            ! don't forget carry bit
1791         stx     $acc1,[$rp+8]
1792         or      $acc3,$t3,$acc3
1793         stx     $acc2,[$rp+16]
1794         retl
1795         stx     $acc3,[$rp+24]
1796 .type   __ecp_nistz256_div_by_2_vis3,#function
1797 .size   __ecp_nistz256_div_by_2_vis3,.-__ecp_nistz256_div_by_2_vis3
1798
1799 ! compared to __ecp_nistz256_mul_mont it's almost 4x smaller and
1800 ! 4x faster [on T4]...
1801 .align  32
1802 __ecp_nistz256_mul_mont_vis3:
1803         mulx    $a0,$bi,$acc0
1804         not     $poly3,$poly3           ! 0xFFFFFFFF00000001
1805         umulxhi $a0,$bi,$t0
1806         mulx    $a1,$bi,$acc1
1807         umulxhi $a1,$bi,$t1
1808         mulx    $a2,$bi,$acc2
1809         umulxhi $a2,$bi,$t2
1810         mulx    $a3,$bi,$acc3
1811         umulxhi $a3,$bi,$t3
1812         ldx     [$bp+8],$bi             ! b[1]
1813
1814         addcc   $acc1,$t0,$acc1         ! accumulate high parts of multiplication
1815          sllx   $acc0,32,$t0
1816         addxccc $acc2,$t1,$acc2
1817          srlx   $acc0,32,$t1
1818         addxccc $acc3,$t2,$acc3
1819         addxc   %g0,$t3,$acc4
1820         mov     0,$acc5
1821 ___
1822 for($i=1;$i<4;$i++) {
1823         # Reduction iteration is normally performed by accumulating
1824         # result of multiplication of modulus by "magic" digit [and
1825         # omitting least significant word, which is guaranteed to
1826         # be 0], but thanks to special form of modulus and "magic"
1827         # digit being equal to least significant word, it can be
1828         # performed with additions and subtractions alone. Indeed:
1829         #
1830         #            ffff0001.00000000.0000ffff.ffffffff
1831         # *                                     abcdefgh
1832         # + xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.abcdefgh
1833         #
1834         # Now observing that ff..ff*x = (2^n-1)*x = 2^n*x-x, we
1835         # rewrite above as:
1836         #
1837         #   xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.abcdefgh
1838         # + abcdefgh.abcdefgh.0000abcd.efgh0000.00000000
1839         # - 0000abcd.efgh0000.00000000.00000000.abcdefgh
1840         #
1841         # or marking redundant operations:
1842         #
1843         #   xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.--------
1844         # + abcdefgh.abcdefgh.0000abcd.efgh0000.--------
1845         # - 0000abcd.efgh0000.--------.--------.--------
1846         #   ^^^^^^^^ but this word is calculated with umulxhi, because
1847         #            there is no subtract with 64-bit borrow:-(
1848
1849 $code.=<<___;
1850         sub     $acc0,$t0,$t2           ! acc0*0xFFFFFFFF00000001, low part
1851         umulxhi $acc0,$poly3,$t3        ! acc0*0xFFFFFFFF00000001, high part
1852         addcc   $acc1,$t0,$acc0         ! +=acc[0]<<96 and omit acc[0]
1853         mulx    $a0,$bi,$t0
1854         addxccc $acc2,$t1,$acc1
1855         mulx    $a1,$bi,$t1
1856         addxccc $acc3,$t2,$acc2         ! +=acc[0]*0xFFFFFFFF00000001
1857         mulx    $a2,$bi,$t2
1858         addxccc $acc4,$t3,$acc3
1859         mulx    $a3,$bi,$t3
1860         addxc   $acc5,%g0,$acc4
1861
1862         addcc   $acc0,$t0,$acc0         ! accumulate low parts of multiplication
1863         umulxhi $a0,$bi,$t0
1864         addxccc $acc1,$t1,$acc1
1865         umulxhi $a1,$bi,$t1
1866         addxccc $acc2,$t2,$acc2
1867         umulxhi $a2,$bi,$t2
1868         addxccc $acc3,$t3,$acc3
1869         umulxhi $a3,$bi,$t3
1870         addxc   $acc4,%g0,$acc4
1871 ___
1872 $code.=<<___    if ($i<3);
1873         ldx     [$bp+8*($i+1)],$bi      ! bp[$i+1]
1874 ___
1875 $code.=<<___;
1876         addcc   $acc1,$t0,$acc1         ! accumulate high parts of multiplication
1877          sllx   $acc0,32,$t0
1878         addxccc $acc2,$t1,$acc2
1879          srlx   $acc0,32,$t1
1880         addxccc $acc3,$t2,$acc3
1881         addxccc $acc4,$t3,$acc4
1882         addxc   %g0,%g0,$acc5
1883 ___
1884 }
1885 $code.=<<___;
1886         sub     $acc0,$t0,$t2           ! acc0*0xFFFFFFFF00000001, low part
1887         umulxhi $acc0,$poly3,$t3        ! acc0*0xFFFFFFFF00000001, high part
1888         addcc   $acc1,$t0,$acc0         ! +=acc[0]<<96 and omit acc[0]
1889         addxccc $acc2,$t1,$acc1
1890         addxccc $acc3,$t2,$acc2         ! +=acc[0]*0xFFFFFFFF00000001
1891         addxccc $acc4,$t3,$acc3
1892         b       .Lmul_final_vis3        ! see below
1893         addxc   $acc5,%g0,$acc4
1894 .type   __ecp_nistz256_mul_mont_vis3,#function
1895 .size   __ecp_nistz256_mul_mont_vis3,.-__ecp_nistz256_mul_mont_vis3
1896
1897 ! compared to above __ecp_nistz256_mul_mont_vis3 it's 21% less
1898 ! instructions, but only 14% faster [on T4]...
1899 .align  32
1900 __ecp_nistz256_sqr_mont_vis3:
1901         !  |  |  |  |  |  |a1*a0|  |
1902         !  |  |  |  |  |a2*a0|  |  |
1903         !  |  |a3*a2|a3*a0|  |  |  |
1904         !  |  |  |  |a2*a1|  |  |  |
1905         !  |  |  |a3*a1|  |  |  |  |
1906         ! *|  |  |  |  |  |  |  | 2|
1907         ! +|a3*a3|a2*a2|a1*a1|a0*a0|
1908         !  |--+--+--+--+--+--+--+--|
1909         !  |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is $accx, i.e. follow $accx
1910         !
1911         !  "can't overflow" below mark carrying into high part of
1912         !  multiplication result, which can't overflow, because it
1913         !  can never be all ones.
1914
1915         mulx    $a1,$a0,$acc1           ! a[1]*a[0]
1916         umulxhi $a1,$a0,$t1
1917         mulx    $a2,$a0,$acc2           ! a[2]*a[0]
1918         umulxhi $a2,$a0,$t2
1919         mulx    $a3,$a0,$acc3           ! a[3]*a[0]
1920         umulxhi $a3,$a0,$acc4
1921
1922         addcc   $acc2,$t1,$acc2         ! accumulate high parts of multiplication
1923         mulx    $a2,$a1,$t0             ! a[2]*a[1]
1924         umulxhi $a2,$a1,$t1
1925         addxccc $acc3,$t2,$acc3
1926         mulx    $a3,$a1,$t2             ! a[3]*a[1]
1927         umulxhi $a3,$a1,$t3
1928         addxc   $acc4,%g0,$acc4         ! can't overflow
1929
1930         mulx    $a3,$a2,$acc5           ! a[3]*a[2]
1931         not     $poly3,$poly3           ! 0xFFFFFFFF00000001
1932         umulxhi $a3,$a2,$acc6
1933
1934         addcc   $t2,$t1,$t1             ! accumulate high parts of multiplication
1935         mulx    $a0,$a0,$acc0           ! a[0]*a[0]
1936         addxc   $t3,%g0,$t2             ! can't overflow
1937
1938         addcc   $acc3,$t0,$acc3         ! accumulate low parts of multiplication
1939         umulxhi $a0,$a0,$a0
1940         addxccc $acc4,$t1,$acc4
1941         mulx    $a1,$a1,$t1             ! a[1]*a[1]
1942         addxccc $acc5,$t2,$acc5
1943         umulxhi $a1,$a1,$a1
1944         addxc   $acc6,%g0,$acc6         ! can't overflow
1945
1946         addcc   $acc1,$acc1,$acc1       ! acc[1-6]*=2
1947         mulx    $a2,$a2,$t2             ! a[2]*a[2]
1948         addxccc $acc2,$acc2,$acc2
1949         umulxhi $a2,$a2,$a2
1950         addxccc $acc3,$acc3,$acc3
1951         mulx    $a3,$a3,$t3             ! a[3]*a[3]
1952         addxccc $acc4,$acc4,$acc4
1953         umulxhi $a3,$a3,$a3
1954         addxccc $acc5,$acc5,$acc5
1955         addxccc $acc6,$acc6,$acc6
1956         addxc   %g0,%g0,$acc7
1957
1958         addcc   $acc1,$a0,$acc1         ! +a[i]*a[i]
1959         addxccc $acc2,$t1,$acc2
1960         addxccc $acc3,$a1,$acc3
1961         addxccc $acc4,$t2,$acc4
1962          sllx   $acc0,32,$t0
1963         addxccc $acc5,$a2,$acc5
1964          srlx   $acc0,32,$t1
1965         addxccc $acc6,$t3,$acc6
1966          sub    $acc0,$t0,$t2           ! acc0*0xFFFFFFFF00000001, low part
1967         addxc   $acc7,$a3,$acc7
1968 ___
1969 for($i=0;$i<3;$i++) {                   # reductions, see commentary
1970                                         # in multiplication for details
1971 $code.=<<___;
1972         umulxhi $acc0,$poly3,$t3        ! acc0*0xFFFFFFFF00000001, high part
1973         addcc   $acc1,$t0,$acc0         ! +=acc[0]<<96 and omit acc[0]
1974          sllx   $acc0,32,$t0
1975         addxccc $acc2,$t1,$acc1
1976          srlx   $acc0,32,$t1
1977         addxccc $acc3,$t2,$acc2         ! +=acc[0]*0xFFFFFFFF00000001
1978          sub    $acc0,$t0,$t2           ! acc0*0xFFFFFFFF00000001, low part
1979         addxc   %g0,$t3,$acc3           ! can't overflow
1980 ___
1981 }
1982 $code.=<<___;
1983         umulxhi $acc0,$poly3,$t3        ! acc0*0xFFFFFFFF00000001, high part
1984         addcc   $acc1,$t0,$acc0         ! +=acc[0]<<96 and omit acc[0]
1985         addxccc $acc2,$t1,$acc1
1986         addxccc $acc3,$t2,$acc2         ! +=acc[0]*0xFFFFFFFF00000001
1987         addxc   %g0,$t3,$acc3           ! can't overflow
1988
1989         addcc   $acc0,$acc4,$acc0       ! accumulate upper half
1990         addxccc $acc1,$acc5,$acc1
1991         addxccc $acc2,$acc6,$acc2
1992         addxccc $acc3,$acc7,$acc3
1993         addxc   %g0,%g0,$acc4
1994
1995 .Lmul_final_vis3:
1996
1997         ! Final step is "if result > mod, subtract mod", but as comparison
1998         ! means subtraction, we do the subtraction and then copy outcome
1999         ! if it didn't borrow. But note that as we [have to] replace
2000         ! subtraction with addition with negative, carry/borrow logic is
2001         ! inverse.
2002
2003         addcc   $acc0,1,$t0             ! add -modulus, i.e. subtract
2004         not     $poly3,$poly3           ! restore 0x00000000FFFFFFFE
2005         addxccc $acc1,$poly1,$t1
2006         addxccc $acc2,$minus1,$t2
2007         addxccc $acc3,$poly3,$t3
2008         addxccc $acc4,$minus1,%g0       ! did it carry?
2009
2010         movcs   %xcc,$t0,$acc0
2011         movcs   %xcc,$t1,$acc1
2012         stx     $acc0,[$rp]
2013         movcs   %xcc,$t2,$acc2
2014         stx     $acc1,[$rp+8]
2015         movcs   %xcc,$t3,$acc3
2016         stx     $acc2,[$rp+16]
2017         retl
2018         stx     $acc3,[$rp+24]
2019 .type   __ecp_nistz256_sqr_mont_vis3,#function
2020 .size   __ecp_nistz256_sqr_mont_vis3,.-__ecp_nistz256_sqr_mont_vis3
2021 ___
2022
2023 ########################################################################
2024 # void ecp_nistz256_point_double(P256_POINT *out,const P256_POINT *inp);
2025 #
2026 {
2027 my ($res_x,$res_y,$res_z,
2028     $in_x,$in_y,$in_z,
2029     $S,$M,$Zsqr,$tmp0)=map(32*$_,(0..9));
2030 # above map() describes stack layout with 10 temporary
2031 # 256-bit vectors on top.
2032
2033 $code.=<<___;
2034 .align  32
2035 ecp_nistz256_point_double_vis3:
2036         save    %sp,-STACK64_FRAME-32*10,%sp
2037
2038         mov     $rp,$rp_real
2039 .Ldouble_shortcut_vis3:
2040         mov     -1,$minus1
2041         mov     -2,$poly3
2042         sllx    $minus1,32,$poly1               ! 0xFFFFFFFF00000000
2043         srl     $poly3,0,$poly3                 ! 0x00000000FFFFFFFE
2044
2045         ! convert input to uint64_t[4]
2046         ld      [$ap],$a0                       ! in_x
2047         ld      [$ap+4],$t0
2048         ld      [$ap+8],$a1
2049         ld      [$ap+12],$t1
2050         ld      [$ap+16],$a2
2051         ld      [$ap+20],$t2
2052         ld      [$ap+24],$a3
2053         ld      [$ap+28],$t3
2054         sllx    $t0,32,$t0
2055         sllx    $t1,32,$t1
2056         ld      [$ap+32],$acc0                  ! in_y
2057         or      $a0,$t0,$a0
2058         ld      [$ap+32+4],$t0
2059         sllx    $t2,32,$t2
2060         ld      [$ap+32+8],$acc1
2061         or      $a1,$t1,$a1
2062         ld      [$ap+32+12],$t1
2063         sllx    $t3,32,$t3
2064         ld      [$ap+32+16],$acc2
2065         or      $a2,$t2,$a2
2066         ld      [$ap+32+20],$t2
2067         or      $a3,$t3,$a3
2068         ld      [$ap+32+24],$acc3
2069         sllx    $t0,32,$t0
2070         ld      [$ap+32+28],$t3
2071         sllx    $t1,32,$t1
2072         stx     $a0,[%sp+LOCALS64+$in_x]
2073         sllx    $t2,32,$t2
2074         stx     $a1,[%sp+LOCALS64+$in_x+8]
2075         sllx    $t3,32,$t3
2076         stx     $a2,[%sp+LOCALS64+$in_x+16]
2077         or      $acc0,$t0,$acc0
2078         stx     $a3,[%sp+LOCALS64+$in_x+24]
2079         or      $acc1,$t1,$acc1
2080         stx     $acc0,[%sp+LOCALS64+$in_y]
2081         or      $acc2,$t2,$acc2
2082         stx     $acc1,[%sp+LOCALS64+$in_y+8]
2083         or      $acc3,$t3,$acc3
2084         stx     $acc2,[%sp+LOCALS64+$in_y+16]
2085         stx     $acc3,[%sp+LOCALS64+$in_y+24]
2086
2087         ld      [$ap+64],$a0                    ! in_z
2088         ld      [$ap+64+4],$t0
2089         ld      [$ap+64+8],$a1
2090         ld      [$ap+64+12],$t1
2091         ld      [$ap+64+16],$a2
2092         ld      [$ap+64+20],$t2
2093         ld      [$ap+64+24],$a3
2094         ld      [$ap+64+28],$t3
2095         sllx    $t0,32,$t0
2096         sllx    $t1,32,$t1
2097         or      $a0,$t0,$a0
2098         sllx    $t2,32,$t2
2099         or      $a1,$t1,$a1
2100         sllx    $t3,32,$t3
2101         or      $a2,$t2,$a2
2102         or      $a3,$t3,$a3
2103         sllx    $t0,32,$t0
2104         sllx    $t1,32,$t1
2105         stx     $a0,[%sp+LOCALS64+$in_z]
2106         sllx    $t2,32,$t2
2107         stx     $a1,[%sp+LOCALS64+$in_z+8]
2108         sllx    $t3,32,$t3
2109         stx     $a2,[%sp+LOCALS64+$in_z+16]
2110         stx     $a3,[%sp+LOCALS64+$in_z+24]
2111
2112         ! in_y is still in $acc0-$acc3
2113         call    __ecp_nistz256_mul_by_2_vis3    ! p256_mul_by_2(S, in_y);
2114         add     %sp,LOCALS64+$S,$rp
2115
2116         ! in_z is still in $a0-$a3
2117         call    __ecp_nistz256_sqr_mont_vis3    ! p256_sqr_mont(Zsqr, in_z);
2118         add     %sp,LOCALS64+$Zsqr,$rp
2119
2120         mov     $acc0,$a0                       ! put Zsqr aside
2121         mov     $acc1,$a1
2122         mov     $acc2,$a2
2123         mov     $acc3,$a3
2124
2125         add     %sp,LOCALS64+$in_x,$bp
2126         call    __ecp_nistz256_add_vis3         ! p256_add(M, Zsqr, in_x);
2127         add     %sp,LOCALS64+$M,$rp
2128
2129         mov     $a0,$acc0                       ! restore Zsqr
2130         ldx     [%sp+LOCALS64+$S],$a0           ! forward load
2131         mov     $a1,$acc1
2132         ldx     [%sp+LOCALS64+$S+8],$a1
2133         mov     $a2,$acc2
2134         ldx     [%sp+LOCALS64+$S+16],$a2
2135         mov     $a3,$acc3
2136         ldx     [%sp+LOCALS64+$S+24],$a3
2137
2138         add     %sp,LOCALS64+$in_x,$bp
2139         call    __ecp_nistz256_sub_morf_vis3    ! p256_sub(Zsqr, in_x, Zsqr);
2140         add     %sp,LOCALS64+$Zsqr,$rp
2141
2142         call    __ecp_nistz256_sqr_mont_vis3    ! p256_sqr_mont(S, S);
2143         add     %sp,LOCALS64+$S,$rp
2144
2145         ldx     [%sp+LOCALS64+$in_z],$bi
2146         ldx     [%sp+LOCALS64+$in_y],$a0
2147         ldx     [%sp+LOCALS64+$in_y+8],$a1
2148         ldx     [%sp+LOCALS64+$in_y+16],$a2
2149         ldx     [%sp+LOCALS64+$in_y+24],$a3
2150         add     %sp,LOCALS64+$in_z,$bp
2151         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(tmp0, in_z, in_y);
2152         add     %sp,LOCALS64+$tmp0,$rp
2153
2154         ldx     [%sp+LOCALS64+$M],$bi           ! forward load
2155         ldx     [%sp+LOCALS64+$Zsqr],$a0
2156         ldx     [%sp+LOCALS64+$Zsqr+8],$a1
2157         ldx     [%sp+LOCALS64+$Zsqr+16],$a2
2158         ldx     [%sp+LOCALS64+$Zsqr+24],$a3
2159
2160         call    __ecp_nistz256_mul_by_2_vis3    ! p256_mul_by_2(res_z, tmp0);
2161         add     %sp,LOCALS64+$res_z,$rp
2162
2163         add     %sp,LOCALS64+$M,$bp
2164         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(M, M, Zsqr);
2165         add     %sp,LOCALS64+$M,$rp
2166
2167         mov     $acc0,$a0                       ! put aside M
2168         mov     $acc1,$a1
2169         mov     $acc2,$a2
2170         mov     $acc3,$a3
2171         call    __ecp_nistz256_mul_by_2_vis3
2172         add     %sp,LOCALS64+$M,$rp
2173         mov     $a0,$t0                         ! copy M
2174         ldx     [%sp+LOCALS64+$S],$a0           ! forward load
2175         mov     $a1,$t1
2176         ldx     [%sp+LOCALS64+$S+8],$a1
2177         mov     $a2,$t2
2178         ldx     [%sp+LOCALS64+$S+16],$a2
2179         mov     $a3,$t3
2180         ldx     [%sp+LOCALS64+$S+24],$a3
2181         call    __ecp_nistz256_add_noload_vis3  ! p256_mul_by_3(M, M);
2182         add     %sp,LOCALS64+$M,$rp
2183
2184         call    __ecp_nistz256_sqr_mont_vis3    ! p256_sqr_mont(tmp0, S);
2185         add     %sp,LOCALS64+$tmp0,$rp
2186
2187         ldx     [%sp+LOCALS64+$S],$bi           ! forward load
2188         ldx     [%sp+LOCALS64+$in_x],$a0
2189         ldx     [%sp+LOCALS64+$in_x+8],$a1
2190         ldx     [%sp+LOCALS64+$in_x+16],$a2
2191         ldx     [%sp+LOCALS64+$in_x+24],$a3
2192
2193         call    __ecp_nistz256_div_by_2_vis3    ! p256_div_by_2(res_y, tmp0);
2194         add     %sp,LOCALS64+$res_y,$rp
2195
2196         add     %sp,LOCALS64+$S,$bp
2197         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(S, S, in_x);
2198         add     %sp,LOCALS64+$S,$rp
2199
2200         ldx     [%sp+LOCALS64+$M],$a0           ! forward load
2201         ldx     [%sp+LOCALS64+$M+8],$a1
2202         ldx     [%sp+LOCALS64+$M+16],$a2
2203         ldx     [%sp+LOCALS64+$M+24],$a3
2204
2205         call    __ecp_nistz256_mul_by_2_vis3    ! p256_mul_by_2(tmp0, S);
2206         add     %sp,LOCALS64+$tmp0,$rp
2207
2208         call    __ecp_nistz256_sqr_mont_vis3    ! p256_sqr_mont(res_x, M);
2209         add     %sp,LOCALS64+$res_x,$rp
2210
2211         add     %sp,LOCALS64+$tmp0,$bp
2212         call    __ecp_nistz256_sub_from_vis3    ! p256_sub(res_x, res_x, tmp0);
2213         add     %sp,LOCALS64+$res_x,$rp
2214
2215         ldx     [%sp+LOCALS64+$M],$a0           ! forward load
2216         ldx     [%sp+LOCALS64+$M+8],$a1
2217         ldx     [%sp+LOCALS64+$M+16],$a2
2218         ldx     [%sp+LOCALS64+$M+24],$a3
2219
2220         add     %sp,LOCALS64+$S,$bp
2221         call    __ecp_nistz256_sub_morf_vis3    ! p256_sub(S, S, res_x);
2222         add     %sp,LOCALS64+$S,$rp
2223
2224         mov     $acc0,$bi
2225         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(S, S, M);
2226         add     %sp,LOCALS64+$S,$rp
2227
2228         ldx     [%sp+LOCALS64+$res_x],$a0       ! forward load
2229         ldx     [%sp+LOCALS64+$res_x+8],$a1
2230         ldx     [%sp+LOCALS64+$res_x+16],$a2
2231         ldx     [%sp+LOCALS64+$res_x+24],$a3
2232
2233         add     %sp,LOCALS64+$res_y,$bp
2234         call    __ecp_nistz256_sub_from_vis3    ! p256_sub(res_y, S, res_y);
2235         add     %sp,LOCALS64+$res_y,$bp
2236
2237         ! convert output to uint_32[8]
2238         srlx    $a0,32,$t0
2239         srlx    $a1,32,$t1
2240         st      $a0,[$rp_real]                  ! res_x
2241         srlx    $a2,32,$t2
2242         st      $t0,[$rp_real+4]
2243         srlx    $a3,32,$t3
2244         st      $a1,[$rp_real+8]
2245         st      $t1,[$rp_real+12]
2246         st      $a2,[$rp_real+16]
2247         st      $t2,[$rp_real+20]
2248         st      $a3,[$rp_real+24]
2249         st      $t3,[$rp_real+28]
2250
2251         ldx     [%sp+LOCALS64+$res_z],$a0       ! forward load
2252         srlx    $acc0,32,$t0
2253         ldx     [%sp+LOCALS64+$res_z+8],$a1
2254         srlx    $acc1,32,$t1
2255         ldx     [%sp+LOCALS64+$res_z+16],$a2
2256         srlx    $acc2,32,$t2
2257         ldx     [%sp+LOCALS64+$res_z+24],$a3
2258         srlx    $acc3,32,$t3
2259         st      $acc0,[$rp_real+32]             ! res_y
2260         st      $t0,  [$rp_real+32+4]
2261         st      $acc1,[$rp_real+32+8]
2262         st      $t1,  [$rp_real+32+12]
2263         st      $acc2,[$rp_real+32+16]
2264         st      $t2,  [$rp_real+32+20]
2265         st      $acc3,[$rp_real+32+24]
2266         st      $t3,  [$rp_real+32+28]
2267
2268         srlx    $a0,32,$t0
2269         srlx    $a1,32,$t1
2270         st      $a0,[$rp_real+64]               ! res_z
2271         srlx    $a2,32,$t2
2272         st      $t0,[$rp_real+64+4]
2273         srlx    $a3,32,$t3
2274         st      $a1,[$rp_real+64+8]
2275         st      $t1,[$rp_real+64+12]
2276         st      $a2,[$rp_real+64+16]
2277         st      $t2,[$rp_real+64+20]
2278         st      $a3,[$rp_real+64+24]
2279         st      $t3,[$rp_real+64+28]
2280
2281         ret
2282         restore
2283 .type   ecp_nistz256_point_double_vis3,#function
2284 .size   ecp_nistz256_point_double_vis3,.-ecp_nistz256_point_double_vis3
2285 ___
2286 }
2287 ########################################################################
2288 # void ecp_nistz256_point_add(P256_POINT *out,const P256_POINT *in1,
2289 #                             const P256_POINT *in2);
2290 {
2291 my ($res_x,$res_y,$res_z,
2292     $in1_x,$in1_y,$in1_z,
2293     $in2_x,$in2_y,$in2_z,
2294     $H,$Hsqr,$R,$Rsqr,$Hcub,
2295     $U1,$U2,$S1,$S2)=map(32*$_,(0..17));
2296 my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
2297
2298 # above map() describes stack layout with 18 temporary
2299 # 256-bit vectors on top. Then we reserve some space for
2300 # !in1infty, !in2infty and result of check for zero.
2301
2302 $code.=<<___;
2303 .align  32
2304 ecp_nistz256_point_add_vis3:
2305         save    %sp,-STACK64_FRAME-32*18-32,%sp
2306
2307         mov     $rp,$rp_real
2308         mov     -1,$minus1
2309         mov     -2,$poly3
2310         sllx    $minus1,32,$poly1               ! 0xFFFFFFFF00000000
2311         srl     $poly3,0,$poly3                 ! 0x00000000FFFFFFFE
2312
2313         ! convert input to uint64_t[4]
2314         ld      [$bp],$a0                       ! in2_x
2315         ld      [$bp+4],$t0
2316         ld      [$bp+8],$a1
2317         ld      [$bp+12],$t1
2318         ld      [$bp+16],$a2
2319         ld      [$bp+20],$t2
2320         ld      [$bp+24],$a3
2321         ld      [$bp+28],$t3
2322         sllx    $t0,32,$t0
2323         sllx    $t1,32,$t1
2324         ld      [$bp+32],$acc0                  ! in2_y
2325         or      $a0,$t0,$a0
2326         ld      [$bp+32+4],$t0
2327         sllx    $t2,32,$t2
2328         ld      [$bp+32+8],$acc1
2329         or      $a1,$t1,$a1
2330         ld      [$bp+32+12],$t1
2331         sllx    $t3,32,$t3
2332         ld      [$bp+32+16],$acc2
2333         or      $a2,$t2,$a2
2334         ld      [$bp+32+20],$t2
2335         or      $a3,$t3,$a3
2336         ld      [$bp+32+24],$acc3
2337         sllx    $t0,32,$t0
2338         ld      [$bp+32+28],$t3
2339         sllx    $t1,32,$t1
2340         stx     $a0,[%sp+LOCALS64+$in2_x]
2341         sllx    $t2,32,$t2
2342         stx     $a1,[%sp+LOCALS64+$in2_x+8]
2343         sllx    $t3,32,$t3
2344         stx     $a2,[%sp+LOCALS64+$in2_x+16]
2345         or      $acc0,$t0,$acc0
2346         stx     $a3,[%sp+LOCALS64+$in2_x+24]
2347         or      $acc1,$t1,$acc1
2348         stx     $acc0,[%sp+LOCALS64+$in2_y]
2349         or      $acc2,$t2,$acc2
2350         stx     $acc1,[%sp+LOCALS64+$in2_y+8]
2351         or      $acc3,$t3,$acc3
2352         stx     $acc2,[%sp+LOCALS64+$in2_y+16]
2353         stx     $acc3,[%sp+LOCALS64+$in2_y+24]
2354
2355         ld      [$bp+64],$acc0                  ! in2_z
2356         ld      [$bp+64+4],$t0
2357         ld      [$bp+64+8],$acc1
2358         ld      [$bp+64+12],$t1
2359         ld      [$bp+64+16],$acc2
2360         ld      [$bp+64+20],$t2
2361         ld      [$bp+64+24],$acc3
2362         ld      [$bp+64+28],$t3
2363         sllx    $t0,32,$t0
2364         sllx    $t1,32,$t1
2365         ld      [$ap],$a0                       ! in1_x
2366         or      $acc0,$t0,$acc0
2367         ld      [$ap+4],$t0
2368         sllx    $t2,32,$t2
2369         ld      [$ap+8],$a1
2370         or      $acc1,$t1,$acc1
2371         ld      [$ap+12],$t1
2372         sllx    $t3,32,$t3
2373         ld      [$ap+16],$a2
2374         or      $acc2,$t2,$acc2
2375         ld      [$ap+20],$t2
2376         or      $acc3,$t3,$acc3
2377         ld      [$ap+24],$a3
2378         sllx    $t0,32,$t0
2379         ld      [$ap+28],$t3
2380         sllx    $t1,32,$t1
2381         stx     $acc0,[%sp+LOCALS64+$in2_z]
2382         sllx    $t2,32,$t2
2383         stx     $acc1,[%sp+LOCALS64+$in2_z+8]
2384         sllx    $t3,32,$t3
2385         stx     $acc2,[%sp+LOCALS64+$in2_z+16]
2386         stx     $acc3,[%sp+LOCALS64+$in2_z+24]
2387
2388         or      $acc1,$acc0,$acc0
2389         or      $acc3,$acc2,$acc2
2390         or      $acc2,$acc0,$acc0
2391         movrnz  $acc0,-1,$acc0                  ! !in2infty
2392         stx     $acc0,[%fp+STACK_BIAS-8]
2393
2394         or      $a0,$t0,$a0
2395         ld      [$ap+32],$acc0                  ! in1_y
2396         or      $a1,$t1,$a1
2397         ld      [$ap+32+4],$t0
2398         or      $a2,$t2,$a2
2399         ld      [$ap+32+8],$acc1
2400         or      $a3,$t3,$a3
2401         ld      [$ap+32+12],$t1
2402         ld      [$ap+32+16],$acc2
2403         ld      [$ap+32+20],$t2
2404         ld      [$ap+32+24],$acc3
2405         sllx    $t0,32,$t0
2406         ld      [$ap+32+28],$t3
2407         sllx    $t1,32,$t1
2408         stx     $a0,[%sp+LOCALS64+$in1_x]
2409         sllx    $t2,32,$t2
2410         stx     $a1,[%sp+LOCALS64+$in1_x+8]
2411         sllx    $t3,32,$t3
2412         stx     $a2,[%sp+LOCALS64+$in1_x+16]
2413         or      $acc0,$t0,$acc0
2414         stx     $a3,[%sp+LOCALS64+$in1_x+24]
2415         or      $acc1,$t1,$acc1
2416         stx     $acc0,[%sp+LOCALS64+$in1_y]
2417         or      $acc2,$t2,$acc2
2418         stx     $acc1,[%sp+LOCALS64+$in1_y+8]
2419         or      $acc3,$t3,$acc3
2420         stx     $acc2,[%sp+LOCALS64+$in1_y+16]
2421         stx     $acc3,[%sp+LOCALS64+$in1_y+24]
2422
2423         ldx     [%sp+LOCALS64+$in2_z],$a0       ! forward load
2424         ldx     [%sp+LOCALS64+$in2_z+8],$a1
2425         ldx     [%sp+LOCALS64+$in2_z+16],$a2
2426         ldx     [%sp+LOCALS64+$in2_z+24],$a3
2427
2428         ld      [$ap+64],$acc0                  ! in1_z
2429         ld      [$ap+64+4],$t0
2430         ld      [$ap+64+8],$acc1
2431         ld      [$ap+64+12],$t1
2432         ld      [$ap+64+16],$acc2
2433         ld      [$ap+64+20],$t2
2434         ld      [$ap+64+24],$acc3
2435         ld      [$ap+64+28],$t3
2436         sllx    $t0,32,$t0
2437         sllx    $t1,32,$t1
2438         or      $acc0,$t0,$acc0
2439         sllx    $t2,32,$t2
2440         or      $acc1,$t1,$acc1
2441         sllx    $t3,32,$t3
2442         stx     $acc0,[%sp+LOCALS64+$in1_z]
2443         or      $acc2,$t2,$acc2
2444         stx     $acc1,[%sp+LOCALS64+$in1_z+8]
2445         or      $acc3,$t3,$acc3
2446         stx     $acc2,[%sp+LOCALS64+$in1_z+16]
2447         stx     $acc3,[%sp+LOCALS64+$in1_z+24]
2448
2449         or      $acc1,$acc0,$acc0
2450         or      $acc3,$acc2,$acc2
2451         or      $acc2,$acc0,$acc0
2452         movrnz  $acc0,-1,$acc0                  ! !in1infty
2453         stx     $acc0,[%fp+STACK_BIAS-16]
2454
2455         call    __ecp_nistz256_sqr_mont_vis3    ! p256_sqr_mont(Z2sqr, in2_z);
2456         add     %sp,LOCALS64+$Z2sqr,$rp
2457
2458         ldx     [%sp+LOCALS64+$in1_z],$a0
2459         ldx     [%sp+LOCALS64+$in1_z+8],$a1
2460         ldx     [%sp+LOCALS64+$in1_z+16],$a2
2461         ldx     [%sp+LOCALS64+$in1_z+24],$a3
2462         call    __ecp_nistz256_sqr_mont_vis3    ! p256_sqr_mont(Z1sqr, in1_z);
2463         add     %sp,LOCALS64+$Z1sqr,$rp
2464
2465         ldx     [%sp+LOCALS64+$Z2sqr],$bi
2466         ldx     [%sp+LOCALS64+$in2_z],$a0
2467         ldx     [%sp+LOCALS64+$in2_z+8],$a1
2468         ldx     [%sp+LOCALS64+$in2_z+16],$a2
2469         ldx     [%sp+LOCALS64+$in2_z+24],$a3
2470         add     %sp,LOCALS64+$Z2sqr,$bp
2471         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(S1, Z2sqr, in2_z);
2472         add     %sp,LOCALS64+$S1,$rp
2473
2474         ldx     [%sp+LOCALS64+$Z1sqr],$bi
2475         ldx     [%sp+LOCALS64+$in1_z],$a0
2476         ldx     [%sp+LOCALS64+$in1_z+8],$a1
2477         ldx     [%sp+LOCALS64+$in1_z+16],$a2
2478         ldx     [%sp+LOCALS64+$in1_z+24],$a3
2479         add     %sp,LOCALS64+$Z1sqr,$bp
2480         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(S2, Z1sqr, in1_z);
2481         add     %sp,LOCALS64+$S2,$rp
2482
2483         ldx     [%sp+LOCALS64+$S1],$bi
2484         ldx     [%sp+LOCALS64+$in1_y],$a0
2485         ldx     [%sp+LOCALS64+$in1_y+8],$a1
2486         ldx     [%sp+LOCALS64+$in1_y+16],$a2
2487         ldx     [%sp+LOCALS64+$in1_y+24],$a3
2488         add     %sp,LOCALS64+$S1,$bp
2489         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(S1, S1, in1_y);
2490         add     %sp,LOCALS64+$S1,$rp
2491
2492         ldx     [%sp+LOCALS64+$S2],$bi
2493         ldx     [%sp+LOCALS64+$in2_y],$a0
2494         ldx     [%sp+LOCALS64+$in2_y+8],$a1
2495         ldx     [%sp+LOCALS64+$in2_y+16],$a2
2496         ldx     [%sp+LOCALS64+$in2_y+24],$a3
2497         add     %sp,LOCALS64+$S2,$bp
2498         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(S2, S2, in2_y);
2499         add     %sp,LOCALS64+$S2,$rp
2500
2501         ldx     [%sp+LOCALS64+$Z2sqr],$bi       ! forward load
2502         ldx     [%sp+LOCALS64+$in1_x],$a0
2503         ldx     [%sp+LOCALS64+$in1_x+8],$a1
2504         ldx     [%sp+LOCALS64+$in1_x+16],$a2
2505         ldx     [%sp+LOCALS64+$in1_x+24],$a3
2506
2507         add     %sp,LOCALS64+$S1,$bp
2508         call    __ecp_nistz256_sub_from_vis3    ! p256_sub(R, S2, S1);
2509         add     %sp,LOCALS64+$R,$rp
2510
2511         or      $acc1,$acc0,$acc0               ! see if result is zero
2512         or      $acc3,$acc2,$acc2
2513         or      $acc2,$acc0,$acc0
2514         stx     $acc0,[%fp+STACK_BIAS-24]
2515
2516         add     %sp,LOCALS64+$Z2sqr,$bp
2517         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(U1, in1_x, Z2sqr);
2518         add     %sp,LOCALS64+$U1,$rp
2519
2520         ldx     [%sp+LOCALS64+$Z1sqr],$bi
2521         ldx     [%sp+LOCALS64+$in2_x],$a0
2522         ldx     [%sp+LOCALS64+$in2_x+8],$a1
2523         ldx     [%sp+LOCALS64+$in2_x+16],$a2
2524         ldx     [%sp+LOCALS64+$in2_x+24],$a3
2525         add     %sp,LOCALS64+$Z1sqr,$bp
2526         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(U2, in2_x, Z1sqr);
2527         add     %sp,LOCALS64+$U2,$rp
2528
2529         ldx     [%sp+LOCALS64+$R],$a0           ! forward load
2530         ldx     [%sp+LOCALS64+$R+8],$a1
2531         ldx     [%sp+LOCALS64+$R+16],$a2
2532         ldx     [%sp+LOCALS64+$R+24],$a3
2533
2534         add     %sp,LOCALS64+$U1,$bp
2535         call    __ecp_nistz256_sub_from_vis3    ! p256_sub(H, U2, U1);
2536         add     %sp,LOCALS64+$H,$rp
2537
2538         or      $acc1,$acc0,$acc0               ! see if result is zero
2539         or      $acc3,$acc2,$acc2
2540         orcc    $acc2,$acc0,$acc0
2541
2542         bne,pt  %xcc,.Ladd_proceed_vis3         ! is_equal(U1,U2)?
2543         nop
2544
2545         ldx     [%fp+STACK_BIAS-8],$t0
2546         ldx     [%fp+STACK_BIAS-16],$t1
2547         ldx     [%fp+STACK_BIAS-24],$t2
2548         andcc   $t0,$t1,%g0
2549         be,pt   %xcc,.Ladd_proceed_vis3         ! (in1infty || in2infty)?
2550         nop
2551         andcc   $t2,$t2,%g0
2552         be,a,pt %xcc,.Ldouble_shortcut_vis3     ! is_equal(S1,S2)?
2553         add     %sp,32*(12-10)+32,%sp           ! difference in frame sizes
2554
2555         st      %g0,[$rp_real]
2556         st      %g0,[$rp_real+4]
2557         st      %g0,[$rp_real+8]
2558         st      %g0,[$rp_real+12]
2559         st      %g0,[$rp_real+16]
2560         st      %g0,[$rp_real+20]
2561         st      %g0,[$rp_real+24]
2562         st      %g0,[$rp_real+28]
2563         st      %g0,[$rp_real+32]
2564         st      %g0,[$rp_real+32+4]
2565         st      %g0,[$rp_real+32+8]
2566         st      %g0,[$rp_real+32+12]
2567         st      %g0,[$rp_real+32+16]
2568         st      %g0,[$rp_real+32+20]
2569         st      %g0,[$rp_real+32+24]
2570         st      %g0,[$rp_real+32+28]
2571         st      %g0,[$rp_real+64]
2572         st      %g0,[$rp_real+64+4]
2573         st      %g0,[$rp_real+64+8]
2574         st      %g0,[$rp_real+64+12]
2575         st      %g0,[$rp_real+64+16]
2576         st      %g0,[$rp_real+64+20]
2577         st      %g0,[$rp_real+64+24]
2578         st      %g0,[$rp_real+64+28]
2579         b       .Ladd_done_vis3
2580         nop
2581
2582 .align  16
2583 .Ladd_proceed_vis3:
2584         call    __ecp_nistz256_sqr_mont_vis3    ! p256_sqr_mont(Rsqr, R);
2585         add     %sp,LOCALS64+$Rsqr,$rp
2586
2587         ldx     [%sp+LOCALS64+$H],$bi
2588         ldx     [%sp+LOCALS64+$in1_z],$a0
2589         ldx     [%sp+LOCALS64+$in1_z+8],$a1
2590         ldx     [%sp+LOCALS64+$in1_z+16],$a2
2591         ldx     [%sp+LOCALS64+$in1_z+24],$a3
2592         add     %sp,LOCALS64+$H,$bp
2593         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(res_z, H, in1_z);
2594         add     %sp,LOCALS64+$res_z,$rp
2595
2596         ldx     [%sp+LOCALS64+$H],$a0
2597         ldx     [%sp+LOCALS64+$H+8],$a1
2598         ldx     [%sp+LOCALS64+$H+16],$a2
2599         ldx     [%sp+LOCALS64+$H+24],$a3
2600         call    __ecp_nistz256_sqr_mont_vis3    ! p256_sqr_mont(Hsqr, H);
2601         add     %sp,LOCALS64+$Hsqr,$rp
2602
2603         ldx     [%sp+LOCALS64+$res_z],$bi
2604         ldx     [%sp+LOCALS64+$in2_z],$a0
2605         ldx     [%sp+LOCALS64+$in2_z+8],$a1
2606         ldx     [%sp+LOCALS64+$in2_z+16],$a2
2607         ldx     [%sp+LOCALS64+$in2_z+24],$a3
2608         add     %sp,LOCALS64+$res_z,$bp
2609         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(res_z, res_z, in2_z);
2610         add     %sp,LOCALS64+$res_z,$rp
2611
2612         ldx     [%sp+LOCALS64+$H],$bi
2613         ldx     [%sp+LOCALS64+$Hsqr],$a0
2614         ldx     [%sp+LOCALS64+$Hsqr+8],$a1
2615         ldx     [%sp+LOCALS64+$Hsqr+16],$a2
2616         ldx     [%sp+LOCALS64+$Hsqr+24],$a3
2617         add     %sp,LOCALS64+$H,$bp
2618         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(Hcub, Hsqr, H);
2619         add     %sp,LOCALS64+$Hcub,$rp
2620
2621         ldx     [%sp+LOCALS64+$U1],$bi
2622         ldx     [%sp+LOCALS64+$Hsqr],$a0
2623         ldx     [%sp+LOCALS64+$Hsqr+8],$a1
2624         ldx     [%sp+LOCALS64+$Hsqr+16],$a2
2625         ldx     [%sp+LOCALS64+$Hsqr+24],$a3
2626         add     %sp,LOCALS64+$U1,$bp
2627         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(U2, U1, Hsqr);
2628         add     %sp,LOCALS64+$U2,$rp
2629
2630         call    __ecp_nistz256_mul_by_2_vis3    ! p256_mul_by_2(Hsqr, U2);
2631         add     %sp,LOCALS64+$Hsqr,$rp
2632
2633         add     %sp,LOCALS64+$Rsqr,$bp
2634         call    __ecp_nistz256_sub_morf_vis3    ! p256_sub(res_x, Rsqr, Hsqr);
2635         add     %sp,LOCALS64+$res_x,$rp
2636
2637         add     %sp,LOCALS64+$Hcub,$bp
2638         call    __ecp_nistz256_sub_from_vis3    !  p256_sub(res_x, res_x, Hcub);
2639         add     %sp,LOCALS64+$res_x,$rp
2640
2641         ldx     [%sp+LOCALS64+$S1],$bi          ! forward load
2642         ldx     [%sp+LOCALS64+$Hcub],$a0
2643         ldx     [%sp+LOCALS64+$Hcub+8],$a1
2644         ldx     [%sp+LOCALS64+$Hcub+16],$a2
2645         ldx     [%sp+LOCALS64+$Hcub+24],$a3
2646
2647         add     %sp,LOCALS64+$U2,$bp
2648         call    __ecp_nistz256_sub_morf_vis3    ! p256_sub(res_y, U2, res_x);
2649         add     %sp,LOCALS64+$res_y,$rp
2650
2651         add     %sp,LOCALS64+$S1,$bp
2652         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(S2, S1, Hcub);
2653         add     %sp,LOCALS64+$S2,$rp
2654
2655         ldx     [%sp+LOCALS64+$R],$bi
2656         ldx     [%sp+LOCALS64+$res_y],$a0
2657         ldx     [%sp+LOCALS64+$res_y+8],$a1
2658         ldx     [%sp+LOCALS64+$res_y+16],$a2
2659         ldx     [%sp+LOCALS64+$res_y+24],$a3
2660         add     %sp,LOCALS64+$R,$bp
2661         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(res_y, res_y, R);
2662         add     %sp,LOCALS64+$res_y,$rp
2663
2664         add     %sp,LOCALS64+$S2,$bp
2665         call    __ecp_nistz256_sub_from_vis3    ! p256_sub(res_y, res_y, S2);
2666         add     %sp,LOCALS64+$res_y,$rp
2667
2668         ldx     [%fp+STACK_BIAS-16],$t1         ! !in1infty
2669         ldx     [%fp+STACK_BIAS-8],$t2          ! !in2infty
2670 ___
2671 for($i=0;$i<96;$i+=16) {                        # conditional moves
2672 $code.=<<___;
2673         ldx     [%sp+LOCALS64+$res_x+$i],$acc0  ! res
2674         ldx     [%sp+LOCALS64+$res_x+$i+8],$acc1
2675         ldx     [%sp+LOCALS64+$in2_x+$i],$acc2  ! in2
2676         ldx     [%sp+LOCALS64+$in2_x+$i+8],$acc3
2677         ldx     [%sp+LOCALS64+$in1_x+$i],$acc4  ! in1
2678         ldx     [%sp+LOCALS64+$in1_x+$i+8],$acc5
2679         movrz   $t1,$acc2,$acc0
2680         movrz   $t1,$acc3,$acc1
2681         movrz   $t2,$acc4,$acc0
2682         movrz   $t2,$acc5,$acc1
2683         srlx    $acc0,32,$acc2
2684         srlx    $acc1,32,$acc3
2685         st      $acc0,[$rp_real+$i]
2686         st      $acc2,[$rp_real+$i+4]
2687         st      $acc1,[$rp_real+$i+8]
2688         st      $acc3,[$rp_real+$i+12]
2689 ___
2690 }
2691 $code.=<<___;
2692 .Ladd_done_vis3:
2693         ret
2694         restore
2695 .type   ecp_nistz256_point_add_vis3,#function
2696 .size   ecp_nistz256_point_add_vis3,.-ecp_nistz256_point_add_vis3
2697 ___
2698 }
2699 ########################################################################
2700 # void ecp_nistz256_point_add_affine(P256_POINT *out,const P256_POINT *in1,
2701 #                                    const P256_POINT_AFFINE *in2);
2702 {
2703 my ($res_x,$res_y,$res_z,
2704     $in1_x,$in1_y,$in1_z,
2705     $in2_x,$in2_y,
2706     $U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr)=map(32*$_,(0..14));
2707 my $Z1sqr = $S2;
2708 # above map() describes stack layout with 15 temporary
2709 # 256-bit vectors on top. Then we reserve some space for
2710 # !in1infty and !in2infty.
2711
2712 $code.=<<___;
2713 .align  32
2714 ecp_nistz256_point_add_affine_vis3:
2715         save    %sp,-STACK64_FRAME-32*15-32,%sp
2716
2717         mov     $rp,$rp_real
2718         mov     -1,$minus1
2719         mov     -2,$poly3
2720         sllx    $minus1,32,$poly1               ! 0xFFFFFFFF00000000
2721         srl     $poly3,0,$poly3                 ! 0x00000000FFFFFFFE
2722
2723         ! convert input to uint64_t[4]
2724         ld      [$bp],$a0                       ! in2_x
2725         ld      [$bp+4],$t0
2726         ld      [$bp+8],$a1
2727         ld      [$bp+12],$t1
2728         ld      [$bp+16],$a2
2729         ld      [$bp+20],$t2
2730         ld      [$bp+24],$a3
2731         ld      [$bp+28],$t3
2732         sllx    $t0,32,$t0
2733         sllx    $t1,32,$t1
2734         ld      [$bp+32],$acc0                  ! in2_y
2735         or      $a0,$t0,$a0
2736         ld      [$bp+32+4],$t0
2737         sllx    $t2,32,$t2
2738         ld      [$bp+32+8],$acc1
2739         or      $a1,$t1,$a1
2740         ld      [$bp+32+12],$t1
2741         sllx    $t3,32,$t3
2742         ld      [$bp+32+16],$acc2
2743         or      $a2,$t2,$a2
2744         ld      [$bp+32+20],$t2
2745         or      $a3,$t3,$a3
2746         ld      [$bp+32+24],$acc3
2747         sllx    $t0,32,$t0
2748         ld      [$bp+32+28],$t3
2749         sllx    $t1,32,$t1
2750         stx     $a0,[%sp+LOCALS64+$in2_x]
2751         sllx    $t2,32,$t2
2752         stx     $a1,[%sp+LOCALS64+$in2_x+8]
2753         sllx    $t3,32,$t3
2754         stx     $a2,[%sp+LOCALS64+$in2_x+16]
2755         or      $acc0,$t0,$acc0
2756         stx     $a3,[%sp+LOCALS64+$in2_x+24]
2757         or      $acc1,$t1,$acc1
2758         stx     $acc0,[%sp+LOCALS64+$in2_y]
2759         or      $acc2,$t2,$acc2
2760         stx     $acc1,[%sp+LOCALS64+$in2_y+8]
2761         or      $acc3,$t3,$acc3
2762         stx     $acc2,[%sp+LOCALS64+$in2_y+16]
2763         stx     $acc3,[%sp+LOCALS64+$in2_y+24]
2764
2765         or      $a1,$a0,$a0
2766         or      $a3,$a2,$a2
2767         or      $acc1,$acc0,$acc0
2768         or      $acc3,$acc2,$acc2
2769         or      $a2,$a0,$a0
2770         or      $acc2,$acc0,$acc0
2771         or      $acc0,$a0,$a0
2772         movrnz  $a0,-1,$a0                      ! !in2infty
2773         stx     $a0,[%fp+STACK_BIAS-8]
2774
2775         ld      [$ap],$a0                       ! in1_x
2776         ld      [$ap+4],$t0
2777         ld      [$ap+8],$a1
2778         ld      [$ap+12],$t1
2779         ld      [$ap+16],$a2
2780         ld      [$ap+20],$t2
2781         ld      [$ap+24],$a3
2782         ld      [$ap+28],$t3
2783         sllx    $t0,32,$t0
2784         sllx    $t1,32,$t1
2785         ld      [$ap+32],$acc0                  ! in1_y
2786         or      $a0,$t0,$a0
2787         ld      [$ap+32+4],$t0
2788         sllx    $t2,32,$t2
2789         ld      [$ap+32+8],$acc1
2790         or      $a1,$t1,$a1
2791         ld      [$ap+32+12],$t1
2792         sllx    $t3,32,$t3
2793         ld      [$ap+32+16],$acc2
2794         or      $a2,$t2,$a2
2795         ld      [$ap+32+20],$t2
2796         or      $a3,$t3,$a3
2797         ld      [$ap+32+24],$acc3
2798         sllx    $t0,32,$t0
2799         ld      [$ap+32+28],$t3
2800         sllx    $t1,32,$t1
2801         stx     $a0,[%sp+LOCALS64+$in1_x]
2802         sllx    $t2,32,$t2
2803         stx     $a1,[%sp+LOCALS64+$in1_x+8]
2804         sllx    $t3,32,$t3
2805         stx     $a2,[%sp+LOCALS64+$in1_x+16]
2806         or      $acc0,$t0,$acc0
2807         stx     $a3,[%sp+LOCALS64+$in1_x+24]
2808         or      $acc1,$t1,$acc1
2809         stx     $acc0,[%sp+LOCALS64+$in1_y]
2810         or      $acc2,$t2,$acc2
2811         stx     $acc1,[%sp+LOCALS64+$in1_y+8]
2812         or      $acc3,$t3,$acc3
2813         stx     $acc2,[%sp+LOCALS64+$in1_y+16]
2814         stx     $acc3,[%sp+LOCALS64+$in1_y+24]
2815
2816         ld      [$ap+64],$a0                    ! in1_z
2817         ld      [$ap+64+4],$t0
2818         ld      [$ap+64+8],$a1
2819         ld      [$ap+64+12],$t1
2820         ld      [$ap+64+16],$a2
2821         ld      [$ap+64+20],$t2
2822         ld      [$ap+64+24],$a3
2823         ld      [$ap+64+28],$t3
2824         sllx    $t0,32,$t0
2825         sllx    $t1,32,$t1
2826         or      $a0,$t0,$a0
2827         sllx    $t2,32,$t2
2828         or      $a1,$t1,$a1
2829         sllx    $t3,32,$t3
2830         stx     $a0,[%sp+LOCALS64+$in1_z]
2831         or      $a2,$t2,$a2
2832         stx     $a1,[%sp+LOCALS64+$in1_z+8]
2833         or      $a3,$t3,$a3
2834         stx     $a2,[%sp+LOCALS64+$in1_z+16]
2835         stx     $a3,[%sp+LOCALS64+$in1_z+24]
2836
2837         or      $a1,$a0,$t0
2838         or      $a3,$a2,$t2
2839         or      $t2,$t0,$t0
2840         movrnz  $t0,-1,$t0                      ! !in1infty
2841         stx     $t0,[%fp+STACK_BIAS-16]
2842
2843         call    __ecp_nistz256_sqr_mont_vis3    ! p256_sqr_mont(Z1sqr, in1_z);
2844         add     %sp,LOCALS64+$Z1sqr,$rp
2845
2846         ldx     [%sp+LOCALS64+$in2_x],$bi
2847         mov     $acc0,$a0
2848         mov     $acc1,$a1
2849         mov     $acc2,$a2
2850         mov     $acc3,$a3
2851         add     %sp,LOCALS64+$in2_x,$bp
2852         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(U2, Z1sqr, in2_x);
2853         add     %sp,LOCALS64+$U2,$rp
2854
2855         ldx     [%sp+LOCALS64+$Z1sqr],$bi       ! forward load
2856         ldx     [%sp+LOCALS64+$in1_z],$a0
2857         ldx     [%sp+LOCALS64+$in1_z+8],$a1
2858         ldx     [%sp+LOCALS64+$in1_z+16],$a2
2859         ldx     [%sp+LOCALS64+$in1_z+24],$a3
2860
2861         add     %sp,LOCALS64+$in1_x,$bp
2862         call    __ecp_nistz256_sub_from_vis3    ! p256_sub(H, U2, in1_x);
2863         add     %sp,LOCALS64+$H,$rp
2864
2865         add     %sp,LOCALS64+$Z1sqr,$bp
2866         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(S2, Z1sqr, in1_z);
2867         add     %sp,LOCALS64+$S2,$rp
2868
2869         ldx     [%sp+LOCALS64+$H],$bi
2870         ldx     [%sp+LOCALS64+$in1_z],$a0
2871         ldx     [%sp+LOCALS64+$in1_z+8],$a1
2872         ldx     [%sp+LOCALS64+$in1_z+16],$a2
2873         ldx     [%sp+LOCALS64+$in1_z+24],$a3
2874         add     %sp,LOCALS64+$H,$bp
2875         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(res_z, H, in1_z);
2876         add     %sp,LOCALS64+$res_z,$rp
2877
2878         ldx     [%sp+LOCALS64+$S2],$bi
2879         ldx     [%sp+LOCALS64+$in2_y],$a0
2880         ldx     [%sp+LOCALS64+$in2_y+8],$a1
2881         ldx     [%sp+LOCALS64+$in2_y+16],$a2
2882         ldx     [%sp+LOCALS64+$in2_y+24],$a3
2883         add     %sp,LOCALS64+$S2,$bp
2884         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(S2, S2, in2_y);
2885         add     %sp,LOCALS64+$S2,$rp
2886
2887         ldx     [%sp+LOCALS64+$H],$a0           ! forward load
2888         ldx     [%sp+LOCALS64+$H+8],$a1
2889         ldx     [%sp+LOCALS64+$H+16],$a2
2890         ldx     [%sp+LOCALS64+$H+24],$a3
2891
2892         add     %sp,LOCALS64+$in1_y,$bp
2893         call    __ecp_nistz256_sub_from_vis3    ! p256_sub(R, S2, in1_y);
2894         add     %sp,LOCALS64+$R,$rp
2895
2896         call    __ecp_nistz256_sqr_mont_vis3    ! p256_sqr_mont(Hsqr, H);
2897         add     %sp,LOCALS64+$Hsqr,$rp
2898
2899         ldx     [%sp+LOCALS64+$R],$a0
2900         ldx     [%sp+LOCALS64+$R+8],$a1
2901         ldx     [%sp+LOCALS64+$R+16],$a2
2902         ldx     [%sp+LOCALS64+$R+24],$a3
2903         call    __ecp_nistz256_sqr_mont_vis3    ! p256_sqr_mont(Rsqr, R);
2904         add     %sp,LOCALS64+$Rsqr,$rp
2905
2906         ldx     [%sp+LOCALS64+$H],$bi
2907         ldx     [%sp+LOCALS64+$Hsqr],$a0
2908         ldx     [%sp+LOCALS64+$Hsqr+8],$a1
2909         ldx     [%sp+LOCALS64+$Hsqr+16],$a2
2910         ldx     [%sp+LOCALS64+$Hsqr+24],$a3
2911         add     %sp,LOCALS64+$H,$bp
2912         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(Hcub, Hsqr, H);
2913         add     %sp,LOCALS64+$Hcub,$rp
2914
2915         ldx     [%sp+LOCALS64+$Hsqr],$bi
2916         ldx     [%sp+LOCALS64+$in1_x],$a0
2917         ldx     [%sp+LOCALS64+$in1_x+8],$a1
2918         ldx     [%sp+LOCALS64+$in1_x+16],$a2
2919         ldx     [%sp+LOCALS64+$in1_x+24],$a3
2920         add     %sp,LOCALS64+$Hsqr,$bp
2921         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(U2, in1_x, Hsqr);
2922         add     %sp,LOCALS64+$U2,$rp
2923
2924         call    __ecp_nistz256_mul_by_2_vis3    ! p256_mul_by_2(Hsqr, U2);
2925         add     %sp,LOCALS64+$Hsqr,$rp
2926
2927         add     %sp,LOCALS64+$Rsqr,$bp
2928         call    __ecp_nistz256_sub_morf_vis3    ! p256_sub(res_x, Rsqr, Hsqr);
2929         add     %sp,LOCALS64+$res_x,$rp
2930
2931         add     %sp,LOCALS64+$Hcub,$bp
2932         call    __ecp_nistz256_sub_from_vis3    !  p256_sub(res_x, res_x, Hcub);
2933         add     %sp,LOCALS64+$res_x,$rp
2934
2935         ldx     [%sp+LOCALS64+$Hcub],$bi        ! forward load
2936         ldx     [%sp+LOCALS64+$in1_y],$a0
2937         ldx     [%sp+LOCALS64+$in1_y+8],$a1
2938         ldx     [%sp+LOCALS64+$in1_y+16],$a2
2939         ldx     [%sp+LOCALS64+$in1_y+24],$a3
2940
2941         add     %sp,LOCALS64+$U2,$bp
2942         call    __ecp_nistz256_sub_morf_vis3    ! p256_sub(res_y, U2, res_x);
2943         add     %sp,LOCALS64+$res_y,$rp
2944
2945         add     %sp,LOCALS64+$Hcub,$bp
2946         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(S2, in1_y, Hcub);
2947         add     %sp,LOCALS64+$S2,$rp
2948
2949         ldx     [%sp+LOCALS64+$R],$bi
2950         ldx     [%sp+LOCALS64+$res_y],$a0
2951         ldx     [%sp+LOCALS64+$res_y+8],$a1
2952         ldx     [%sp+LOCALS64+$res_y+16],$a2
2953         ldx     [%sp+LOCALS64+$res_y+24],$a3
2954         add     %sp,LOCALS64+$R,$bp
2955         call    __ecp_nistz256_mul_mont_vis3    ! p256_mul_mont(res_y, res_y, R);
2956         add     %sp,LOCALS64+$res_y,$rp
2957
2958         add     %sp,LOCALS64+$S2,$bp
2959         call    __ecp_nistz256_sub_from_vis3    ! p256_sub(res_y, res_y, S2);
2960         add     %sp,LOCALS64+$res_y,$rp
2961
2962         ldx     [%fp+STACK_BIAS-16],$t1         ! !in1infty
2963         ldx     [%fp+STACK_BIAS-8],$t2          ! !in2infty
2964 1:      call    .+8
2965         add     %o7,.Lone_mont_vis3-1b,$bp
2966 ___
2967 for($i=0;$i<64;$i+=16) {                        # conditional moves
2968 $code.=<<___;
2969         ldx     [%sp+LOCALS64+$res_x+$i],$acc0  ! res
2970         ldx     [%sp+LOCALS64+$res_x+$i+8],$acc1
2971         ldx     [%sp+LOCALS64+$in2_x+$i],$acc2  ! in2
2972         ldx     [%sp+LOCALS64+$in2_x+$i+8],$acc3
2973         ldx     [%sp+LOCALS64+$in1_x+$i],$acc4  ! in1
2974         ldx     [%sp+LOCALS64+$in1_x+$i+8],$acc5
2975         movrz   $t1,$acc2,$acc0
2976         movrz   $t1,$acc3,$acc1
2977         movrz   $t2,$acc4,$acc0
2978         movrz   $t2,$acc5,$acc1
2979         srlx    $acc0,32,$acc2
2980         srlx    $acc1,32,$acc3
2981         st      $acc0,[$rp_real+$i]
2982         st      $acc2,[$rp_real+$i+4]
2983         st      $acc1,[$rp_real+$i+8]
2984         st      $acc3,[$rp_real+$i+12]
2985 ___
2986 }
2987 for(;$i<96;$i+=16) {
2988 $code.=<<___;
2989         ldx     [%sp+LOCALS64+$res_x+$i],$acc0  ! res
2990         ldx     [%sp+LOCALS64+$res_x+$i+8],$acc1
2991         ldx     [$bp+$i-64],$acc2               ! "in2"
2992         ldx     [$bp+$i-64+8],$acc3
2993         ldx     [%sp+LOCALS64+$in1_x+$i],$acc4  ! in1
2994         ldx     [%sp+LOCALS64+$in1_x+$i+8],$acc5
2995         movrz   $t1,$acc2,$acc0
2996         movrz   $t1,$acc3,$acc1
2997         movrz   $t2,$acc4,$acc0
2998         movrz   $t2,$acc5,$acc1
2999         srlx    $acc0,32,$acc2
3000         srlx    $acc1,32,$acc3
3001         st      $acc0,[$rp_real+$i]
3002         st      $acc2,[$rp_real+$i+4]
3003         st      $acc1,[$rp_real+$i+8]
3004         st      $acc3,[$rp_real+$i+12]
3005 ___
3006 }
3007 $code.=<<___;
3008         ret
3009         restore
3010 .type   ecp_nistz256_point_add_affine_vis3,#function
3011 .size   ecp_nistz256_point_add_affine_vis3,.-ecp_nistz256_point_add_affine_vis3
3012 .align  64
3013 .Lone_mont_vis3:
3014 .long   0x00000000,0x00000001, 0xffffffff,0x00000000
3015 .long   0xffffffff,0xffffffff, 0x00000000,0xfffffffe
3016 .align  64
3017 ___
3018 }                                                               }}}
3019 \f
3020 # Purpose of these subroutines is to explicitly encode VIS instructions,
3021 # so that one can compile the module without having to specify VIS
3022 # extensions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a.
3023 # Idea is to reserve for option to produce "universal" binary and let
3024 # programmer detect if current CPU is VIS capable at run-time.
3025 sub unvis3 {
3026 my ($mnemonic,$rs1,$rs2,$rd)=@_;
3027 my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 );
3028 my ($ref,$opf);
3029 my %visopf = (  "addxc"         => 0x011,
3030                 "addxccc"       => 0x013,
3031                 "umulxhi"       => 0x016        );
3032
3033     $ref = "$mnemonic\t$rs1,$rs2,$rd";
3034
3035     if ($opf=$visopf{$mnemonic}) {
3036         foreach ($rs1,$rs2,$rd) {
3037             return $ref if (!/%([goli])([0-9])/);
3038             $_=$bias{$1}+$2;
3039         }
3040
3041         return  sprintf ".word\t0x%08x !%s",
3042                         0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2,
3043                         $ref;
3044     } else {
3045         return $ref;
3046     }
3047 }
3048
3049 foreach (split("\n",$code)) {
3050         s/\`([^\`]*)\`/eval $1/ge;
3051
3052         s/\b(umulxhi|addxc[c]{0,2})\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/
3053                 &unvis3($1,$2,$3,$4)
3054          /ge;
3055
3056         print $_,"\n";
3057 }
3058
3059 close STDOUT or die "error closing STDOUT: $!";