crypto/bn/asm/ppc64-mont.pl

   1 #!/usr/bin/env perl
   2
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9
  10 # December 2007
  11
  12 # The reason for undertaken effort is basically following. Even though
  13 # Power 6 CPU operates at incredible 4.7GHz clock frequency, its PKI
  14 # performance was observed to be less than impressive, essentially as
  15 # fast as 1.8GHz PPC970, or 2.6 times(!) slower than one would hope.
  16 # Well, it's not surprising that IBM had to make some sacrifices to
  17 # boost the clock frequency that much, but no overall improvement?
  18 # Having observed how much difference did switching to FPU make on
  19 # UltraSPARC, playing same stunt on Power 6 appeared appropriate...
  20 # Unfortunately the resulting performance improvement is not as
  21 # impressive, ~30%, and in absolute terms is still very far from what
  22 # one would expect from 4.7GHz CPU. There is a chance that I'm doing
  23 # something wrong, but in the lack of assembler level micro-profiling
  24 # data or at least decent platform guide I can't tell... Or better
  25 # results might be achieved with VMX... Anyway, this module provides
  26 # *worse* performance on other PowerPC implementations, ~40-15% slower
  27 # on PPC970 depending on key length and ~40% slower on Power 5 for all
  28 # key lengths. As it's obviously inappropriate as "best all-round"
  29 # alternative, it has to be complemented with run-time CPU family
  30 # detection. Oh! It should also be noted that unlike other PowerPC
  31 # implementation IALU ppc-mont.pl module performs *suboptimaly* on
  32 # >=1024-bit key lengths on Power 6. It should also be noted that
  33 # *everything* said so far applies to 64-bit builds! As far as 32-bit
  34 # application executed on 64-bit CPU goes, this module is likely to
  35 # become preferred choice, because it's easy to adapt it for such
  36 # case and *is* faster than 32-bit ppc-mont.pl on *all* processors.
  37
  38 # February 2008
  39
  40 # Micro-profiling assisted optimization results in ~15% improvement
  41 # over original ppc64-mont.pl version, or overall ~50% improvement
  42 # over ppc.pl module on Power 6. If compared to ppc-mont.pl on same
  43 # Power 6 CPU, this module is 5-150% faster depending on key length,
  44 # [hereafter] more for longer keys. But if compared to ppc-mont.pl
  45 # on 1.8GHz PPC970, it's only 5-55% faster. Still far from impressive
  46 # in absolute terms, but it's apparently the way Power 6 is...
  47
  48 # December 2009
  49
  50 # Adapted for 32-bit build this module delivers 25-120%, yes, more
  51 # than *twice* for longer keys, performance improvement over 32-bit
  52 # ppc-mont.pl on 1.8GHz PPC970. However! This implementation utilizes
  53 # even 64-bit integer operations and the trouble is that most PPC
  54 # operating systems don't preserve upper halves of general purpose
  55 # registers upon 32-bit signal delivery. They do preserve them upon
  56 # context switch, but not signalling:-( This means that asynchronous
  57 # signals have to be blocked upon entry to this subroutine. Signal
  58 # masking (and of course complementary unmasking) has quite an impact
  59 # on performance, naturally larger for shorter keys. It's so severe
  60 # that 512-bit key performance can be as low as 1/3 of expected one.
  61 # This is why this routine can be engaged for longer key operations
  62 # only on these OSes, see crypto/ppccap.c for further details. MacOS X
  63 # is an exception from this and doesn't require signal masking, and
  64 # that's where above improvement coefficients were collected. For
  65 # others alternative would be to break dependence on upper halves of
  66 # GPRs by sticking to 32-bit integer operations...
  67
  68 # December 2012
  69
  70 # Remove above mentioned dependence on GPRs' upper halves in 32-bit
  71 # build. No signal masking overhead, but integer instructions are
  72 # *more* numerous... It's still "universally" faster than 32-bit
  73 # ppc-mont.pl, but improvement coefficient is not as impressive
  74 # for longer keys...
  75
  76 $flavour = shift;
  77
  78 if ($flavour =~ /32/) {
  79         $SIZE_T=4;
  80         $RZONE= 224;
  81         $fname= "bn_mul_mont_fpu64";
  82
  83         $STUX=  "stwux";        # store indexed and update
  84         $PUSH=  "stw";
  85         $POP=   "lwz";
  86 } elsif ($flavour =~ /64/) {
  87         $SIZE_T=8;
  88         $RZONE= 288;
  89         $fname= "bn_mul_mont_fpu64";
  90
  91         # same as above, but 64-bit mnemonics...
  92         $STUX=  "stdux";        # store indexed and update
  93         $PUSH=  "std";
  94         $POP=   "ld";
  95 } else { die "nonsense $flavour"; }
  96
  97 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  98 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
  99 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
 100 die "can't locate ppc-xlate.pl";
 101
 102 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
 103
 104 $FRAME=64;      # padded frame header
 105 $TRANSFER=16*8;
 106
 107 $carry="r0";
 108 $sp="r1";
 109 $toc="r2";
 110 $rp="r3";       $ovf="r3";
 111 $ap="r4";
 112 $bp="r5";
 113 $np="r6";
 114 $n0="r7";
 115 $num="r8";
 116 $rp="r9";       # $rp is reassigned
 117 $tp="r10";
 118 $j="r11";
 119 $i="r12";
 120 # non-volatile registers
 121 $c1="r19";
 122 $n1="r20";
 123 $a1="r21";
 124 $nap_d="r22";   # interleaved ap and np in double format
 125 $a0="r23";      # ap[0]
 126 $t0="r24";      # temporary registers
 127 $t1="r25";
 128 $t2="r26";
 129 $t3="r27";
 130 $t4="r28";
 131 $t5="r29";
 132 $t6="r30";
 133 $t7="r31";
 134
 135 # PPC offers enough register bank capacity to unroll inner loops twice
 136 #
 137 #     ..A3A2A1A0
 138 #           dcba
 139 #    -----------
 140 #            A0a
 141 #           A0b
 142 #          A0c
 143 #         A0d
 144 #          A1a
 145 #         A1b
 146 #        A1c
 147 #       A1d
 148 #        A2a
 149 #       A2b
 150 #      A2c
 151 #     A2d
 152 #      A3a
 153 #     A3b
 154 #    A3c
 155 #   A3d
 156 #    ..a
 157 #   ..b
 158 #
 159 $ba="f0";       $bb="f1";       $bc="f2";       $bd="f3";
 160 $na="f4";       $nb="f5";       $nc="f6";       $nd="f7";
 161 $dota="f8";     $dotb="f9";
 162 $A0="f10";      $A1="f11";      $A2="f12";      $A3="f13";
 163 $N0="f20";      $N1="f21";      $N2="f22";      $N3="f23";
 164 $T0a="f24";     $T0b="f25";
 165 $T1a="f26";     $T1b="f27";
 166 $T2a="f28";     $T2b="f29";
 167 $T3a="f30";     $T3b="f31";
 168 \f
 169 # sp----------->+-------------------------------+
 170 #               | saved sp                      |
 171 #               +-------------------------------+
 172 #               .                               .
 173 #   +64         +-------------------------------+
 174 #               | 16 gpr<->fpr transfer zone    |
 175 #               .                               .
 176 #               .                               .
 177 #   +16*8       +-------------------------------+
 178 #               | __int64 tmp[-1]               |
 179 #               +-------------------------------+
 180 #               | __int64 tmp[num]              |
 181 #               .                               .
 182 #               .                               .
 183 #               .                               .
 184 #   +(num+1)*8  +-------------------------------+
 185 #               | padding to 64 byte boundary   |
 186 #               .                               .
 187 #   +X          +-------------------------------+
 188 #               | double nap_d[4*num]           |
 189 #               .                               .
 190 #               .                               .
 191 #               .                               .
 192 #               +-------------------------------+
 193 #               .                               .
 194 #   -13*size_t  +-------------------------------+
 195 #               | 13 saved gpr, r19-r31         |
 196 #               .                               .
 197 #               .                               .
 198 #   -12*8       +-------------------------------+
 199 #               | 12 saved fpr, f20-f31         |
 200 #               .                               .
 201 #               .                               .
 202 #               +-------------------------------+
 203 \f
 204 $code=<<___;
 205 .machine "any"
 206 .text
 207
 208 .globl  .$fname
 209 .align  5
 210 .$fname:
 211         cmpwi   $num,`3*8/$SIZE_T`
 212         mr      $rp,r3          ; $rp is reassigned
 213         li      r3,0            ; possible "not handled" return code
 214         bltlr-
 215         andi.   r0,$num,`16/$SIZE_T-1`          ; $num has to be "even"
 216         bnelr-
 217
 218         slwi    $num,$num,`log($SIZE_T)/log(2)` ; num*=sizeof(BN_LONG)
 219         li      $i,-4096
 220         slwi    $tp,$num,2      ; place for {an}p_{lh}[num], i.e. 4*num
 221         add     $tp,$tp,$num    ; place for tp[num+1]
 222         addi    $tp,$tp,`$FRAME+$TRANSFER+8+64+$RZONE`
 223         subf    $tp,$tp,$sp     ; $sp-$tp
 224         and     $tp,$tp,$i      ; minimize TLB usage
 225         subf    $tp,$sp,$tp     ; $tp-$sp
 226         mr      $i,$sp
 227         $STUX   $sp,$sp,$tp     ; alloca
 228
 229         $PUSH   r19,`-12*8-13*$SIZE_T`($i)
 230         $PUSH   r20,`-12*8-12*$SIZE_T`($i)
 231         $PUSH   r21,`-12*8-11*$SIZE_T`($i)
 232         $PUSH   r22,`-12*8-10*$SIZE_T`($i)
 233         $PUSH   r23,`-12*8-9*$SIZE_T`($i)
 234         $PUSH   r24,`-12*8-8*$SIZE_T`($i)
 235         $PUSH   r25,`-12*8-7*$SIZE_T`($i)
 236         $PUSH   r26,`-12*8-6*$SIZE_T`($i)
 237         $PUSH   r27,`-12*8-5*$SIZE_T`($i)
 238         $PUSH   r28,`-12*8-4*$SIZE_T`($i)
 239         $PUSH   r29,`-12*8-3*$SIZE_T`($i)
 240         $PUSH   r30,`-12*8-2*$SIZE_T`($i)
 241         $PUSH   r31,`-12*8-1*$SIZE_T`($i)
 242         stfd    f20,`-12*8`($i)
 243         stfd    f21,`-11*8`($i)
 244         stfd    f22,`-10*8`($i)
 245         stfd    f23,`-9*8`($i)
 246         stfd    f24,`-8*8`($i)
 247         stfd    f25,`-7*8`($i)
 248         stfd    f26,`-6*8`($i)
 249         stfd    f27,`-5*8`($i)
 250         stfd    f28,`-4*8`($i)
 251         stfd    f29,`-3*8`($i)
 252         stfd    f30,`-2*8`($i)
 253         stfd    f31,`-1*8`($i)
 254
 255         addi    $tp,$sp,`$FRAME+$TRANSFER+8+64`
 256         li      $i,-64
 257         add     $nap_d,$tp,$num
 258         and     $nap_d,$nap_d,$i        ; align to 64 bytes
 259         ; nap_d is off by 1, because it's used with stfdu/lfdu
 260         addi    $nap_d,$nap_d,-8
 261         srwi    $j,$num,`3+1`   ; counter register, num/2
 262         addi    $j,$j,-1
 263         addi    $tp,$sp,`$FRAME+$TRANSFER-8`
 264         li      $carry,0
 265         mtctr   $j
 266 ___
 267 \f
 268 $code.=<<___ if ($SIZE_T==8);
 269         ld      $a0,0($ap)              ; pull ap[0] value
 270         ld      $t3,0($bp)              ; bp[0]
 271         ld      $n0,0($n0)              ; pull n0[0] value
 272
 273         mulld   $t7,$a0,$t3             ; ap[0]*bp[0]
 274         ; transfer bp[0] to FPU as 4x16-bit values
 275         extrdi  $t0,$t3,16,48
 276         extrdi  $t1,$t3,16,32
 277         extrdi  $t2,$t3,16,16
 278         extrdi  $t3,$t3,16,0
 279         std     $t0,`$FRAME+0`($sp)
 280         std     $t1,`$FRAME+8`($sp)
 281         std     $t2,`$FRAME+16`($sp)
 282         std     $t3,`$FRAME+24`($sp)
 283
 284         mulld   $t7,$t7,$n0             ; tp[0]*n0
 285         ; transfer (ap[0]*bp[0])*n0 to FPU as 4x16-bit values
 286         extrdi  $t4,$t7,16,48
 287         extrdi  $t5,$t7,16,32
 288         extrdi  $t6,$t7,16,16
 289         extrdi  $t7,$t7,16,0
 290         std     $t4,`$FRAME+32`($sp)
 291         std     $t5,`$FRAME+40`($sp)
 292         std     $t6,`$FRAME+48`($sp)
 293         std     $t7,`$FRAME+56`($sp)
 294
 295         extrdi  $t0,$a0,32,32           ; lwz   $t0,4($ap)
 296         extrdi  $t1,$a0,32,0            ; lwz   $t1,0($ap)
 297         lwz     $t2,12($ap)             ; load a[1] as 32-bit word pair
 298         lwz     $t3,8($ap)
 299         lwz     $t4,4($np)              ; load n[0] as 32-bit word pair
 300         lwz     $t5,0($np)
 301         lwz     $t6,12($np)             ; load n[1] as 32-bit word pair
 302         lwz     $t7,8($np)
 303 ___
 304 $code.=<<___ if ($SIZE_T==4);
 305         lwz     $a0,0($ap)              ; pull ap[0,1] value
 306         mr      $n1,$n0
 307         lwz     $a1,4($ap)
 308         li      $c1,0
 309         lwz     $t1,0($bp)              ; bp[0,1]
 310         lwz     $t3,4($bp)
 311         lwz     $n0,0($n1)              ; pull n0[0,1] value
 312         lwz     $n1,4($n1)
 313
 314         mullw   $t4,$a0,$t1             ; mulld ap[0]*bp[0]
 315         mulhwu  $t5,$a0,$t1
 316         mullw   $t6,$a1,$t1
 317         mullw   $t7,$a0,$t3
 318         add     $t5,$t5,$t6
 319         add     $t5,$t5,$t7
 320         ; transfer bp[0] to FPU as 4x16-bit values
 321         extrwi  $t0,$t1,16,16
 322         extrwi  $t1,$t1,16,0
 323         extrwi  $t2,$t3,16,16
 324         extrwi  $t3,$t3,16,0
 325         std     $t0,`$FRAME+0`($sp)     ; yes, std in 32-bit build
 326         std     $t1,`$FRAME+8`($sp)
 327         std     $t2,`$FRAME+16`($sp)
 328         std     $t3,`$FRAME+24`($sp)
 329
 330         mullw   $t0,$t4,$n0             ; mulld tp[0]*n0
 331         mulhwu  $t1,$t4,$n0
 332         mullw   $t2,$t5,$n0
 333         mullw   $t3,$t4,$n1
 334         add     $t1,$t1,$t2
 335         add     $t1,$t1,$t3
 336         ; transfer (ap[0]*bp[0])*n0 to FPU as 4x16-bit values
 337         extrwi  $t4,$t0,16,16
 338         extrwi  $t5,$t0,16,0
 339         extrwi  $t6,$t1,16,16
 340         extrwi  $t7,$t1,16,0
 341         std     $t4,`$FRAME+32`($sp)    ; yes, std in 32-bit build
 342         std     $t5,`$FRAME+40`($sp)
 343         std     $t6,`$FRAME+48`($sp)
 344         std     $t7,`$FRAME+56`($sp)
 345
 346         mr      $t0,$a0                 ; lwz   $t0,0($ap)
 347         mr      $t1,$a1                 ; lwz   $t1,4($ap)
 348         lwz     $t2,8($ap)              ; load a[j..j+3] as 32-bit word pairs
 349         lwz     $t3,12($ap)
 350         lwz     $t4,0($np)              ; load n[j..j+3] as 32-bit word pairs
 351         lwz     $t5,4($np)
 352         lwz     $t6,8($np)
 353         lwz     $t7,12($np)
 354 ___
 355 $code.=<<___;
 356         lfd     $ba,`$FRAME+0`($sp)
 357         lfd     $bb,`$FRAME+8`($sp)
 358         lfd     $bc,`$FRAME+16`($sp)
 359         lfd     $bd,`$FRAME+24`($sp)
 360         lfd     $na,`$FRAME+32`($sp)
 361         lfd     $nb,`$FRAME+40`($sp)
 362         lfd     $nc,`$FRAME+48`($sp)
 363         lfd     $nd,`$FRAME+56`($sp)
 364         std     $t0,`$FRAME+64`($sp)    ; yes, std even in 32-bit build
 365         std     $t1,`$FRAME+72`($sp)
 366         std     $t2,`$FRAME+80`($sp)
 367         std     $t3,`$FRAME+88`($sp)
 368         std     $t4,`$FRAME+96`($sp)
 369         std     $t5,`$FRAME+104`($sp)
 370         std     $t6,`$FRAME+112`($sp)
 371         std     $t7,`$FRAME+120`($sp)
 372         fcfid   $ba,$ba
 373         fcfid   $bb,$bb
 374         fcfid   $bc,$bc
 375         fcfid   $bd,$bd
 376         fcfid   $na,$na
 377         fcfid   $nb,$nb
 378         fcfid   $nc,$nc
 379         fcfid   $nd,$nd
 380
 381         lfd     $A0,`$FRAME+64`($sp)
 382         lfd     $A1,`$FRAME+72`($sp)
 383         lfd     $A2,`$FRAME+80`($sp)
 384         lfd     $A3,`$FRAME+88`($sp)
 385         lfd     $N0,`$FRAME+96`($sp)
 386         lfd     $N1,`$FRAME+104`($sp)
 387         lfd     $N2,`$FRAME+112`($sp)
 388         lfd     $N3,`$FRAME+120`($sp)
 389         fcfid   $A0,$A0
 390         fcfid   $A1,$A1
 391         fcfid   $A2,$A2
 392         fcfid   $A3,$A3
 393         fcfid   $N0,$N0
 394         fcfid   $N1,$N1
 395         fcfid   $N2,$N2
 396         fcfid   $N3,$N3
 397         addi    $ap,$ap,16
 398         addi    $np,$np,16
 399
 400         fmul    $T1a,$A1,$ba
 401         fmul    $T1b,$A1,$bb
 402         stfd    $A0,8($nap_d)           ; save a[j] in double format
 403         stfd    $A1,16($nap_d)
 404         fmul    $T2a,$A2,$ba
 405         fmul    $T2b,$A2,$bb
 406         stfd    $A2,24($nap_d)          ; save a[j+1] in double format
 407         stfd    $A3,32($nap_d)
 408         fmul    $T3a,$A3,$ba
 409         fmul    $T3b,$A3,$bb
 410         stfd    $N0,40($nap_d)          ; save n[j] in double format
 411         stfd    $N1,48($nap_d)
 412         fmul    $T0a,$A0,$ba
 413         fmul    $T0b,$A0,$bb
 414         stfd    $N2,56($nap_d)          ; save n[j+1] in double format
 415         stfdu   $N3,64($nap_d)
 416
 417         fmadd   $T1a,$A0,$bc,$T1a
 418         fmadd   $T1b,$A0,$bd,$T1b
 419         fmadd   $T2a,$A1,$bc,$T2a
 420         fmadd   $T2b,$A1,$bd,$T2b
 421         fmadd   $T3a,$A2,$bc,$T3a
 422         fmadd   $T3b,$A2,$bd,$T3b
 423         fmul    $dota,$A3,$bc
 424         fmul    $dotb,$A3,$bd
 425
 426         fmadd   $T1a,$N1,$na,$T1a
 427         fmadd   $T1b,$N1,$nb,$T1b
 428         fmadd   $T2a,$N2,$na,$T2a
 429         fmadd   $T2b,$N2,$nb,$T2b
 430         fmadd   $T3a,$N3,$na,$T3a
 431         fmadd   $T3b,$N3,$nb,$T3b
 432         fmadd   $T0a,$N0,$na,$T0a
 433         fmadd   $T0b,$N0,$nb,$T0b
 434
 435         fmadd   $T1a,$N0,$nc,$T1a
 436         fmadd   $T1b,$N0,$nd,$T1b
 437         fmadd   $T2a,$N1,$nc,$T2a
 438         fmadd   $T2b,$N1,$nd,$T2b
 439         fmadd   $T3a,$N2,$nc,$T3a
 440         fmadd   $T3b,$N2,$nd,$T3b
 441         fmadd   $dota,$N3,$nc,$dota
 442         fmadd   $dotb,$N3,$nd,$dotb
 443
 444         fctid   $T0a,$T0a
 445         fctid   $T0b,$T0b
 446         fctid   $T1a,$T1a
 447         fctid   $T1b,$T1b
 448         fctid   $T2a,$T2a
 449         fctid   $T2b,$T2b
 450         fctid   $T3a,$T3a
 451         fctid   $T3b,$T3b
 452
 453         stfd    $T0a,`$FRAME+0`($sp)
 454         stfd    $T0b,`$FRAME+8`($sp)
 455         stfd    $T1a,`$FRAME+16`($sp)
 456         stfd    $T1b,`$FRAME+24`($sp)
 457         stfd    $T2a,`$FRAME+32`($sp)
 458         stfd    $T2b,`$FRAME+40`($sp)
 459         stfd    $T3a,`$FRAME+48`($sp)
 460         stfd    $T3b,`$FRAME+56`($sp)
 461 \f
 462 .align  5
 463 L1st:
 464 ___
 465 $code.=<<___ if ($SIZE_T==8);
 466         lwz     $t0,4($ap)              ; load a[j] as 32-bit word pair
 467         lwz     $t1,0($ap)
 468         lwz     $t2,12($ap)             ; load a[j+1] as 32-bit word pair
 469         lwz     $t3,8($ap)
 470         lwz     $t4,4($np)              ; load n[j] as 32-bit word pair
 471         lwz     $t5,0($np)
 472         lwz     $t6,12($np)             ; load n[j+1] as 32-bit word pair
 473         lwz     $t7,8($np)
 474 ___
 475 $code.=<<___ if ($SIZE_T==4);
 476         lwz     $t0,0($ap)              ; load a[j..j+3] as 32-bit word pairs
 477         lwz     $t1,4($ap)
 478         lwz     $t2,8($ap)
 479         lwz     $t3,12($ap)
 480         lwz     $t4,0($np)              ; load n[j..j+3] as 32-bit word pairs
 481         lwz     $t5,4($np)
 482         lwz     $t6,8($np)
 483         lwz     $t7,12($np)
 484 ___
 485 $code.=<<___;
 486         std     $t0,`$FRAME+64`($sp)    ; yes, std even in 32-bit build
 487         std     $t1,`$FRAME+72`($sp)
 488         std     $t2,`$FRAME+80`($sp)
 489         std     $t3,`$FRAME+88`($sp)
 490         std     $t4,`$FRAME+96`($sp)
 491         std     $t5,`$FRAME+104`($sp)
 492         std     $t6,`$FRAME+112`($sp)
 493         std     $t7,`$FRAME+120`($sp)
 494 ___
 495 if ($SIZE_T==8 or $flavour =~ /osx/) {
 496 $code.=<<___;
 497         ld      $t0,`$FRAME+0`($sp)
 498         ld      $t1,`$FRAME+8`($sp)
 499         ld      $t2,`$FRAME+16`($sp)
 500         ld      $t3,`$FRAME+24`($sp)
 501         ld      $t4,`$FRAME+32`($sp)
 502         ld      $t5,`$FRAME+40`($sp)
 503         ld      $t6,`$FRAME+48`($sp)
 504         ld      $t7,`$FRAME+56`($sp)
 505 ___
 506 } else {
 507 $code.=<<___;
 508         lwz     $t1,`$FRAME+0`($sp)
 509         lwz     $t0,`$FRAME+4`($sp)
 510         lwz     $t3,`$FRAME+8`($sp)
 511         lwz     $t2,`$FRAME+12`($sp)
 512         lwz     $t5,`$FRAME+16`($sp)
 513         lwz     $t4,`$FRAME+20`($sp)
 514         lwz     $t7,`$FRAME+24`($sp)
 515         lwz     $t6,`$FRAME+28`($sp)
 516 ___
 517 }
 518 $code.=<<___;
 519         lfd     $A0,`$FRAME+64`($sp)
 520         lfd     $A1,`$FRAME+72`($sp)
 521         lfd     $A2,`$FRAME+80`($sp)
 522         lfd     $A3,`$FRAME+88`($sp)
 523         lfd     $N0,`$FRAME+96`($sp)
 524         lfd     $N1,`$FRAME+104`($sp)
 525         lfd     $N2,`$FRAME+112`($sp)
 526         lfd     $N3,`$FRAME+120`($sp)
 527         fcfid   $A0,$A0
 528         fcfid   $A1,$A1
 529         fcfid   $A2,$A2
 530         fcfid   $A3,$A3
 531         fcfid   $N0,$N0
 532         fcfid   $N1,$N1
 533         fcfid   $N2,$N2
 534         fcfid   $N3,$N3
 535         addi    $ap,$ap,16
 536         addi    $np,$np,16
 537
 538         fmul    $T1a,$A1,$ba
 539         fmul    $T1b,$A1,$bb
 540         fmul    $T2a,$A2,$ba
 541         fmul    $T2b,$A2,$bb
 542         stfd    $A0,8($nap_d)           ; save a[j] in double format
 543         stfd    $A1,16($nap_d)
 544         fmul    $T3a,$A3,$ba
 545         fmul    $T3b,$A3,$bb
 546         fmadd   $T0a,$A0,$ba,$dota
 547         fmadd   $T0b,$A0,$bb,$dotb
 548         stfd    $A2,24($nap_d)          ; save a[j+1] in double format
 549         stfd    $A3,32($nap_d)
 550 ___
 551 if ($SIZE_T==8 or $flavour =~ /osx/) {
 552 $code.=<<___;
 553         fmadd   $T1a,$A0,$bc,$T1a
 554         fmadd   $T1b,$A0,$bd,$T1b
 555         fmadd   $T2a,$A1,$bc,$T2a
 556         fmadd   $T2b,$A1,$bd,$T2b
 557         stfd    $N0,40($nap_d)          ; save n[j] in double format
 558         stfd    $N1,48($nap_d)
 559         fmadd   $T3a,$A2,$bc,$T3a
 560         fmadd   $T3b,$A2,$bd,$T3b
 561          add    $t0,$t0,$carry          ; can not overflow
 562         fmul    $dota,$A3,$bc
 563         fmul    $dotb,$A3,$bd
 564         stfd    $N2,56($nap_d)          ; save n[j+1] in double format
 565         stfdu   $N3,64($nap_d)
 566          srdi   $carry,$t0,16
 567          add    $t1,$t1,$carry
 568          srdi   $carry,$t1,16
 569
 570         fmadd   $T1a,$N1,$na,$T1a
 571         fmadd   $T1b,$N1,$nb,$T1b
 572          insrdi $t0,$t1,16,32
 573         fmadd   $T2a,$N2,$na,$T2a
 574         fmadd   $T2b,$N2,$nb,$T2b
 575          add    $t2,$t2,$carry
 576         fmadd   $T3a,$N3,$na,$T3a
 577         fmadd   $T3b,$N3,$nb,$T3b
 578          srdi   $carry,$t2,16
 579         fmadd   $T0a,$N0,$na,$T0a
 580         fmadd   $T0b,$N0,$nb,$T0b
 581          insrdi $t0,$t2,16,16
 582          add    $t3,$t3,$carry
 583          srdi   $carry,$t3,16
 584
 585         fmadd   $T1a,$N0,$nc,$T1a
 586         fmadd   $T1b,$N0,$nd,$T1b
 587          insrdi $t0,$t3,16,0            ; 0..63 bits
 588         fmadd   $T2a,$N1,$nc,$T2a
 589         fmadd   $T2b,$N1,$nd,$T2b
 590          add    $t4,$t4,$carry
 591         fmadd   $T3a,$N2,$nc,$T3a
 592         fmadd   $T3b,$N2,$nd,$T3b
 593          srdi   $carry,$t4,16
 594         fmadd   $dota,$N3,$nc,$dota
 595         fmadd   $dotb,$N3,$nd,$dotb
 596          add    $t5,$t5,$carry
 597          srdi   $carry,$t5,16
 598          insrdi $t4,$t5,16,32
 599
 600         fctid   $T0a,$T0a
 601         fctid   $T0b,$T0b
 602          add    $t6,$t6,$carry
 603         fctid   $T1a,$T1a
 604         fctid   $T1b,$T1b
 605          srdi   $carry,$t6,16
 606         fctid   $T2a,$T2a
 607         fctid   $T2b,$T2b
 608          insrdi $t4,$t6,16,16
 609         fctid   $T3a,$T3a
 610         fctid   $T3b,$T3b
 611          add    $t7,$t7,$carry
 612          insrdi $t4,$t7,16,0            ; 64..127 bits
 613          srdi   $carry,$t7,16           ; upper 33 bits
 614
 615         stfd    $T0a,`$FRAME+0`($sp)
 616         stfd    $T0b,`$FRAME+8`($sp)
 617         stfd    $T1a,`$FRAME+16`($sp)
 618         stfd    $T1b,`$FRAME+24`($sp)
 619         stfd    $T2a,`$FRAME+32`($sp)
 620         stfd    $T2b,`$FRAME+40`($sp)
 621         stfd    $T3a,`$FRAME+48`($sp)
 622         stfd    $T3b,`$FRAME+56`($sp)
 623          std    $t0,8($tp)              ; tp[j-1]
 624          stdu   $t4,16($tp)             ; tp[j]
 625 ___
 626 } else {
 627 $code.=<<___;
 628         fmadd   $T1a,$A0,$bc,$T1a
 629         fmadd   $T1b,$A0,$bd,$T1b
 630          addc   $t0,$t0,$carry
 631          adde   $t1,$t1,$c1
 632          srwi   $carry,$t0,16
 633         fmadd   $T2a,$A1,$bc,$T2a
 634         fmadd   $T2b,$A1,$bd,$T2b
 635         stfd    $N0,40($nap_d)          ; save n[j] in double format
 636         stfd    $N1,48($nap_d)
 637          srwi   $c1,$t1,16
 638          insrwi $carry,$t1,16,0
 639         fmadd   $T3a,$A2,$bc,$T3a
 640         fmadd   $T3b,$A2,$bd,$T3b
 641          addc   $t2,$t2,$carry
 642          adde   $t3,$t3,$c1
 643          srwi   $carry,$t2,16
 644         fmul    $dota,$A3,$bc
 645         fmul    $dotb,$A3,$bd
 646         stfd    $N2,56($nap_d)          ; save n[j+1] in double format
 647         stfdu   $N3,64($nap_d)
 648          insrwi $t0,$t2,16,0            ; 0..31 bits
 649          srwi   $c1,$t3,16
 650          insrwi $carry,$t3,16,0
 651
 652         fmadd   $T1a,$N1,$na,$T1a
 653         fmadd   $T1b,$N1,$nb,$T1b
 654          lwz    $t3,`$FRAME+32`($sp)    ; permuted $t1
 655          lwz    $t2,`$FRAME+36`($sp)    ; permuted $t0
 656          addc   $t4,$t4,$carry
 657          adde   $t5,$t5,$c1
 658          srwi   $carry,$t4,16
 659         fmadd   $T2a,$N2,$na,$T2a
 660         fmadd   $T2b,$N2,$nb,$T2b
 661          srwi   $c1,$t5,16
 662          insrwi $carry,$t5,16,0
 663         fmadd   $T3a,$N3,$na,$T3a
 664         fmadd   $T3b,$N3,$nb,$T3b
 665          addc   $t6,$t6,$carry
 666          adde   $t7,$t7,$c1
 667          srwi   $carry,$t6,16
 668         fmadd   $T0a,$N0,$na,$T0a
 669         fmadd   $T0b,$N0,$nb,$T0b
 670          insrwi $t4,$t6,16,0            ; 32..63 bits
 671          srwi   $c1,$t7,16
 672          insrwi $carry,$t7,16,0
 673
 674         fmadd   $T1a,$N0,$nc,$T1a
 675         fmadd   $T1b,$N0,$nd,$T1b
 676          lwz    $t7,`$FRAME+40`($sp)    ; permuted $t3
 677          lwz    $t6,`$FRAME+44`($sp)    ; permuted $t2
 678          addc   $t2,$t2,$carry
 679          adde   $t3,$t3,$c1
 680          srwi   $carry,$t2,16
 681         fmadd   $T2a,$N1,$nc,$T2a
 682         fmadd   $T2b,$N1,$nd,$T2b
 683          stw    $t0,12($tp)             ; tp[j-1]
 684          stw    $t4,8($tp)
 685          srwi   $c1,$t3,16
 686          insrwi $carry,$t3,16,0
 687         fmadd   $T3a,$N2,$nc,$T3a
 688         fmadd   $T3b,$N2,$nd,$T3b
 689          lwz    $t1,`$FRAME+48`($sp)    ; permuted $t5
 690          lwz    $t0,`$FRAME+52`($sp)    ; permuted $t4
 691          addc   $t6,$t6,$carry
 692          adde   $t7,$t7,$c1
 693          srwi   $carry,$t6,16
 694         fmadd   $dota,$N3,$nc,$dota
 695         fmadd   $dotb,$N3,$nd,$dotb
 696          insrwi $t2,$t6,16,0            ; 64..95 bits
 697          srwi   $c1,$t7,16
 698          insrwi $carry,$t7,16,0
 699
 700         fctid   $T0a,$T0a
 701         fctid   $T0b,$T0b
 702          lwz    $t5,`$FRAME+56`($sp)    ; permuted $t7
 703          lwz    $t4,`$FRAME+60`($sp)    ; permuted $t6
 704          addc   $t0,$t0,$carry
 705          adde   $t1,$t1,$c1
 706          srwi   $carry,$t0,16
 707         fctid   $T1a,$T1a
 708         fctid   $T1b,$T1b
 709          srwi   $c1,$t1,16
 710          insrwi $carry,$t1,16,0
 711         fctid   $T2a,$T2a
 712         fctid   $T2b,$T2b
 713          addc   $t4,$t4,$carry
 714          adde   $t5,$t5,$c1
 715          srwi   $carry,$t4,16
 716         fctid   $T3a,$T3a
 717         fctid   $T3b,$T3b
 718          insrwi $t0,$t4,16,0            ; 96..127 bits
 719          srwi   $c1,$t5,16
 720          insrwi $carry,$t5,16,0
 721
 722         stfd    $T0a,`$FRAME+0`($sp)
 723         stfd    $T0b,`$FRAME+8`($sp)
 724         stfd    $T1a,`$FRAME+16`($sp)
 725         stfd    $T1b,`$FRAME+24`($sp)
 726         stfd    $T2a,`$FRAME+32`($sp)
 727         stfd    $T2b,`$FRAME+40`($sp)
 728         stfd    $T3a,`$FRAME+48`($sp)
 729         stfd    $T3b,`$FRAME+56`($sp)
 730          stw    $t2,20($tp)             ; tp[j]
 731          stwu   $t0,16($tp)
 732 ___
 733 }
 734 $code.=<<___;
 735         bdnz-   L1st
 736 \f
 737         fctid   $dota,$dota
 738         fctid   $dotb,$dotb
 739 ___
 740 if ($SIZE_T==8 or $flavour =~ /osx/) {
 741 $code.=<<___;
 742         ld      $t0,`$FRAME+0`($sp)
 743         ld      $t1,`$FRAME+8`($sp)
 744         ld      $t2,`$FRAME+16`($sp)
 745         ld      $t3,`$FRAME+24`($sp)
 746         ld      $t4,`$FRAME+32`($sp)
 747         ld      $t5,`$FRAME+40`($sp)
 748         ld      $t6,`$FRAME+48`($sp)
 749         ld      $t7,`$FRAME+56`($sp)
 750         stfd    $dota,`$FRAME+64`($sp)
 751         stfd    $dotb,`$FRAME+72`($sp)
 752
 753         add     $t0,$t0,$carry          ; can not overflow
 754         srdi    $carry,$t0,16
 755         add     $t1,$t1,$carry
 756         srdi    $carry,$t1,16
 757         insrdi  $t0,$t1,16,32
 758         add     $t2,$t2,$carry
 759         srdi    $carry,$t2,16
 760         insrdi  $t0,$t2,16,16
 761         add     $t3,$t3,$carry
 762         srdi    $carry,$t3,16
 763         insrdi  $t0,$t3,16,0            ; 0..63 bits
 764         add     $t4,$t4,$carry
 765         srdi    $carry,$t4,16
 766         add     $t5,$t5,$carry
 767         srdi    $carry,$t5,16
 768         insrdi  $t4,$t5,16,32
 769         add     $t6,$t6,$carry
 770         srdi    $carry,$t6,16
 771         insrdi  $t4,$t6,16,16
 772         add     $t7,$t7,$carry
 773         insrdi  $t4,$t7,16,0            ; 64..127 bits
 774         srdi    $carry,$t7,16           ; upper 33 bits
 775         ld      $t6,`$FRAME+64`($sp)
 776         ld      $t7,`$FRAME+72`($sp)
 777
 778         std     $t0,8($tp)              ; tp[j-1]
 779         stdu    $t4,16($tp)             ; tp[j]
 780
 781         add     $t6,$t6,$carry          ; can not overflow
 782         srdi    $carry,$t6,16
 783         add     $t7,$t7,$carry
 784         insrdi  $t6,$t7,48,0
 785         srdi    $ovf,$t7,48
 786         std     $t6,8($tp)              ; tp[num-1]
 787 ___
 788 } else {
 789 $code.=<<___;
 790         lwz     $t1,`$FRAME+0`($sp)
 791         lwz     $t0,`$FRAME+4`($sp)
 792         lwz     $t3,`$FRAME+8`($sp)
 793         lwz     $t2,`$FRAME+12`($sp)
 794         lwz     $t5,`$FRAME+16`($sp)
 795         lwz     $t4,`$FRAME+20`($sp)
 796         lwz     $t7,`$FRAME+24`($sp)
 797         lwz     $t6,`$FRAME+28`($sp)
 798         stfd    $dota,`$FRAME+64`($sp)
 799         stfd    $dotb,`$FRAME+72`($sp)
 800
 801         addc    $t0,$t0,$carry
 802         adde    $t1,$t1,$c1
 803         srwi    $carry,$t0,16
 804         insrwi  $carry,$t1,16,0
 805         srwi    $c1,$t1,16
 806         addc    $t2,$t2,$carry
 807         adde    $t3,$t3,$c1
 808         srwi    $carry,$t2,16
 809          insrwi $t0,$t2,16,0            ; 0..31 bits
 810         insrwi  $carry,$t3,16,0
 811         srwi    $c1,$t3,16
 812         addc    $t4,$t4,$carry
 813         adde    $t5,$t5,$c1
 814         srwi    $carry,$t4,16
 815         insrwi  $carry,$t5,16,0
 816         srwi    $c1,$t5,16
 817         addc    $t6,$t6,$carry
 818         adde    $t7,$t7,$c1
 819         srwi    $carry,$t6,16
 820          insrwi $t4,$t6,16,0            ; 32..63 bits
 821         insrwi  $carry,$t7,16,0
 822         srwi    $c1,$t7,16
 823          stw    $t0,12($tp)             ; tp[j-1]
 824          stw    $t4,8($tp)
 825
 826         lwz     $t3,`$FRAME+32`($sp)    ; permuted $t1
 827         lwz     $t2,`$FRAME+36`($sp)    ; permuted $t0
 828         lwz     $t7,`$FRAME+40`($sp)    ; permuted $t3
 829         lwz     $t6,`$FRAME+44`($sp)    ; permuted $t2
 830         lwz     $t1,`$FRAME+48`($sp)    ; permuted $t5
 831         lwz     $t0,`$FRAME+52`($sp)    ; permuted $t4
 832         lwz     $t5,`$FRAME+56`($sp)    ; permuted $t7
 833         lwz     $t4,`$FRAME+60`($sp)    ; permuted $t6
 834
 835         addc    $t2,$t2,$carry
 836         adde    $t3,$t3,$c1
 837         srwi    $carry,$t2,16
 838         insrwi  $carry,$t3,16,0
 839         srwi    $c1,$t3,16
 840         addc    $t6,$t6,$carry
 841         adde    $t7,$t7,$c1
 842         srwi    $carry,$t6,16
 843          insrwi $t2,$t6,16,0            ; 64..95 bits
 844         insrwi  $carry,$t7,16,0
 845         srwi    $c1,$t7,16
 846         addc    $t0,$t0,$carry
 847         adde    $t1,$t1,$c1
 848         srwi    $carry,$t0,16
 849         insrwi  $carry,$t1,16,0
 850         srwi    $c1,$t1,16
 851         addc    $t4,$t4,$carry
 852         adde    $t5,$t5,$c1
 853         srwi    $carry,$t4,16
 854          insrwi $t0,$t4,16,0            ; 96..127 bits
 855         insrwi  $carry,$t5,16,0
 856         srwi    $c1,$t5,16
 857          stw    $t2,20($tp)             ; tp[j]
 858          stwu   $t0,16($tp)
 859
 860         lwz     $t7,`$FRAME+64`($sp)
 861         lwz     $t6,`$FRAME+68`($sp)
 862         lwz     $t5,`$FRAME+72`($sp)
 863         lwz     $t4,`$FRAME+76`($sp)
 864
 865         addc    $t6,$t6,$carry
 866         adde    $t7,$t7,$c1
 867         srwi    $carry,$t6,16
 868         insrwi  $carry,$t7,16,0
 869         srwi    $c1,$t7,16
 870         addc    $t4,$t4,$carry
 871         adde    $t5,$t5,$c1
 872
 873         insrwi  $t6,$t4,16,0
 874         srwi    $t4,$t4,16
 875         insrwi  $t4,$t5,16,0
 876         srwi    $ovf,$t5,16
 877         stw     $t6,12($tp)             ; tp[num-1]
 878         stw     $t4,8($tp)
 879 ___
 880 }
 881 $code.=<<___;
 882         slwi    $t7,$num,2
 883         subf    $nap_d,$t7,$nap_d       ; rewind pointer
 884 \f
 885         li      $i,8                    ; i=1
 886 .align  5
 887 Louter:
 888         addi    $tp,$sp,`$FRAME+$TRANSFER`
 889         li      $carry,0
 890         mtctr   $j
 891 ___
 892 $code.=<<___ if ($SIZE_T==8);
 893         ldx     $t3,$bp,$i              ; bp[i]
 894
 895         ld      $t6,`$FRAME+$TRANSFER+8`($sp)   ; tp[0]
 896         mulld   $t7,$a0,$t3             ; ap[0]*bp[i]
 897         add     $t7,$t7,$t6             ; ap[0]*bp[i]+tp[0]
 898         ; transfer bp[i] to FPU as 4x16-bit values
 899         extrdi  $t0,$t3,16,48
 900         extrdi  $t1,$t3,16,32
 901         extrdi  $t2,$t3,16,16
 902         extrdi  $t3,$t3,16,0
 903         std     $t0,`$FRAME+0`($sp)
 904         std     $t1,`$FRAME+8`($sp)
 905         std     $t2,`$FRAME+16`($sp)
 906         std     $t3,`$FRAME+24`($sp)
 907
 908         mulld   $t7,$t7,$n0             ; tp[0]*n0
 909         ; transfer (ap[0]*bp[i]+tp[0])*n0 to FPU as 4x16-bit values
 910         extrdi  $t4,$t7,16,48
 911         extrdi  $t5,$t7,16,32
 912         extrdi  $t6,$t7,16,16
 913         extrdi  $t7,$t7,16,0
 914         std     $t4,`$FRAME+32`($sp)
 915         std     $t5,`$FRAME+40`($sp)
 916         std     $t6,`$FRAME+48`($sp)
 917         std     $t7,`$FRAME+56`($sp)
 918 ___
 919 $code.=<<___ if ($SIZE_T==4);
 920         add     $t0,$bp,$i
 921         li      $c1,0
 922         lwz     $t1,0($t0)              ; bp[i,i+1]
 923         lwz     $t3,4($t0)
 924
 925         mullw   $t4,$a0,$t1             ; ap[0]*bp[i]
 926         lwz     $t0,`$FRAME+$TRANSFER+8+4`($sp) ; tp[0]
 927         mulhwu  $t5,$a0,$t1
 928         lwz     $t2,`$FRAME+$TRANSFER+8`($sp)   ; tp[0]
 929         mullw   $t6,$a1,$t1
 930         mullw   $t7,$a0,$t3
 931         add     $t5,$t5,$t6
 932         add     $t5,$t5,$t7
 933         addc    $t4,$t4,$t0             ; ap[0]*bp[i]+tp[0]
 934         adde    $t5,$t5,$t2
 935         ; transfer bp[i] to FPU as 4x16-bit values
 936         extrwi  $t0,$t1,16,16
 937         extrwi  $t1,$t1,16,0
 938         extrwi  $t2,$t3,16,16
 939         extrwi  $t3,$t3,16,0
 940         std     $t0,`$FRAME+0`($sp)     ; yes, std in 32-bit build
 941         std     $t1,`$FRAME+8`($sp)
 942         std     $t2,`$FRAME+16`($sp)
 943         std     $t3,`$FRAME+24`($sp)
 944
 945         mullw   $t0,$t4,$n0             ; mulld tp[0]*n0
 946         mulhwu  $t1,$t4,$n0
 947         mullw   $t2,$t5,$n0
 948         mullw   $t3,$t4,$n1
 949         add     $t1,$t1,$t2
 950         add     $t1,$t1,$t3
 951         ; transfer (ap[0]*bp[i]+tp[0])*n0 to FPU as 4x16-bit values
 952         extrwi  $t4,$t0,16,16
 953         extrwi  $t5,$t0,16,0
 954         extrwi  $t6,$t1,16,16
 955         extrwi  $t7,$t1,16,0
 956         std     $t4,`$FRAME+32`($sp)    ; yes, std in 32-bit build
 957         std     $t5,`$FRAME+40`($sp)
 958         std     $t6,`$FRAME+48`($sp)
 959         std     $t7,`$FRAME+56`($sp)
 960 ___
 961 $code.=<<___;
 962         lfd     $A0,8($nap_d)           ; load a[j] in double format
 963         lfd     $A1,16($nap_d)
 964         lfd     $A2,24($nap_d)          ; load a[j+1] in double format
 965         lfd     $A3,32($nap_d)
 966         lfd     $N0,40($nap_d)          ; load n[j] in double format
 967         lfd     $N1,48($nap_d)
 968         lfd     $N2,56($nap_d)          ; load n[j+1] in double format
 969         lfdu    $N3,64($nap_d)
 970
 971         lfd     $ba,`$FRAME+0`($sp)
 972         lfd     $bb,`$FRAME+8`($sp)
 973         lfd     $bc,`$FRAME+16`($sp)
 974         lfd     $bd,`$FRAME+24`($sp)
 975         lfd     $na,`$FRAME+32`($sp)
 976         lfd     $nb,`$FRAME+40`($sp)
 977         lfd     $nc,`$FRAME+48`($sp)
 978         lfd     $nd,`$FRAME+56`($sp)
 979
 980         fcfid   $ba,$ba
 981         fcfid   $bb,$bb
 982         fcfid   $bc,$bc
 983         fcfid   $bd,$bd
 984         fcfid   $na,$na
 985         fcfid   $nb,$nb
 986         fcfid   $nc,$nc
 987         fcfid   $nd,$nd
 988
 989         fmul    $T1a,$A1,$ba
 990         fmul    $T1b,$A1,$bb
 991         fmul    $T2a,$A2,$ba
 992         fmul    $T2b,$A2,$bb
 993         fmul    $T3a,$A3,$ba
 994         fmul    $T3b,$A3,$bb
 995         fmul    $T0a,$A0,$ba
 996         fmul    $T0b,$A0,$bb
 997
 998         fmadd   $T1a,$A0,$bc,$T1a
 999         fmadd   $T1b,$A0,$bd,$T1b
1000         fmadd   $T2a,$A1,$bc,$T2a
1001         fmadd   $T2b,$A1,$bd,$T2b
1002         fmadd   $T3a,$A2,$bc,$T3a
1003         fmadd   $T3b,$A2,$bd,$T3b
1004         fmul    $dota,$A3,$bc
1005         fmul    $dotb,$A3,$bd
1006
1007         fmadd   $T1a,$N1,$na,$T1a
1008         fmadd   $T1b,$N1,$nb,$T1b
1009          lfd    $A0,8($nap_d)           ; load a[j] in double format
1010          lfd    $A1,16($nap_d)
1011         fmadd   $T2a,$N2,$na,$T2a
1012         fmadd   $T2b,$N2,$nb,$T2b
1013          lfd    $A2,24($nap_d)          ; load a[j+1] in double format
1014          lfd    $A3,32($nap_d)
1015         fmadd   $T3a,$N3,$na,$T3a
1016         fmadd   $T3b,$N3,$nb,$T3b
1017         fmadd   $T0a,$N0,$na,$T0a
1018         fmadd   $T0b,$N0,$nb,$T0b
1019
1020         fmadd   $T1a,$N0,$nc,$T1a
1021         fmadd   $T1b,$N0,$nd,$T1b
1022         fmadd   $T2a,$N1,$nc,$T2a
1023         fmadd   $T2b,$N1,$nd,$T2b
1024         fmadd   $T3a,$N2,$nc,$T3a
1025         fmadd   $T3b,$N2,$nd,$T3b
1026         fmadd   $dota,$N3,$nc,$dota
1027         fmadd   $dotb,$N3,$nd,$dotb
1028
1029         fctid   $T0a,$T0a
1030         fctid   $T0b,$T0b
1031         fctid   $T1a,$T1a
1032         fctid   $T1b,$T1b
1033         fctid   $T2a,$T2a
1034         fctid   $T2b,$T2b
1035         fctid   $T3a,$T3a
1036         fctid   $T3b,$T3b
1037
1038         stfd    $T0a,`$FRAME+0`($sp)
1039         stfd    $T0b,`$FRAME+8`($sp)
1040         stfd    $T1a,`$FRAME+16`($sp)
1041         stfd    $T1b,`$FRAME+24`($sp)
1042         stfd    $T2a,`$FRAME+32`($sp)
1043         stfd    $T2b,`$FRAME+40`($sp)
1044         stfd    $T3a,`$FRAME+48`($sp)
1045         stfd    $T3b,`$FRAME+56`($sp)
1046 \f
1047 .align  5
1048 Linner:
1049         fmul    $T1a,$A1,$ba
1050         fmul    $T1b,$A1,$bb
1051         fmul    $T2a,$A2,$ba
1052         fmul    $T2b,$A2,$bb
1053         lfd     $N0,40($nap_d)          ; load n[j] in double format
1054         lfd     $N1,48($nap_d)
1055         fmul    $T3a,$A3,$ba
1056         fmul    $T3b,$A3,$bb
1057         fmadd   $T0a,$A0,$ba,$dota
1058         fmadd   $T0b,$A0,$bb,$dotb
1059         lfd     $N2,56($nap_d)          ; load n[j+1] in double format
1060         lfdu    $N3,64($nap_d)
1061
1062         fmadd   $T1a,$A0,$bc,$T1a
1063         fmadd   $T1b,$A0,$bd,$T1b
1064         fmadd   $T2a,$A1,$bc,$T2a
1065         fmadd   $T2b,$A1,$bd,$T2b
1066          lfd    $A0,8($nap_d)           ; load a[j] in double format
1067          lfd    $A1,16($nap_d)
1068         fmadd   $T3a,$A2,$bc,$T3a
1069         fmadd   $T3b,$A2,$bd,$T3b
1070         fmul    $dota,$A3,$bc
1071         fmul    $dotb,$A3,$bd
1072          lfd    $A2,24($nap_d)          ; load a[j+1] in double format
1073          lfd    $A3,32($nap_d)
1074 ___
1075 if ($SIZE_T==8 or $flavour =~ /osx/) {
1076 $code.=<<___;
1077         fmadd   $T1a,$N1,$na,$T1a
1078         fmadd   $T1b,$N1,$nb,$T1b
1079          ld     $t0,`$FRAME+0`($sp)
1080          ld     $t1,`$FRAME+8`($sp)
1081         fmadd   $T2a,$N2,$na,$T2a
1082         fmadd   $T2b,$N2,$nb,$T2b
1083          ld     $t2,`$FRAME+16`($sp)
1084          ld     $t3,`$FRAME+24`($sp)
1085         fmadd   $T3a,$N3,$na,$T3a
1086         fmadd   $T3b,$N3,$nb,$T3b
1087          add    $t0,$t0,$carry          ; can not overflow
1088          ld     $t4,`$FRAME+32`($sp)
1089          ld     $t5,`$FRAME+40`($sp)
1090         fmadd   $T0a,$N0,$na,$T0a
1091         fmadd   $T0b,$N0,$nb,$T0b
1092          srdi   $carry,$t0,16
1093          add    $t1,$t1,$carry
1094          srdi   $carry,$t1,16
1095          ld     $t6,`$FRAME+48`($sp)
1096          ld     $t7,`$FRAME+56`($sp)
1097
1098         fmadd   $T1a,$N0,$nc,$T1a
1099         fmadd   $T1b,$N0,$nd,$T1b
1100          insrdi $t0,$t1,16,32
1101          ld     $t1,8($tp)              ; tp[j]
1102         fmadd   $T2a,$N1,$nc,$T2a
1103         fmadd   $T2b,$N1,$nd,$T2b
1104          add    $t2,$t2,$carry
1105         fmadd   $T3a,$N2,$nc,$T3a
1106         fmadd   $T3b,$N2,$nd,$T3b
1107          srdi   $carry,$t2,16
1108          insrdi $t0,$t2,16,16
1109         fmadd   $dota,$N3,$nc,$dota
1110         fmadd   $dotb,$N3,$nd,$dotb
1111          add    $t3,$t3,$carry
1112          ldu    $t2,16($tp)             ; tp[j+1]
1113          srdi   $carry,$t3,16
1114          insrdi $t0,$t3,16,0            ; 0..63 bits
1115          add    $t4,$t4,$carry
1116
1117         fctid   $T0a,$T0a
1118         fctid   $T0b,$T0b
1119          srdi   $carry,$t4,16
1120         fctid   $T1a,$T1a
1121         fctid   $T1b,$T1b
1122          add    $t5,$t5,$carry
1123         fctid   $T2a,$T2a
1124         fctid   $T2b,$T2b
1125          srdi   $carry,$t5,16
1126          insrdi $t4,$t5,16,32
1127         fctid   $T3a,$T3a
1128         fctid   $T3b,$T3b
1129          add    $t6,$t6,$carry
1130          srdi   $carry,$t6,16
1131          insrdi $t4,$t6,16,16
1132
1133         stfd    $T0a,`$FRAME+0`($sp)
1134         stfd    $T0b,`$FRAME+8`($sp)
1135          add    $t7,$t7,$carry
1136          addc   $t3,$t0,$t1
1137 ___
1138 $code.=<<___ if ($SIZE_T==4);           # adjust XER[CA]
1139         extrdi  $t0,$t0,32,0
1140         extrdi  $t1,$t1,32,0
1141         adde    $t0,$t0,$t1
1142 ___
1143 $code.=<<___;
1144         stfd    $T1a,`$FRAME+16`($sp)
1145         stfd    $T1b,`$FRAME+24`($sp)
1146          insrdi $t4,$t7,16,0            ; 64..127 bits
1147          srdi   $carry,$t7,16           ; upper 33 bits
1148         stfd    $T2a,`$FRAME+32`($sp)
1149         stfd    $T2b,`$FRAME+40`($sp)
1150          adde   $t5,$t4,$t2
1151 ___
1152 $code.=<<___ if ($SIZE_T==4);           # adjust XER[CA]
1153         extrdi  $t4,$t4,32,0
1154         extrdi  $t2,$t2,32,0
1155         adde    $t4,$t4,$t2
1156 ___
1157 $code.=<<___;
1158         stfd    $T3a,`$FRAME+48`($sp)
1159         stfd    $T3b,`$FRAME+56`($sp)
1160          addze  $carry,$carry
1161          std    $t3,-16($tp)            ; tp[j-1]
1162          std    $t5,-8($tp)             ; tp[j]
1163 ___
1164 } else {
1165 $code.=<<___;
1166         fmadd   $T1a,$N1,$na,$T1a
1167         fmadd   $T1b,$N1,$nb,$T1b
1168          lwz    $t1,`$FRAME+0`($sp)
1169          lwz    $t0,`$FRAME+4`($sp)
1170         fmadd   $T2a,$N2,$na,$T2a
1171         fmadd   $T2b,$N2,$nb,$T2b
1172          lwz    $t3,`$FRAME+8`($sp)
1173          lwz    $t2,`$FRAME+12`($sp)
1174         fmadd   $T3a,$N3,$na,$T3a
1175         fmadd   $T3b,$N3,$nb,$T3b
1176          lwz    $t5,`$FRAME+16`($sp)
1177          lwz    $t4,`$FRAME+20`($sp)
1178          addc   $t0,$t0,$carry
1179          adde   $t1,$t1,$c1
1180          srwi   $carry,$t0,16
1181         fmadd   $T0a,$N0,$na,$T0a
1182         fmadd   $T0b,$N0,$nb,$T0b
1183          lwz    $t7,`$FRAME+24`($sp)
1184          lwz    $t6,`$FRAME+28`($sp)
1185          srwi   $c1,$t1,16
1186          insrwi $carry,$t1,16,0
1187
1188         fmadd   $T1a,$N0,$nc,$T1a
1189         fmadd   $T1b,$N0,$nd,$T1b
1190          addc   $t2,$t2,$carry
1191          adde   $t3,$t3,$c1
1192          srwi   $carry,$t2,16
1193         fmadd   $T2a,$N1,$nc,$T2a
1194         fmadd   $T2b,$N1,$nd,$T2b
1195          insrwi $t0,$t2,16,0            ; 0..31 bits
1196          srwi   $c1,$t3,16
1197          insrwi $carry,$t3,16,0
1198         fmadd   $T3a,$N2,$nc,$T3a
1199         fmadd   $T3b,$N2,$nd,$T3b
1200          lwz    $t2,12($tp)             ; tp[j]
1201          lwz    $t3,8($tp)
1202          addc   $t4,$t4,$carry
1203          adde   $t5,$t5,$c1
1204          srwi   $carry,$t4,16
1205         fmadd   $dota,$N3,$nc,$dota
1206         fmadd   $dotb,$N3,$nd,$dotb
1207          srwi   $c1,$t5,16
1208          insrwi $carry,$t5,16,0
1209
1210         fctid   $T0a,$T0a
1211          addc   $t6,$t6,$carry
1212          adde   $t7,$t7,$c1
1213          srwi   $carry,$t6,16
1214         fctid   $T0b,$T0b
1215          insrwi $t4,$t6,16,0            ; 32..63 bits
1216          srwi   $c1,$t7,16
1217          insrwi $carry,$t7,16,0
1218         fctid   $T1a,$T1a
1219          addc   $t0,$t0,$t2
1220          adde   $t4,$t4,$t3
1221          lwz    $t3,`$FRAME+32`($sp)    ; permuted $t1
1222          lwz    $t2,`$FRAME+36`($sp)    ; permuted $t0
1223         fctid   $T1b,$T1b
1224          addze  $carry,$carry
1225          addze  $c1,$c1
1226          stw    $t0,4($tp)              ; tp[j-1]
1227          stw    $t4,0($tp)
1228         fctid   $T2a,$T2a
1229          addc   $t2,$t2,$carry
1230          adde   $t3,$t3,$c1
1231          srwi   $carry,$t2,16
1232          lwz    $t7,`$FRAME+40`($sp)    ; permuted $t3
1233          lwz    $t6,`$FRAME+44`($sp)    ; permuted $t2
1234         fctid   $T2b,$T2b
1235          srwi   $c1,$t3,16
1236          insrwi $carry,$t3,16,0
1237          lwz    $t1,`$FRAME+48`($sp)    ; permuted $t5
1238          lwz    $t0,`$FRAME+52`($sp)    ; permuted $t4
1239         fctid   $T3a,$T3a
1240          addc   $t6,$t6,$carry
1241          adde   $t7,$t7,$c1
1242          srwi   $carry,$t6,16
1243          lwz    $t5,`$FRAME+56`($sp)    ; permuted $t7
1244          lwz    $t4,`$FRAME+60`($sp)    ; permuted $t6
1245         fctid   $T3b,$T3b
1246
1247          insrwi $t2,$t6,16,0            ; 64..95 bits
1248         insrwi  $carry,$t7,16,0
1249         srwi    $c1,$t7,16
1250          lwz    $t6,20($tp)
1251          lwzu   $t7,16($tp)
1252         addc    $t0,$t0,$carry
1253          stfd   $T0a,`$FRAME+0`($sp)
1254         adde    $t1,$t1,$c1
1255         srwi    $carry,$t0,16
1256          stfd   $T0b,`$FRAME+8`($sp)
1257         insrwi  $carry,$t1,16,0
1258         srwi    $c1,$t1,16
1259         addc    $t4,$t4,$carry
1260          stfd   $T1a,`$FRAME+16`($sp)
1261         adde    $t5,$t5,$c1
1262         srwi    $carry,$t4,16
1263          insrwi $t0,$t4,16,0            ; 96..127 bits
1264          stfd   $T1b,`$FRAME+24`($sp)
1265         insrwi  $carry,$t5,16,0
1266         srwi    $c1,$t5,16
1267
1268         addc    $t2,$t2,$t6
1269          stfd   $T2a,`$FRAME+32`($sp)
1270         adde    $t0,$t0,$t7
1271          stfd   $T2b,`$FRAME+40`($sp)
1272         addze   $carry,$carry
1273          stfd   $T3a,`$FRAME+48`($sp)
1274         addze   $c1,$c1
1275          stfd   $T3b,`$FRAME+56`($sp)
1276          stw    $t2,-4($tp)             ; tp[j]
1277          stw    $t0,-8($tp)
1278 ___
1279 }
1280 $code.=<<___;
1281         bdnz-   Linner
1282 \f
1283         fctid   $dota,$dota
1284         fctid   $dotb,$dotb
1285 ___
1286 if ($SIZE_T==8 or $flavour =~ /osx/) {
1287 $code.=<<___;
1288         ld      $t0,`$FRAME+0`($sp)
1289         ld      $t1,`$FRAME+8`($sp)
1290         ld      $t2,`$FRAME+16`($sp)
1291         ld      $t3,`$FRAME+24`($sp)
1292         ld      $t4,`$FRAME+32`($sp)
1293         ld      $t5,`$FRAME+40`($sp)
1294         ld      $t6,`$FRAME+48`($sp)
1295         ld      $t7,`$FRAME+56`($sp)
1296         stfd    $dota,`$FRAME+64`($sp)
1297         stfd    $dotb,`$FRAME+72`($sp)
1298
1299         add     $t0,$t0,$carry          ; can not overflow
1300         srdi    $carry,$t0,16
1301         add     $t1,$t1,$carry
1302         srdi    $carry,$t1,16
1303         insrdi  $t0,$t1,16,32
1304         add     $t2,$t2,$carry
1305         ld      $t1,8($tp)              ; tp[j]
1306         srdi    $carry,$t2,16
1307         insrdi  $t0,$t2,16,16
1308         add     $t3,$t3,$carry
1309         ldu     $t2,16($tp)             ; tp[j+1]
1310         srdi    $carry,$t3,16
1311         insrdi  $t0,$t3,16,0            ; 0..63 bits
1312         add     $t4,$t4,$carry
1313         srdi    $carry,$t4,16
1314         add     $t5,$t5,$carry
1315         srdi    $carry,$t5,16
1316         insrdi  $t4,$t5,16,32
1317         add     $t6,$t6,$carry
1318         srdi    $carry,$t6,16
1319         insrdi  $t4,$t6,16,16
1320         add     $t7,$t7,$carry
1321         insrdi  $t4,$t7,16,0            ; 64..127 bits
1322         srdi    $carry,$t7,16           ; upper 33 bits
1323         ld      $t6,`$FRAME+64`($sp)
1324         ld      $t7,`$FRAME+72`($sp)
1325
1326         addc    $t3,$t0,$t1
1327 ___
1328 $code.=<<___ if ($SIZE_T==4);           # adjust XER[CA]
1329         extrdi  $t0,$t0,32,0
1330         extrdi  $t1,$t1,32,0
1331         adde    $t0,$t0,$t1
1332 ___
1333 $code.=<<___;
1334         adde    $t5,$t4,$t2
1335 ___
1336 $code.=<<___ if ($SIZE_T==4);           # adjust XER[CA]
1337         extrdi  $t4,$t4,32,0
1338         extrdi  $t2,$t2,32,0
1339         adde    $t4,$t4,$t2
1340 ___
1341 $code.=<<___;
1342         addze   $carry,$carry
1343
1344         std     $t3,-16($tp)            ; tp[j-1]
1345         std     $t5,-8($tp)             ; tp[j]
1346
1347         add     $carry,$carry,$ovf      ; comsume upmost overflow
1348         add     $t6,$t6,$carry          ; can not overflow
1349         srdi    $carry,$t6,16
1350         add     $t7,$t7,$carry
1351         insrdi  $t6,$t7,48,0
1352         srdi    $ovf,$t7,48
1353         std     $t6,0($tp)              ; tp[num-1]
1354 ___
1355 } else {
1356 $code.=<<___;
1357         lwz     $t1,`$FRAME+0`($sp)
1358         lwz     $t0,`$FRAME+4`($sp)
1359         lwz     $t3,`$FRAME+8`($sp)
1360         lwz     $t2,`$FRAME+12`($sp)
1361         lwz     $t5,`$FRAME+16`($sp)
1362         lwz     $t4,`$FRAME+20`($sp)
1363         lwz     $t7,`$FRAME+24`($sp)
1364         lwz     $t6,`$FRAME+28`($sp)
1365         stfd    $dota,`$FRAME+64`($sp)
1366         stfd    $dotb,`$FRAME+72`($sp)
1367
1368         addc    $t0,$t0,$carry
1369         adde    $t1,$t1,$c1
1370         srwi    $carry,$t0,16
1371         insrwi  $carry,$t1,16,0
1372         srwi    $c1,$t1,16
1373         addc    $t2,$t2,$carry
1374         adde    $t3,$t3,$c1
1375         srwi    $carry,$t2,16
1376          insrwi $t0,$t2,16,0            ; 0..31 bits
1377          lwz    $t2,12($tp)             ; tp[j]
1378         insrwi  $carry,$t3,16,0
1379         srwi    $c1,$t3,16
1380          lwz    $t3,8($tp)
1381         addc    $t4,$t4,$carry
1382         adde    $t5,$t5,$c1
1383         srwi    $carry,$t4,16
1384         insrwi  $carry,$t5,16,0
1385         srwi    $c1,$t5,16
1386         addc    $t6,$t6,$carry
1387         adde    $t7,$t7,$c1
1388         srwi    $carry,$t6,16
1389          insrwi $t4,$t6,16,0            ; 32..63 bits
1390         insrwi  $carry,$t7,16,0
1391         srwi    $c1,$t7,16
1392
1393         addc    $t0,$t0,$t2
1394         adde    $t4,$t4,$t3
1395         addze   $carry,$carry
1396         addze   $c1,$c1
1397          stw    $t0,4($tp)              ; tp[j-1]
1398          stw    $t4,0($tp)
1399
1400         lwz     $t3,`$FRAME+32`($sp)    ; permuted $t1
1401         lwz     $t2,`$FRAME+36`($sp)    ; permuted $t0
1402         lwz     $t7,`$FRAME+40`($sp)    ; permuted $t3
1403         lwz     $t6,`$FRAME+44`($sp)    ; permuted $t2
1404         lwz     $t1,`$FRAME+48`($sp)    ; permuted $t5
1405         lwz     $t0,`$FRAME+52`($sp)    ; permuted $t4
1406         lwz     $t5,`$FRAME+56`($sp)    ; permuted $t7
1407         lwz     $t4,`$FRAME+60`($sp)    ; permuted $t6
1408
1409         addc    $t2,$t2,$carry
1410         adde    $t3,$t3,$c1
1411         srwi    $carry,$t2,16
1412         insrwi  $carry,$t3,16,0
1413         srwi    $c1,$t3,16
1414         addc    $t6,$t6,$carry
1415         adde    $t7,$t7,$c1
1416         srwi    $carry,$t6,16
1417          insrwi $t2,$t6,16,0            ; 64..95 bits
1418          lwz    $t6,20($tp)
1419         insrwi  $carry,$t7,16,0
1420         srwi    $c1,$t7,16
1421          lwzu   $t7,16($tp)
1422         addc    $t0,$t0,$carry
1423         adde    $t1,$t1,$c1
1424         srwi    $carry,$t0,16
1425         insrwi  $carry,$t1,16,0
1426         srwi    $c1,$t1,16
1427         addc    $t4,$t4,$carry
1428         adde    $t5,$t5,$c1
1429         srwi    $carry,$t4,16
1430          insrwi $t0,$t4,16,0            ; 96..127 bits
1431         insrwi  $carry,$t5,16,0
1432         srwi    $c1,$t5,16
1433
1434         addc    $t2,$t2,$t6
1435         adde    $t0,$t0,$t7
1436          lwz    $t7,`$FRAME+64`($sp)
1437          lwz    $t6,`$FRAME+68`($sp)
1438         addze   $carry,$carry
1439         addze   $c1,$c1
1440          lwz    $t5,`$FRAME+72`($sp)
1441          lwz    $t4,`$FRAME+76`($sp)
1442
1443         addc    $t6,$t6,$carry
1444         adde    $t7,$t7,$c1
1445          stw    $t2,-4($tp)             ; tp[j]
1446          stw    $t0,-8($tp)
1447         addc    $t6,$t6,$ovf
1448         addze   $t7,$t7
1449         srwi    $carry,$t6,16
1450         insrwi  $carry,$t7,16,0
1451         srwi    $c1,$t7,16
1452         addc    $t4,$t4,$carry
1453         adde    $t5,$t5,$c1
1454
1455         insrwi  $t6,$t4,16,0
1456         srwi    $t4,$t4,16
1457         insrwi  $t4,$t5,16,0
1458         srwi    $ovf,$t5,16
1459         stw     $t6,4($tp)              ; tp[num-1]
1460         stw     $t4,0($tp)
1461 ___
1462 }
1463 $code.=<<___;
1464         slwi    $t7,$num,2
1465         addi    $i,$i,8
1466         subf    $nap_d,$t7,$nap_d       ; rewind pointer
1467         cmpw    $i,$num
1468         blt-    Louter
1469 ___
1470 \f
1471 $code.=<<___ if ($SIZE_T==8);
1472         subf    $np,$num,$np    ; rewind np
1473         addi    $j,$j,1         ; restore counter
1474         subfc   $i,$i,$i        ; j=0 and "clear" XER[CA]
1475         addi    $tp,$sp,`$FRAME+$TRANSFER+8`
1476         addi    $t4,$sp,`$FRAME+$TRANSFER+16`
1477         addi    $t5,$np,8
1478         addi    $t6,$rp,8
1479         mtctr   $j
1480
1481 .align  4
1482 Lsub:   ldx     $t0,$tp,$i
1483         ldx     $t1,$np,$i
1484         ldx     $t2,$t4,$i
1485         ldx     $t3,$t5,$i
1486         subfe   $t0,$t1,$t0     ; tp[j]-np[j]
1487         subfe   $t2,$t3,$t2     ; tp[j+1]-np[j+1]
1488         stdx    $t0,$rp,$i
1489         stdx    $t2,$t6,$i
1490         addi    $i,$i,16
1491         bdnz-   Lsub
1492
1493         li      $i,0
1494         subfe   $ovf,$i,$ovf    ; handle upmost overflow bit
1495         and     $ap,$tp,$ovf
1496         andc    $np,$rp,$ovf
1497         or      $ap,$ap,$np     ; ap=borrow?tp:rp
1498         addi    $t7,$ap,8
1499         mtctr   $j
1500
1501 .align  4
1502 Lcopy:                          ; copy or in-place refresh
1503         ldx     $t0,$ap,$i
1504         ldx     $t1,$t7,$i
1505         std     $i,8($nap_d)    ; zap nap_d
1506         std     $i,16($nap_d)
1507         std     $i,24($nap_d)
1508         std     $i,32($nap_d)
1509         std     $i,40($nap_d)
1510         std     $i,48($nap_d)
1511         std     $i,56($nap_d)
1512         stdu    $i,64($nap_d)
1513         stdx    $t0,$rp,$i
1514         stdx    $t1,$t6,$i
1515         stdx    $i,$tp,$i       ; zap tp at once
1516         stdx    $i,$t4,$i
1517         addi    $i,$i,16
1518         bdnz-   Lcopy
1519 ___
1520 $code.=<<___ if ($SIZE_T==4);
1521         subf    $np,$num,$np    ; rewind np
1522         addi    $j,$j,1         ; restore counter
1523         subfc   $i,$i,$i        ; j=0 and "clear" XER[CA]
1524         addi    $tp,$sp,`$FRAME+$TRANSFER`
1525         addi    $np,$np,-4
1526         addi    $rp,$rp,-4
1527         addi    $ap,$sp,`$FRAME+$TRANSFER+4`
1528         mtctr   $j
1529
1530 .align  4
1531 Lsub:   lwz     $t0,12($tp)     ; load tp[j..j+3] in 64-bit word order
1532         lwz     $t1,8($tp)
1533         lwz     $t2,20($tp)
1534         lwzu    $t3,16($tp)
1535         lwz     $t4,4($np)      ; load np[j..j+3] in 32-bit word order
1536         lwz     $t5,8($np)
1537         lwz     $t6,12($np)
1538         lwzu    $t7,16($np)
1539         subfe   $t4,$t4,$t0     ; tp[j]-np[j]
1540          stw    $t0,4($ap)      ; save tp[j..j+3] in 32-bit word order
1541         subfe   $t5,$t5,$t1     ; tp[j+1]-np[j+1]
1542          stw    $t1,8($ap)
1543         subfe   $t6,$t6,$t2     ; tp[j+2]-np[j+2]
1544          stw    $t2,12($ap)
1545         subfe   $t7,$t7,$t3     ; tp[j+3]-np[j+3]
1546          stwu   $t3,16($ap)
1547         stw     $t4,4($rp)
1548         stw     $t5,8($rp)
1549         stw     $t6,12($rp)
1550         stwu    $t7,16($rp)
1551         bdnz-   Lsub
1552
1553         li      $i,0
1554         subfe   $ovf,$i,$ovf    ; handle upmost overflow bit
1555         addi    $tp,$sp,`$FRAME+$TRANSFER+4`
1556         subf    $rp,$num,$rp    ; rewind rp
1557         and     $ap,$tp,$ovf
1558         andc    $np,$rp,$ovf
1559         or      $ap,$ap,$np     ; ap=borrow?tp:rp
1560         addi    $tp,$sp,`$FRAME+$TRANSFER`
1561         mtctr   $j
1562
1563 .align  4
1564 Lcopy:                          ; copy or in-place refresh
1565         lwz     $t0,4($ap)
1566         lwz     $t1,8($ap)
1567         lwz     $t2,12($ap)
1568         lwzu    $t3,16($ap)
1569         std     $i,8($nap_d)    ; zap nap_d
1570         std     $i,16($nap_d)
1571         std     $i,24($nap_d)
1572         std     $i,32($nap_d)
1573         std     $i,40($nap_d)
1574         std     $i,48($nap_d)
1575         std     $i,56($nap_d)
1576         stdu    $i,64($nap_d)
1577         stw     $t0,4($rp)
1578         stw     $t1,8($rp)
1579         stw     $t2,12($rp)
1580         stwu    $t3,16($rp)
1581         std     $i,8($tp)       ; zap tp at once
1582         stdu    $i,16($tp)
1583         bdnz-   Lcopy
1584 ___
1585 \f
1586 $code.=<<___;
1587         $POP    $i,0($sp)
1588         li      r3,1    ; signal "handled"
1589         $POP    r19,`-12*8-13*$SIZE_T`($i)
1590         $POP    r20,`-12*8-12*$SIZE_T`($i)
1591         $POP    r21,`-12*8-11*$SIZE_T`($i)
1592         $POP    r22,`-12*8-10*$SIZE_T`($i)
1593         $POP    r23,`-12*8-9*$SIZE_T`($i)
1594         $POP    r24,`-12*8-8*$SIZE_T`($i)
1595         $POP    r25,`-12*8-7*$SIZE_T`($i)
1596         $POP    r26,`-12*8-6*$SIZE_T`($i)
1597         $POP    r27,`-12*8-5*$SIZE_T`($i)
1598         $POP    r28,`-12*8-4*$SIZE_T`($i)
1599         $POP    r29,`-12*8-3*$SIZE_T`($i)
1600         $POP    r30,`-12*8-2*$SIZE_T`($i)
1601         $POP    r31,`-12*8-1*$SIZE_T`($i)
1602         lfd     f20,`-12*8`($i)
1603         lfd     f21,`-11*8`($i)
1604         lfd     f22,`-10*8`($i)
1605         lfd     f23,`-9*8`($i)
1606         lfd     f24,`-8*8`($i)
1607         lfd     f25,`-7*8`($i)
1608         lfd     f26,`-6*8`($i)
1609         lfd     f27,`-5*8`($i)
1610         lfd     f28,`-4*8`($i)
1611         lfd     f29,`-3*8`($i)
1612         lfd     f30,`-2*8`($i)
1613         lfd     f31,`-1*8`($i)
1614         mr      $sp,$i
1615         blr
1616         .long   0
1617         .byte   0,12,4,0,0x8c,13,6,0
1618         .long   0
1619 .size   .$fname,.-.$fname
1620
1621 .asciz  "Montgomery Multiplication for PPC64, CRYPTOGAMS by <appro\@openssl.org>"
1622 ___
1623
1624 $code =~ s/\`([^\`]*)\`/eval $1/gem;
1625 print $code;
1626 close STDOUT;