crypto/poly1305/asm/poly1305-sparcv9.pl

   1 #!/usr/bin/env perl
   2 #
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9 #
  10 # This module implements Poly1305 hash for SPARCv9, vanilla, as well
  11 # as VIS3 and FMA extensions.
  12 #
  13 # May, August 2015
  14 #
  15 # Numbers are cycles per processed byte with poly1305_blocks alone.
  16 #
  17 #                       IALU(*)         FMA
  18 #
  19 # UltraSPARC III        11.9(**)
  20 # SPARC T3              7.85
  21 # SPARC T4              1.67(***)       6.55
  22 # SPARC64 X             5.54            3.64
  23 #
  24 # (*)   Comparison to compiler-generated code is really problematic,
  25 #       because latter's performance varies too much depending on too
  26 #       many variables. For example, one can measure from 5x to 15x
  27 #       improvement on T4 for gcc-4.6. Well, in T4 case it's a bit
  28 #       unfair comparison, because compiler doesn't use VIS3, but
  29 #       given same initial conditions coefficient varies from 3x to 9x.
  30 # (**)  Pre-III performance should be even worse; floating-point
  31 #       performance for UltraSPARC I-IV on the other hand is reported
  32 #       to be 4.25 for hand-coded assembly, but they are just too old
  33 #       to care about.
  34 # (***) Multi-process benchmark saturates at ~12.5x single-process
  35 #       result on 8-core processor, or ~21GBps per 2.85GHz socket.
  36
  37 my $output = pop;
  38 open STDOUT,">$output";
  39
  40 my ($ctx,$inp,$len,$padbit,$shl,$shr)   = map("%i$_",(0..5));
  41 my ($r0,$r1,$r2,$r3,$s1,$s2,$s3,$h4)    = map("%l$_",(0..7));
  42 my ($h0,$h1,$h2,$h3, $t0,$t1,$t2)       = map("%o$_",(0..5,7));
  43 my ($d0,$d1,$d2,$d3)                    = map("%g$_",(1..4));
  44
  45 my $output = pop;
  46 open STDOUT,">$stdout";
  47
  48 $code.=<<___;
  49 #include "sparc_arch.h"
  50
  51 #ifdef  __arch64__
  52 .register       %g2,#scratch
  53 .register       %g3,#scratch
  54 # define        STPTR   stx
  55 # define        SIZE_T  8
  56 #else
  57 # define        STPTR   st
  58 # define        SIZE_T  4
  59 #endif
  60 #define LOCALS  (STACK_BIAS+STACK_FRAME)
  61
  62 .section        ".text",#alloc,#execinstr
  63
  64 #ifdef __PIC__
  65 SPARC_PIC_THUNK(%g1)
  66 #endif
  67
  68 .globl  poly1305_init
  69 .align  32
  70 poly1305_init:
  71         save    %sp,-STACK_FRAME-16,%sp
  72         nop
  73
  74         SPARC_LOAD_ADDRESS(OPENSSL_sparcv9cap_P,%g1)
  75         ld      [%g1],%g1
  76
  77         and     %g1,SPARCV9_FMADD|SPARCV9_PREFER_FPU|SPARCV9_VIS3,%g1
  78         cmp     %g1,SPARCV9_FMADD|SPARCV9_PREFER_FPU
  79         be      .Lpoly1305_init_fma
  80         nop
  81
  82         stx     %g0,[$ctx+0]
  83         stx     %g0,[$ctx+8]            ! zero hash value
  84         brz,pn  $inp,.Lno_key
  85         stx     %g0,[$ctx+16]
  86
  87         and     $inp,7,$shr             ! alignment factor
  88         andn    $inp,7,$inp
  89         sll     $shr,3,$shr             ! *8
  90         neg     $shr,$shl
  91
  92         sethi   %hi(0x0ffffffc),$t0
  93         set     8,$h1
  94         or      $t0,%lo(0x0ffffffc),$t0
  95         set     16,$h2
  96         sllx    $t0,32,$t1
  97         or      $t0,$t1,$t1             ! 0x0ffffffc0ffffffc
  98         or      $t1,3,$t0               ! 0x0ffffffc0fffffff
  99
 100         ldxa    [$inp+%g0]0x88,$h0      ! load little-endian key
 101         brz,pt  $shr,.Lkey_aligned
 102         ldxa    [$inp+$h1]0x88,$h1
 103
 104         ldxa    [$inp+$h2]0x88,$h2
 105         srlx    $h0,$shr,$h0
 106         sllx    $h1,$shl,$t2
 107         srlx    $h1,$shr,$h1
 108         or      $t2,$h0,$h0
 109         sllx    $h2,$shl,$h2
 110         or      $h2,$h1,$h1
 111
 112 .Lkey_aligned:
 113         and     $t0,$h0,$h0
 114         and     $t1,$h1,$h1
 115         stx     $h0,[$ctx+32+0]         ! store key
 116         stx     $h1,[$ctx+32+8]
 117
 118         andcc   %g1,SPARCV9_VIS3,%g0
 119         be      .Lno_key
 120         nop
 121
 122 1:      call    .+8
 123         add     %o7,poly1305_blocks_vis3-1b,%o7
 124
 125         add     %o7,poly1305_emit-poly1305_blocks_vis3,%o5
 126         STPTR   %o7,[%i2]
 127         STPTR   %o5,[%i2+SIZE_T]
 128
 129         ret
 130         restore %g0,1,%o0               ! return 1
 131
 132 .Lno_key:
 133         ret
 134         restore %g0,%g0,%o0             ! return 0
 135 .size   poly1305_init,.-poly1305_init
 136
 137 .globl  poly1305_blocks
 138 .align  32
 139 poly1305_blocks:
 140         save    %sp,-STACK_FRAME,%sp
 141         andn    $len,15,$len
 142
 143         brz,pn  $len,.Lno_data
 144         nop
 145
 146         ld      [$ctx+32+0],$r1         ! load key
 147         ld      [$ctx+32+4],$r0
 148         ld      [$ctx+32+8],$r3
 149         ld      [$ctx+32+12],$r2
 150
 151         ld      [$ctx+0],$h1            ! load hash value
 152         ld      [$ctx+4],$h0
 153         ld      [$ctx+8],$h3
 154         ld      [$ctx+12],$h2
 155         ld      [$ctx+16],$h4
 156
 157         and     $inp,7,$shr             ! alignment factor
 158         andn    $inp,7,$inp
 159         set     8,$d1
 160         sll     $shr,3,$shr             ! *8
 161         set     16,$d2
 162         neg     $shr,$shl
 163
 164         srl     $r1,2,$s1
 165         srl     $r2,2,$s2
 166         add     $r1,$s1,$s1
 167         srl     $r3,2,$s3
 168         add     $r2,$s2,$s2
 169         add     $r3,$s3,$s3
 170
 171 .Loop:
 172         ldxa    [$inp+%g0]0x88,$d0      ! load little-endian input
 173         brz,pt  $shr,.Linp_aligned
 174         ldxa    [$inp+$d1]0x88,$d1
 175
 176         ldxa    [$inp+$d2]0x88,$d2
 177         srlx    $d0,$shr,$d0
 178         sllx    $d1,$shl,$t1
 179         srlx    $d1,$shr,$d1
 180         or      $t1,$d0,$d0
 181         sllx    $d2,$shl,$d2
 182         or      $d2,$d1,$d1
 183
 184 .Linp_aligned:
 185         srlx    $d0,32,$t0
 186         addcc   $d0,$h0,$h0             ! accumulate input
 187         srlx    $d1,32,$t1
 188         addccc  $t0,$h1,$h1
 189         addccc  $d1,$h2,$h2
 190         addccc  $t1,$h3,$h3
 191         addc    $padbit,$h4,$h4
 192
 193         umul    $r0,$h0,$d0
 194         umul    $r1,$h0,$d1
 195         umul    $r2,$h0,$d2
 196         umul    $r3,$h0,$d3
 197          sub    $len,16,$len
 198          add    $inp,16,$inp
 199
 200         umul    $s3,$h1,$t0
 201         umul    $r0,$h1,$t1
 202         umul    $r1,$h1,$t2
 203         add     $t0,$d0,$d0
 204         add     $t1,$d1,$d1
 205         umul    $r2,$h1,$t0
 206         add     $t2,$d2,$d2
 207         add     $t0,$d3,$d3
 208
 209         umul    $s2,$h2,$t1
 210         umul    $s3,$h2,$t2
 211         umul    $r0,$h2,$t0
 212         add     $t1,$d0,$d0
 213         add     $t2,$d1,$d1
 214         umul    $r1,$h2,$t1
 215         add     $t0,$d2,$d2
 216         add     $t1,$d3,$d3
 217
 218         umul    $s1,$h3,$t2
 219         umul    $s2,$h3,$t0
 220         umul    $s3,$h3,$t1
 221         add     $t2,$d0,$d0
 222         add     $t0,$d1,$d1
 223         umul    $r0,$h3,$t2
 224         add     $t1,$d2,$d2
 225         add     $t2,$d3,$d3
 226
 227         umul    $s1,$h4,$t0
 228         umul    $s2,$h4,$t1
 229         umul    $s3,$h4,$t2
 230         umul    $r0,$h4,$h4
 231         add     $t0,$d1,$d1
 232         add     $t1,$d2,$d2
 233         srlx    $d0,32,$h1
 234         add     $t2,$d3,$d3
 235         srlx    $d1,32,$h2
 236
 237         addcc   $d1,$h1,$h1
 238         srlx    $d2,32,$h3
 239          set    8,$d1
 240         addccc  $d2,$h2,$h2
 241         srlx    $d3,32,$t0
 242          set    16,$d2
 243         addccc  $d3,$h3,$h3
 244         addc    $t0,$h4,$h4
 245
 246         srl     $h4,2,$t0               ! final reduction step
 247         andn    $h4,3,$t1
 248         and     $h4,3,$h4
 249         add     $t1,$t0,$t0
 250
 251         addcc   $t0,$d0,$h0
 252         addccc  %g0,$h1,$h1
 253         addccc  %g0,$h2,$h2
 254         brnz,pt $len,.Loop
 255         addc    %g0,$h3,$h3
 256
 257         st      $h1,[$ctx+0]            ! store hash value
 258         st      $h0,[$ctx+4]
 259         st      $h3,[$ctx+8]
 260         st      $h2,[$ctx+12]
 261         st      $h4,[$ctx+16]
 262
 263 .Lno_data:
 264         ret
 265         restore
 266 .size   poly1305_blocks,.-poly1305_blocks
 267 ___
 268 ########################################################################
 269 # VIS3 has umulxhi and addxc...
 270 {
 271 my ($H0,$H1,$H2,$R0,$R1,$S1,$T1) = map("%o$_",(0..5,7));
 272 my ($D0,$D1,$D2,$T0) = map("%g$_",(1..4));
 273
 274 $code.=<<___;
 275 .align  32
 276 poly1305_blocks_vis3:
 277         save    %sp,-STACK_FRAME,%sp
 278         andn    $len,15,$len
 279
 280         brz,pn  $len,.Lno_data
 281         nop
 282
 283         ldx     [$ctx+32+0],$R0         ! load key
 284         ldx     [$ctx+32+8],$R1
 285
 286         ldx     [$ctx+0],$H0            ! load hash value
 287         ldx     [$ctx+8],$H1
 288         ld      [$ctx+16],$H2
 289
 290         and     $inp,7,$shr             ! alignment factor
 291         andn    $inp,7,$inp
 292         set     8,$r1
 293         sll     $shr,3,$shr             ! *8
 294         set     16,$r2
 295         neg     $shr,$shl
 296
 297         srlx    $R1,2,$S1
 298         add     $R1,$S1,$S1
 299
 300 .Loop_vis3:
 301         ldxa    [$inp+%g0]0x88,$D0      ! load little-endian input
 302         brz,pt  $shr,.Linp_aligned_vis3
 303         ldxa    [$inp+$r1]0x88,$D1
 304
 305         ldxa    [$inp+$r2]0x88,$D2
 306         srlx    $D0,$shr,$D0
 307         sllx    $D1,$shl,$T1
 308         srlx    $D1,$shr,$D1
 309         or      $T1,$D0,$D0
 310         sllx    $D2,$shl,$D2
 311         or      $D2,$D1,$D1
 312
 313 .Linp_aligned_vis3:
 314         addcc   $D0,$H0,$H0             ! accumulate input
 315          sub    $len,16,$len
 316         addxccc $D1,$H1,$H1
 317          add    $inp,16,$inp
 318
 319         mulx    $R0,$H0,$D0             ! r0*h0
 320         addxc   $padbit,$H2,$H2
 321         umulxhi $R0,$H0,$D1
 322         mulx    $S1,$H1,$T0             ! s1*h1
 323         umulxhi $S1,$H1,$T1
 324         addcc   $T0,$D0,$D0
 325         mulx    $R1,$H0,$T0             ! r1*h0
 326         addxc   $T1,$D1,$D1
 327         umulxhi $R1,$H0,$D2
 328         addcc   $T0,$D1,$D1
 329         mulx    $R0,$H1,$T0             ! r0*h1
 330         addxc   %g0,$D2,$D2
 331         umulxhi $R0,$H1,$T1
 332         addcc   $T0,$D1,$D1
 333         mulx    $S1,$H2,$T0             ! s1*h2
 334         addxc   $T1,$D2,$D2
 335         mulx    $R0,$H2,$T1             ! r0*h2
 336         addcc   $T0,$D1,$D1
 337         addxc   $T1,$D2,$D2
 338
 339         srlx    $D2,2,$T0               ! final reduction step
 340         andn    $D2,3,$T1
 341         and     $D2,3,$H2
 342         add     $T1,$T0,$T0
 343
 344         addcc   $T0,$D0,$H0
 345         brnz,pt $len,.Loop_vis3
 346         addxc   %g0,$D1,$H1
 347
 348         stx     $H0,[$ctx+0]            ! store hash value
 349         stx     $H1,[$ctx+8]
 350         st      $H2,[$ctx+16]
 351
 352         ret
 353         restore
 354 .size   poly1305_blocks_vis3,.-poly1305_blocks_vis3
 355 ___
 356 }
 357 my ($mac,$nonce) = ($inp,$len);
 358
 359 $code.=<<___;
 360 .globl  poly1305_emit
 361 .align  32
 362 poly1305_emit:
 363         save    %sp,-STACK_FRAME,%sp
 364
 365         ld      [$ctx+0],$h1            ! load hash value
 366         ld      [$ctx+4],$h0
 367         ld      [$ctx+8],$h3
 368         ld      [$ctx+12],$h2
 369         ld      [$ctx+16],$h4
 370
 371         addcc   $h0,5,$r0               ! compare to modulus
 372         addccc  $h1,0,$r1
 373         addccc  $h2,0,$r2
 374         addccc  $h3,0,$r3
 375         addc    $h4,0,$h4
 376         andcc   $h4,4,%g0               ! did it carry/borrow?
 377
 378         movnz   %icc,$r0,$h0
 379         ld      [$nonce+0],$r0          ! load nonce
 380         movnz   %icc,$r1,$h1
 381         ld      [$nonce+4],$r1
 382         movnz   %icc,$r2,$h2
 383         ld      [$nonce+8],$r2
 384         movnz   %icc,$r3,$h3
 385         ld      [$nonce+12],$r3
 386
 387         addcc   $r0,$h0,$h0             ! accumulate nonce
 388         addccc  $r1,$h1,$h1
 389         addccc  $r2,$h2,$h2
 390         addc    $r3,$h3,$h3
 391
 392         srl     $h0,8,$r0
 393         stb     $h0,[$mac+0]            ! store little-endian result
 394         srl     $h0,16,$r1
 395         stb     $r0,[$mac+1]
 396         srl     $h0,24,$r2
 397         stb     $r1,[$mac+2]
 398         stb     $r2,[$mac+3]
 399
 400         srl     $h1,8,$r0
 401         stb     $h1,[$mac+4]
 402         srl     $h1,16,$r1
 403         stb     $r0,[$mac+5]
 404         srl     $h1,24,$r2
 405         stb     $r1,[$mac+6]
 406         stb     $r2,[$mac+7]
 407
 408         srl     $h2,8,$r0
 409         stb     $h2,[$mac+8]
 410         srl     $h2,16,$r1
 411         stb     $r0,[$mac+9]
 412         srl     $h2,24,$r2
 413         stb     $r1,[$mac+10]
 414         stb     $r2,[$mac+11]
 415
 416         srl     $h3,8,$r0
 417         stb     $h3,[$mac+12]
 418         srl     $h3,16,$r1
 419         stb     $r0,[$mac+13]
 420         srl     $h3,24,$r2
 421         stb     $r1,[$mac+14]
 422         stb     $r2,[$mac+15]
 423
 424         ret
 425         restore
 426 .size   poly1305_emit,.-poly1305_emit
 427 ___
 428
 429 {
 430 my ($ctx,$inp,$len,$padbit) = map("%i$_",(0..3));
 431 my ($in0,$in1,$in2,$in3,$in4) = map("%o$_",(0..4));
 432 my ($i1,$step,$shr,$shl) = map("%l$_",(0..7));
 433 my $i2=$step;
 434
 435 my ($h0lo,$h0hi,$h1lo,$h1hi,$h2lo,$h2hi,$h3lo,$h3hi,
 436     $two0,$two32,$two64,$two96,$two130,$five_two130,
 437     $r0lo,$r0hi,$r1lo,$r1hi,$r2lo,$r2hi,
 438     $s2lo,$s2hi,$s3lo,$s3hi,
 439     $c0lo,$c0hi,$c1lo,$c1hi,$c2lo,$c2hi,$c3lo,$c3hi) = map("%f".2*$_,(0..31));
 440 # borrowings
 441 my ($r3lo,$r3hi,$s1lo,$s1hi) = ($c0lo,$c0hi,$c1lo,$c1hi);
 442 my ($x0,$x1,$x2,$x3) = ($c2lo,$c2hi,$c3lo,$c3hi);
 443 my ($y0,$y1,$y2,$y3) = ($c1lo,$c1hi,$c3hi,$c3lo);
 444
 445 $code.=<<___;
 446 .align  32
 447 poly1305_init_fma:
 448         save    %sp,-STACK_FRAME-16,%sp
 449         nop
 450
 451 .Lpoly1305_init_fma:
 452 1:      call    .+8
 453         add     %o7,.Lconsts_fma-1b,%o7
 454
 455         ldd     [%o7+8*0],$two0                 ! load constants
 456         ldd     [%o7+8*1],$two32
 457         ldd     [%o7+8*2],$two64
 458         ldd     [%o7+8*3],$two96
 459         ldd     [%o7+8*5],$five_two130
 460
 461         std     $two0,[$ctx+8*0]                ! initial hash value, biased 0
 462         std     $two32,[$ctx+8*1]
 463         std     $two64,[$ctx+8*2]
 464         std     $two96,[$ctx+8*3]
 465
 466         brz,pn  $inp,.Lno_key_fma
 467         nop
 468
 469         stx     %fsr,[%sp+LOCALS]               ! save original %fsr
 470         ldx     [%o7+8*6],%fsr                  ! load new %fsr
 471
 472         std     $two0,[$ctx+8*4]                ! key "template"
 473         std     $two32,[$ctx+8*5]
 474         std     $two64,[$ctx+8*6]
 475         std     $two96,[$ctx+8*7]
 476
 477         and     $inp,7,$shr
 478         andn    $inp,7,$inp                     ! align pointer
 479         mov     8,$i1
 480         sll     $shr,3,$shr
 481         mov     16,$i2
 482         neg     $shr,$shl
 483
 484         ldxa    [$inp+%g0]0x88,$in0             ! load little-endian key
 485         ldxa    [$inp+$i1]0x88,$in2
 486
 487         brz     $shr,.Lkey_aligned_fma
 488         sethi   %hi(0xf0000000),$i1             !   0xf0000000
 489
 490         ldxa    [$inp+$i2]0x88,$in4
 491
 492         srlx    $in0,$shr,$in0                  ! align data
 493         sllx    $in2,$shl,$in1
 494         srlx    $in2,$shr,$in2
 495         or      $in1,$in0,$in0
 496         sllx    $in4,$shl,$in3
 497         or      $in3,$in2,$in2
 498
 499 .Lkey_aligned_fma:
 500         or      $i1,3,$i2                       !   0xf0000003
 501         srlx    $in0,32,$in1
 502         andn    $in0,$i1,$in0                   ! &=0x0fffffff
 503         andn    $in1,$i2,$in1                   ! &=0x0ffffffc
 504         srlx    $in2,32,$in3
 505         andn    $in2,$i2,$in2
 506         andn    $in3,$i2,$in3
 507
 508         st      $in0,[$ctx+`8*4+4`]             ! fill "template"
 509         st      $in1,[$ctx+`8*5+4`]
 510         st      $in2,[$ctx+`8*6+4`]
 511         st      $in3,[$ctx+`8*7+4`]
 512
 513         ldd     [$ctx+8*4],$h0lo                ! load [biased] key
 514         ldd     [$ctx+8*5],$h1lo
 515         ldd     [$ctx+8*6],$h2lo
 516         ldd     [$ctx+8*7],$h3lo
 517
 518         fsubd   $h0lo,$two0, $h0lo              ! r0
 519          ldd    [%o7+8*7],$two0                 ! more constants
 520         fsubd   $h1lo,$two32,$h1lo              ! r1
 521          ldd    [%o7+8*8],$two32
 522         fsubd   $h2lo,$two64,$h2lo              ! r2
 523          ldd    [%o7+8*9],$two64
 524         fsubd   $h3lo,$two96,$h3lo              ! r3
 525          ldd    [%o7+8*10],$two96
 526
 527         fmuld   $five_two130,$h1lo,$s1lo        ! s1
 528         fmuld   $five_two130,$h2lo,$s2lo        ! s2
 529         fmuld   $five_two130,$h3lo,$s3lo        ! s3
 530
 531         faddd   $h0lo,$two0, $h0hi
 532         faddd   $h1lo,$two32,$h1hi
 533         faddd   $h2lo,$two64,$h2hi
 534         faddd   $h3lo,$two96,$h3hi
 535
 536         fsubd   $h0hi,$two0, $h0hi
 537          ldd    [%o7+8*11],$two0                ! more constants
 538         fsubd   $h1hi,$two32,$h1hi
 539          ldd    [%o7+8*12],$two32
 540         fsubd   $h2hi,$two64,$h2hi
 541          ldd    [%o7+8*13],$two64
 542         fsubd   $h3hi,$two96,$h3hi
 543
 544         fsubd   $h0lo,$h0hi,$h0lo
 545          std    $h0hi,[$ctx+8*5]                ! r0hi
 546         fsubd   $h1lo,$h1hi,$h1lo
 547          std    $h1hi,[$ctx+8*7]                ! r1hi
 548         fsubd   $h2lo,$h2hi,$h2lo
 549          std    $h2hi,[$ctx+8*9]                ! r2hi
 550         fsubd   $h3lo,$h3hi,$h3lo
 551          std    $h3hi,[$ctx+8*11]               ! r3hi
 552
 553         faddd   $s1lo,$two0, $s1hi
 554         faddd   $s2lo,$two32,$s2hi
 555         faddd   $s3lo,$two64,$s3hi
 556
 557         fsubd   $s1hi,$two0, $s1hi
 558         fsubd   $s2hi,$two32,$s2hi
 559         fsubd   $s3hi,$two64,$s3hi
 560
 561         fsubd   $s1lo,$s1hi,$s1lo
 562         fsubd   $s2lo,$s2hi,$s2lo
 563         fsubd   $s3lo,$s3hi,$s3lo
 564
 565         ldx     [%sp+LOCALS],%fsr               ! restore %fsr
 566
 567         std     $h0lo,[$ctx+8*4]                ! r0lo
 568         std     $h1lo,[$ctx+8*6]                ! r1lo
 569         std     $h2lo,[$ctx+8*8]                ! r2lo
 570         std     $h3lo,[$ctx+8*10]               ! r3lo
 571
 572         std     $s1hi,[$ctx+8*13]
 573         std     $s2hi,[$ctx+8*15]
 574         std     $s3hi,[$ctx+8*17]
 575
 576         std     $s1lo,[$ctx+8*12]
 577         std     $s2lo,[$ctx+8*14]
 578         std     $s3lo,[$ctx+8*16]
 579
 580         add     %o7,poly1305_blocks_fma-.Lconsts_fma,%o0
 581         add     %o7,poly1305_emit_fma-.Lconsts_fma,%o1
 582         STPTR   %o0,[%i2]
 583         STPTR   %o1,[%i2+SIZE_T]
 584
 585         ret
 586         restore %g0,1,%o0                       ! return 1
 587
 588 .Lno_key_fma:
 589         ret
 590         restore %g0,%g0,%o0                     ! return 0
 591 .size   poly1305_init_fma,.-poly1305_init_fma
 592
 593 .align  32
 594 poly1305_blocks_fma:
 595         save    %sp,-STACK_FRAME-48,%sp
 596         srlx    $len,4,$len
 597
 598         brz,pn  $len,.Labort
 599         sub     $len,1,$len
 600
 601 1:      call    .+8
 602         add     %o7,.Lconsts_fma-1b,%o7
 603
 604         ldd     [%o7+8*0],$two0                 ! load constants
 605         ldd     [%o7+8*1],$two32
 606         ldd     [%o7+8*2],$two64
 607         ldd     [%o7+8*3],$two96
 608         ldd     [%o7+8*4],$two130
 609         ldd     [%o7+8*5],$five_two130
 610
 611         ldd     [$ctx+8*0],$h0lo                ! load [biased] hash value
 612         ldd     [$ctx+8*1],$h1lo
 613         ldd     [$ctx+8*2],$h2lo
 614         ldd     [$ctx+8*3],$h3lo
 615
 616         std     $two0,[%sp+LOCALS+8*0]          ! input "template"
 617         sethi   %hi((1023+52+96)<<20),$in3
 618         std     $two32,[%sp+LOCALS+8*1]
 619         or      $padbit,$in3,$in3
 620         std     $two64,[%sp+LOCALS+8*2]
 621         st      $in3,[%sp+LOCALS+8*3]
 622
 623         and     $inp,7,$shr
 624         andn    $inp,7,$inp                     ! align pointer
 625         mov     8,$i1
 626         sll     $shr,3,$shr
 627         mov     16,$step
 628         neg     $shr,$shl
 629
 630         ldxa    [$inp+%g0]0x88,$in0             ! load little-endian input
 631         brz     $shr,.Linp_aligned_fma
 632         ldxa    [$inp+$i1]0x88,$in2
 633
 634         ldxa    [$inp+$step]0x88,$in4
 635         add     $inp,8,$inp
 636
 637         srlx    $in0,$shr,$in0                  ! align data
 638         sllx    $in2,$shl,$in1
 639         srlx    $in2,$shr,$in2
 640         or      $in1,$in0,$in0
 641         sllx    $in4,$shl,$in3
 642         srlx    $in4,$shr,$in4                  ! pre-shift
 643         or      $in3,$in2,$in2
 644
 645 .Linp_aligned_fma:
 646         srlx    $in0,32,$in1
 647         movrz   $len,0,$step
 648         srlx    $in2,32,$in3
 649         add     $step,$inp,$inp                 ! conditional advance
 650
 651         st      $in0,[%sp+LOCALS+8*0+4]         ! fill "template"
 652         st      $in1,[%sp+LOCALS+8*1+4]
 653         st      $in2,[%sp+LOCALS+8*2+4]
 654         st      $in3,[%sp+LOCALS+8*3+4]
 655
 656         ldd     [$ctx+8*4],$r0lo                ! load key
 657         ldd     [$ctx+8*5],$r0hi
 658         ldd     [$ctx+8*6],$r1lo
 659         ldd     [$ctx+8*7],$r1hi
 660         ldd     [$ctx+8*8],$r2lo
 661         ldd     [$ctx+8*9],$r2hi
 662         ldd     [$ctx+8*10],$r3lo
 663         ldd     [$ctx+8*11],$r3hi
 664         ldd     [$ctx+8*12],$s1lo
 665         ldd     [$ctx+8*13],$s1hi
 666         ldd     [$ctx+8*14],$s2lo
 667         ldd     [$ctx+8*15],$s2hi
 668         ldd     [$ctx+8*16],$s3lo
 669         ldd     [$ctx+8*17],$s3hi
 670
 671         stx     %fsr,[%sp+LOCALS+8*4]           ! save original %fsr
 672         ldx     [%o7+8*6],%fsr                  ! load new %fsr
 673
 674         subcc   $len,1,$len
 675         movrz   $len,0,$step
 676
 677         ldd     [%sp+LOCALS+8*0],$x0            ! load biased input
 678         ldd     [%sp+LOCALS+8*1],$x1
 679         ldd     [%sp+LOCALS+8*2],$x2
 680         ldd     [%sp+LOCALS+8*3],$x3
 681
 682         fsubd   $h0lo,$two0, $h0lo              ! de-bias hash value
 683         fsubd   $h1lo,$two32,$h1lo
 684          ldxa   [$inp+%g0]0x88,$in0             ! modulo-scheduled input load
 685         fsubd   $h2lo,$two64,$h2lo
 686         fsubd   $h3lo,$two96,$h3lo
 687          ldxa   [$inp+$i1]0x88,$in2
 688
 689         fsubd   $x0,$two0, $x0                  ! de-bias input
 690         fsubd   $x1,$two32,$x1
 691         fsubd   $x2,$two64,$x2
 692         fsubd   $x3,$two96,$x3
 693
 694         brz     $shr,.Linp_aligned_fma2
 695         add     $step,$inp,$inp                 ! conditional advance
 696
 697         sllx    $in0,$shl,$in1                  ! align data
 698         srlx    $in0,$shr,$in3
 699         or      $in1,$in4,$in0
 700         sllx    $in2,$shl,$in1
 701         srlx    $in2,$shr,$in4                  ! pre-shift
 702         or      $in3,$in1,$in2
 703 .Linp_aligned_fma2:
 704         srlx    $in0,32,$in1
 705         srlx    $in2,32,$in3
 706
 707         faddd   $h0lo,$x0,$x0                   ! accumulate input
 708          stw    $in0,[%sp+LOCALS+8*0+4]
 709         faddd   $h1lo,$x1,$x1
 710          stw    $in1,[%sp+LOCALS+8*1+4]
 711         faddd   $h2lo,$x2,$x2
 712          stw    $in2,[%sp+LOCALS+8*2+4]
 713         faddd   $h3lo,$x3,$x3
 714          stw    $in3,[%sp+LOCALS+8*3+4]
 715
 716         b       .Lentry_fma
 717         nop
 718
 719 .align  16
 720 .Loop_fma:
 721         ldxa    [$inp+%g0]0x88,$in0             ! modulo-scheduled input load
 722         ldxa    [$inp+$i1]0x88,$in2
 723         movrz   $len,0,$step
 724
 725         faddd   $y0,$h0lo,$h0lo                 ! accumulate input
 726         faddd   $y1,$h0hi,$h0hi
 727         faddd   $y2,$h2lo,$h2lo
 728         faddd   $y3,$h2hi,$h2hi
 729
 730         brz,pn  $shr,.Linp_aligned_fma3
 731         add     $step,$inp,$inp                 ! conditional advance
 732
 733         sllx    $in0,$shl,$in1                  ! align data
 734         srlx    $in0,$shr,$in3
 735         or      $in1,$in4,$in0
 736         sllx    $in2,$shl,$in1
 737         srlx    $in2,$shr,$in4                  ! pre-shift
 738         or      $in3,$in1,$in2
 739
 740 .Linp_aligned_fma3:
 741         !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! base 2^48 -> base 2^32
 742         faddd   $two64,$h1lo,$c1lo
 743          srlx   $in0,32,$in1
 744         faddd   $two64,$h1hi,$c1hi
 745          srlx   $in2,32,$in3
 746         faddd   $two130,$h3lo,$c3lo
 747          st     $in0,[%sp+LOCALS+8*0+4]         ! fill "template"
 748         faddd   $two130,$h3hi,$c3hi
 749          st     $in1,[%sp+LOCALS+8*1+4]
 750         faddd   $two32,$h0lo,$c0lo
 751          st     $in2,[%sp+LOCALS+8*2+4]
 752         faddd   $two32,$h0hi,$c0hi
 753          st     $in3,[%sp+LOCALS+8*3+4]
 754         faddd   $two96,$h2lo,$c2lo
 755         faddd   $two96,$h2hi,$c2hi
 756
 757         fsubd   $c1lo,$two64,$c1lo
 758         fsubd   $c1hi,$two64,$c1hi
 759         fsubd   $c3lo,$two130,$c3lo
 760         fsubd   $c3hi,$two130,$c3hi
 761         fsubd   $c0lo,$two32,$c0lo
 762         fsubd   $c0hi,$two32,$c0hi
 763         fsubd   $c2lo,$two96,$c2lo
 764         fsubd   $c2hi,$two96,$c2hi
 765
 766         fsubd   $h1lo,$c1lo,$h1lo
 767         fsubd   $h1hi,$c1hi,$h1hi
 768         fsubd   $h3lo,$c3lo,$h3lo
 769         fsubd   $h3hi,$c3hi,$h3hi
 770         fsubd   $h2lo,$c2lo,$h2lo
 771         fsubd   $h2hi,$c2hi,$h2hi
 772         fsubd   $h0lo,$c0lo,$h0lo
 773         fsubd   $h0hi,$c0hi,$h0hi
 774
 775         faddd   $h1lo,$c0lo,$h1lo
 776         faddd   $h1hi,$c0hi,$h1hi
 777         faddd   $h3lo,$c2lo,$h3lo
 778         faddd   $h3hi,$c2hi,$h3hi
 779         faddd   $h2lo,$c1lo,$h2lo
 780         faddd   $h2hi,$c1hi,$h2hi
 781         fmaddd  $five_two130,$c3lo,$h0lo,$h0lo
 782         fmaddd  $five_two130,$c3hi,$h0hi,$h0hi
 783
 784         faddd   $h1lo,$h1hi,$x1
 785          ldd    [$ctx+8*12],$s1lo               ! reload constants
 786         faddd   $h3lo,$h3hi,$x3
 787          ldd    [$ctx+8*13],$s1hi
 788         faddd   $h2lo,$h2hi,$x2
 789          ldd    [$ctx+8*10],$r3lo
 790         faddd   $h0lo,$h0hi,$x0
 791          ldd    [$ctx+8*11],$r3hi
 792
 793 .Lentry_fma:
 794         fmuld   $x1,$s3lo,$h0lo
 795         fmuld   $x1,$s3hi,$h0hi
 796         fmuld   $x1,$r1lo,$h2lo
 797         fmuld   $x1,$r1hi,$h2hi
 798         fmuld   $x1,$r0lo,$h1lo
 799         fmuld   $x1,$r0hi,$h1hi
 800         fmuld   $x1,$r2lo,$h3lo
 801         fmuld   $x1,$r2hi,$h3hi
 802
 803         fmaddd  $x3,$s1lo,$h0lo,$h0lo
 804         fmaddd  $x3,$s1hi,$h0hi,$h0hi
 805         fmaddd  $x3,$s3lo,$h2lo,$h2lo
 806         fmaddd  $x3,$s3hi,$h2hi,$h2hi
 807         fmaddd  $x3,$s2lo,$h1lo,$h1lo
 808         fmaddd  $x3,$s2hi,$h1hi,$h1hi
 809         fmaddd  $x3,$r0lo,$h3lo,$h3lo
 810         fmaddd  $x3,$r0hi,$h3hi,$h3hi
 811
 812         fmaddd  $x2,$s2lo,$h0lo,$h0lo
 813         fmaddd  $x2,$s2hi,$h0hi,$h0hi
 814         fmaddd  $x2,$r0lo,$h2lo,$h2lo
 815         fmaddd  $x2,$r0hi,$h2hi,$h2hi
 816         fmaddd  $x2,$s3lo,$h1lo,$h1lo
 817          ldd    [%sp+LOCALS+8*0],$y0            ! load [biased] input
 818         fmaddd  $x2,$s3hi,$h1hi,$h1hi
 819          ldd    [%sp+LOCALS+8*1],$y1
 820         fmaddd  $x2,$r1lo,$h3lo,$h3lo
 821          ldd    [%sp+LOCALS+8*2],$y2
 822         fmaddd  $x2,$r1hi,$h3hi,$h3hi
 823          ldd    [%sp+LOCALS+8*3],$y3
 824
 825         fmaddd  $x0,$r0lo,$h0lo,$h0lo
 826          fsubd  $y0,$two0, $y0                  ! de-bias input
 827         fmaddd  $x0,$r0hi,$h0hi,$h0hi
 828          fsubd  $y1,$two32,$y1
 829         fmaddd  $x0,$r2lo,$h2lo,$h2lo
 830          fsubd  $y2,$two64,$y2
 831         fmaddd  $x0,$r2hi,$h2hi,$h2hi
 832          fsubd  $y3,$two96,$y3
 833         fmaddd  $x0,$r1lo,$h1lo,$h1lo
 834         fmaddd  $x0,$r1hi,$h1hi,$h1hi
 835         fmaddd  $x0,$r3lo,$h3lo,$h3lo
 836         fmaddd  $x0,$r3hi,$h3hi,$h3hi
 837
 838         bcc     SIZE_T_CC,.Loop_fma
 839         subcc   $len,1,$len
 840
 841         !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! base 2^48 -> base 2^32
 842         faddd   $h0lo,$two32,$c0lo
 843         faddd   $h0hi,$two32,$c0hi
 844         faddd   $h2lo,$two96,$c2lo
 845         faddd   $h2hi,$two96,$c2hi
 846         faddd   $h1lo,$two64,$c1lo
 847         faddd   $h1hi,$two64,$c1hi
 848         faddd   $h3lo,$two130,$c3lo
 849         faddd   $h3hi,$two130,$c3hi
 850
 851         fsubd   $c0lo,$two32,$c0lo
 852         fsubd   $c0hi,$two32,$c0hi
 853         fsubd   $c2lo,$two96,$c2lo
 854         fsubd   $c2hi,$two96,$c2hi
 855         fsubd   $c1lo,$two64,$c1lo
 856         fsubd   $c1hi,$two64,$c1hi
 857         fsubd   $c3lo,$two130,$c3lo
 858         fsubd   $c3hi,$two130,$c3hi
 859
 860         fsubd   $h1lo,$c1lo,$h1lo
 861         fsubd   $h1hi,$c1hi,$h1hi
 862         fsubd   $h3lo,$c3lo,$h3lo
 863         fsubd   $h3hi,$c3hi,$h3hi
 864         fsubd   $h2lo,$c2lo,$h2lo
 865         fsubd   $h2hi,$c2hi,$h2hi
 866         fsubd   $h0lo,$c0lo,$h0lo
 867         fsubd   $h0hi,$c0hi,$h0hi
 868
 869         faddd   $h1lo,$c0lo,$h1lo
 870         faddd   $h1hi,$c0hi,$h1hi
 871         faddd   $h3lo,$c2lo,$h3lo
 872         faddd   $h3hi,$c2hi,$h3hi
 873         faddd   $h2lo,$c1lo,$h2lo
 874         faddd   $h2hi,$c1hi,$h2hi
 875         fmaddd  $five_two130,$c3lo,$h0lo,$h0lo
 876         fmaddd  $five_two130,$c3hi,$h0hi,$h0hi
 877
 878         faddd   $h1lo,$h1hi,$x1
 879         faddd   $h3lo,$h3hi,$x3
 880         faddd   $h2lo,$h2hi,$x2
 881         faddd   $h0lo,$h0hi,$x0
 882
 883         faddd   $x1,$two32,$x1                  ! bias
 884         faddd   $x3,$two96,$x3
 885         faddd   $x2,$two64,$x2
 886         faddd   $x0,$two0, $x0
 887
 888         ldx     [%sp+LOCALS+8*4],%fsr           ! restore saved %fsr
 889
 890         std     $x1,[$ctx+8*1]                  ! store [biased] hash value
 891         std     $x3,[$ctx+8*3]
 892         std     $x2,[$ctx+8*2]
 893         std     $x0,[$ctx+8*0]
 894
 895 .Labort:
 896         ret
 897         restore
 898 .size   poly1305_blocks_fma,.-poly1305_blocks_fma
 899 ___
 900 {
 901 my ($mac,$nonce)=($inp,$len);
 902
 903 my ($h0,$h1,$h2,$h3,$h4, $d0,$d1,$d2,$d3, $mask
 904    ) = (map("%l$_",(0..5)),map("%o$_",(0..4)));
 905
 906 $code.=<<___;
 907 .align  32
 908 poly1305_emit_fma:
 909         save    %sp,-STACK_FRAME,%sp
 910
 911         ld      [$ctx+8*0+0],$d0                ! load hash
 912         ld      [$ctx+8*0+4],$h0
 913         ld      [$ctx+8*1+0],$d1
 914         ld      [$ctx+8*1+4],$h1
 915         ld      [$ctx+8*2+0],$d2
 916         ld      [$ctx+8*2+4],$h2
 917         ld      [$ctx+8*3+0],$d3
 918         ld      [$ctx+8*3+4],$h3
 919
 920         sethi   %hi(0xfff00000),$mask
 921         andn    $d0,$mask,$d0                   ! mask exponent
 922         andn    $d1,$mask,$d1
 923         andn    $d2,$mask,$d2
 924         andn    $d3,$mask,$d3                   ! can be partially reduced...
 925         mov     3,$mask
 926
 927         srl     $d3,2,$padbit                   ! ... so reduce
 928         and     $d3,$mask,$h4
 929         andn    $d3,$mask,$d3
 930         add     $padbit,$d3,$d3
 931
 932         addcc   $d3,$h0,$h0
 933         addccc  $d0,$h1,$h1
 934         addccc  $d1,$h2,$h2
 935         addccc  $d2,$h3,$h3
 936         addc    %g0,$h4,$h4
 937
 938         addcc   $h0,5,$d0                       ! compare to modulus
 939         addccc  $h1,0,$d1
 940         addccc  $h2,0,$d2
 941         addccc  $h3,0,$d3
 942         addc    $h4,0,$mask
 943
 944         srl     $mask,2,$mask                   ! did it carry/borrow?
 945         neg     $mask,$mask
 946         sra     $mask,31,$mask                  ! mask
 947
 948         andn    $h0,$mask,$h0
 949         and     $d0,$mask,$d0
 950         andn    $h1,$mask,$h1
 951         and     $d1,$mask,$d1
 952         or      $d0,$h0,$h0
 953         ld      [$nonce+0],$d0                  ! load nonce
 954         andn    $h2,$mask,$h2
 955         and     $d2,$mask,$d2
 956         or      $d1,$h1,$h1
 957         ld      [$nonce+4],$d1
 958         andn    $h3,$mask,$h3
 959         and     $d3,$mask,$d3
 960         or      $d2,$h2,$h2
 961         ld      [$nonce+8],$d2
 962         or      $d3,$h3,$h3
 963         ld      [$nonce+12],$d3
 964
 965         addcc   $d0,$h0,$h0                     ! accumulate nonce
 966         addccc  $d1,$h1,$h1
 967         addccc  $d2,$h2,$h2
 968         addc    $d3,$h3,$h3
 969
 970         stb     $h0,[$mac+0]                    ! write little-endian result
 971         srl     $h0,8,$h0
 972         stb     $h1,[$mac+4]
 973         srl     $h1,8,$h1
 974         stb     $h2,[$mac+8]
 975         srl     $h2,8,$h2
 976         stb     $h3,[$mac+12]
 977         srl     $h3,8,$h3
 978
 979         stb     $h0,[$mac+1]
 980         srl     $h0,8,$h0
 981         stb     $h1,[$mac+5]
 982         srl     $h1,8,$h1
 983         stb     $h2,[$mac+9]
 984         srl     $h2,8,$h2
 985         stb     $h3,[$mac+13]
 986         srl     $h3,8,$h3
 987
 988         stb     $h0,[$mac+2]
 989         srl     $h0,8,$h0
 990         stb     $h1,[$mac+6]
 991         srl     $h1,8,$h1
 992         stb     $h2,[$mac+10]
 993         srl     $h2,8,$h2
 994         stb     $h3,[$mac+14]
 995         srl     $h3,8,$h3
 996
 997         stb     $h0,[$mac+3]
 998         stb     $h1,[$mac+7]
 999         stb     $h2,[$mac+11]
1000         stb     $h3,[$mac+15]
1001
1002         ret
1003         restore
1004 .size   poly1305_emit_fma,.-poly1305_emit_fma
1005 ___
1006 }
1007
1008 $code.=<<___;
1009 .align  64
1010 .Lconsts_fma:
1011 .word   0x43300000,0x00000000           ! 2^(52+0)
1012 .word   0x45300000,0x00000000           ! 2^(52+32)
1013 .word   0x47300000,0x00000000           ! 2^(52+64)
1014 .word   0x49300000,0x00000000           ! 2^(52+96)
1015 .word   0x4b500000,0x00000000           ! 2^(52+130)
1016
1017 .word   0x37f40000,0x00000000           ! 5/2^130
1018 .word   0,1<<30                         ! fsr: truncate, no exceptions
1019
1020 .word   0x44300000,0x00000000           ! 2^(52+16+0)
1021 .word   0x46300000,0x00000000           ! 2^(52+16+32)
1022 .word   0x48300000,0x00000000           ! 2^(52+16+64)
1023 .word   0x4a300000,0x00000000           ! 2^(52+16+96)
1024 .word   0x3e300000,0x00000000           ! 2^(52+16+0-96)
1025 .word   0x40300000,0x00000000           ! 2^(52+16+32-96)
1026 .word   0x42300000,0x00000000           ! 2^(52+16+64-96)
1027 .asciz  "Poly1305 for SPARCv9/VIS3/FMA, CRYPTOGAMS by <appro\@openssl.org>"
1028 .align  4
1029 ___
1030 }
1031 \f
1032 # Purpose of these subroutines is to explicitly encode VIS instructions,
1033 # so that one can compile the module without having to specify VIS
1034 # extensions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a.
1035 # Idea is to reserve for option to produce "universal" binary and let
1036 # programmer detect if current CPU is VIS capable at run-time.
1037 sub unvis3 {
1038 my ($mnemonic,$rs1,$rs2,$rd)=@_;
1039 my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 );
1040 my ($ref,$opf);
1041 my %visopf = (  "addxc"         => 0x011,
1042                 "addxccc"       => 0x013,
1043                 "umulxhi"       => 0x016        );
1044
1045     $ref = "$mnemonic\t$rs1,$rs2,$rd";
1046
1047     if ($opf=$visopf{$mnemonic}) {
1048         foreach ($rs1,$rs2,$rd) {
1049             return $ref if (!/%([goli])([0-9])/);
1050             $_=$bias{$1}+$2;
1051         }
1052
1053         return  sprintf ".word\t0x%08x !%s",
1054                         0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2,
1055                         $ref;
1056     } else {
1057         return $ref;
1058     }
1059 }
1060
1061 sub unfma {
1062 my ($mnemonic,$rs1,$rs2,$rs3,$rd)=@_;
1063 my ($ref,$opf);
1064 my %fmaopf = (  "fmadds"        => 0x1,
1065                 "fmaddd"        => 0x2,
1066                 "fmsubs"        => 0x5,
1067                 "fmsubd"        => 0x6          );
1068
1069     $ref = "$mnemonic\t$rs1,$rs2,$rs3,$rd";
1070
1071     if ($opf=$fmaopf{$mnemonic}) {
1072         foreach ($rs1,$rs2,$rs3,$rd) {
1073             return $ref if (!/%f([0-9]{1,2})/);
1074             $_=$1;
1075             if ($1>=32) {
1076                 return $ref if ($1&1);
1077                 # re-encode for upper double register addressing
1078                 $_=($1|$1>>5)&31;
1079             }
1080         }
1081
1082         return  sprintf ".word\t0x%08x !%s",
1083                         0x81b80000|$rd<<25|$rs1<<14|$rs3<<9|$opf<<5|$rs2,
1084                         $ref;
1085     } else {
1086         return $ref;
1087     }
1088 }
1089
1090 foreach (split("\n",$code)) {
1091         s/\`([^\`]*)\`/eval $1/ge;
1092
1093         s/\b(umulxhi|addxc[c]{0,2})\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/
1094                 &unvis3($1,$2,$3,$4)
1095          /ge    or
1096         s/\b(fmadd[sd])\s+(%f[0-9]+),\s*(%f[0-9]+),\s*(%f[0-9]+),\s*(%f[0-9]+)/
1097                 &unfma($1,$2,$3,$4,$5)
1098          /ge;
1099
1100         print $_,"\n";
1101 }
1102
1103 close STDOUT;