crypto/chacha/asm/chacha-ia64.pl

   1 #!/usr/bin/env perl
   2 #
   3 # ====================================================================
   4 # Written by Andy Polyakov, @dot-asm, initially for use with OpenSSL.
   5 # ====================================================================
   6 #
   7 # ChaCha20 for Itanium.
   8 #
   9 # March 2019
  10 #
  11 # Itanium 9xxx, which has pair of shifters, manages to process one byte
  12 # in 9.3 cycles. This aligns perfectly with theoretical estimate.
  13 # On the other hand, pre-9000 CPU has single shifter and each extr/dep
  14 # pairs below takes additional cycle. Then final input->xor->output
  15 # pass runs slower than expected... Overall result is 15.6 cpb, two
  16 # cycles more than theoretical estimate.
  17
  18 $output = pop;
  19 open STDOUT, ">$output" if $output;
  20
  21 my @k = map("r$_",(16..31));
  22 my @x = map("r$_",(38..53));
  23 my @y = map("r$_",(8..11));
  24 my @z = map("r$_",(15,35..37));
  25 my ($out,$inp,$len,$key,$counter) = map("r$_",(32..36));
  26
  27 $code.=<<___;
  28 #if defined(_HPUX_SOURCE)
  29 # if !defined(_LP64)
  30 #  define ADDP  addp4
  31 # else
  32 #  define ADDP  add
  33 # endif
  34 #else
  35 # define ADDP   add
  36 #endif
  37
  38 .text
  39
  40 .global ChaCha20_ctr32#
  41 .proc   ChaCha20_ctr32#
  42 .align  32
  43 ChaCha20_ctr32:
  44         .prologue
  45         .save           ar.pfs,r2
  46 { .mmi; alloc           r2=ar.pfs,5,17,0,0
  47         ADDP            @k[11]=4,$key
  48         .save           ar.lc,r3
  49         mov             r3=ar.lc                }
  50 { .mmi; ADDP            $key=0,$key
  51         ADDP            $counter=0,$counter
  52         .save           pr,r14
  53         mov             r14=pr                  };;
  54
  55         .body
  56 { .mlx; ld4             @k[4]=[$key],8
  57         movl            @k[0]=0x61707865        }
  58 { .mlx; ld4             @k[5]=[@k[11]],8
  59         movl            @k[1]=0x3320646e        };;
  60 { .mlx; ld4             @k[6]=[$key],8
  61         movl            @k[2]=0x79622d32        }
  62 { .mlx; ld4             @k[7]=[@k[11]],8
  63         movl            @k[3]=0x6b206574        };;
  64 { .mmi; ld4             @k[8]=[$key],8
  65         ld4             @k[9]=[@k[11]],8
  66         add             @k[15]=4,$counter       };;
  67 { .mmi; ld4             @k[10]=[$key]
  68         ld4             @k[11]=[@k[11]]
  69         mov             @x[0]=@k[0]             };;
  70 { .mmi; ld4             @k[12]=[$counter],8
  71         ld4             @k[13]=[@k[15]],8
  72         mov             @x[1]=@k[1]             };;
  73 { .mmi; ld4             @k[14]=[$counter]
  74         ld4             @k[15]=[@k[15]]
  75         mov             @x[2]=@k[2]             }
  76 { .mmi; mov             @x[3]=@k[3]
  77         mov             @x[4]=@k[4]
  78         mov             @x[5]=@k[5]             };;
  79 { .mmi; mov             @x[6]=@k[6]
  80         mov             @x[7]=@k[7]
  81         mov             @x[8]=@k[8]             }
  82 { .mmi; mov             @x[9]=@k[9]
  83         mov             @x[10]=@k[10]
  84         mov             @x[11]=@k[11]           }
  85 { .mmi; mov             @x[12]=@k[12]
  86         mov             @x[13]=@k[13]
  87         mov             @x[14]=@k[14]           };;
  88
  89 .Loop_outer:
  90 { .mii; mov             @x[15]=@k[15]
  91         mov             ar.lc=9
  92         mov             ar.ec=1                 }
  93 { .mmb; cmp.geu         p6,p0=64,$len
  94         sub             @z[1]=64,$len
  95         brp.loop.imp    .Loop_top,.Loop_end-16  };;
  96
  97 .Loop_top:
  98 ___
  99 sub ROUND {
 100 my ($a0,$b0,$c0,$d0)=@_;
 101 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
 102 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
 103 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
 104
 105 $code.=<<___;
 106 { .mmi; add             @x[$a0]=@x[$a0],@x[$b0]
 107         add             @x[$a1]=@x[$a1],@x[$b1]
 108         add             @x[$a2]=@x[$a2],@x[$b2]         };;
 109 { .mmi; add             @x[$a3]=@x[$a3],@x[$b3]
 110         xor             @x[$d0]=@x[$d0],@x[$a0]
 111         xor             @x[$d1]=@x[$d1],@x[$a1]         };;
 112 { .mmi; xor             @x[$d2]=@x[$d2],@x[$a2]
 113         xor             @x[$d3]=@x[$d3],@x[$a3]
 114         extr.u          @y[0]=@x[$d0],16,16             };;
 115 { .mii; extr.u          @y[1]=@x[$d1],16,16
 116         dep             @x[$d0]=@x[$d0],@y[0],16,16     };;
 117 { .mii;  add            @x[$c0]=@x[$c0],@x[$d0]
 118         extr.u          @y[2]=@x[$d2],16,16
 119         dep             @x[$d1]=@x[$d1],@y[1],16,16     };;
 120 { .mii;  add            @x[$c1]=@x[$c1],@x[$d1]
 121          xor            @x[$b0]=@x[$b0],@x[$c0]
 122         extr.u          @y[3]=@x[$d3],16,16             };;
 123 { .mii;  xor            @x[$b1]=@x[$b1],@x[$c1]
 124         dep             @x[$d2]=@x[$d2],@y[2],16,16
 125         dep             @x[$d3]=@x[$d3],@y[3],16,16     };;
 126 { .mmi;  add            @x[$c2]=@x[$c2],@x[$d2]
 127          add            @x[$c3]=@x[$c3],@x[$d3]
 128          extr.u         @y[0]=@x[$b0],20,12             };;
 129 { .mmi;  xor            @x[$b2]=@x[$b2],@x[$c2]
 130          xor            @x[$b3]=@x[$b3],@x[$c3]
 131          dep.z          @x[$b0]=@x[$b0],12,20           };;
 132 { .mii;  or             @x[$b0]=@x[$b0],@y[0]
 133          extr.u         @y[1]=@x[$b1],20,12
 134          dep.z          @x[$b1]=@x[$b1],12,20           };;
 135 { .mii; add             @x[$a0]=@x[$a0],@x[$b0]
 136          extr.u         @y[2]=@x[$b2],20,12
 137          extr.u         @y[3]=@x[$b3],20,12             }
 138 { .mii;  or             @x[$b1]=@x[$b1],@y[1]
 139          dep.z          @x[$b2]=@x[$b2],12,20
 140          dep.z          @x[$b3]=@x[$b3],12,20           };;
 141 { .mmi;  or             @x[$b2]=@x[$b2],@y[2]
 142          or             @x[$b3]=@x[$b3],@y[3]
 143         add             @x[$a1]=@x[$a1],@x[$b1]         };;
 144 { .mmi; add             @x[$a2]=@x[$a2],@x[$b2]
 145         add             @x[$a3]=@x[$a3],@x[$b3]
 146         xor             @x[$d0]=@x[$d0],@x[$a0]         };;
 147 { .mii; xor             @x[$d1]=@x[$d1],@x[$a1]
 148         extr.u          @y[0]=@x[$d0],24,8
 149         dep.z           @x[$d0]=@x[$d0],8,24            };;
 150 { .mii; or              @x[$d0]=@x[$d0],@y[0]
 151         extr.u          @y[1]=@x[$d1],24,8
 152         dep.z           @x[$d1]=@x[$d1],8,24            };;
 153 { .mmi; or              @x[$d1]=@x[$d1],@y[1]
 154         xor             @x[$d2]=@x[$d2],@x[$a2]
 155         xor             @x[$d3]=@x[$d3],@x[$a3]         };;
 156 { .mii;  add            @x[$c0]=@x[$c0],@x[$d0]
 157         extr.u          @y[2]=@x[$d2],24,8
 158         dep.z           @x[$d2]=@x[$d2],8,24            };;
 159 { .mii;  xor            @x[$b0]=@x[$b0],@x[$c0]
 160         extr.u          @y[3]=@x[$d3],24,8
 161         dep.z           @x[$d3]=@x[$d3],8,24            };;
 162 { .mmi; or              @x[$d2]=@x[$d2],@y[2]
 163         or              @x[$d3]=@x[$d3],@y[3]
 164          extr.u         @y[0]=@x[$b0],25,7              };;
 165 { .mmi;  add            @x[$c1]=@x[$c1],@x[$d1]
 166          add            @x[$c2]=@x[$c2],@x[$d2]
 167          dep.z          @x[$b0]=@x[$b0],7,25            };;
 168 { .mmi;  xor            @x[$b1]=@x[$b1],@x[$c1]
 169          xor            @x[$b2]=@x[$b2],@x[$c2]
 170          add            @x[$c3]=@x[$c3],@x[$d3]         };;
 171 { .mii;  xor            @x[$b3]=@x[$b3],@x[$c3]
 172          extr.u         @y[1]=@x[$b1],25,7
 173          dep.z          @x[$b1]=@x[$b1],7,25            };;
 174 { .mii;  or             @x[$b0]=@x[$b0],@y[0]
 175          extr.u         @y[2]=@x[$b2],25,7
 176          dep.z          @x[$b2]=@x[$b2],7,25            };;
 177 { .mii;  or             @x[$b1]=@x[$b1],@y[1]
 178          extr.u         @y[3]=@x[$b3],25,7
 179          dep.z          @x[$b3]=@x[$b3],7,25            };;
 180 ___
 181 $code.=<<___            if ($d0 == 12);
 182 { .mmi;  or             @x[$b2]=@x[$b2],@y[2]
 183          or             @x[$b3]=@x[$b3],@y[3]
 184         mov             @z[0]=-1                        };;
 185 ___
 186 $code.=<<___            if ($d0 == 15);
 187 { .mmb;  or             @x[$b2]=@x[$b2],@y[2]
 188          or             @x[$b3]=@x[$b3],@y[3]
 189         br.ctop.sptk    .Loop_top                       };;
 190 ___
 191 }
 192         &ROUND(0, 4, 8, 12);
 193         &ROUND(0, 5, 10, 15);
 194 $code.=<<___;
 195 .Loop_end:
 196
 197 { .mmi; add             @x[0]=@x[0],@k[0]
 198         add             @x[1]=@x[1],@k[1]
 199 (p6)    shr.u           @z[0]=@z[0],@z[1]               }
 200 { .mmb; add             @x[2]=@x[2],@k[2]
 201         add             @x[3]=@x[3],@k[3]
 202         clrrrb.pr                                       };;
 203 { .mmi; add             @x[4]=@x[4],@k[4]
 204         add             @x[5]=@x[5],@k[5]
 205         add             @x[6]=@x[6],@k[6]               }
 206 { .mmi; add             @x[7]=@x[7],@k[7]
 207         add             @x[8]=@x[8],@k[8]
 208         add             @x[9]=@x[9],@k[9]               }
 209 { .mmi; add             @x[10]=@x[10],@k[10]
 210         add             @x[11]=@x[11],@k[11]
 211         add             @x[12]=@x[12],@k[12]            }
 212 { .mmi; add             @x[13]=@x[13],@k[13]
 213         add             @x[14]=@x[14],@k[14]
 214         add             @x[15]=@x[15],@k[15]            }
 215 { .mmi; add             @k[12]=1,@k[12]                 // next counter
 216         mov             pr=@z[0],0x1ffff                };;
 217
 218 //////////////////////////////////////////////////////////////////
 219 // Each predicate bit corresponds to byte to be processed. Note
 220 // that p0 is wired to 1, but it works out, because there always
 221 // is at least one byte to process...
 222 { .mmi; (p0)    ld1             @z[0]=[$inp],1
 223                 shr.u           @y[1]=@x[0],8           };;
 224 { .mmi; (p1)    ld1             @z[1]=[$inp],1
 225         (p2)    shr.u           @y[2]=@x[0],16          };;
 226 { .mmi; (p2)    ld1             @z[2]=[$inp],1
 227         (p0)    xor             @z[0]=@z[0],@x[0]
 228         (p3)    shr.u           @y[3]=@x[0],24          };;
 229 ___
 230 for(my $i0=0; $i0<60; $i0+=4) {
 231 my ($i1, $i2, $i3, $i4, $i5, $i6, $i7) = map($i0+$_,(1..7));
 232 my $k = $i0/4+1;
 233
 234 $code.=<<___;
 235 { .mmi; (p$i3)  ld1             @z[3]=[$inp],1
 236         (p$i0)  st1             [$out]=@z[0],1
 237         (p$i1)  xor             @z[1]=@z[1],@y[1]       };;
 238 { .mmi; (p$i4)  ld1             @z[0]=[$inp],1
 239         (p$i5)  shr.u           @y[1]=@x[$k],8          }
 240 { .mmi; (p$i1)  st1             [$out]=@z[1],1
 241         (p$i2)  xor             @z[2]=@z[2],@y[2]
 242         (p1)    mov             @x[$k-1]=@k[$k-1]       };;
 243 { .mfi; (p$i5)  ld1             @z[1]=[$inp],1
 244         (p$i6)  shr.u           @y[2]=@x[$k],16         }
 245 { .mfi; (p$i2)  st1             [$out]=@z[2],1
 246         (p$i3)  xor             @z[3]=@z[3],@y[3]       };;
 247 { .mfi; (p$i6)  ld1             @z[2]=[$inp],1
 248         (p$i7)  shr.u           @y[3]=@x[$k],24         }
 249 ___
 250 $code.=<<___    if ($i0==0);    # p1,p2 are available for reuse in first round
 251 { .mmi; (p$i3)  st1             [$out]=@z[3],1
 252         (p$i4)  xor             @z[0]=@z[0],@x[$k]
 253                 cmp.ltu         p1,p2=64,$len           };;
 254 ___
 255 $code.=<<___    if ($i0>0);
 256 { .mfi; (p$i3)  st1             [$out]=@z[3],1
 257         (p$i4)  xor             @z[0]=@z[0],@x[$k]      };;
 258 ___
 259 }
 260 $code.=<<___;
 261 { .mmi; (p63)   ld1             @z[3]=[$inp],1
 262         (p60)   st1             [$out]=@z[0],1
 263         (p61)   xor             @z[1]=@z[1],@y[1]       };;
 264 { .mmi; (p61)   st1             [$out]=@z[1],1
 265         (p62)   xor             @z[2]=@z[2],@y[2]       };;
 266 { .mmi; (p62)   st1             [$out]=@z[2],1
 267         (p63)   xor             @z[3]=@z[3],@y[3]
 268         (p2)    mov             ar.lc=r3                };;
 269 { .mib; (p63)   st1             [$out]=@z[3],1
 270         (p1)    add             $len=-64,$len
 271 (p1)    br.dptk.many            .Loop_outer             };;
 272
 273 { .mmi; mov                     @k[4]=0                 // wipe key material
 274         mov                     @k[5]=0
 275         mov                     @k[6]=0                 }
 276 { .mmi; mov                     @k[7]=0
 277         mov                     @k[8]=0
 278         mov                     @k[9]=0                 }
 279 { .mmi; mov                     @k[10]=0
 280         mov                     @k[11]=0
 281         mov                     @k[12]=0                }
 282 { .mmi; mov                     @k[13]=0
 283         mov                     @k[14]=0
 284         mov                     @k[15]=0                }
 285 { .mib; mov                     pr=r14,0x1ffff
 286         br.ret.sptk.many        b0                      };;
 287 .endp   ChaCha20_ctr32#
 288 stringz "ChaCha20 for IA64, CRYPTOGAMS by \@dot-asm"
 289 ___
 290
 291 print $code;
 292 close STDOUT;