crypto/chacha/asm/chacha-ia64.pl

   1 #!/usr/bin/env perl
   2 #
   3 # ====================================================================
   4 # Written by Andy Polyakov, @dot-asm, initially for use with OpenSSL.
   5 # ====================================================================
   6 #
   7 # ChaCha20 for Itanium.
   8 #
   9 # March 2019
  10 #
  11 # Itanium 9xxx, which has pair of shifters, manages to process one byte
  12 # in 9.3 cycles. This aligns perfectly with theoretical estimate.
  13 # On the other hand, pre-9000 CPU has single shifter and each extr/dep
  14 # pairs below takes additional cycle. Then final input->xor->output
  15 # pass runs slower than expected... Overall result is 15.6 cpb, two
  16 # cycles more than theoretical estimate.
  17
  18 $output = pop and open STDOUT, ">$output";
  19
  20 my @k = map("r$_",(16..31));
  21 my @x = map("r$_",(38..53));
  22 my @y = map("r$_",(8..11));
  23 my @z = map("r$_",(15,35..37));
  24 my ($out,$inp,$len,$key,$counter) = map("r$_",(32..36));
  25
  26 $code.=<<___;
  27 #if defined(_HPUX_SOURCE)
  28 # if !defined(_LP64)
  29 #  define ADDP  addp4
  30 # else
  31 #  define ADDP  add
  32 # endif
  33 #else
  34 # define ADDP   add
  35 #endif
  36
  37 .text
  38
  39 .global ChaCha20_ctr32#
  40 .proc   ChaCha20_ctr32#
  41 .align  32
  42 ChaCha20_ctr32:
  43         .prologue
  44         .save           ar.pfs,r2
  45 { .mmi; alloc           r2=ar.pfs,5,17,0,0
  46         ADDP            @k[11]=4,$key
  47         .save           ar.lc,r3
  48         mov             r3=ar.lc                }
  49 { .mmi; ADDP            $key=0,$key
  50         ADDP            $counter=0,$counter
  51         .save           pr,r14
  52         mov             r14=pr                  };;
  53
  54         .body
  55 { .mlx; ld4             @k[4]=[$key],8
  56         movl            @k[0]=0x61707865        }
  57 { .mlx; ld4             @k[5]=[@k[11]],8
  58         movl            @k[1]=0x3320646e        };;
  59 { .mlx; ld4             @k[6]=[$key],8
  60         movl            @k[2]=0x79622d32        }
  61 { .mlx; ld4             @k[7]=[@k[11]],8
  62         movl            @k[3]=0x6b206574        };;
  63 { .mmi; ld4             @k[8]=[$key],8
  64         ld4             @k[9]=[@k[11]],8
  65         add             @k[15]=4,$counter       };;
  66 { .mmi; ld4             @k[10]=[$key]
  67         ld4             @k[11]=[@k[11]]
  68         mov             @x[0]=@k[0]             };;
  69 { .mmi; ld4             @k[12]=[$counter],8
  70         ld4             @k[13]=[@k[15]],8
  71         mov             @x[1]=@k[1]             };;
  72 { .mmi; ld4             @k[14]=[$counter]
  73         ld4             @k[15]=[@k[15]]
  74         mov             @x[2]=@k[2]             }
  75 { .mmi; mov             @x[3]=@k[3]
  76         mov             @x[4]=@k[4]
  77         mov             @x[5]=@k[5]             };;
  78 { .mmi; mov             @x[6]=@k[6]
  79         mov             @x[7]=@k[7]
  80         mov             @x[8]=@k[8]             }
  81 { .mmi; mov             @x[9]=@k[9]
  82         mov             @x[10]=@k[10]
  83         mov             @x[11]=@k[11]           }
  84 { .mmi; mov             @x[12]=@k[12]
  85         mov             @x[13]=@k[13]
  86         mov             @x[14]=@k[14]           };;
  87
  88 .Loop_outer:
  89 { .mii; mov             @x[15]=@k[15]
  90         mov             ar.lc=9
  91         mov             ar.ec=1                 }
  92 { .mmb; cmp.geu         p6,p0=64,$len
  93         sub             @z[1]=64,$len
  94         brp.loop.imp    .Loop_top,.Loop_end-16  };;
  95
  96 .Loop_top:
  97 ___
  98 sub ROUND {
  99 my ($a0,$b0,$c0,$d0)=@_;
 100 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
 101 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
 102 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
 103
 104 $code.=<<___;
 105 { .mmi; add             @x[$a0]=@x[$a0],@x[$b0]
 106         add             @x[$a1]=@x[$a1],@x[$b1]
 107         add             @x[$a2]=@x[$a2],@x[$b2]         };;
 108 { .mmi; add             @x[$a3]=@x[$a3],@x[$b3]
 109         xor             @x[$d0]=@x[$d0],@x[$a0]
 110         xor             @x[$d1]=@x[$d1],@x[$a1]         };;
 111 { .mmi; xor             @x[$d2]=@x[$d2],@x[$a2]
 112         xor             @x[$d3]=@x[$d3],@x[$a3]
 113         extr.u          @y[0]=@x[$d0],16,16             };;
 114 { .mii; extr.u          @y[1]=@x[$d1],16,16
 115         dep             @x[$d0]=@x[$d0],@y[0],16,16     };;
 116 { .mii;  add            @x[$c0]=@x[$c0],@x[$d0]
 117         extr.u          @y[2]=@x[$d2],16,16
 118         dep             @x[$d1]=@x[$d1],@y[1],16,16     };;
 119 { .mii;  add            @x[$c1]=@x[$c1],@x[$d1]
 120          xor            @x[$b0]=@x[$b0],@x[$c0]
 121         extr.u          @y[3]=@x[$d3],16,16             };;
 122 { .mii;  xor            @x[$b1]=@x[$b1],@x[$c1]
 123         dep             @x[$d2]=@x[$d2],@y[2],16,16
 124         dep             @x[$d3]=@x[$d3],@y[3],16,16     };;
 125 { .mmi;  add            @x[$c2]=@x[$c2],@x[$d2]
 126          add            @x[$c3]=@x[$c3],@x[$d3]
 127          extr.u         @y[0]=@x[$b0],20,12             };;
 128 { .mmi;  xor            @x[$b2]=@x[$b2],@x[$c2]
 129          xor            @x[$b3]=@x[$b3],@x[$c3]
 130          dep.z          @x[$b0]=@x[$b0],12,20           };;
 131 { .mii;  or             @x[$b0]=@x[$b0],@y[0]
 132          extr.u         @y[1]=@x[$b1],20,12
 133          dep.z          @x[$b1]=@x[$b1],12,20           };;
 134 { .mii; add             @x[$a0]=@x[$a0],@x[$b0]
 135          extr.u         @y[2]=@x[$b2],20,12
 136          extr.u         @y[3]=@x[$b3],20,12             }
 137 { .mii;  or             @x[$b1]=@x[$b1],@y[1]
 138          dep.z          @x[$b2]=@x[$b2],12,20
 139          dep.z          @x[$b3]=@x[$b3],12,20           };;
 140 { .mmi;  or             @x[$b2]=@x[$b2],@y[2]
 141          or             @x[$b3]=@x[$b3],@y[3]
 142         add             @x[$a1]=@x[$a1],@x[$b1]         };;
 143 { .mmi; add             @x[$a2]=@x[$a2],@x[$b2]
 144         add             @x[$a3]=@x[$a3],@x[$b3]
 145         xor             @x[$d0]=@x[$d0],@x[$a0]         };;
 146 { .mii; xor             @x[$d1]=@x[$d1],@x[$a1]
 147         extr.u          @y[0]=@x[$d0],24,8
 148         dep.z           @x[$d0]=@x[$d0],8,24            };;
 149 { .mii; or              @x[$d0]=@x[$d0],@y[0]
 150         extr.u          @y[1]=@x[$d1],24,8
 151         dep.z           @x[$d1]=@x[$d1],8,24            };;
 152 { .mmi; or              @x[$d1]=@x[$d1],@y[1]
 153         xor             @x[$d2]=@x[$d2],@x[$a2]
 154         xor             @x[$d3]=@x[$d3],@x[$a3]         };;
 155 { .mii;  add            @x[$c0]=@x[$c0],@x[$d0]
 156         extr.u          @y[2]=@x[$d2],24,8
 157         dep.z           @x[$d2]=@x[$d2],8,24            };;
 158 { .mii;  xor            @x[$b0]=@x[$b0],@x[$c0]
 159         extr.u          @y[3]=@x[$d3],24,8
 160         dep.z           @x[$d3]=@x[$d3],8,24            };;
 161 { .mmi; or              @x[$d2]=@x[$d2],@y[2]
 162         or              @x[$d3]=@x[$d3],@y[3]
 163          extr.u         @y[0]=@x[$b0],25,7              };;
 164 { .mmi;  add            @x[$c1]=@x[$c1],@x[$d1]
 165          add            @x[$c2]=@x[$c2],@x[$d2]
 166          dep.z          @x[$b0]=@x[$b0],7,25            };;
 167 { .mmi;  xor            @x[$b1]=@x[$b1],@x[$c1]
 168          xor            @x[$b2]=@x[$b2],@x[$c2]
 169          add            @x[$c3]=@x[$c3],@x[$d3]         };;
 170 { .mii;  xor            @x[$b3]=@x[$b3],@x[$c3]
 171          extr.u         @y[1]=@x[$b1],25,7
 172          dep.z          @x[$b1]=@x[$b1],7,25            };;
 173 { .mii;  or             @x[$b0]=@x[$b0],@y[0]
 174          extr.u         @y[2]=@x[$b2],25,7
 175          dep.z          @x[$b2]=@x[$b2],7,25            };;
 176 { .mii;  or             @x[$b1]=@x[$b1],@y[1]
 177          extr.u         @y[3]=@x[$b3],25,7
 178          dep.z          @x[$b3]=@x[$b3],7,25            };;
 179 ___
 180 $code.=<<___            if ($d0 == 12);
 181 { .mmi;  or             @x[$b2]=@x[$b2],@y[2]
 182          or             @x[$b3]=@x[$b3],@y[3]
 183         mov             @z[0]=-1                        };;
 184 ___
 185 $code.=<<___            if ($d0 == 15);
 186 { .mmb;  or             @x[$b2]=@x[$b2],@y[2]
 187          or             @x[$b3]=@x[$b3],@y[3]
 188         br.ctop.sptk    .Loop_top                       };;
 189 ___
 190 }
 191         &ROUND(0, 4, 8, 12);
 192         &ROUND(0, 5, 10, 15);
 193 $code.=<<___;
 194 .Loop_end:
 195
 196 { .mmi; add             @x[0]=@x[0],@k[0]
 197         add             @x[1]=@x[1],@k[1]
 198 (p6)    shr.u           @z[0]=@z[0],@z[1]               }
 199 { .mmb; add             @x[2]=@x[2],@k[2]
 200         add             @x[3]=@x[3],@k[3]
 201         clrrrb.pr                                       };;
 202 { .mmi; add             @x[4]=@x[4],@k[4]
 203         add             @x[5]=@x[5],@k[5]
 204         add             @x[6]=@x[6],@k[6]               }
 205 { .mmi; add             @x[7]=@x[7],@k[7]
 206         add             @x[8]=@x[8],@k[8]
 207         add             @x[9]=@x[9],@k[9]               }
 208 { .mmi; add             @x[10]=@x[10],@k[10]
 209         add             @x[11]=@x[11],@k[11]
 210         add             @x[12]=@x[12],@k[12]            }
 211 { .mmi; add             @x[13]=@x[13],@k[13]
 212         add             @x[14]=@x[14],@k[14]
 213         add             @x[15]=@x[15],@k[15]            }
 214 { .mmi; add             @k[12]=1,@k[12]                 // next counter
 215         mov             pr=@z[0],0x1ffff                };;
 216
 217 //////////////////////////////////////////////////////////////////
 218 // Each predicate bit corresponds to byte to be processed. Note
 219 // that p0 is wired to 1, but it works out, because there always
 220 // is at least one byte to process...
 221 { .mmi; (p0)    ld1             @z[0]=[$inp],1
 222                 shr.u           @y[1]=@x[0],8           };;
 223 { .mmi; (p1)    ld1             @z[1]=[$inp],1
 224         (p2)    shr.u           @y[2]=@x[0],16          };;
 225 { .mmi; (p2)    ld1             @z[2]=[$inp],1
 226         (p0)    xor             @z[0]=@z[0],@x[0]
 227         (p3)    shr.u           @y[3]=@x[0],24          };;
 228 ___
 229 for(my $i0=0; $i0<60; $i0+=4) {
 230 my ($i1, $i2, $i3, $i4, $i5, $i6, $i7) = map($i0+$_,(1..7));
 231 my $k = $i0/4+1;
 232
 233 $code.=<<___;
 234 { .mmi; (p$i3)  ld1             @z[3]=[$inp],1
 235         (p$i0)  st1             [$out]=@z[0],1
 236         (p$i1)  xor             @z[1]=@z[1],@y[1]       };;
 237 { .mmi; (p$i4)  ld1             @z[0]=[$inp],1
 238         (p$i5)  shr.u           @y[1]=@x[$k],8          }
 239 { .mmi; (p$i1)  st1             [$out]=@z[1],1
 240         (p$i2)  xor             @z[2]=@z[2],@y[2]
 241         (p1)    mov             @x[$k-1]=@k[$k-1]       };;
 242 { .mfi; (p$i5)  ld1             @z[1]=[$inp],1
 243         (p$i6)  shr.u           @y[2]=@x[$k],16         }
 244 { .mfi; (p$i2)  st1             [$out]=@z[2],1
 245         (p$i3)  xor             @z[3]=@z[3],@y[3]       };;
 246 { .mfi; (p$i6)  ld1             @z[2]=[$inp],1
 247         (p$i7)  shr.u           @y[3]=@x[$k],24         }
 248 ___
 249 $code.=<<___    if ($i0==0);    # p1,p2 are available for reuse in first round
 250 { .mmi; (p$i3)  st1             [$out]=@z[3],1
 251         (p$i4)  xor             @z[0]=@z[0],@x[$k]
 252                 cmp.ltu         p1,p2=64,$len           };;
 253 ___
 254 $code.=<<___    if ($i0>0);
 255 { .mfi; (p$i3)  st1             [$out]=@z[3],1
 256         (p$i4)  xor             @z[0]=@z[0],@x[$k]      };;
 257 ___
 258 }
 259 $code.=<<___;
 260 { .mmi; (p63)   ld1             @z[3]=[$inp],1
 261         (p60)   st1             [$out]=@z[0],1
 262         (p61)   xor             @z[1]=@z[1],@y[1]       };;
 263 { .mmi; (p61)   st1             [$out]=@z[1],1
 264         (p62)   xor             @z[2]=@z[2],@y[2]       };;
 265 { .mmi; (p62)   st1             [$out]=@z[2],1
 266         (p63)   xor             @z[3]=@z[3],@y[3]
 267         (p2)    mov             ar.lc=r3                };;
 268 { .mib; (p63)   st1             [$out]=@z[3],1
 269         (p1)    add             $len=-64,$len
 270 (p1)    br.dptk.many            .Loop_outer             };;
 271
 272 { .mmi; mov                     @k[4]=0                 // wipe key material
 273         mov                     @k[5]=0
 274         mov                     @k[6]=0                 }
 275 { .mmi; mov                     @k[7]=0
 276         mov                     @k[8]=0
 277         mov                     @k[9]=0                 }
 278 { .mmi; mov                     @k[10]=0
 279         mov                     @k[11]=0
 280         mov                     @k[12]=0                }
 281 { .mmi; mov                     @k[13]=0
 282         mov                     @k[14]=0
 283         mov                     @k[15]=0                }
 284 { .mib; mov                     pr=r14,0x1ffff
 285         br.ret.sptk.many        b0                      };;
 286 .endp   ChaCha20_ctr32#
 287 stringz "ChaCha20 for IA64, CRYPTOGAMS by \@dot-asm"
 288 ___
 289
 290 print $code;
 291 close STDOUT;