crypto/sha/asm/sha512-sse2.pl

   1 #!/usr/bin/env perl
   2 #
   3 # SHA512_Transform_SSE2.
   4 #
   5 # As the name suggests, this is an IA-32 SSE2 implementation of
   6 # SHA512_Transform. Motivating factor for the undertaken effort was that
   7 # SHA512 was observed to *consistently* perform *significantly* poorer
   8 # than SHA256 [2x and slower is common] on 32-bit platforms. On 64-bit
   9 # platforms on the other hand SHA512 tend to outperform SHA256 [~50%
  10 # seem to be common improvement factor]. All this is perfectly natural,
  11 # as SHA512 is a 64-bit algorithm. But isn't IA-32 SSE2 essentially
  12 # a 64-bit instruction set? Is it rich enough to implement SHA512?
  13 # If answer was "no," then you wouldn't have been reading this...
  14 #
  15 # [Preliminary] throughput numbers (larger is better):
  16 #
  17 #               2.4GHz P4       1.4GHz AMD32    1.4GHz AMD64
  18 # SHA256/gcc    38              36              46
  19 # SHA512/gcc    9               15              72
  20 # SHA512/sse2   53(*)           51
  21 # SHA512/icc    21              21
  22 # SHA256/icc    52              42
  23 #
  24 #  (*)  I.e. it gives ~6x speed-up on P4 if compared to code generated
  25 #       by gcc, and 2.5x over icc. It was worth it:-) Well, one can
  26 #       argue that handcoded *non*-SSE2 implementation would perform
  27 #       better than compiler generated one, and comparison therefore
  28 #       is not exactly fair. As SHA512 puts enormous pressure on IA-32
  29 #       GP register bank, I reckon handcoded version wouldn't perform
  30 #       significantly better than one compiled with icc, ~20% perhaps.
  31 #       So that this code would still outperform it with distinguishing
  32 #       marginal. But feel free to prove me wrong:-)
  33 #
  34 #                                               <appro@fy.chalmers.se>
  35 push(@INC,"perlasm","../../perlasm");
  36 require "x86asm.pl";
  37
  38 &asm_init($ARGV[0],"sha512-sse2.pl",$ARGV[$#ARGV] eq "386");
  39
  40 $K512="esi";    # K512[80] table, found at the end...
  41 #$W512="esp";   # $W512 is not just W512[16]: it comprises *two* copies
  42                 # of W512[16] and a copy of A-H variables...
  43 $W512_SZ=8*(16+16+8);   # see above...
  44 #$Kidx="ebx";   # index in K512 table, advances from 0 to 80...
  45 $Widx="edx";    # index in W512, wraps around at 16...
  46 $data="edi";    # 16 qwords of input data...
  47 $A="mm0";       # B-D and
  48 $E="mm1";       # F-H are allocated dynamically...
  49 $Aoff=256+0;    # A-H offsets relative to $W512...
  50 $Boff=256+8;
  51 $Coff=256+16;
  52 $Doff=256+24;
  53 $Eoff=256+32;
  54 $Foff=256+40;
  55 $Goff=256+48;
  56 $Hoff=256+56;
  57
  58 sub SHA2_ROUND()
  59 { local ($kidx,$widx)=@_;
  60
  61         # One can argue that one could reorder instructions for better
  62         # performance. Well, I tried and it doesn't seem to make any
  63         # noticeable difference. Modern out-of-order execution cores
  64         # reorder instructions to their liking in either case and they
  65         # apparently do decent job. So we can keep the code more
  66         # readable/regular/comprehensible:-)
  67
  68         # I adhere to 64-bit %mmX registers in order to avoid/not care
  69         # about #GP exceptions on misaligned 128-bit access, most
  70         # notably in paddq with memory operand.
  71
  72         &movq   ("mm4",&QWP($Foff,$W512));      # load f
  73         &movq   ("mm5",&QWP($Goff,$W512));      # load g
  74         &movq   ("mm6",&QWP($Hoff,$W512));      # load h
  75         &movq   (&QWP($Foff,$W512),$E);         # f = e
  76         &movq   (&QWP($Goff,$W512),"mm4");      # g = f
  77         &movq   (&QWP($Hoff,$W512),"mm5");      # h = g
  78
  79         &movq   ("mm2",$E);                     # %mm2 is sliding right
  80         &movq   ("mm3",$E);                     # %mm3 is sliding left
  81         &psrlq  ("mm2",14);
  82         &psllq  ("mm3",23);
  83         &movq   ("mm7","mm2");                  # %mm7 is T1
  84         &pxor   ("mm7","mm3");
  85         &psrlq  ("mm2",4);
  86         &psllq  ("mm3",23);
  87         &pxor   ("mm7","mm2");
  88         &pxor   ("mm7","mm3");
  89         &psrlq  ("mm2",23);
  90         &psllq  ("mm3",4);
  91         &pxor   ("mm7","mm2");
  92         &pxor   ("mm7","mm3");                  # T1=Sigma1_512(e)
  93
  94         &pxor   ("mm4","mm5");                  # f^=g
  95         &pand   ("mm4",$E);                     # f&=e
  96         &pxor   ("mm4","mm5");                  # f^=g
  97         &paddq  ("mm7","mm4");                  # T1+=Ch(e,f,g)
  98
  99         &movq   ("mm2",&QWP($Boff,$W512));      # load b
 100         &movq   ("mm3",&QWP($Coff,$W512));      # load c
 101         &movq   ($E,&QWP($Doff,$W512));         # e = d
 102         &movq   (&QWP($Boff,$W512),$A);         # b = a
 103         &movq   (&QWP($Coff,$W512),"mm2");      # c = b
 104         &movq   (&QWP($Doff,$W512),"mm3");      # d = c
 105
 106         &paddq  ("mm7","mm6");                  # T1+=h
 107         &paddq  ("mm7",&QWP(0,$K512,$kidx,8));  # T1+=K512[i]
 108         &paddq  ("mm7",&QWP(0,$W512,$widx,8));  # T1+=W512[i]
 109         &paddq  ($E,"mm7");                     # e += T1
 110
 111         &movq   ("mm4",$A);                     # %mm4 is sliding right
 112         &movq   ("mm5",$A);                     # %mm5 is sliding left
 113         &psrlq  ("mm4",28);
 114         &psllq  ("mm5",25);
 115         &movq   ("mm6","mm4");                  # %mm6 is T2
 116         &pxor   ("mm6","mm5");
 117         &psrlq  ("mm4",6);
 118         &psllq  ("mm5",5);
 119         &pxor   ("mm6","mm4");
 120         &pxor   ("mm6","mm5");
 121         &psrlq  ("mm4",5);
 122         &psllq  ("mm5",6);
 123         &pxor   ("mm6","mm4");
 124         &pxor   ("mm6","mm5");                  # T2=Sigma0_512(a)
 125
 126         &movq   ("mm4","mm2");                  # %mm4=b
 127         &pand   ("mm2",$A);                     # b&=a
 128         &pand   ("mm4","mm3");                  # %mm4&=c
 129         &pand   ("mm3",$A);                     # c&=a
 130         &pxor   ("mm4","mm2");                  # %mm4^=b&a
 131         &pxor   ("mm4","mm3");                  # %mm4^=c&a
 132         &paddq  ("mm6","mm4");                  # T2+=Maj(a,b,c)
 133
 134         &movq   ($A,"mm7");                     # a=T1
 135         &paddq  ($A,"mm6");                     # a+=T2
 136 }
 137
 138 $func="SHA512_Transform_SSE2";
 139
 140 &function_begin_B($func);
 141         if (0) {# Caller is expected to check if it's appropriate to
 142                 # call this routine. Below 3 lines are retained for
 143                 # debugging purposes...
 144                 &picmeup("eax","OPENSSL_ia32cap");
 145                 &bt     (&DWP(0,"eax"),26);
 146                 &jnc    ("SHA512_Transform");
 147         }
 148
 149         &push   ("ebp");
 150         &mov    ("ebp","esp");
 151         &push   ("ebx");
 152         &push   ("esi");
 153         &push   ("edi");
 154
 155         &mov    ($Widx,&DWP(8,"ebp"));          # A-H state, 1st arg
 156         &mov    ($data,&DWP(12,"ebp"));         # input data, 2nd arg
 157         &call   (&label("pic_point"));          # make it PIC!
 158 &set_label("pic_point");
 159         &blindpop($K512);
 160         &lea    ($K512,&DWP(&label("K512")."-".&label("pic_point"),$K512));
 161
 162         $W512 = "esp";                  # start using %esp as W512
 163         &sub    ($W512,$W512_SZ);
 164         &and    ($W512,-16);            # ensure 128-bit alignment
 165
 166         # make private copy of A-H
 167         #     v assume the worst and stick to unaligned load
 168         &movdqu ("xmm0",&QWP(0,$Widx));
 169         &movdqu ("xmm1",&QWP(16,$Widx));
 170         &movdqu ("xmm2",&QWP(32,$Widx));
 171         &movdqu ("xmm3",&QWP(48,$Widx));
 172         &movdqa (&QWP($Aoff,$W512),"xmm0");     # a,b
 173         &movdqa (&QWP($Coff,$W512),"xmm1");     # c,d
 174         &movdqa (&QWP($Eoff,$W512),"xmm2");     # e,f
 175         &movdqa (&QWP($Goff,$W512),"xmm3");     # g,h
 176
 177         &xor    ($Widx,$Widx);
 178
 179         &movdq2q($A,"xmm0");                    # load a
 180         &movdq2q($E,"xmm2");                    # load e
 181
 182         # Why aren't loops unrolled? It makes sense to unroll if
 183         # execution time for loop body is comparable with branch
 184         # penalties and/or if whole data-set resides in register
 185         # bank. Neither is case here...
 186
 187 &align(8);
 188 &set_label("_1st_loop");                # 0-15
 189         # flip input stream byte order...
 190         &mov    ("eax",&DWP(0,$data,$Widx,8));
 191         &mov    ("ebx",&DWP(4,$data,$Widx,8));
 192         &bswap  ("eax");
 193         &bswap  ("ebx");
 194         &mov    (&DWP(0,$W512,$Widx,8),"ebx");          # W512[i]
 195         &mov    (&DWP(4,$W512,$Widx,8),"eax");
 196         &mov    (&DWP(128+0,$W512,$Widx,8),"ebx");      # copy of W512[i]
 197         &mov    (&DWP(128+4,$W512,$Widx,8),"eax");
 198
 199         &SHA2_ROUND($Widx,$Widx); &inc($Widx);
 200
 201 &cmp    ($Widx,16)
 202 &jl     (&label("_1st_loop"));
 203
 204         $Kidx = "ebx";                  # start using %ebx as Kidx
 205         &mov    ($Kidx,$Widx);
 206
 207 &align(8);
 208 &set_label("_2nd_loop");                # 16-79
 209         &and($Widx,0xf);
 210
 211         # 128-bit fragment! I update W512[i] and W512[i+1] in
 212         # parallel:-) Note that I refer to W512[(i&0xf)+N] and not to
 213         # W512[(i+N)&0xf]! This is exactly what I maintain the second
 214         # copy of W512[16] for...
 215         &movdqu ("xmm0",&QWP(8*1,$W512,$Widx,8));       # s0=W512[i+1]
 216         &movdqa ("xmm2","xmm0");                # %xmm2 is sliding right
 217         &movdqa ("xmm3","xmm0");                # %xmm3 is sliding left
 218         &psrlq  ("xmm2",1);
 219         &psllq  ("xmm3",56);
 220         &movdqa ("xmm0","xmm2");
 221         &pxor   ("xmm0","xmm3");
 222         &psrlq  ("xmm2",6);
 223         &psllq  ("xmm3",7);
 224         &pxor   ("xmm0","xmm2");
 225         &pxor   ("xmm0","xmm3");
 226         &psrlq  ("xmm2",1);
 227         &pxor   ("xmm0","xmm2");                # s0 = sigma0_512(s0);
 228
 229         &movdqa ("xmm1",&QWP(8*14,$W512,$Widx,8));      # s1=W512[i+14]
 230         &movdqa ("xmm4","xmm1");                # %xmm4 is sliding right
 231         &movdqa ("xmm5","xmm1");                # %xmm5 is sliding left
 232         &psrlq  ("xmm4",6);
 233         &psllq  ("xmm5",3);
 234         &movdqa ("xmm1","xmm4");
 235         &pxor   ("xmm1","xmm5");
 236         &psrlq  ("xmm4",13);
 237         &psllq  ("xmm5",42);
 238         &pxor   ("xmm1","xmm4");
 239         &pxor   ("xmm1","xmm5");
 240         &psrlq  ("xmm4",42);
 241         &pxor   ("xmm1","xmm4");                # s1 = sigma1_512(s1);
 242
 243         #     + have to explictly load W512[i+9] as it's not 128-bit
 244         #     v aligned and paddq would throw an exception...
 245         &movdqu ("xmm6",&QWP(8*9,$W512,$Widx,8));
 246         &paddq  ("xmm0","xmm1");                # s0 += s1
 247         &paddq  ("xmm0","xmm6");                # s0 += W512[i+9]
 248         &paddq  ("xmm0",&QWP(0,$W512,$Widx,8)); # s0 += W512[i]
 249
 250         &movdqa (&QWP(0,$W512,$Widx,8),"xmm0");         # W512[i] = s0
 251         &movdqa (&QWP(16*8,$W512,$Widx,8),"xmm0");      # copy of W512[i]
 252
 253         # as the above fragment was 128-bit, we "owe" 2 rounds...
 254         &SHA2_ROUND($Kidx,$Widx); &inc($Kidx); &inc($Widx);
 255         &SHA2_ROUND($Kidx,$Widx); &inc($Kidx); &inc($Widx);
 256
 257 &cmp    ($Kidx,80);
 258 &jl     (&label("_2nd_loop"));
 259
 260         # update A-H state
 261         &mov    ($Widx,&DWP(8,"ebp"));          # A-H state, 1st arg
 262         &movq   (&QWP($Aoff,$W512),$A);         # write out a
 263         &movq   (&QWP($Eoff,$W512),$E);         # write out e
 264         &movdqu ("xmm0",&QWP(0,$Widx));
 265         &movdqu ("xmm1",&QWP(16,$Widx));
 266         &movdqu ("xmm2",&QWP(32,$Widx));
 267         &movdqu ("xmm3",&QWP(48,$Widx));
 268         &paddq  ("xmm0",&QWP($Aoff,$W512));     # 128-bit additions...
 269         &paddq  ("xmm1",&QWP($Coff,$W512));
 270         &paddq  ("xmm2",&QWP($Eoff,$W512));
 271         &paddq  ("xmm3",&QWP($Goff,$W512));
 272         &movdqu (&QWP(0,$Widx),"xmm0");
 273         &movdqu (&QWP(16,$Widx),"xmm1");
 274         &movdqu (&QWP(32,$Widx),"xmm2");
 275         &movdqu (&QWP(48,$Widx),"xmm3");
 276
 277         # epilogue
 278         &emms   ();     # required for at least ELF and Win32 ABIs
 279         &mov    ("edi",&DWP(-12,"ebp"));
 280         &mov    ("esi",&DWP(-8,"ebp"));
 281         &mov    ("ebx",&DWP(-4,"ebp"));
 282         &leave  ();
 283 &ret    ();
 284
 285 &align(16);
 286 &set_label("K512");     # Yes! I keep it in the code segment!
 287         &data_word(0xd728ae22,0x428a2f98);      # u64
 288         &data_word(0x23ef65cd,0x71374491);      # u64
 289         &data_word(0xec4d3b2f,0xb5c0fbcf);      # u64
 290         &data_word(0x8189dbbc,0xe9b5dba5);      # u64
 291         &data_word(0xf348b538,0x3956c25b);      # u64
 292         &data_word(0xb605d019,0x59f111f1);      # u64
 293         &data_word(0xaf194f9b,0x923f82a4);      # u64
 294         &data_word(0xda6d8118,0xab1c5ed5);      # u64
 295         &data_word(0xa3030242,0xd807aa98);      # u64
 296         &data_word(0x45706fbe,0x12835b01);      # u64
 297         &data_word(0x4ee4b28c,0x243185be);      # u64
 298         &data_word(0xd5ffb4e2,0x550c7dc3);      # u64
 299         &data_word(0xf27b896f,0x72be5d74);      # u64
 300         &data_word(0x3b1696b1,0x80deb1fe);      # u64
 301         &data_word(0x25c71235,0x9bdc06a7);      # u64
 302         &data_word(0xcf692694,0xc19bf174);      # u64
 303         &data_word(0x9ef14ad2,0xe49b69c1);      # u64
 304         &data_word(0x384f25e3,0xefbe4786);      # u64
 305         &data_word(0x8b8cd5b5,0x0fc19dc6);      # u64
 306         &data_word(0x77ac9c65,0x240ca1cc);      # u64
 307         &data_word(0x592b0275,0x2de92c6f);      # u64
 308         &data_word(0x6ea6e483,0x4a7484aa);      # u64
 309         &data_word(0xbd41fbd4,0x5cb0a9dc);      # u64
 310         &data_word(0x831153b5,0x76f988da);      # u64
 311         &data_word(0xee66dfab,0x983e5152);      # u64
 312         &data_word(0x2db43210,0xa831c66d);      # u64
 313         &data_word(0x98fb213f,0xb00327c8);      # u64
 314         &data_word(0xbeef0ee4,0xbf597fc7);      # u64
 315         &data_word(0x3da88fc2,0xc6e00bf3);      # u64
 316         &data_word(0x930aa725,0xd5a79147);      # u64
 317         &data_word(0xe003826f,0x06ca6351);      # u64
 318         &data_word(0x0a0e6e70,0x14292967);      # u64
 319         &data_word(0x46d22ffc,0x27b70a85);      # u64
 320         &data_word(0x5c26c926,0x2e1b2138);      # u64
 321         &data_word(0x5ac42aed,0x4d2c6dfc);      # u64
 322         &data_word(0x9d95b3df,0x53380d13);      # u64
 323         &data_word(0x8baf63de,0x650a7354);      # u64
 324         &data_word(0x3c77b2a8,0x766a0abb);      # u64
 325         &data_word(0x47edaee6,0x81c2c92e);      # u64
 326         &data_word(0x1482353b,0x92722c85);      # u64
 327         &data_word(0x4cf10364,0xa2bfe8a1);      # u64
 328         &data_word(0xbc423001,0xa81a664b);      # u64
 329         &data_word(0xd0f89791,0xc24b8b70);      # u64
 330         &data_word(0x0654be30,0xc76c51a3);      # u64
 331         &data_word(0xd6ef5218,0xd192e819);      # u64
 332         &data_word(0x5565a910,0xd6990624);      # u64
 333         &data_word(0x5771202a,0xf40e3585);      # u64
 334         &data_word(0x32bbd1b8,0x106aa070);      # u64
 335         &data_word(0xb8d2d0c8,0x19a4c116);      # u64
 336         &data_word(0x5141ab53,0x1e376c08);      # u64
 337         &data_word(0xdf8eeb99,0x2748774c);      # u64
 338         &data_word(0xe19b48a8,0x34b0bcb5);      # u64
 339         &data_word(0xc5c95a63,0x391c0cb3);      # u64
 340         &data_word(0xe3418acb,0x4ed8aa4a);      # u64
 341         &data_word(0x7763e373,0x5b9cca4f);      # u64
 342         &data_word(0xd6b2b8a3,0x682e6ff3);      # u64
 343         &data_word(0x5defb2fc,0x748f82ee);      # u64
 344         &data_word(0x43172f60,0x78a5636f);      # u64
 345         &data_word(0xa1f0ab72,0x84c87814);      # u64
 346         &data_word(0x1a6439ec,0x8cc70208);      # u64
 347         &data_word(0x23631e28,0x90befffa);      # u64
 348         &data_word(0xde82bde9,0xa4506ceb);      # u64
 349         &data_word(0xb2c67915,0xbef9a3f7);      # u64
 350         &data_word(0xe372532b,0xc67178f2);      # u64
 351         &data_word(0xea26619c,0xca273ece);      # u64
 352         &data_word(0x21c0c207,0xd186b8c7);      # u64
 353         &data_word(0xcde0eb1e,0xeada7dd6);      # u64
 354         &data_word(0xee6ed178,0xf57d4f7f);      # u64
 355         &data_word(0x72176fba,0x06f067aa);      # u64
 356         &data_word(0xa2c898a6,0x0a637dc5);      # u64
 357         &data_word(0xbef90dae,0x113f9804);      # u64
 358         &data_word(0x131c471b,0x1b710b35);      # u64
 359         &data_word(0x23047d84,0x28db77f5);      # u64
 360         &data_word(0x40c72493,0x32caab7b);      # u64
 361         &data_word(0x15c9bebc,0x3c9ebe0a);      # u64
 362         &data_word(0x9c100d4c,0x431d67c4);      # u64
 363         &data_word(0xcb3e42b6,0x4cc5d4be);      # u64
 364         &data_word(0xfc657e2a,0x597f299c);      # u64
 365         &data_word(0x3ad6faec,0x5fcb6fab);      # u64
 366         &data_word(0x4a475817,0x6c44198c);      # u64
 367
 368 &function_end_B($func);
 369
 370 &asm_finish();