SSE2 accelerated bn_mul_add_words. Code is currently disabled till proper
[openssl.git] / crypto / bn / asm / bn-586.pl
1 #!/usr/local/bin/perl
2
3 push(@INC,"perlasm","../../perlasm");
4 require "x86asm.pl";
5
6 &asm_init($ARGV[0],$0);
7
8 $sse2=0;        # SSE2 is temporarily disabled...
9
10 &bn_mul_add_words("bn_mul_add_words");
11 &bn_mul_words("bn_mul_words");
12 &bn_sqr_words("bn_sqr_words");
13 &bn_div_words("bn_div_words");
14 &bn_add_words("bn_add_words");
15 &bn_sub_words("bn_sub_words");
16 &bn_sub_part_words("bn_sub_part_words");
17
18 &asm_finish();
19
20 sub bn_mul_add_words
21         {
22         local($name)=@_;
23
24         &function_begin($name,"");
25
26         &comment("");
27         $Low="eax";
28         $High="edx";
29         $a="ebx";
30         $w="ebp";
31         $r="edi";
32         $c="esi";
33
34         &xor($c,$c);            # clear carry
35         &mov($r,&wparam(0));    #
36
37         &mov("ecx",&wparam(2)); #
38         &mov($a,&wparam(1));    #
39
40         &and("ecx",0xfffffff8); # num / 8
41         &mov($w,&wparam(3));    #
42
43         &push("ecx");           # Up the stack for a tmp variable
44
45         &jz(&label("maw_finish"));
46
47         if ($sse2) {
48                 &picmeup("eax","OPENSSL_ia32cap");
49                 &bt(&DWP(0,"eax"),26);
50                 &jnc(&label("maw_loop"));
51
52                 &movd("mm0",$w);                # mm0 = w
53                 &pxor("mm1","mm1");             # mm1 = carry_in
54
55                 &set_label("maw_sse2_loop",0);
56                 &movd("mm3",&DWP(0,$r,"",0));   # mm3 = r[0]
57                 &paddq("mm1","mm3");            # mm1 = carry_in + r[0]
58                 &movd("mm2",&DWP(0,$a,"",0));   # mm2 = a[0]
59                 &pmuludq("mm2","mm0");          # mm2 = w*a[0]
60                 &movd("mm4",&DWP(4,$a,"",0));   # mm4 = a[1]
61                 &pmuludq("mm4","mm0");          # mm4 = w*a[1]
62                 &movd("mm6",&DWP(8,$a,"",0));   # mm6 = a[2]
63                 &pmuludq("mm6","mm0");          # mm6 = w*a[2]
64                 &movd("mm7",&DWP(12,$a,"",0));  # mm7 = a[3]
65                 &pmuludq("mm7","mm0");          # mm7 = w*a[3]
66                 &paddq("mm1","mm2");            # mm1 = carry_in + r[0] + w*a[0]
67                 &movd("mm3",&DWP(4,$r,"",0));   # mm3 = r[1]
68                 &paddq("mm3","mm4");            # mm3 = r[1] + w*a[1]
69                 &movd("mm5",&DWP(8,$r,"",0));   # mm5 = r[2]
70                 &paddq("mm5","mm6");            # mm5 = r[2] + w*a[2]
71                 &movd("mm4",&DWP(12,$r,"",0));  # mm4 = r[3]
72                 &paddq("mm7","mm4");            # mm7 = r[3] + w*a[3]
73                 &movd(&DWP(0,$r,"",0),"mm1");
74                 &movd("mm2",&DWP(16,$a,"",0));  # mm2 = a[4]
75                 &pmuludq("mm2","mm0");          # mm2 = w*a[4]
76                 &psrlq("mm1",32);               # mm1 = carry0
77                 &movd("mm4",&DWP(20,$a,"",0));  # mm4 = a[5]
78                 &pmuludq("mm4","mm0");          # mm4 = w*a[5]
79                 &paddq("mm1","mm3");            # mm1 = carry0 + r[1] + w*a[1]
80                 &movd("mm6",&DWP(24,$a,"",0));  # mm6 = a[6]
81                 &pmuludq("mm6","mm0");          # mm6 = w*a[6]
82                 &movd(&DWP(4,$r,"",0),"mm1");
83                 &psrlq("mm1",32);               # mm1 = carry1
84                 &movd("mm3",&DWP(28,$a,"",0));  # mm3 = a[7]
85                 &add($a,32);
86                 &pmuludq("mm3","mm0");          # mm3 = w*a[7]
87                 &paddq("mm1","mm5");            # mm1 = carry1 + r[2] + w*a[2]
88                 &movd("mm5",&DWP(16,$r,"",0));  # mm5 = r[4]
89                 &paddq("mm2","mm5");            # mm2 = r[4] + w*a[4]
90                 &movd(&DWP(8,$r,"",0),"mm1");
91                 &psrlq("mm1",32);               # mm1 = carry2
92                 &paddq("mm1","mm7");            # mm1 = carry2 + r[3] + w*a[3]
93                 &movd("mm5",&DWP(20,$r,"",0));  # mm5 = r[5]
94                 &paddq("mm4","mm5");            # mm4 = r[5] + w*a[5]
95                 &movd(&DWP(12,$r,"",0),"mm1");
96                 &psrlq("mm1",32);               # mm1 = carry3
97                 &paddq("mm1","mm2");            # mm1 = carry3 + r[4] + w*a[4]
98                 &movd("mm5",&DWP(24,$r,"",0));  # mm5 = r[6]
99                 &paddq("mm6","mm5");            # mm6 = r[6] + w*a[6]
100                 &movd(&DWP(16,$r,"",0),"mm1");
101                 &psrlq("mm1",32);               # mm1 = carry4
102                 &paddq("mm1","mm4");            # mm1 = carry4 + r[5] + w*a[5]
103                 &movd("mm5",&DWP(28,$r,"",0));  # mm5 = r[7]
104                 &paddq("mm3","mm5");            # mm3 = r[7] + w*a[7]
105                 &movd(&DWP(20,$r,"",0),"mm1");
106                 &psrlq("mm1",32);               # mm1 = carry5
107                 &paddq("mm1","mm6");            # mm1 = carry5 + r[6] + w*a[6]
108                 &movd(&DWP(24,$r,"",0),"mm1");
109                 &psrlq("mm1",32);               # mm1 = carry6
110                 &paddq("mm1","mm3");            # mm1 = carry6 + r[7] + w*a[7]
111                 &movd(&DWP(28,$r,"",0),"mm1");
112                 &add($r,32);
113                 &psrlq("mm1",32);               # mm1 = carry_out
114
115                 &sub("ecx",8);
116                 &jnz(&label("maw_sse2_loop"));
117
118                 &movd($c,"mm1");                # c = carry_out
119                 &emms();
120
121                 &jmp(&label("maw_finish"));
122         }
123
124         &set_label("maw_loop",0);
125
126         &mov(&swtmp(0),"ecx");  #
127
128         for ($i=0; $i<32; $i+=4)
129                 {
130                 &comment("Round $i");
131
132                  &mov("eax",&DWP($i,$a,"",0));  # *a
133                 &mul($w);                       # *a * w
134                 &add("eax",$c);         # L(t)+= *r
135                  &mov($c,&DWP($i,$r,"",0));     # L(t)+= *r
136                 &adc("edx",0);                  # H(t)+=carry
137                  &add("eax",$c);                # L(t)+=c
138                 &adc("edx",0);                  # H(t)+=carry
139                  &mov(&DWP($i,$r,"",0),"eax");  # *r= L(t);
140                 &mov($c,"edx");                 # c=  H(t);
141                 }
142
143         &comment("");
144         &mov("ecx",&swtmp(0));  #
145         &add($a,32);
146         &add($r,32);
147         &sub("ecx",8);
148         &jnz(&label("maw_loop"));
149
150         &set_label("maw_finish",0);
151         &mov("ecx",&wparam(2)); # get num
152         &and("ecx",7);
153         &jnz(&label("maw_finish2"));    # helps branch prediction
154         &jmp(&label("maw_end"));
155
156         &set_label("maw_finish2",1);
157         for ($i=0; $i<7; $i++)
158                 {
159                 &comment("Tail Round $i");
160                  &mov("eax",&DWP($i*4,$a,"",0));# *a
161                 &mul($w);                       # *a * w
162                 &add("eax",$c);                 # L(t)+=c
163                  &mov($c,&DWP($i*4,$r,"",0));   # L(t)+= *r
164                 &adc("edx",0);                  # H(t)+=carry
165                  &add("eax",$c);
166                 &adc("edx",0);                  # H(t)+=carry
167                  &dec("ecx") if ($i != 7-1);
168                 &mov(&DWP($i*4,$r,"",0),"eax"); # *r= L(t);
169                  &mov($c,"edx");                        # c=  H(t);
170                 &jz(&label("maw_end")) if ($i != 7-1);
171                 }
172         &set_label("maw_end",0);
173         &mov("eax",$c);
174
175         &pop("ecx");    # clear variable from
176
177         &function_end($name);
178         }
179
180 sub bn_mul_words
181         {
182         local($name)=@_;
183
184         &function_begin($name,"");
185
186         &comment("");
187         $Low="eax";
188         $High="edx";
189         $a="ebx";
190         $w="ecx";
191         $r="edi";
192         $c="esi";
193         $num="ebp";
194
195         &xor($c,$c);            # clear carry
196         &mov($r,&wparam(0));    #
197         &mov($a,&wparam(1));    #
198         &mov($num,&wparam(2));  #
199         &mov($w,&wparam(3));    #
200
201         &and($num,0xfffffff8);  # num / 8
202         &jz(&label("mw_finish"));
203
204         &set_label("mw_loop",0);
205         for ($i=0; $i<32; $i+=4)
206                 {
207                 &comment("Round $i");
208
209                  &mov("eax",&DWP($i,$a,"",0));  # *a
210                 &mul($w);                       # *a * w
211                 &add("eax",$c);                 # L(t)+=c
212                  # XXX
213
214                 &adc("edx",0);                  # H(t)+=carry
215                  &mov(&DWP($i,$r,"",0),"eax");  # *r= L(t);
216
217                 &mov($c,"edx");                 # c=  H(t);
218                 }
219
220         &comment("");
221         &add($a,32);
222         &add($r,32);
223         &sub($num,8);
224         &jz(&label("mw_finish"));
225         &jmp(&label("mw_loop"));
226
227         &set_label("mw_finish",0);
228         &mov($num,&wparam(2));  # get num
229         &and($num,7);
230         &jnz(&label("mw_finish2"));
231         &jmp(&label("mw_end"));
232
233         &set_label("mw_finish2",1);
234         for ($i=0; $i<7; $i++)
235                 {
236                 &comment("Tail Round $i");
237                  &mov("eax",&DWP($i*4,$a,"",0));# *a
238                 &mul($w);                       # *a * w
239                 &add("eax",$c);                 # L(t)+=c
240                  # XXX
241                 &adc("edx",0);                  # H(t)+=carry
242                  &mov(&DWP($i*4,$r,"",0),"eax");# *r= L(t);
243                 &mov($c,"edx");                 # c=  H(t);
244                  &dec($num) if ($i != 7-1);
245                 &jz(&label("mw_end")) if ($i != 7-1);
246                 }
247         &set_label("mw_end",0);
248         &mov("eax",$c);
249
250         &function_end($name);
251         }
252
253 sub bn_sqr_words
254         {
255         local($name)=@_;
256
257         &function_begin($name,"");
258
259         &comment("");
260         $r="esi";
261         $a="edi";
262         $num="ebx";
263
264         &mov($r,&wparam(0));    #
265         &mov($a,&wparam(1));    #
266         &mov($num,&wparam(2));  #
267
268         &and($num,0xfffffff8);  # num / 8
269         &jz(&label("sw_finish"));
270
271         &set_label("sw_loop",0);
272         for ($i=0; $i<32; $i+=4)
273                 {
274                 &comment("Round $i");
275                 &mov("eax",&DWP($i,$a,"",0));   # *a
276                  # XXX
277                 &mul("eax");                    # *a * *a
278                 &mov(&DWP($i*2,$r,"",0),"eax"); #
279                  &mov(&DWP($i*2+4,$r,"",0),"edx");#
280                 }
281
282         &comment("");
283         &add($a,32);
284         &add($r,64);
285         &sub($num,8);
286         &jnz(&label("sw_loop"));
287
288         &set_label("sw_finish",0);
289         &mov($num,&wparam(2));  # get num
290         &and($num,7);
291         &jz(&label("sw_end"));
292
293         for ($i=0; $i<7; $i++)
294                 {
295                 &comment("Tail Round $i");
296                 &mov("eax",&DWP($i*4,$a,"",0)); # *a
297                  # XXX
298                 &mul("eax");                    # *a * *a
299                 &mov(&DWP($i*8,$r,"",0),"eax"); #
300                  &dec($num) if ($i != 7-1);
301                 &mov(&DWP($i*8+4,$r,"",0),"edx");
302                  &jz(&label("sw_end")) if ($i != 7-1);
303                 }
304         &set_label("sw_end",0);
305
306         &function_end($name);
307         }
308
309 sub bn_div_words
310         {
311         local($name)=@_;
312
313         &function_begin($name,"");
314         &mov("edx",&wparam(0)); #
315         &mov("eax",&wparam(1)); #
316         &mov("ebx",&wparam(2)); #
317         &div("ebx");
318         &function_end($name);
319         }
320
321 sub bn_add_words
322         {
323         local($name)=@_;
324
325         &function_begin($name,"");
326
327         &comment("");
328         $a="esi";
329         $b="edi";
330         $c="eax";
331         $r="ebx";
332         $tmp1="ecx";
333         $tmp2="edx";
334         $num="ebp";
335
336         &mov($r,&wparam(0));    # get r
337          &mov($a,&wparam(1));   # get a
338         &mov($b,&wparam(2));    # get b
339          &mov($num,&wparam(3)); # get num
340         &xor($c,$c);            # clear carry
341          &and($num,0xfffffff8); # num / 8
342
343         &jz(&label("aw_finish"));
344
345         &set_label("aw_loop",0);
346         for ($i=0; $i<8; $i++)
347                 {
348                 &comment("Round $i");
349
350                 &mov($tmp1,&DWP($i*4,$a,"",0));         # *a
351                  &mov($tmp2,&DWP($i*4,$b,"",0));        # *b
352                 &add($tmp1,$c);
353                  &mov($c,0);
354                 &adc($c,$c);
355                  &add($tmp1,$tmp2);
356                 &adc($c,0);
357                  &mov(&DWP($i*4,$r,"",0),$tmp1);        # *r
358                 }
359
360         &comment("");
361         &add($a,32);
362          &add($b,32);
363         &add($r,32);
364          &sub($num,8);
365         &jnz(&label("aw_loop"));
366
367         &set_label("aw_finish",0);
368         &mov($num,&wparam(3));  # get num
369         &and($num,7);
370          &jz(&label("aw_end"));
371
372         for ($i=0; $i<7; $i++)
373                 {
374                 &comment("Tail Round $i");
375                 &mov($tmp1,&DWP($i*4,$a,"",0)); # *a
376                  &mov($tmp2,&DWP($i*4,$b,"",0));# *b
377                 &add($tmp1,$c);
378                  &mov($c,0);
379                 &adc($c,$c);
380                  &add($tmp1,$tmp2);
381                 &adc($c,0);
382                  &dec($num) if ($i != 6);
383                 &mov(&DWP($i*4,$r,"",0),$tmp1); # *r
384                  &jz(&label("aw_end")) if ($i != 6);
385                 }
386         &set_label("aw_end",0);
387
388 #       &mov("eax",$c);         # $c is "eax"
389
390         &function_end($name);
391         }
392
393 sub bn_sub_words
394         {
395         local($name)=@_;
396
397         &function_begin($name,"");
398
399         &comment("");
400         $a="esi";
401         $b="edi";
402         $c="eax";
403         $r="ebx";
404         $tmp1="ecx";
405         $tmp2="edx";
406         $num="ebp";
407
408         &mov($r,&wparam(0));    # get r
409          &mov($a,&wparam(1));   # get a
410         &mov($b,&wparam(2));    # get b
411          &mov($num,&wparam(3)); # get num
412         &xor($c,$c);            # clear carry
413          &and($num,0xfffffff8); # num / 8
414
415         &jz(&label("aw_finish"));
416
417         &set_label("aw_loop",0);
418         for ($i=0; $i<8; $i++)
419                 {
420                 &comment("Round $i");
421
422                 &mov($tmp1,&DWP($i*4,$a,"",0));         # *a
423                  &mov($tmp2,&DWP($i*4,$b,"",0));        # *b
424                 &sub($tmp1,$c);
425                  &mov($c,0);
426                 &adc($c,$c);
427                  &sub($tmp1,$tmp2);
428                 &adc($c,0);
429                  &mov(&DWP($i*4,$r,"",0),$tmp1);        # *r
430                 }
431
432         &comment("");
433         &add($a,32);
434          &add($b,32);
435         &add($r,32);
436          &sub($num,8);
437         &jnz(&label("aw_loop"));
438
439         &set_label("aw_finish",0);
440         &mov($num,&wparam(3));  # get num
441         &and($num,7);
442          &jz(&label("aw_end"));
443
444         for ($i=0; $i<7; $i++)
445                 {
446                 &comment("Tail Round $i");
447                 &mov($tmp1,&DWP($i*4,$a,"",0)); # *a
448                  &mov($tmp2,&DWP($i*4,$b,"",0));# *b
449                 &sub($tmp1,$c);
450                  &mov($c,0);
451                 &adc($c,$c);
452                  &sub($tmp1,$tmp2);
453                 &adc($c,0);
454                  &dec($num) if ($i != 6);
455                 &mov(&DWP($i*4,$r,"",0),$tmp1); # *r
456                  &jz(&label("aw_end")) if ($i != 6);
457                 }
458         &set_label("aw_end",0);
459
460 #       &mov("eax",$c);         # $c is "eax"
461
462         &function_end($name);
463         }
464
465 sub bn_sub_part_words
466         {
467         local($name)=@_;
468
469         &function_begin($name,"");
470
471         &comment("");
472         $a="esi";
473         $b="edi";
474         $c="eax";
475         $r="ebx";
476         $tmp1="ecx";
477         $tmp2="edx";
478         $num="ebp";
479
480         &mov($r,&wparam(0));    # get r
481          &mov($a,&wparam(1));   # get a
482         &mov($b,&wparam(2));    # get b
483          &mov($num,&wparam(3)); # get num
484         &xor($c,$c);            # clear carry
485          &and($num,0xfffffff8); # num / 8
486
487         &jz(&label("aw_finish"));
488
489         &set_label("aw_loop",0);
490         for ($i=0; $i<8; $i++)
491                 {
492                 &comment("Round $i");
493
494                 &mov($tmp1,&DWP($i*4,$a,"",0));         # *a
495                  &mov($tmp2,&DWP($i*4,$b,"",0));        # *b
496                 &sub($tmp1,$c);
497                  &mov($c,0);
498                 &adc($c,$c);
499                  &sub($tmp1,$tmp2);
500                 &adc($c,0);
501                  &mov(&DWP($i*4,$r,"",0),$tmp1);        # *r
502                 }
503
504         &comment("");
505         &add($a,32);
506          &add($b,32);
507         &add($r,32);
508          &sub($num,8);
509         &jnz(&label("aw_loop"));
510
511         &set_label("aw_finish",0);
512         &mov($num,&wparam(3));  # get num
513         &and($num,7);
514          &jz(&label("aw_end"));
515
516         for ($i=0; $i<7; $i++)
517                 {
518                 &comment("Tail Round $i");
519                 &mov($tmp1,&DWP(0,$a,"",0));    # *a
520                  &mov($tmp2,&DWP(0,$b,"",0));# *b
521                 &sub($tmp1,$c);
522                  &mov($c,0);
523                 &adc($c,$c);
524                  &sub($tmp1,$tmp2);
525                 &adc($c,0);
526                 &mov(&DWP(0,$r,"",0),$tmp1);    # *r
527                 &add($a, 4);
528                 &add($b, 4);
529                 &add($r, 4);
530                  &dec($num) if ($i != 6);
531                  &jz(&label("aw_end")) if ($i != 6);
532                 }
533         &set_label("aw_end",0);
534
535         &cmp(&wparam(4),0);
536         &je(&label("pw_end"));
537
538         &mov($num,&wparam(4));  # get dl
539         &cmp($num,0);
540         &je(&label("pw_end"));
541         &jge(&label("pw_pos"));
542
543         &comment("pw_neg");
544         &mov($tmp2,0);
545         &sub($tmp2,$num);
546         &mov($num,$tmp2);
547         &and($num,0xfffffff8);  # num / 8
548         &jz(&label("pw_neg_finish"));
549
550         &set_label("pw_neg_loop",0);
551         for ($i=0; $i<8; $i++)
552         {
553             &comment("dl<0 Round $i");
554
555             &mov($tmp1,0);
556             &mov($tmp2,&DWP($i*4,$b,"",0));     # *b
557             &sub($tmp1,$c);
558             &mov($c,0);
559             &adc($c,$c);
560             &sub($tmp1,$tmp2);
561             &adc($c,0);
562             &mov(&DWP($i*4,$r,"",0),$tmp1);     # *r
563         }
564             
565         &comment("");
566         &add($b,32);
567         &add($r,32);
568         &sub($num,8);
569         &jnz(&label("pw_neg_loop"));
570             
571         &set_label("pw_neg_finish",0);
572         &mov($tmp2,&wparam(4)); # get dl
573         &mov($num,0);
574         &sub($num,$tmp2);
575         &and($num,7);
576         &jz(&label("pw_end"));
577             
578         for ($i=0; $i<7; $i++)
579         {
580             &comment("dl<0 Tail Round $i");
581             &mov($tmp1,0);
582             &mov($tmp2,&DWP($i*4,$b,"",0));# *b
583             &sub($tmp1,$c);
584             &mov($c,0);
585             &adc($c,$c);
586             &sub($tmp1,$tmp2);
587             &adc($c,0);
588             &dec($num) if ($i != 6);
589             &mov(&DWP($i*4,$r,"",0),$tmp1);     # *r
590             &jz(&label("pw_end")) if ($i != 6);
591         }
592
593         &jmp(&label("pw_end"));
594         
595         &set_label("pw_pos",0);
596         
597         &and($num,0xfffffff8);  # num / 8
598         &jz(&label("pw_pos_finish"));
599
600         &set_label("pw_pos_loop",0);
601
602         for ($i=0; $i<8; $i++)
603         {
604             &comment("dl>0 Round $i");
605
606             &mov($tmp1,&DWP($i*4,$a,"",0));     # *a
607             &sub($tmp1,$c);
608             &mov(&DWP($i*4,$r,"",0),$tmp1);     # *r
609             &jnc(&label("pw_nc".$i));
610         }
611             
612         &comment("");
613         &add($a,32);
614         &add($r,32);
615         &sub($num,8);
616         &jnz(&label("pw_pos_loop"));
617             
618         &set_label("pw_pos_finish",0);
619         &mov($num,&wparam(4));  # get dl
620         &and($num,7);
621         &jz(&label("pw_end"));
622             
623         for ($i=0; $i<7; $i++)
624         {
625             &comment("dl>0 Tail Round $i");
626             &mov($tmp1,&DWP($i*4,$a,"",0));     # *a
627             &sub($tmp1,$c);
628             &mov(&DWP($i*4,$r,"",0),$tmp1);     # *r
629             &jnc(&label("pw_tail_nc".$i));
630             &dec($num) if ($i != 6);
631             &jz(&label("pw_end")) if ($i != 6);
632         }
633         &mov($c,1);
634         &jmp(&label("pw_end"));
635
636         &set_label("pw_nc_loop",0);
637         for ($i=0; $i<8; $i++)
638         {
639             &mov($tmp1,&DWP($i*4,$a,"",0));     # *a
640             &mov(&DWP($i*4,$r,"",0),$tmp1);     # *r
641             &set_label("pw_nc".$i,0);
642         }
643             
644         &comment("");
645         &add($a,32);
646         &add($r,32);
647         &sub($num,8);
648         &jnz(&label("pw_nc_loop"));
649             
650         &mov($num,&wparam(4));  # get dl
651         &and($num,7);
652         &jz(&label("pw_nc_end"));
653             
654         for ($i=0; $i<7; $i++)
655         {
656             &mov($tmp1,&DWP($i*4,$a,"",0));     # *a
657             &mov(&DWP($i*4,$r,"",0),$tmp1);     # *r
658             &set_label("pw_tail_nc".$i,0);
659             &dec($num) if ($i != 6);
660             &jz(&label("pw_nc_end")) if ($i != 6);
661         }
662
663         &set_label("pw_nc_end",0);
664         &mov($c,0);
665
666         &set_label("pw_end",0);
667
668 #       &mov("eax",$c);         # $c is "eax"
669
670         &function_end($name);
671         }
672