crypto/ec/asm/ecp_nistz256-x86.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2015-2016 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the OpenSSL license (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9
  10 # ====================================================================
  11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  12 # project. The module is, however, dual licensed under OpenSSL and
  13 # CRYPTOGAMS licenses depending on where you obtain it. For further
  14 # details see http://www.openssl.org/~appro/cryptogams/.
  15 # ====================================================================
  16 #
  17 # ECP_NISTZ256 module for x86/SSE2.
  18 #
  19 # October 2014.
  20 #
  21 # Original ECP_NISTZ256 submission targeting x86_64 is detailed in
  22 # http://eprint.iacr.org/2013/816. In the process of adaptation
  23 # original .c module was made 32-bit savvy in order to make this
  24 # implementation possible.
  25 #
  26 #               with/without -DECP_NISTZ256_ASM
  27 # Pentium       +66-163%
  28 # PIII          +72-172%
  29 # P4            +65-132%
  30 # Core2         +90-215%
  31 # Sandy Bridge  +105-265% (contemporary i[57]-* are all close to this)
  32 # Atom          +65-155%
  33 # Opteron       +54-110%
  34 # Bulldozer     +99-240%
  35 # VIA Nano      +93-290%
  36 #
  37 # Ranges denote minimum and maximum improvement coefficients depending
  38 # on benchmark. Lower coefficients are for ECDSA sign, server-side
  39 # operation. Keep in mind that +200% means 3x improvement.
  40
  41 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  42 push(@INC,"${dir}","${dir}../../perlasm");
  43 require "x86asm.pl";
  44
  45 $output=pop;
  46 open STDOUT,">$output";
  47
  48 &asm_init($ARGV[0],"ecp_nistz256-x86.pl",$ARGV[$#ARGV] eq "386");
  49
  50 $sse2=0;
  51 for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
  52
  53 &external_label("OPENSSL_ia32cap_P") if ($sse2);
  54
  55
  56 ########################################################################
  57 # Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7
  58 #
  59 open TABLE,"<ecp_nistz256_table.c"              or
  60 open TABLE,"<${dir}../ecp_nistz256_table.c"     or
  61 die "failed to open ecp_nistz256_table.c:",$!;
  62
  63 use integer;
  64
  65 foreach(<TABLE>) {
  66         s/TOBN\(\s*(0x[0-9a-f]+),\s*(0x[0-9a-f]+)\s*\)/push @arr,hex($2),hex($1)/geo;
  67 }
  68 close TABLE;
  69
  70 # See ecp_nistz256_table.c for explanation for why it's 64*16*37.
  71 # 64*16*37-1 is because $#arr returns last valid index or @arr, not
  72 # amount of elements.
  73 die "insane number of elements" if ($#arr != 64*16*37-1);
  74
  75 &public_label("ecp_nistz256_precomputed");
  76 &align(4096);
  77 &set_label("ecp_nistz256_precomputed");
  78
  79 ########################################################################
  80 # this conversion smashes P256_POINT_AFFINE by individual bytes with
  81 # 64 byte interval, similar to
  82 #       1111222233334444
  83 #       1234123412341234
  84 for(1..37) {
  85         @tbl = splice(@arr,0,64*16);
  86         for($i=0;$i<64;$i++) {
  87                 undef @line;
  88                 for($j=0;$j<64;$j++) {
  89                         push @line,(@tbl[$j*16+$i/4]>>(($i%4)*8))&0xff;
  90                 }
  91                 &data_byte(join(',',map { sprintf "0x%02x",$_} @line));
  92         }
  93 }
  94
  95 ########################################################################
  96 # Keep in mind that constants are stored least to most significant word
  97 &static_label("RR");
  98 &set_label("RR",64);
  99 &data_word(3,0,-1,-5,-2,-1,-3,4);       # 2^512 mod P-256
 100
 101 &static_label("ONE_mont");
 102 &set_label("ONE_mont");
 103 &data_word(1,0,0,-1,-1,-1,-2,0);
 104
 105 &static_label("ONE");
 106 &set_label("ONE");
 107 &data_word(1,0,0,0,0,0,0,0);
 108 &asciz("ECP_NISZ256 for x86/SSE2, CRYPTOGAMS by <appro\@openssl.org>");
 109 &align(64);
 110
 111 ########################################################################
 112 # void ecp_nistz256_mul_by_2(BN_ULONG edi[8],const BN_ULONG esi[8]);
 113 &function_begin("ecp_nistz256_mul_by_2");
 114         &mov    ("esi",&wparam(1));
 115         &mov    ("edi",&wparam(0));
 116         &mov    ("ebp","esi");
 117 ########################################################################
 118 # common pattern for internal functions is that %edi is result pointer,
 119 # %esi and %ebp are input ones, %ebp being optional. %edi is preserved.
 120         &call   ("_ecp_nistz256_add");
 121 &function_end("ecp_nistz256_mul_by_2");
 122
 123 ########################################################################
 124 # void ecp_nistz256_mul_by_3(BN_ULONG edi[8],const BN_ULONG esi[8]);
 125 &function_begin("ecp_nistz256_mul_by_3");
 126         &mov    ("esi",&wparam(1));
 127                                         # multiplication by 3 is performed
 128                                         # as 2*n+n, but we can't use output
 129                                         # to store 2*n, because if output
 130                                         # pointer equals to input, then
 131                                         # we'll get 2*n+2*n.
 132         &stack_push(8);                 # therefore we need to allocate
 133                                         # 256-bit intermediate buffer.
 134         &mov    ("edi","esp");
 135         &mov    ("ebp","esi");
 136         &call   ("_ecp_nistz256_add");
 137         &lea    ("esi",&DWP(0,"edi"));
 138         &mov    ("ebp",&wparam(1));
 139         &mov    ("edi",&wparam(0));
 140         &call   ("_ecp_nistz256_add");
 141         &stack_pop(8);
 142 &function_end("ecp_nistz256_mul_by_3");
 143
 144 ########################################################################
 145 # void ecp_nistz256_div_by_2(BN_ULONG edi[8],const BN_ULONG esi[8]);
 146 &function_begin("ecp_nistz256_div_by_2");
 147         &mov    ("esi",&wparam(1));
 148         &mov    ("edi",&wparam(0));
 149         &call   ("_ecp_nistz256_div_by_2");
 150 &function_end("ecp_nistz256_div_by_2");
 151
 152 &function_begin_B("_ecp_nistz256_div_by_2");
 153         # tmp = a is odd ? a+mod : a
 154         #
 155         # note that because mod has special form, i.e. consists of
 156         # 0xffffffff, 1 and 0s, we can conditionally synthesize it by
 157         # assigning least significant bit of input to one register,
 158         # %ebp, and its negative to another, %edx.
 159
 160         &mov    ("ebp",&DWP(0,"esi"));
 161         &xor    ("edx","edx");
 162         &mov    ("ebx",&DWP(4,"esi"));
 163         &mov    ("eax","ebp");
 164         &and    ("ebp",1);
 165         &mov    ("ecx",&DWP(8,"esi"));
 166         &sub    ("edx","ebp");
 167
 168         &add    ("eax","edx");
 169         &adc    ("ebx","edx");
 170         &mov    (&DWP(0,"edi"),"eax");
 171         &adc    ("ecx","edx");
 172         &mov    (&DWP(4,"edi"),"ebx");
 173         &mov    (&DWP(8,"edi"),"ecx");
 174
 175         &mov    ("eax",&DWP(12,"esi"));
 176         &mov    ("ebx",&DWP(16,"esi"));
 177         &adc    ("eax",0);
 178         &mov    ("ecx",&DWP(20,"esi"));
 179         &adc    ("ebx",0);
 180         &mov    (&DWP(12,"edi"),"eax");
 181         &adc    ("ecx",0);
 182         &mov    (&DWP(16,"edi"),"ebx");
 183         &mov    (&DWP(20,"edi"),"ecx");
 184
 185         &mov    ("eax",&DWP(24,"esi"));
 186         &mov    ("ebx",&DWP(28,"esi"));
 187         &adc    ("eax","ebp");
 188         &adc    ("ebx","edx");
 189         &mov    (&DWP(24,"edi"),"eax");
 190         &sbb    ("esi","esi");                  # broadcast carry bit
 191         &mov    (&DWP(28,"edi"),"ebx");
 192
 193         # ret = tmp >> 1
 194
 195         &mov    ("eax",&DWP(0,"edi"));
 196         &mov    ("ebx",&DWP(4,"edi"));
 197         &mov    ("ecx",&DWP(8,"edi"));
 198         &mov    ("edx",&DWP(12,"edi"));
 199
 200         &shr    ("eax",1);
 201         &mov    ("ebp","ebx");
 202         &shl    ("ebx",31);
 203         &or     ("eax","ebx");
 204
 205         &shr    ("ebp",1);
 206         &mov    ("ebx","ecx");
 207         &shl    ("ecx",31);
 208         &mov    (&DWP(0,"edi"),"eax");
 209         &or     ("ebp","ecx");
 210         &mov    ("eax",&DWP(16,"edi"));
 211
 212         &shr    ("ebx",1);
 213         &mov    ("ecx","edx");
 214         &shl    ("edx",31);
 215         &mov    (&DWP(4,"edi"),"ebp");
 216         &or     ("ebx","edx");
 217         &mov    ("ebp",&DWP(20,"edi"));
 218
 219         &shr    ("ecx",1);
 220         &mov    ("edx","eax");
 221         &shl    ("eax",31);
 222         &mov    (&DWP(8,"edi"),"ebx");
 223         &or     ("ecx","eax");
 224         &mov    ("ebx",&DWP(24,"edi"));
 225
 226         &shr    ("edx",1);
 227         &mov    ("eax","ebp");
 228         &shl    ("ebp",31);
 229         &mov    (&DWP(12,"edi"),"ecx");
 230         &or     ("edx","ebp");
 231         &mov    ("ecx",&DWP(28,"edi"));
 232
 233         &shr    ("eax",1);
 234         &mov    ("ebp","ebx");
 235         &shl    ("ebx",31);
 236         &mov    (&DWP(16,"edi"),"edx");
 237         &or     ("eax","ebx");
 238
 239         &shr    ("ebp",1);
 240         &mov    ("ebx","ecx");
 241         &shl    ("ecx",31);
 242         &mov    (&DWP(20,"edi"),"eax");
 243         &or     ("ebp","ecx");
 244
 245         &shr    ("ebx",1);
 246         &shl    ("esi",31);
 247         &mov    (&DWP(24,"edi"),"ebp");
 248         &or     ("ebx","esi");                  # handle top-most carry bit
 249         &mov    (&DWP(28,"edi"),"ebx");
 250
 251         &ret    ();
 252 &function_end_B("_ecp_nistz256_div_by_2");
 253
 254 ########################################################################
 255 # void ecp_nistz256_add(BN_ULONG edi[8],const BN_ULONG esi[8],
 256 #                                       const BN_ULONG ebp[8]);
 257 &function_begin("ecp_nistz256_add");
 258         &mov    ("esi",&wparam(1));
 259         &mov    ("ebp",&wparam(2));
 260         &mov    ("edi",&wparam(0));
 261         &call   ("_ecp_nistz256_add");
 262 &function_end("ecp_nistz256_add");
 263
 264 &function_begin_B("_ecp_nistz256_add");
 265         &mov    ("eax",&DWP(0,"esi"));
 266         &mov    ("ebx",&DWP(4,"esi"));
 267         &mov    ("ecx",&DWP(8,"esi"));
 268         &add    ("eax",&DWP(0,"ebp"));
 269         &mov    ("edx",&DWP(12,"esi"));
 270         &adc    ("ebx",&DWP(4,"ebp"));
 271         &mov    (&DWP(0,"edi"),"eax");
 272         &adc    ("ecx",&DWP(8,"ebp"));
 273         &mov    (&DWP(4,"edi"),"ebx");
 274         &adc    ("edx",&DWP(12,"ebp"));
 275         &mov    (&DWP(8,"edi"),"ecx");
 276         &mov    (&DWP(12,"edi"),"edx");
 277
 278         &mov    ("eax",&DWP(16,"esi"));
 279         &mov    ("ebx",&DWP(20,"esi"));
 280         &mov    ("ecx",&DWP(24,"esi"));
 281         &adc    ("eax",&DWP(16,"ebp"));
 282         &mov    ("edx",&DWP(28,"esi"));
 283         &adc    ("ebx",&DWP(20,"ebp"));
 284         &mov    (&DWP(16,"edi"),"eax");
 285         &adc    ("ecx",&DWP(24,"ebp"));
 286         &mov    (&DWP(20,"edi"),"ebx");
 287         &adc    ("edx",&DWP(28,"ebp"));
 288         &mov    (&DWP(24,"edi"),"ecx");
 289         &sbb    ("esi","esi");                  # broadcast carry bit
 290         &mov    (&DWP(28,"edi"),"edx");
 291
 292         # if a+b carries, subtract modulus.
 293         #
 294         # Note that because mod has special form, i.e. consists of
 295         # 0xffffffff, 1 and 0s, we can conditionally synthesize it by
 296         # assigning carry bit to one register, %ebp, and its negative
 297         # to another, %esi. But we started by calculating %esi...
 298
 299         &mov    ("eax",&DWP(0,"edi"));
 300         &mov    ("ebp","esi");
 301         &mov    ("ebx",&DWP(4,"edi"));
 302         &shr    ("ebp",31);
 303         &mov    ("ecx",&DWP(8,"edi"));
 304         &sub    ("eax","esi");
 305         &mov    ("edx",&DWP(12,"edi"));
 306         &sbb    ("ebx","esi");
 307         &mov    (&DWP(0,"edi"),"eax");
 308         &sbb    ("ecx","esi");
 309         &mov    (&DWP(4,"edi"),"ebx");
 310         &sbb    ("edx",0);
 311         &mov    (&DWP(8,"edi"),"ecx");
 312         &mov    (&DWP(12,"edi"),"edx");
 313
 314         &mov    ("eax",&DWP(16,"edi"));
 315         &mov    ("ebx",&DWP(20,"edi"));
 316         &mov    ("ecx",&DWP(24,"edi"));
 317         &sbb    ("eax",0);
 318         &mov    ("edx",&DWP(28,"edi"));
 319         &sbb    ("ebx",0);
 320         &mov    (&DWP(16,"edi"),"eax");
 321         &sbb    ("ecx","ebp");
 322         &mov    (&DWP(20,"edi"),"ebx");
 323         &sbb    ("edx","esi");
 324         &mov    (&DWP(24,"edi"),"ecx");
 325         &mov    (&DWP(28,"edi"),"edx");
 326
 327         &ret    ();
 328 &function_end_B("_ecp_nistz256_add");
 329
 330 ########################################################################
 331 # void ecp_nistz256_sub(BN_ULONG edi[8],const BN_ULONG esi[8],
 332 #                                       const BN_ULONG ebp[8]);
 333 &function_begin("ecp_nistz256_sub");
 334         &mov    ("esi",&wparam(1));
 335         &mov    ("ebp",&wparam(2));
 336         &mov    ("edi",&wparam(0));
 337         &call   ("_ecp_nistz256_sub");
 338 &function_end("ecp_nistz256_sub");
 339
 340 &function_begin_B("_ecp_nistz256_sub");
 341         &mov    ("eax",&DWP(0,"esi"));
 342         &mov    ("ebx",&DWP(4,"esi"));
 343         &mov    ("ecx",&DWP(8,"esi"));
 344         &sub    ("eax",&DWP(0,"ebp"));
 345         &mov    ("edx",&DWP(12,"esi"));
 346         &sbb    ("ebx",&DWP(4,"ebp"));
 347         &mov    (&DWP(0,"edi"),"eax");
 348         &sbb    ("ecx",&DWP(8,"ebp"));
 349         &mov    (&DWP(4,"edi"),"ebx");
 350         &sbb    ("edx",&DWP(12,"ebp"));
 351         &mov    (&DWP(8,"edi"),"ecx");
 352         &mov    (&DWP(12,"edi"),"edx");
 353
 354         &mov    ("eax",&DWP(16,"esi"));
 355         &mov    ("ebx",&DWP(20,"esi"));
 356         &mov    ("ecx",&DWP(24,"esi"));
 357         &sbb    ("eax",&DWP(16,"ebp"));
 358         &mov    ("edx",&DWP(28,"esi"));
 359         &sbb    ("ebx",&DWP(20,"ebp"));
 360         &sbb    ("ecx",&DWP(24,"ebp"));
 361         &mov    (&DWP(16,"edi"),"eax");
 362         &sbb    ("edx",&DWP(28,"ebp"));
 363         &mov    (&DWP(20,"edi"),"ebx");
 364         &sbb    ("esi","esi");                  # broadcast borrow bit
 365         &mov    (&DWP(24,"edi"),"ecx");
 366         &mov    (&DWP(28,"edi"),"edx");
 367
 368         # if a-b borrows, add modulus.
 369         #
 370         # Note that because mod has special form, i.e. consists of
 371         # 0xffffffff, 1 and 0s, we can conditionally synthesize it by
 372         # assigning borrow bit to one register, %ebp, and its negative
 373         # to another, %esi. But we started by calculating %esi...
 374
 375         &mov    ("eax",&DWP(0,"edi"));
 376         &mov    ("ebp","esi");
 377         &mov    ("ebx",&DWP(4,"edi"));
 378         &shr    ("ebp",31);
 379         &mov    ("ecx",&DWP(8,"edi"));
 380         &add    ("eax","esi");
 381         &mov    ("edx",&DWP(12,"edi"));
 382         &adc    ("ebx","esi");
 383         &mov    (&DWP(0,"edi"),"eax");
 384         &adc    ("ecx","esi");
 385         &mov    (&DWP(4,"edi"),"ebx");
 386         &adc    ("edx",0);
 387         &mov    (&DWP(8,"edi"),"ecx");
 388         &mov    (&DWP(12,"edi"),"edx");
 389
 390         &mov    ("eax",&DWP(16,"edi"));
 391         &mov    ("ebx",&DWP(20,"edi"));
 392         &mov    ("ecx",&DWP(24,"edi"));
 393         &adc    ("eax",0);
 394         &mov    ("edx",&DWP(28,"edi"));
 395         &adc    ("ebx",0);
 396         &mov    (&DWP(16,"edi"),"eax");
 397         &adc    ("ecx","ebp");
 398         &mov    (&DWP(20,"edi"),"ebx");
 399         &adc    ("edx","esi");
 400         &mov    (&DWP(24,"edi"),"ecx");
 401         &mov    (&DWP(28,"edi"),"edx");
 402
 403         &ret    ();
 404 &function_end_B("_ecp_nistz256_sub");
 405
 406 ########################################################################
 407 # void ecp_nistz256_neg(BN_ULONG edi[8],const BN_ULONG esi[8]);
 408 &function_begin("ecp_nistz256_neg");
 409         &mov    ("ebp",&wparam(1));
 410         &mov    ("edi",&wparam(0));
 411
 412         &xor    ("eax","eax");
 413         &stack_push(8);
 414         &mov    (&DWP(0,"esp"),"eax");
 415         &mov    ("esi","esp");
 416         &mov    (&DWP(4,"esp"),"eax");
 417         &mov    (&DWP(8,"esp"),"eax");
 418         &mov    (&DWP(12,"esp"),"eax");
 419         &mov    (&DWP(16,"esp"),"eax");
 420         &mov    (&DWP(20,"esp"),"eax");
 421         &mov    (&DWP(24,"esp"),"eax");
 422         &mov    (&DWP(28,"esp"),"eax");
 423
 424         &call   ("_ecp_nistz256_sub");
 425
 426         &stack_pop(8);
 427 &function_end("ecp_nistz256_neg");
 428
 429 &function_begin_B("_picup_eax");
 430         &mov    ("eax",&DWP(0,"esp"));
 431         &ret    ();
 432 &function_end_B("_picup_eax");
 433
 434 ########################################################################
 435 # void ecp_nistz256_to_mont(BN_ULONG edi[8],const BN_ULONG esi[8]);
 436 &function_begin("ecp_nistz256_to_mont");
 437         &mov    ("esi",&wparam(1));
 438         &call   ("_picup_eax");
 439     &set_label("pic");
 440         &lea    ("ebp",&DWP(&label("RR")."-".&label("pic"),"eax"));
 441                                                 if ($sse2) {
 442         &picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
 443         &mov    ("eax",&DWP(0,"eax"));          }
 444         &mov    ("edi",&wparam(0));
 445         &call   ("_ecp_nistz256_mul_mont");
 446 &function_end("ecp_nistz256_to_mont");
 447
 448 ########################################################################
 449 # void ecp_nistz256_from_mont(BN_ULONG edi[8],const BN_ULONG esi[8]);
 450 &function_begin("ecp_nistz256_from_mont");
 451         &mov    ("esi",&wparam(1));
 452         &call   ("_picup_eax");
 453     &set_label("pic");
 454         &lea    ("ebp",&DWP(&label("ONE")."-".&label("pic"),"eax"));
 455                                                 if ($sse2) {
 456         &picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
 457         &mov    ("eax",&DWP(0,"eax"));          }
 458         &mov    ("edi",&wparam(0));
 459         &call   ("_ecp_nistz256_mul_mont");
 460 &function_end("ecp_nistz256_from_mont");
 461
 462 ########################################################################
 463 # void ecp_nistz256_mul_mont(BN_ULONG edi[8],const BN_ULONG esi[8],
 464 #                                            const BN_ULONG ebp[8]);
 465 &function_begin("ecp_nistz256_mul_mont");
 466         &mov    ("esi",&wparam(1));
 467         &mov    ("ebp",&wparam(2));
 468                                                 if ($sse2) {
 469         &call   ("_picup_eax");
 470     &set_label("pic");
 471         &picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
 472         &mov    ("eax",&DWP(0,"eax"));          }
 473         &mov    ("edi",&wparam(0));
 474         &call   ("_ecp_nistz256_mul_mont");
 475 &function_end("ecp_nistz256_mul_mont");
 476
 477 ########################################################################
 478 # void ecp_nistz256_sqr_mont(BN_ULONG edi[8],const BN_ULONG esi[8]);
 479 &function_begin("ecp_nistz256_sqr_mont");
 480         &mov    ("esi",&wparam(1));
 481                                                 if ($sse2) {
 482         &call   ("_picup_eax");
 483     &set_label("pic");
 484         &picmeup("eax","OPENSSL_ia32cap_P","eax",&label("pic"));
 485         &mov    ("eax",&DWP(0,"eax"));          }
 486         &mov    ("edi",&wparam(0));
 487         &mov    ("ebp","esi");
 488         &call   ("_ecp_nistz256_mul_mont");
 489 &function_end("ecp_nistz256_sqr_mont");
 490
 491 &function_begin_B("_ecp_nistz256_mul_mont");
 492                                                 if ($sse2) {
 493         &and    ("eax",1<<24|1<<26);
 494         &cmp    ("eax",1<<24|1<<26);            # see if XMM+SSE2 is on
 495         &jne    (&label("mul_mont_ialu"));
 496
 497         ########################################
 498         # SSE2 code path featuring 32x16-bit
 499         # multiplications is ~2x faster than
 500         # IALU counterpart (except on Atom)...
 501         ########################################
 502         # stack layout:
 503         # +------------------------------------+< %esp
 504         # | 7 16-byte temporary XMM words,     |
 505         # | "sliding" toward lower address     |
 506         # .                                    .
 507         # +------------------------------------+
 508         # | unused XMM word                    |
 509         # +------------------------------------+< +128,%ebx
 510         # | 8 16-byte XMM words holding copies |
 511         # | of a[i]<<64|a[i]                   |
 512         # .                                    .
 513         # .                                    .
 514         # +------------------------------------+< +256
 515         &mov    ("edx","esp");
 516         &sub    ("esp",0x100);
 517
 518         &movd   ("xmm7",&DWP(0,"ebp"));         # b[0] -> 0000.00xy
 519         &lea    ("ebp",&DWP(4,"ebp"));
 520         &pcmpeqd("xmm6","xmm6");
 521         &psrlq  ("xmm6",48);                    # compose 0xffff<<64|0xffff
 522
 523         &pshuflw("xmm7","xmm7",0b11011100);     # 0000.00xy -> 0000.0x0y
 524         &and    ("esp",-64);
 525         &pshufd ("xmm7","xmm7",0b11011100);     # 0000.0x0y -> 000x.000y
 526         &lea    ("ebx",&DWP(0x80,"esp"));
 527
 528         &movd   ("xmm0",&DWP(4*0,"esi"));       # a[0] -> 0000.00xy
 529         &pshufd ("xmm0","xmm0",0b11001100);     # 0000.00xy -> 00xy.00xy
 530         &movd   ("xmm1",&DWP(4*1,"esi"));       # a[1] -> ...
 531         &movdqa (&QWP(0x00,"ebx"),"xmm0");      # offload converted a[0]
 532         &pmuludq("xmm0","xmm7");                # a[0]*b[0]
 533
 534         &movd   ("xmm2",&DWP(4*2,"esi"));
 535         &pshufd ("xmm1","xmm1",0b11001100);
 536         &movdqa (&QWP(0x10,"ebx"),"xmm1");
 537         &pmuludq("xmm1","xmm7");                # a[1]*b[0]
 538
 539          &movq  ("xmm4","xmm0");                # clear upper 64 bits
 540          &pslldq("xmm4",6);
 541          &paddq ("xmm4","xmm0");
 542          &movdqa("xmm5","xmm4");
 543          &psrldq("xmm4",10);                    # upper 32 bits of a[0]*b[0]
 544          &pand  ("xmm5","xmm6");                # lower 32 bits of a[0]*b[0]
 545
 546         # Upper half of a[0]*b[i] is carried into next multiplication
 547         # iteration, while lower one "participates" in actual reduction.
 548         # Normally latter is done by accumulating result of multiplication
 549         # of modulus by "magic" digit, but thanks to special form of modulus
 550         # and "magic" digit it can be performed only with additions and
 551         # subtractions (see note in IALU section below). Note that we are
 552         # not bothered with carry bits, they are accumulated in "flatten"
 553         # phase after all multiplications and reductions.
 554
 555         &movd   ("xmm3",&DWP(4*3,"esi"));
 556         &pshufd ("xmm2","xmm2",0b11001100);
 557         &movdqa (&QWP(0x20,"ebx"),"xmm2");
 558         &pmuludq("xmm2","xmm7");                # a[2]*b[0]
 559          &paddq ("xmm1","xmm4");                # a[1]*b[0]+hw(a[0]*b[0]), carry
 560         &movdqa (&QWP(0x00,"esp"),"xmm1");      # t[0]
 561
 562         &movd   ("xmm0",&DWP(4*4,"esi"));
 563         &pshufd ("xmm3","xmm3",0b11001100);
 564         &movdqa (&QWP(0x30,"ebx"),"xmm3");
 565         &pmuludq("xmm3","xmm7");                # a[3]*b[0]
 566         &movdqa (&QWP(0x10,"esp"),"xmm2");
 567
 568         &movd   ("xmm1",&DWP(4*5,"esi"));
 569         &pshufd ("xmm0","xmm0",0b11001100);
 570         &movdqa (&QWP(0x40,"ebx"),"xmm0");
 571         &pmuludq("xmm0","xmm7");                # a[4]*b[0]
 572          &paddq ("xmm3","xmm5");                # a[3]*b[0]+lw(a[0]*b[0]), reduction step
 573         &movdqa (&QWP(0x20,"esp"),"xmm3");
 574
 575         &movd   ("xmm2",&DWP(4*6,"esi"));
 576         &pshufd ("xmm1","xmm1",0b11001100);
 577         &movdqa (&QWP(0x50,"ebx"),"xmm1");
 578         &pmuludq("xmm1","xmm7");                # a[5]*b[0]
 579         &movdqa (&QWP(0x30,"esp"),"xmm0");
 580          &pshufd("xmm4","xmm5",0b10110001);     # xmm4 = xmm5<<32, reduction step
 581
 582         &movd   ("xmm3",&DWP(4*7,"esi"));
 583         &pshufd ("xmm2","xmm2",0b11001100);
 584         &movdqa (&QWP(0x60,"ebx"),"xmm2");
 585         &pmuludq("xmm2","xmm7");                # a[6]*b[0]
 586         &movdqa (&QWP(0x40,"esp"),"xmm1");
 587          &psubq ("xmm4","xmm5");                # xmm4 = xmm5*0xffffffff, reduction step
 588
 589         &movd   ("xmm0",&DWP(0,"ebp"));         # b[1] -> 0000.00xy
 590         &pshufd ("xmm3","xmm3",0b11001100);
 591         &movdqa (&QWP(0x70,"ebx"),"xmm3");
 592         &pmuludq("xmm3","xmm7");                # a[7]*b[0]
 593
 594         &pshuflw("xmm7","xmm0",0b11011100);     # 0000.00xy -> 0000.0x0y
 595         &movdqa ("xmm0",&QWP(0x00,"ebx"));      # pre-load converted a[0]
 596         &pshufd ("xmm7","xmm7",0b11011100);     # 0000.0x0y -> 000x.000y
 597
 598         &mov    ("ecx",6);
 599         &lea    ("ebp",&DWP(4,"ebp"));
 600         &jmp    (&label("madd_sse2"));
 601
 602 &set_label("madd_sse2",16);
 603          &paddq ("xmm2","xmm5");                # a[6]*b[i-1]+lw(a[0]*b[i-1]), reduction step [modulo-scheduled]
 604          &paddq ("xmm3","xmm4");                # a[7]*b[i-1]+lw(a[0]*b[i-1])*0xffffffff, reduction step [modulo-scheduled]
 605         &movdqa ("xmm1",&QWP(0x10,"ebx"));
 606         &pmuludq("xmm0","xmm7");                # a[0]*b[i]
 607          &movdqa(&QWP(0x50,"esp"),"xmm2");
 608
 609         &movdqa ("xmm2",&QWP(0x20,"ebx"));
 610         &pmuludq("xmm1","xmm7");                # a[1]*b[i]
 611          &movdqa(&QWP(0x60,"esp"),"xmm3");
 612         &paddq  ("xmm0",&QWP(0x00,"esp"));
 613
 614         &movdqa ("xmm3",&QWP(0x30,"ebx"));
 615         &pmuludq("xmm2","xmm7");                # a[2]*b[i]
 616          &movq  ("xmm4","xmm0");                # clear upper 64 bits
 617          &pslldq("xmm4",6);
 618         &paddq  ("xmm1",&QWP(0x10,"esp"));
 619          &paddq ("xmm4","xmm0");
 620          &movdqa("xmm5","xmm4");
 621          &psrldq("xmm4",10);                    # upper 33 bits of a[0]*b[i]+t[0]
 622
 623         &movdqa ("xmm0",&QWP(0x40,"ebx"));
 624         &pmuludq("xmm3","xmm7");                # a[3]*b[i]
 625          &paddq ("xmm1","xmm4");                # a[1]*b[i]+hw(a[0]*b[i]), carry
 626         &paddq  ("xmm2",&QWP(0x20,"esp"));
 627         &movdqa (&QWP(0x00,"esp"),"xmm1");
 628
 629         &movdqa ("xmm1",&QWP(0x50,"ebx"));
 630         &pmuludq("xmm0","xmm7");                # a[4]*b[i]
 631         &paddq  ("xmm3",&QWP(0x30,"esp"));
 632         &movdqa (&QWP(0x10,"esp"),"xmm2");
 633          &pand  ("xmm5","xmm6");                # lower 32 bits of a[0]*b[i]
 634
 635         &movdqa ("xmm2",&QWP(0x60,"ebx"));
 636         &pmuludq("xmm1","xmm7");                # a[5]*b[i]
 637          &paddq ("xmm3","xmm5");                # a[3]*b[i]+lw(a[0]*b[i]), reduction step
 638         &paddq  ("xmm0",&QWP(0x40,"esp"));
 639         &movdqa (&QWP(0x20,"esp"),"xmm3");
 640          &pshufd("xmm4","xmm5",0b10110001);     # xmm4 = xmm5<<32, reduction step
 641
 642         &movdqa ("xmm3","xmm7");
 643         &pmuludq("xmm2","xmm7");                # a[6]*b[i]
 644          &movd  ("xmm7",&DWP(0,"ebp"));         # b[i++] -> 0000.00xy
 645          &lea   ("ebp",&DWP(4,"ebp"));
 646         &paddq  ("xmm1",&QWP(0x50,"esp"));
 647          &psubq ("xmm4","xmm5");                # xmm4 = xmm5*0xffffffff, reduction step
 648         &movdqa (&QWP(0x30,"esp"),"xmm0");
 649          &pshuflw("xmm7","xmm7",0b11011100);    # 0000.00xy -> 0000.0x0y
 650
 651         &pmuludq("xmm3",&QWP(0x70,"ebx"));      # a[7]*b[i]
 652          &pshufd("xmm7","xmm7",0b11011100);     # 0000.0x0y -> 000x.000y
 653          &movdqa("xmm0",&QWP(0x00,"ebx"));      # pre-load converted a[0]
 654         &movdqa (&QWP(0x40,"esp"),"xmm1");
 655         &paddq  ("xmm2",&QWP(0x60,"esp"));
 656
 657         &dec    ("ecx");
 658         &jnz    (&label("madd_sse2"));
 659
 660          &paddq ("xmm2","xmm5");                # a[6]*b[6]+lw(a[0]*b[6]), reduction step [modulo-scheduled]
 661          &paddq ("xmm3","xmm4");                # a[7]*b[6]+lw(a[0]*b[6])*0xffffffff, reduction step [modulo-scheduled]
 662         &movdqa ("xmm1",&QWP(0x10,"ebx"));
 663         &pmuludq("xmm0","xmm7");                # a[0]*b[7]
 664          &movdqa(&QWP(0x50,"esp"),"xmm2");
 665
 666         &movdqa ("xmm2",&QWP(0x20,"ebx"));
 667         &pmuludq("xmm1","xmm7");                # a[1]*b[7]
 668          &movdqa(&QWP(0x60,"esp"),"xmm3");
 669         &paddq  ("xmm0",&QWP(0x00,"esp"));
 670
 671         &movdqa ("xmm3",&QWP(0x30,"ebx"));
 672         &pmuludq("xmm2","xmm7");                # a[2]*b[7]
 673          &movq  ("xmm4","xmm0");                # clear upper 64 bits
 674          &pslldq("xmm4",6);
 675         &paddq  ("xmm1",&QWP(0x10,"esp"));
 676          &paddq ("xmm4","xmm0");
 677          &movdqa("xmm5","xmm4");
 678          &psrldq("xmm4",10);                    # upper 33 bits of a[0]*b[i]+t[0]
 679
 680         &movdqa ("xmm0",&QWP(0x40,"ebx"));
 681         &pmuludq("xmm3","xmm7");                # a[3]*b[7]
 682          &paddq ("xmm1","xmm4");                # a[1]*b[7]+hw(a[0]*b[7]), carry
 683         &paddq  ("xmm2",&QWP(0x20,"esp"));
 684         &movdqa (&QWP(0x00,"esp"),"xmm1");
 685
 686         &movdqa ("xmm1",&QWP(0x50,"ebx"));
 687         &pmuludq("xmm0","xmm7");                # a[4]*b[7]
 688         &paddq  ("xmm3",&QWP(0x30,"esp"));
 689         &movdqa (&QWP(0x10,"esp"),"xmm2");
 690          &pand  ("xmm5","xmm6");                # lower 32 bits of a[0]*b[i]
 691
 692         &movdqa ("xmm2",&QWP(0x60,"ebx"));
 693         &pmuludq("xmm1","xmm7");                # a[5]*b[7]
 694          &paddq ("xmm3","xmm5");                # reduction step
 695         &paddq  ("xmm0",&QWP(0x40,"esp"));
 696         &movdqa (&QWP(0x20,"esp"),"xmm3");
 697          &pshufd("xmm4","xmm5",0b10110001);     # xmm4 = xmm5<<32, reduction step
 698
 699         &movdqa ("xmm3",&QWP(0x70,"ebx"));
 700         &pmuludq("xmm2","xmm7");                # a[6]*b[7]
 701         &paddq  ("xmm1",&QWP(0x50,"esp"));
 702          &psubq ("xmm4","xmm5");                # xmm4 = xmm5*0xffffffff, reduction step
 703         &movdqa (&QWP(0x30,"esp"),"xmm0");
 704
 705         &pmuludq("xmm3","xmm7");                # a[7]*b[7]
 706         &pcmpeqd("xmm7","xmm7");
 707         &movdqa ("xmm0",&QWP(0x00,"esp"));
 708         &pslldq ("xmm7",8);
 709         &movdqa (&QWP(0x40,"esp"),"xmm1");
 710         &paddq  ("xmm2",&QWP(0x60,"esp"));
 711
 712          &paddq ("xmm2","xmm5");                # a[6]*b[7]+lw(a[0]*b[7]), reduction step
 713          &paddq ("xmm3","xmm4");                # a[6]*b[7]+lw(a[0]*b[7])*0xffffffff, reduction step
 714          &movdqa(&QWP(0x50,"esp"),"xmm2");
 715          &movdqa(&QWP(0x60,"esp"),"xmm3");
 716
 717         &movdqa ("xmm1",&QWP(0x10,"esp"));
 718         &movdqa ("xmm2",&QWP(0x20,"esp"));
 719         &movdqa ("xmm3",&QWP(0x30,"esp"));
 720
 721         &movq   ("xmm4","xmm0");                # "flatten"
 722         &pand   ("xmm0","xmm7");
 723         &xor    ("ebp","ebp");
 724         &pslldq ("xmm4",6);
 725          &movq  ("xmm5","xmm1");
 726         &paddq  ("xmm0","xmm4");
 727          &pand  ("xmm1","xmm7");
 728         &psrldq ("xmm0",6);
 729         &movd   ("eax","xmm0");
 730         &psrldq ("xmm0",4);
 731
 732         &paddq  ("xmm5","xmm0");
 733         &movdqa ("xmm0",&QWP(0x40,"esp"));
 734         &sub    ("eax",-1);                     # start subtracting modulus,
 735                                                 # this is used to determine
 736                                                 # if result is larger/smaller
 737                                                 # than modulus (see below)
 738         &pslldq ("xmm5",6);
 739          &movq  ("xmm4","xmm2");
 740         &paddq  ("xmm1","xmm5");
 741          &pand  ("xmm2","xmm7");
 742         &psrldq ("xmm1",6);
 743         &mov    (&DWP(4*0,"edi"),"eax");
 744         &movd   ("eax","xmm1");
 745         &psrldq ("xmm1",4);
 746
 747         &paddq  ("xmm4","xmm1");
 748         &movdqa ("xmm1",&QWP(0x50,"esp"));
 749         &sbb    ("eax",-1);
 750         &pslldq ("xmm4",6);
 751          &movq  ("xmm5","xmm3");
 752         &paddq  ("xmm2","xmm4");
 753          &pand  ("xmm3","xmm7");
 754         &psrldq ("xmm2",6);
 755         &mov    (&DWP(4*1,"edi"),"eax");
 756         &movd   ("eax","xmm2");
 757         &psrldq ("xmm2",4);
 758
 759         &paddq  ("xmm5","xmm2");
 760         &movdqa ("xmm2",&QWP(0x60,"esp"));
 761         &sbb    ("eax",-1);
 762         &pslldq ("xmm5",6);
 763          &movq  ("xmm4","xmm0");
 764         &paddq  ("xmm3","xmm5");
 765          &pand  ("xmm0","xmm7");
 766         &psrldq ("xmm3",6);
 767         &mov    (&DWP(4*2,"edi"),"eax");
 768         &movd   ("eax","xmm3");
 769         &psrldq ("xmm3",4);
 770
 771         &paddq  ("xmm4","xmm3");
 772         &sbb    ("eax",0);
 773         &pslldq ("xmm4",6);
 774          &movq  ("xmm5","xmm1");
 775         &paddq  ("xmm0","xmm4");
 776          &pand  ("xmm1","xmm7");
 777         &psrldq ("xmm0",6);
 778         &mov    (&DWP(4*3,"edi"),"eax");
 779         &movd   ("eax","xmm0");
 780         &psrldq ("xmm0",4);
 781
 782         &paddq  ("xmm5","xmm0");
 783         &sbb    ("eax",0);
 784         &pslldq ("xmm5",6);
 785          &movq  ("xmm4","xmm2");
 786         &paddq  ("xmm1","xmm5");
 787          &pand  ("xmm2","xmm7");
 788         &psrldq ("xmm1",6);
 789         &movd   ("ebx","xmm1");
 790         &psrldq ("xmm1",4);
 791         &mov    ("esp","edx");
 792
 793         &paddq  ("xmm4","xmm1");
 794         &pslldq ("xmm4",6);
 795         &paddq  ("xmm2","xmm4");
 796         &psrldq ("xmm2",6);
 797         &movd   ("ecx","xmm2");
 798         &psrldq ("xmm2",4);
 799         &sbb    ("ebx",0);
 800         &movd   ("edx","xmm2");
 801         &pextrw ("esi","xmm2",2);               # top-most overflow bit
 802         &sbb    ("ecx",1);
 803         &sbb    ("edx",-1);
 804         &sbb    ("esi",0);                      # borrow from subtraction
 805
 806         # Final step is "if result > mod, subtract mod", and at this point
 807         # we have result - mod written to output buffer, as well as borrow
 808         # bit from this subtraction, and if borrow bit is set, we add
 809         # modulus back.
 810         #
 811         # Note that because mod has special form, i.e. consists of
 812         # 0xffffffff, 1 and 0s, we can conditionally synthesize it by
 813         # assigning borrow bit to one register, %ebp, and its negative
 814         # to another, %esi. But we started by calculating %esi...
 815
 816         &sub    ("ebp","esi");
 817         &add    (&DWP(4*0,"edi"),"esi");        # add modulus or zero
 818         &adc    (&DWP(4*1,"edi"),"esi");
 819         &adc    (&DWP(4*2,"edi"),"esi");
 820         &adc    (&DWP(4*3,"edi"),0);
 821         &adc    ("eax",0);
 822         &adc    ("ebx",0);
 823         &mov    (&DWP(4*4,"edi"),"eax");
 824         &adc    ("ecx","ebp");
 825         &mov    (&DWP(4*5,"edi"),"ebx");
 826         &adc    ("edx","esi");
 827         &mov    (&DWP(4*6,"edi"),"ecx");
 828         &mov    (&DWP(4*7,"edi"),"edx");
 829
 830         &ret    ();
 831
 832 &set_label("mul_mont_ialu",16);                 }
 833
 834         ########################################
 835         # IALU code path suitable for all CPUs.
 836         ########################################
 837         # stack layout:
 838         # +------------------------------------+< %esp
 839         # | 8 32-bit temporary words, accessed |
 840         # | as circular buffer                 |
 841         # .                                    .
 842         # .                                    .
 843         # +------------------------------------+< +32
 844         # | offloaded destination pointer      |
 845         # +------------------------------------+
 846         # | unused                             |
 847         # +------------------------------------+< +40
 848         &sub    ("esp",10*4);
 849
 850         &mov    ("eax",&DWP(0*4,"esi"));                # a[0]
 851         &mov    ("ebx",&DWP(0*4,"ebp"));                # b[0]
 852         &mov    (&DWP(8*4,"esp"),"edi");                # off-load dst ptr
 853
 854         &mul    ("ebx");                                # a[0]*b[0]
 855         &mov    (&DWP(0*4,"esp"),"eax");                # t[0]
 856         &mov    ("eax",&DWP(1*4,"esi"));
 857         &mov    ("ecx","edx")
 858
 859         &mul    ("ebx");                                # a[1]*b[0]
 860         &add    ("ecx","eax");
 861         &mov    ("eax",&DWP(2*4,"esi"));
 862         &adc    ("edx",0);
 863         &mov    (&DWP(1*4,"esp"),"ecx");                # t[1]
 864         &mov    ("ecx","edx");
 865
 866         &mul    ("ebx");                                # a[2]*b[0]
 867         &add    ("ecx","eax");
 868         &mov    ("eax",&DWP(3*4,"esi"));
 869         &adc    ("edx",0);
 870         &mov    (&DWP(2*4,"esp"),"ecx");                # t[2]
 871         &mov    ("ecx","edx");
 872
 873         &mul    ("ebx");                                # a[3]*b[0]
 874         &add    ("ecx","eax");
 875         &mov    ("eax",&DWP(4*4,"esi"));
 876         &adc    ("edx",0);
 877         &mov    (&DWP(3*4,"esp"),"ecx");                # t[3]
 878         &mov    ("ecx","edx");
 879
 880         &mul    ("ebx");                                # a[4]*b[0]
 881         &add    ("ecx","eax");
 882         &mov    ("eax",&DWP(5*4,"esi"));
 883         &adc    ("edx",0);
 884         &mov    (&DWP(4*4,"esp"),"ecx");                # t[4]
 885         &mov    ("ecx","edx");
 886
 887         &mul    ("ebx");                                # a[5]*b[0]
 888         &add    ("ecx","eax");
 889         &mov    ("eax",&DWP(6*4,"esi"));
 890         &adc    ("edx",0);
 891         &mov    (&DWP(5*4,"esp"),"ecx");                # t[5]
 892         &mov    ("ecx","edx");
 893
 894         &mul    ("ebx");                                # a[6]*b[0]
 895         &add    ("ecx","eax");
 896         &mov    ("eax",&DWP(7*4,"esi"));
 897         &adc    ("edx",0);
 898         &mov    (&DWP(6*4,"esp"),"ecx");                # t[6]
 899         &mov    ("ecx","edx");
 900
 901         &xor    ("edi","edi");                          # initial top-most carry
 902         &mul    ("ebx");                                # a[7]*b[0]
 903         &add    ("ecx","eax");                          # t[7]
 904         &mov    ("eax",&DWP(0*4,"esp"));                # t[0]
 905         &adc    ("edx",0);                              # t[8]
 906
 907 for ($i=0;$i<7;$i++) {
 908         my $j=$i+1;
 909
 910         # Reduction iteration is normally performed by accumulating
 911         # result of multiplication of modulus by "magic" digit [and
 912         # omitting least significant word, which is guaranteed to
 913         # be 0], but thanks to special form of modulus and "magic"
 914         # digit being equal to least significant word, it can be
 915         # performed with additions and subtractions alone. Indeed:
 916         #
 917         #        ffff.0001.0000.0000.0000.ffff.ffff.ffff
 918         # *                                         abcd
 919         # + xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.abcd
 920         #
 921         # Now observing that ff..ff*x = (2^n-1)*x = 2^n*x-x, we
 922         # rewrite above as:
 923         #
 924         #   xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.abcd
 925         # + abcd.0000.abcd.0000.0000.abcd.0000.0000.0000
 926         # -      abcd.0000.0000.0000.0000.0000.0000.abcd
 927         #
 928         # or marking redundant operations:
 929         #
 930         #   xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.xxxx.----
 931         # + abcd.0000.abcd.0000.0000.abcd.----.----.----
 932         # -      abcd.----.----.----.----.----.----.----
 933
 934         &add    (&DWP((($i+3)%8)*4,"esp"),"eax");       # t[3]+=t[0]
 935         &adc    (&DWP((($i+4)%8)*4,"esp"),0);           # t[4]+=0
 936         &adc    (&DWP((($i+5)%8)*4,"esp"),0);           # t[5]+=0
 937         &adc    (&DWP((($i+6)%8)*4,"esp"),"eax");       # t[6]+=t[0]
 938         &adc    ("ecx",0);                              # t[7]+=0
 939         &adc    ("edx","eax");                          # t[8]+=t[0]
 940         &adc    ("edi",0);                              # top-most carry
 941          &mov   ("ebx",&DWP($j*4,"ebp"));               # b[i]
 942         &sub    ("ecx","eax");                          # t[7]-=t[0]
 943          &mov   ("eax",&DWP(0*4,"esi"));                # a[0]
 944         &sbb    ("edx",0);                              # t[8]-=0
 945         &mov    (&DWP((($i+7)%8)*4,"esp"),"ecx");
 946         &sbb    ("edi",0);                              # top-most carry,
 947                                                         # keep in mind that
 948                                                         # netto result is
 949                                                         # *addition* of value
 950                                                         # with (abcd<<32)-abcd
 951                                                         # on top, so that
 952                                                         # underflow is
 953                                                         # impossible, because
 954                                                         # (abcd<<32)-abcd
 955                                                         # doesn't underflow
 956         &mov    (&DWP((($i+8)%8)*4,"esp"),"edx");
 957
 958         &mul    ("ebx");                                # a[0]*b[i]
 959         &add    ("eax",&DWP((($j+0)%8)*4,"esp"));
 960         &adc    ("edx",0);
 961         &mov    (&DWP((($j+0)%8)*4,"esp"),"eax");
 962         &mov    ("eax",&DWP(1*4,"esi"));
 963         &mov    ("ecx","edx")
 964
 965         &mul    ("ebx");                                # a[1]*b[i]
 966         &add    ("ecx",&DWP((($j+1)%8)*4,"esp"));
 967         &adc    ("edx",0);
 968         &add    ("ecx","eax");
 969         &adc    ("edx",0);
 970         &mov    ("eax",&DWP(2*4,"esi"));
 971         &mov    (&DWP((($j+1)%8)*4,"esp"),"ecx");
 972         &mov    ("ecx","edx");
 973
 974         &mul    ("ebx");                                # a[2]*b[i]
 975         &add    ("ecx",&DWP((($j+2)%8)*4,"esp"));
 976         &adc    ("edx",0);
 977         &add    ("ecx","eax");
 978         &adc    ("edx",0);
 979         &mov    ("eax",&DWP(3*4,"esi"));
 980         &mov    (&DWP((($j+2)%8)*4,"esp"),"ecx");
 981         &mov    ("ecx","edx");
 982
 983         &mul    ("ebx");                                # a[3]*b[i]
 984         &add    ("ecx",&DWP((($j+3)%8)*4,"esp"));
 985         &adc    ("edx",0);
 986         &add    ("ecx","eax");
 987         &adc    ("edx",0);
 988         &mov    ("eax",&DWP(4*4,"esi"));
 989         &mov    (&DWP((($j+3)%8)*4,"esp"),"ecx");
 990         &mov    ("ecx","edx");
 991
 992         &mul    ("ebx");                                # a[4]*b[i]
 993         &add    ("ecx",&DWP((($j+4)%8)*4,"esp"));
 994         &adc    ("edx",0);
 995         &add    ("ecx","eax");
 996         &adc    ("edx",0);
 997         &mov    ("eax",&DWP(5*4,"esi"));
 998         &mov    (&DWP((($j+4)%8)*4,"esp"),"ecx");
 999         &mov    ("ecx","edx");
1000
1001         &mul    ("ebx");                                # a[5]*b[i]
1002         &add    ("ecx",&DWP((($j+5)%8)*4,"esp"));
1003         &adc    ("edx",0);
1004         &add    ("ecx","eax");
1005         &adc    ("edx",0);
1006         &mov    ("eax",&DWP(6*4,"esi"));
1007         &mov    (&DWP((($j+5)%8)*4,"esp"),"ecx");
1008         &mov    ("ecx","edx");
1009
1010         &mul    ("ebx");                                # a[6]*b[i]
1011         &add    ("ecx",&DWP((($j+6)%8)*4,"esp"));
1012         &adc    ("edx",0);
1013         &add    ("ecx","eax");
1014         &adc    ("edx",0);
1015         &mov    ("eax",&DWP(7*4,"esi"));
1016         &mov    (&DWP((($j+6)%8)*4,"esp"),"ecx");
1017         &mov    ("ecx","edx");
1018
1019         &mul    ("ebx");                                # a[7]*b[i]
1020         &add    ("ecx",&DWP((($j+7)%8)*4,"esp"));
1021         &adc    ("edx",0);
1022         &add    ("ecx","eax");                          # t[7]
1023         &mov    ("eax",&DWP((($j+0)%8)*4,"esp"));       # t[0]
1024         &adc    ("edx","edi");                          # t[8]
1025         &mov    ("edi",0);
1026         &adc    ("edi",0);                              # top-most carry
1027 }
1028         &mov    ("ebp",&DWP(8*4,"esp"));                # restore dst ptr
1029         &xor    ("esi","esi");
1030         my $j=$i+1;
1031
1032         # last multiplication-less reduction
1033         &add    (&DWP((($i+3)%8)*4,"esp"),"eax");       # t[3]+=t[0]
1034         &adc    (&DWP((($i+4)%8)*4,"esp"),0);           # t[4]+=0
1035         &adc    (&DWP((($i+5)%8)*4,"esp"),0);           # t[5]+=0
1036         &adc    (&DWP((($i+6)%8)*4,"esp"),"eax");       # t[6]+=t[0]
1037         &adc    ("ecx",0);                              # t[7]+=0
1038         &adc    ("edx","eax");                          # t[8]+=t[0]
1039         &adc    ("edi",0);                              # top-most carry
1040          &mov   ("ebx",&DWP((($j+1)%8)*4,"esp"));
1041         &sub    ("ecx","eax");                          # t[7]-=t[0]
1042          &mov   ("eax",&DWP((($j+0)%8)*4,"esp"));
1043         &sbb    ("edx",0);                              # t[8]-=0
1044         &mov    (&DWP((($i+7)%8)*4,"esp"),"ecx");
1045         &sbb    ("edi",0);                              # top-most carry
1046         &mov    (&DWP((($i+8)%8)*4,"esp"),"edx");
1047
1048         # Final step is "if result > mod, subtract mod", but we do it
1049         # "other way around", namely write result - mod to output buffer
1050         # and if subtraction borrowed, add modulus back.
1051
1052         &mov    ("ecx",&DWP((($j+2)%8)*4,"esp"));
1053         &sub    ("eax",-1);
1054         &mov    ("edx",&DWP((($j+3)%8)*4,"esp"));
1055         &sbb    ("ebx",-1);
1056         &mov    (&DWP(0*4,"ebp"),"eax");
1057         &sbb    ("ecx",-1);
1058         &mov    (&DWP(1*4,"ebp"),"ebx");
1059         &sbb    ("edx",0);
1060         &mov    (&DWP(2*4,"ebp"),"ecx");
1061         &mov    (&DWP(3*4,"ebp"),"edx");
1062
1063         &mov    ("eax",&DWP((($j+4)%8)*4,"esp"));
1064         &mov    ("ebx",&DWP((($j+5)%8)*4,"esp"));
1065         &mov    ("ecx",&DWP((($j+6)%8)*4,"esp"));
1066         &sbb    ("eax",0);
1067         &mov    ("edx",&DWP((($j+7)%8)*4,"esp"));
1068         &sbb    ("ebx",0);
1069         &sbb    ("ecx",1);
1070         &sbb    ("edx",-1);
1071         &sbb    ("edi",0);
1072
1073         # Note that because mod has special form, i.e. consists of
1074         # 0xffffffff, 1 and 0s, we can conditionally synthesize it by
1075         # assigning borrow bit to one register, %ebp, and its negative
1076         # to another, %esi. But we started by calculating %esi...
1077
1078         &sub    ("esi","edi");
1079         &add    (&DWP(0*4,"ebp"),"edi");                # add modulus or zero
1080         &adc    (&DWP(1*4,"ebp"),"edi");
1081         &adc    (&DWP(2*4,"ebp"),"edi");
1082         &adc    (&DWP(3*4,"ebp"),0);
1083         &adc    ("eax",0);
1084         &adc    ("ebx",0);
1085         &mov    (&DWP(4*4,"ebp"),"eax");
1086         &adc    ("ecx","esi");
1087         &mov    (&DWP(5*4,"ebp"),"ebx");
1088         &adc    ("edx","edi");
1089         &mov    (&DWP(6*4,"ebp"),"ecx");
1090         &mov    ("edi","ebp");                          # fulfill contract
1091         &mov    (&DWP(7*4,"ebp"),"edx");
1092
1093         &add    ("esp",10*4);
1094         &ret    ();
1095 &function_end_B("_ecp_nistz256_mul_mont");
1096
1097 ########################################################################
1098 # void ecp_nistz256_scatter_w5(void *edi,const P256_POINT *esi,
1099 #                                        int ebp);
1100 &function_begin("ecp_nistz256_scatter_w5");
1101         &mov    ("edi",&wparam(0));
1102         &mov    ("esi",&wparam(1));
1103         &mov    ("ebp",&wparam(2));
1104
1105         &lea    ("edi",&DWP(128-4,"edi","ebp",4));
1106         &mov    ("ebp",96/16);
1107 &set_label("scatter_w5_loop");
1108         &mov    ("eax",&DWP(0,"esi"));
1109         &mov    ("ebx",&DWP(4,"esi"));
1110         &mov    ("ecx",&DWP(8,"esi"));
1111         &mov    ("edx",&DWP(12,"esi"));
1112         &lea    ("esi",&DWP(16,"esi"));
1113         &mov    (&DWP(64*0-128,"edi"),"eax");
1114         &mov    (&DWP(64*1-128,"edi"),"ebx");
1115         &mov    (&DWP(64*2-128,"edi"),"ecx");
1116         &mov    (&DWP(64*3-128,"edi"),"edx");
1117         &lea    ("edi",&DWP(64*4,"edi"));
1118         &dec    ("ebp");
1119         &jnz    (&label("scatter_w5_loop"));
1120 &function_end("ecp_nistz256_scatter_w5");
1121
1122 ########################################################################
1123 # void ecp_nistz256_gather_w5(P256_POINT *edi,const void *esi,
1124 #                                             int ebp);
1125 &function_begin("ecp_nistz256_gather_w5");
1126         &mov    ("esi",&wparam(1));
1127         &mov    ("ebp",&wparam(2));
1128
1129         &lea    ("esi",&DWP(0,"esi","ebp",4));
1130         &neg    ("ebp");
1131         &sar    ("ebp",31);
1132         &mov    ("edi",&wparam(0));
1133         &lea    ("esi",&DWP(0,"esi","ebp",4));
1134
1135     for($i=0;$i<24;$i+=4) {
1136         &mov    ("eax",&DWP(64*($i+0),"esi"));
1137         &mov    ("ebx",&DWP(64*($i+1),"esi"));
1138         &mov    ("ecx",&DWP(64*($i+2),"esi"));
1139         &mov    ("edx",&DWP(64*($i+3),"esi"));
1140         &and    ("eax","ebp");
1141         &and    ("ebx","ebp");
1142         &and    ("ecx","ebp");
1143         &and    ("edx","ebp");
1144         &mov    (&DWP(4*($i+0),"edi"),"eax");
1145         &mov    (&DWP(4*($i+1),"edi"),"ebx");
1146         &mov    (&DWP(4*($i+2),"edi"),"ecx");
1147         &mov    (&DWP(4*($i+3),"edi"),"edx");
1148     }
1149 &function_end("ecp_nistz256_gather_w5");
1150
1151 ########################################################################
1152 # void ecp_nistz256_scatter_w7(void *edi,const P256_POINT_AFFINE *esi,
1153 #                                        int ebp);
1154 &function_begin("ecp_nistz256_scatter_w7");
1155         &mov    ("edi",&wparam(0));
1156         &mov    ("esi",&wparam(1));
1157         &mov    ("ebp",&wparam(2));
1158
1159         &lea    ("edi",&DWP(-1,"edi","ebp"));
1160         &mov    ("ebp",64/4);
1161 &set_label("scatter_w7_loop");
1162         &mov    ("eax",&DWP(0,"esi"));
1163         &lea    ("esi",&DWP(4,"esi"));
1164         &mov    (&BP(64*0,"edi"),"al");
1165         &mov    (&BP(64*1,"edi"),"ah");
1166         &shr    ("eax",16);
1167         &mov    (&BP(64*2,"edi"),"al");
1168         &mov    (&BP(64*3,"edi"),"ah");
1169         &lea    ("edi",&DWP(64*4,"edi"));
1170         &dec    ("ebp");
1171         &jnz    (&label("scatter_w7_loop"));
1172 &function_end("ecp_nistz256_scatter_w7");
1173
1174 ########################################################################
1175 # void ecp_nistz256_gather_w7(P256_POINT_AFFINE *edi,const void *esi,
1176 #                                                    int ebp);
1177 &function_begin("ecp_nistz256_gather_w7");
1178         &mov    ("esi",&wparam(1));
1179         &mov    ("ebp",&wparam(2));
1180
1181         &add    ("esi","ebp");
1182         &neg    ("ebp"),
1183         &sar    ("ebp",31);
1184         &mov    ("edi",&wparam(0));
1185         &lea    ("esi",&DWP(0,"esi","ebp"));
1186
1187     for($i=0;$i<64;$i+=4) {
1188         &movz   ("eax",&BP(64*($i+0),"esi"));
1189         &movz   ("ebx",&BP(64*($i+1),"esi"));
1190         &movz   ("ecx",&BP(64*($i+2),"esi"));
1191         &and    ("eax","ebp");
1192         &movz   ("edx",&BP(64*($i+3),"esi"));
1193         &and    ("ebx","ebp");
1194         &mov    (&BP($i+0,"edi"),"al");
1195         &and    ("ecx","ebp");
1196         &mov    (&BP($i+1,"edi"),"bl");
1197         &and    ("edx","ebp");
1198         &mov    (&BP($i+2,"edi"),"cl");
1199         &mov    (&BP($i+3,"edi"),"dl");
1200     }
1201 &function_end("ecp_nistz256_gather_w7");
1202
1203 ########################################################################
1204 # following subroutines are "literal" implementation of those found in
1205 # ecp_nistz256.c
1206 #
1207 ########################################################################
1208 # void ecp_nistz256_point_double(P256_POINT *out,const P256_POINT *inp);
1209 #
1210 &static_label("point_double_shortcut");
1211 &function_begin("ecp_nistz256_point_double");
1212 {   my ($S,$M,$Zsqr,$in_x,$tmp0)=map(32*$_,(0..4));
1213
1214         &mov    ("esi",&wparam(1));
1215
1216         # above map() describes stack layout with 5 temporary
1217         # 256-bit vectors on top, then we take extra word for
1218         # OPENSSL_ia32cap_P copy.
1219         &stack_push(8*5+1);
1220                                                 if ($sse2) {
1221         &call   ("_picup_eax");
1222     &set_label("pic");
1223         &picmeup("edx","OPENSSL_ia32cap_P","eax",&label("pic"));
1224         &mov    ("ebp",&DWP(0,"edx"));          }
1225
1226 &set_label("point_double_shortcut");
1227         &mov    ("eax",&DWP(0,"esi"));          # copy in_x
1228         &mov    ("ebx",&DWP(4,"esi"));
1229         &mov    ("ecx",&DWP(8,"esi"));
1230         &mov    ("edx",&DWP(12,"esi"));
1231         &mov    (&DWP($in_x+0,"esp"),"eax");
1232         &mov    (&DWP($in_x+4,"esp"),"ebx");
1233         &mov    (&DWP($in_x+8,"esp"),"ecx");
1234         &mov    (&DWP($in_x+12,"esp"),"edx");
1235         &mov    ("eax",&DWP(16,"esi"));
1236         &mov    ("ebx",&DWP(20,"esi"));
1237         &mov    ("ecx",&DWP(24,"esi"));
1238         &mov    ("edx",&DWP(28,"esi"));
1239         &mov    (&DWP($in_x+16,"esp"),"eax");
1240         &mov    (&DWP($in_x+20,"esp"),"ebx");
1241         &mov    (&DWP($in_x+24,"esp"),"ecx");
1242         &mov    (&DWP($in_x+28,"esp"),"edx");
1243         &mov    (&DWP(32*5,"esp"),"ebp");       # OPENSSL_ia32cap_P copy
1244
1245         &lea    ("ebp",&DWP(32,"esi"));
1246         &lea    ("esi",&DWP(32,"esi"));
1247         &lea    ("edi",&DWP($S,"esp"));
1248         &call   ("_ecp_nistz256_add");          # p256_mul_by_2(S, in_y);
1249
1250         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1251         &mov    ("esi",64);
1252         &add    ("esi",&wparam(1));
1253         &lea    ("edi",&DWP($Zsqr,"esp"));
1254         &mov    ("ebp","esi");
1255         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Zsqr, in_z);
1256
1257         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1258         &lea    ("esi",&DWP($S,"esp"));
1259         &lea    ("ebp",&DWP($S,"esp"));
1260         &lea    ("edi",&DWP($S,"esp"));
1261         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(S, S);
1262
1263         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1264         &mov    ("ebp",&wparam(1));
1265         &lea    ("esi",&DWP(32,"ebp"));
1266         &lea    ("ebp",&DWP(64,"ebp"));
1267         &lea    ("edi",&DWP($tmp0,"esp"));
1268         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(tmp0, in_z, in_y);
1269
1270         &lea    ("esi",&DWP($in_x,"esp"));
1271         &lea    ("ebp",&DWP($Zsqr,"esp"));
1272         &lea    ("edi",&DWP($M,"esp"));
1273         &call   ("_ecp_nistz256_add");          # p256_add(M, in_x, Zsqr);
1274
1275         &mov    ("edi",64);
1276         &lea    ("esi",&DWP($tmp0,"esp"));
1277         &lea    ("ebp",&DWP($tmp0,"esp"));
1278         &add    ("edi",&wparam(0));
1279         &call   ("_ecp_nistz256_add");          # p256_mul_by_2(res_z, tmp0);
1280
1281         &lea    ("esi",&DWP($in_x,"esp"));
1282         &lea    ("ebp",&DWP($Zsqr,"esp"));
1283         &lea    ("edi",&DWP($Zsqr,"esp"));
1284         &call   ("_ecp_nistz256_sub");          # p256_sub(Zsqr, in_x, Zsqr);
1285
1286         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1287         &lea    ("esi",&DWP($S,"esp"));
1288         &lea    ("ebp",&DWP($S,"esp"));
1289         &lea    ("edi",&DWP($tmp0,"esp"));
1290         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(tmp0, S);
1291
1292         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1293         &lea    ("esi",&DWP($M,"esp"));
1294         &lea    ("ebp",&DWP($Zsqr,"esp"));
1295         &lea    ("edi",&DWP($M,"esp"));
1296         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(M, M, Zsqr);
1297
1298         &mov    ("edi",32);
1299         &lea    ("esi",&DWP($tmp0,"esp"));
1300         &add    ("edi",&wparam(0));
1301         &call   ("_ecp_nistz256_div_by_2");     # p256_div_by_2(res_y, tmp0);
1302
1303         &lea    ("esi",&DWP($M,"esp"));
1304         &lea    ("ebp",&DWP($M,"esp"));
1305         &lea    ("edi",&DWP($tmp0,"esp"));
1306         &call   ("_ecp_nistz256_add");          # 1/2 p256_mul_by_3(M, M);
1307
1308         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1309         &lea    ("esi",&DWP($in_x,"esp"));
1310         &lea    ("ebp",&DWP($S,"esp"));
1311         &lea    ("edi",&DWP($S,"esp"));
1312         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S, S, in_x);
1313
1314         &lea    ("esi",&DWP($tmp0,"esp"));
1315         &lea    ("ebp",&DWP($M,"esp"));
1316         &lea    ("edi",&DWP($M,"esp"));
1317         &call   ("_ecp_nistz256_add");          # 2/2 p256_mul_by_3(M, M);
1318
1319         &lea    ("esi",&DWP($S,"esp"));
1320         &lea    ("ebp",&DWP($S,"esp"));
1321         &lea    ("edi",&DWP($tmp0,"esp"));
1322         &call   ("_ecp_nistz256_add");          # p256_mul_by_2(tmp0, S);
1323
1324         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1325         &lea    ("esi",&DWP($M,"esp"));
1326         &lea    ("ebp",&DWP($M,"esp"));
1327         &mov    ("edi",&wparam(0));
1328         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(res_x, M);
1329
1330         &mov    ("esi","edi");                  # %edi is still res_x here
1331         &lea    ("ebp",&DWP($tmp0,"esp"));
1332         &call   ("_ecp_nistz256_sub");          # p256_sub(res_x, res_x, tmp0);
1333
1334         &lea    ("esi",&DWP($S,"esp"));
1335         &mov    ("ebp","edi");                  # %edi is still res_x
1336         &lea    ("edi",&DWP($S,"esp"));
1337         &call   ("_ecp_nistz256_sub");          # p256_sub(S, S, res_x);
1338
1339         &mov    ("eax",&DWP(32*5,"esp"));       # OPENSSL_ia32cap_P copy
1340         &mov    ("esi","edi");                  # %edi is still &S
1341         &lea    ("ebp",&DWP($M,"esp"));
1342         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S, S, M);
1343
1344         &mov    ("ebp",32);
1345         &lea    ("esi",&DWP($S,"esp"));
1346         &add    ("ebp",&wparam(0));
1347         &mov    ("edi","ebp");
1348         &call   ("_ecp_nistz256_sub");          # p256_sub(res_y, S, res_y);
1349
1350         &stack_pop(8*5+1);
1351 } &function_end("ecp_nistz256_point_double");
1352
1353 ########################################################################
1354 # void ecp_nistz256_point_add(P256_POINT *out,const P256_POINT *in1,
1355 #                                             const P256_POINT *in2);
1356 &function_begin("ecp_nistz256_point_add");
1357 {   my ($res_x,$res_y,$res_z,
1358         $in1_x,$in1_y,$in1_z,
1359         $in2_x,$in2_y,$in2_z,
1360         $H,$Hsqr,$R,$Rsqr,$Hcub,
1361         $U1,$U2,$S1,$S2)=map(32*$_,(0..17));
1362     my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
1363
1364         &mov    ("esi",&wparam(2));
1365
1366         # above map() describes stack layout with 18 temporary
1367         # 256-bit vectors on top, then we take extra words for
1368         # !in1infty, !in2infty, result of check for zero and
1369         # OPENSSL_ia32cap_P copy. [one unused word for padding]
1370         &stack_push(8*18+5);
1371                                                 if ($sse2) {
1372         &call   ("_picup_eax");
1373     &set_label("pic");
1374         &picmeup("edx","OPENSSL_ia32cap_P","eax",&label("pic"));
1375         &mov    ("ebp",&DWP(0,"edx"));          }
1376
1377         &lea    ("edi",&DWP($in2_x,"esp"));
1378     for($i=0;$i<96;$i+=16) {
1379         &mov    ("eax",&DWP($i+0,"esi"));       # copy in2
1380         &mov    ("ebx",&DWP($i+4,"esi"));
1381         &mov    ("ecx",&DWP($i+8,"esi"));
1382         &mov    ("edx",&DWP($i+12,"esi"));
1383         &mov    (&DWP($i+0,"edi"),"eax");
1384         &mov    (&DWP(32*18+12,"esp"),"ebp")    if ($i==0);
1385         &mov    ("ebp","eax")                   if ($i==0);
1386         &or     ("ebp","eax")                   if ($i!=0 && $i<64);
1387         &mov    (&DWP($i+4,"edi"),"ebx");
1388         &or     ("ebp","ebx")                   if ($i<64);
1389         &mov    (&DWP($i+8,"edi"),"ecx");
1390         &or     ("ebp","ecx")                   if ($i<64);
1391         &mov    (&DWP($i+12,"edi"),"edx");
1392         &or     ("ebp","edx")                   if ($i<64);
1393     }
1394         &xor    ("eax","eax");
1395         &mov    ("esi",&wparam(1));
1396         &sub    ("eax","ebp");
1397         &or     ("ebp","eax");
1398         &sar    ("ebp",31);
1399         &mov    (&DWP(32*18+4,"esp"),"ebp");    # !in2infty
1400
1401         &lea    ("edi",&DWP($in1_x,"esp"));
1402     for($i=0;$i<96;$i+=16) {
1403         &mov    ("eax",&DWP($i+0,"esi"));       # copy in1
1404         &mov    ("ebx",&DWP($i+4,"esi"));
1405         &mov    ("ecx",&DWP($i+8,"esi"));
1406         &mov    ("edx",&DWP($i+12,"esi"));
1407         &mov    (&DWP($i+0,"edi"),"eax");
1408         &mov    ("ebp","eax")                   if ($i==0);
1409         &or     ("ebp","eax")                   if ($i!=0 && $i<64);
1410         &mov    (&DWP($i+4,"edi"),"ebx");
1411         &or     ("ebp","ebx")                   if ($i<64);
1412         &mov    (&DWP($i+8,"edi"),"ecx");
1413         &or     ("ebp","ecx")                   if ($i<64);
1414         &mov    (&DWP($i+12,"edi"),"edx");
1415         &or     ("ebp","edx")                   if ($i<64);
1416     }
1417         &xor    ("eax","eax");
1418         &sub    ("eax","ebp");
1419         &or     ("ebp","eax");
1420         &sar    ("ebp",31);
1421         &mov    (&DWP(32*18+0,"esp"),"ebp");    # !in1infty
1422
1423         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1424         &lea    ("esi",&DWP($in2_z,"esp"));
1425         &lea    ("ebp",&DWP($in2_z,"esp"));
1426         &lea    ("edi",&DWP($Z2sqr,"esp"));
1427         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Z2sqr, in2_z);
1428
1429         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1430         &lea    ("esi",&DWP($in1_z,"esp"));
1431         &lea    ("ebp",&DWP($in1_z,"esp"));
1432         &lea    ("edi",&DWP($Z1sqr,"esp"));
1433         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Z1sqr, in1_z);
1434
1435         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1436         &lea    ("esi",&DWP($Z2sqr,"esp"));
1437         &lea    ("ebp",&DWP($in2_z,"esp"));
1438         &lea    ("edi",&DWP($S1,"esp"));
1439         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S1, Z2sqr, in2_z);
1440
1441         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1442         &lea    ("esi",&DWP($Z1sqr,"esp"));
1443         &lea    ("ebp",&DWP($in1_z,"esp"));
1444         &lea    ("edi",&DWP($S2,"esp"));
1445         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S2, Z1sqr, in1_z);
1446
1447         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1448         &lea    ("esi",&DWP($in1_y,"esp"));
1449         &lea    ("ebp",&DWP($S1,"esp"));
1450         &lea    ("edi",&DWP($S1,"esp"));
1451         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S1, S1, in1_y);
1452
1453         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1454         &lea    ("esi",&DWP($in2_y,"esp"));
1455         &lea    ("ebp",&DWP($S2,"esp"));
1456         &lea    ("edi",&DWP($S2,"esp"));
1457         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S2, S2, in2_y);
1458
1459         &lea    ("esi",&DWP($S2,"esp"));
1460         &lea    ("ebp",&DWP($S1,"esp"));
1461         &lea    ("edi",&DWP($R,"esp"));
1462         &call   ("_ecp_nistz256_sub");          # p256_sub(R, S2, S1);
1463
1464         &or     ("ebx","eax");                  # see if result is zero
1465         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1466         &or     ("ebx","ecx");
1467         &or     ("ebx","edx");
1468         &or     ("ebx",&DWP(0,"edi"));
1469         &or     ("ebx",&DWP(4,"edi"));
1470          &lea   ("esi",&DWP($in1_x,"esp"));
1471         &or     ("ebx",&DWP(8,"edi"));
1472          &lea   ("ebp",&DWP($Z2sqr,"esp"));
1473         &or     ("ebx",&DWP(12,"edi"));
1474          &lea   ("edi",&DWP($U1,"esp"));
1475         &mov    (&DWP(32*18+8,"esp"),"ebx");
1476
1477         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(U1, in1_x, Z2sqr);
1478
1479         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1480         &lea    ("esi",&DWP($in2_x,"esp"));
1481         &lea    ("ebp",&DWP($Z1sqr,"esp"));
1482         &lea    ("edi",&DWP($U2,"esp"));
1483         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(U2, in2_x, Z1sqr);
1484
1485         &lea    ("esi",&DWP($U2,"esp"));
1486         &lea    ("ebp",&DWP($U1,"esp"));
1487         &lea    ("edi",&DWP($H,"esp"));
1488         &call   ("_ecp_nistz256_sub");          # p256_sub(H, U2, U1);
1489
1490         &or     ("eax","ebx");                  # see if result is zero
1491         &or     ("eax","ecx");
1492         &or     ("eax","edx");
1493         &or     ("eax",&DWP(0,"edi"));
1494         &or     ("eax",&DWP(4,"edi"));
1495         &or     ("eax",&DWP(8,"edi"));
1496         &or     ("eax",&DWP(12,"edi"));
1497
1498         &data_byte(0x3e);                       # predict taken
1499         &jnz    (&label("add_proceed"));        # is_equal(U1,U2)?
1500
1501         &mov    ("eax",&DWP(32*18+0,"esp"));
1502         &and    ("eax",&DWP(32*18+4,"esp"));
1503         &mov    ("ebx",&DWP(32*18+8,"esp"));
1504         &jz     (&label("add_proceed"));        # (in1infty || in2infty)?
1505         &test   ("ebx","ebx");
1506         &jz     (&label("add_double"));         # is_equal(S1,S2)?
1507
1508         &mov    ("edi",&wparam(0));
1509         &xor    ("eax","eax");
1510         &mov    ("ecx",96/4);
1511         &data_byte(0xfc,0xf3,0xab);             # cld; stosd
1512         &jmp    (&label("add_done"));
1513
1514 &set_label("add_double",16);
1515         &mov    ("esi",&wparam(1));
1516         &mov    ("ebp",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1517         &add    ("esp",4*((8*18+5)-(8*5+1)));   # difference in frame sizes
1518         &jmp    (&label("point_double_shortcut"));
1519
1520 &set_label("add_proceed",16);
1521         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1522         &lea    ("esi",&DWP($R,"esp"));
1523         &lea    ("ebp",&DWP($R,"esp"));
1524         &lea    ("edi",&DWP($Rsqr,"esp"));
1525         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Rsqr, R);
1526
1527         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1528         &lea    ("esi",&DWP($H,"esp"));
1529         &lea    ("ebp",&DWP($in1_z,"esp"));
1530         &lea    ("edi",&DWP($res_z,"esp"));
1531         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(res_z, H, in1_z);
1532
1533         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1534         &lea    ("esi",&DWP($H,"esp"));
1535         &lea    ("ebp",&DWP($H,"esp"));
1536         &lea    ("edi",&DWP($Hsqr,"esp"));
1537         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Hsqr, H);
1538
1539         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1540         &lea    ("esi",&DWP($in2_z,"esp"));
1541         &lea    ("ebp",&DWP($res_z,"esp"));
1542         &lea    ("edi",&DWP($res_z,"esp"));
1543         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(res_z, res_z, in2_z);
1544
1545         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1546         &lea    ("esi",&DWP($Hsqr,"esp"));
1547         &lea    ("ebp",&DWP($U1,"esp"));
1548         &lea    ("edi",&DWP($U2,"esp"));
1549         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(U2, U1, Hsqr);
1550
1551         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1552         &lea    ("esi",&DWP($H,"esp"));
1553         &lea    ("ebp",&DWP($Hsqr,"esp"));
1554         &lea    ("edi",&DWP($Hcub,"esp"));
1555         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(Hcub, Hsqr, H);
1556
1557         &lea    ("esi",&DWP($U2,"esp"));
1558         &lea    ("ebp",&DWP($U2,"esp"));
1559         &lea    ("edi",&DWP($Hsqr,"esp"));
1560         &call   ("_ecp_nistz256_add");          # p256_mul_by_2(Hsqr, U2);
1561
1562         &lea    ("esi",&DWP($Rsqr,"esp"));
1563         &lea    ("ebp",&DWP($Hsqr,"esp"));
1564         &lea    ("edi",&DWP($res_x,"esp"));
1565         &call   ("_ecp_nistz256_sub");          # p256_sub(res_x, Rsqr, Hsqr);
1566
1567         &lea    ("esi",&DWP($res_x,"esp"));
1568         &lea    ("ebp",&DWP($Hcub,"esp"));
1569         &lea    ("edi",&DWP($res_x,"esp"));
1570         &call   ("_ecp_nistz256_sub");          # p256_sub(res_x, res_x, Hcub);
1571
1572         &lea    ("esi",&DWP($U2,"esp"));
1573         &lea    ("ebp",&DWP($res_x,"esp"));
1574         &lea    ("edi",&DWP($res_y,"esp"));
1575         &call   ("_ecp_nistz256_sub");          # p256_sub(res_y, U2, res_x);
1576
1577         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1578         &lea    ("esi",&DWP($Hcub,"esp"));
1579         &lea    ("ebp",&DWP($S1,"esp"));
1580         &lea    ("edi",&DWP($S2,"esp"));
1581         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S2, S1, Hcub);
1582
1583         &mov    ("eax",&DWP(32*18+12,"esp"));   # OPENSSL_ia32cap_P copy
1584         &lea    ("esi",&DWP($R,"esp"));
1585         &lea    ("ebp",&DWP($res_y,"esp"));
1586         &lea    ("edi",&DWP($res_y,"esp"));
1587         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(res_y, R, res_y);
1588
1589         &lea    ("esi",&DWP($res_y,"esp"));
1590         &lea    ("ebp",&DWP($S2,"esp"));
1591         &lea    ("edi",&DWP($res_y,"esp"));
1592         &call   ("_ecp_nistz256_sub");          # p256_sub(res_y, res_y, S2);
1593
1594         &mov    ("ebp",&DWP(32*18+0,"esp"));    # !in1infty
1595         &mov    ("esi",&DWP(32*18+4,"esp"));    # !in2infty
1596         &mov    ("edi",&wparam(0));
1597         &mov    ("edx","ebp");
1598         &not    ("ebp");
1599         &and    ("edx","esi");
1600         &and    ("ebp","esi");
1601         &not    ("esi");
1602
1603         ########################################
1604         # conditional moves
1605     for($i=64;$i<96;$i+=4) {
1606         &mov    ("eax","edx");
1607         &and    ("eax",&DWP($res_x+$i,"esp"));
1608         &mov    ("ebx","ebp");
1609         &and    ("ebx",&DWP($in2_x+$i,"esp"));
1610         &mov    ("ecx","esi");
1611         &and    ("ecx",&DWP($in1_x+$i,"esp"));
1612         &or     ("eax","ebx");
1613         &or     ("eax","ecx");
1614         &mov    (&DWP($i,"edi"),"eax");
1615     }
1616     for($i=0;$i<64;$i+=4) {
1617         &mov    ("eax","edx");
1618         &and    ("eax",&DWP($res_x+$i,"esp"));
1619         &mov    ("ebx","ebp");
1620         &and    ("ebx",&DWP($in2_x+$i,"esp"));
1621         &mov    ("ecx","esi");
1622         &and    ("ecx",&DWP($in1_x+$i,"esp"));
1623         &or     ("eax","ebx");
1624         &or     ("eax","ecx");
1625         &mov    (&DWP($i,"edi"),"eax");
1626     }
1627     &set_label("add_done");
1628         &stack_pop(8*18+5);
1629 } &function_end("ecp_nistz256_point_add");
1630
1631 ########################################################################
1632 # void ecp_nistz256_point_add_affine(P256_POINT *out,
1633 #                                    const P256_POINT *in1,
1634 #                                    const P256_POINT_AFFINE *in2);
1635 &function_begin("ecp_nistz256_point_add_affine");
1636 {
1637     my ($res_x,$res_y,$res_z,
1638         $in1_x,$in1_y,$in1_z,
1639         $in2_x,$in2_y,
1640         $U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr)=map(32*$_,(0..14));
1641     my $Z1sqr = $S2;
1642     my @ONE_mont=(1,0,0,-1,-1,-1,-2,0);
1643
1644         &mov    ("esi",&wparam(1));
1645
1646         # above map() describes stack layout with 15 temporary
1647         # 256-bit vectors on top, then we take extra words for
1648         # !in1infty, !in2infty, and OPENSSL_ia32cap_P copy.
1649         &stack_push(8*15+3);
1650                                                 if ($sse2) {
1651         &call   ("_picup_eax");
1652     &set_label("pic");
1653         &picmeup("edx","OPENSSL_ia32cap_P","eax",&label("pic"));
1654         &mov    ("ebp",&DWP(0,"edx"));          }
1655
1656         &lea    ("edi",&DWP($in1_x,"esp"));
1657     for($i=0;$i<96;$i+=16) {
1658         &mov    ("eax",&DWP($i+0,"esi"));       # copy in1
1659         &mov    ("ebx",&DWP($i+4,"esi"));
1660         &mov    ("ecx",&DWP($i+8,"esi"));
1661         &mov    ("edx",&DWP($i+12,"esi"));
1662         &mov    (&DWP($i+0,"edi"),"eax");
1663         &mov    (&DWP(32*15+8,"esp"),"ebp")     if ($i==0);
1664         &mov    ("ebp","eax")                   if ($i==0);
1665         &or     ("ebp","eax")                   if ($i!=0 && $i<64);
1666         &mov    (&DWP($i+4,"edi"),"ebx");
1667         &or     ("ebp","ebx")                   if ($i<64);
1668         &mov    (&DWP($i+8,"edi"),"ecx");
1669         &or     ("ebp","ecx")                   if ($i<64);
1670         &mov    (&DWP($i+12,"edi"),"edx");
1671         &or     ("ebp","edx")                   if ($i<64);
1672     }
1673         &xor    ("eax","eax");
1674         &mov    ("esi",&wparam(2));
1675         &sub    ("eax","ebp");
1676         &or     ("ebp","eax");
1677         &sar    ("ebp",31);
1678         &mov    (&DWP(32*15+0,"esp"),"ebp");    # !in1infty
1679
1680         &lea    ("edi",&DWP($in2_x,"esp"));
1681     for($i=0;$i<64;$i+=16) {
1682         &mov    ("eax",&DWP($i+0,"esi"));       # copy in2
1683         &mov    ("ebx",&DWP($i+4,"esi"));
1684         &mov    ("ecx",&DWP($i+8,"esi"));
1685         &mov    ("edx",&DWP($i+12,"esi"));
1686         &mov    (&DWP($i+0,"edi"),"eax");
1687         &mov    ("ebp","eax")                   if ($i==0);
1688         &or     ("ebp","eax")                   if ($i!=0);
1689         &mov    (&DWP($i+4,"edi"),"ebx");
1690         &or     ("ebp","ebx");
1691         &mov    (&DWP($i+8,"edi"),"ecx");
1692         &or     ("ebp","ecx");
1693         &mov    (&DWP($i+12,"edi"),"edx");
1694         &or     ("ebp","edx");
1695     }
1696         &xor    ("ebx","ebx");
1697         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1698         &sub    ("ebx","ebp");
1699          &lea   ("esi",&DWP($in1_z,"esp"));
1700         &or     ("ebx","ebp");
1701          &lea   ("ebp",&DWP($in1_z,"esp"));
1702         &sar    ("ebx",31);
1703          &lea   ("edi",&DWP($Z1sqr,"esp"));
1704         &mov    (&DWP(32*15+4,"esp"),"ebx");    # !in2infty
1705
1706         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Z1sqr, in1_z);
1707
1708         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1709         &lea    ("esi",&DWP($in2_x,"esp"));
1710         &mov    ("ebp","edi");                  # %esi is stull &Z1sqr
1711         &lea    ("edi",&DWP($U2,"esp"));
1712         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(U2, Z1sqr, in2_x);
1713
1714         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1715         &lea    ("esi",&DWP($in1_z,"esp"));
1716         &lea    ("ebp",&DWP($Z1sqr,"esp"));
1717         &lea    ("edi",&DWP($S2,"esp"));
1718         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S2, Z1sqr, in1_z);
1719
1720         &lea    ("esi",&DWP($U2,"esp"));
1721         &lea    ("ebp",&DWP($in1_x,"esp"));
1722         &lea    ("edi",&DWP($H,"esp"));
1723         &call   ("_ecp_nistz256_sub");          # p256_sub(H, U2, in1_x);
1724
1725         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1726         &lea    ("esi",&DWP($in2_y,"esp"));
1727         &lea    ("ebp",&DWP($S2,"esp"));
1728         &lea    ("edi",&DWP($S2,"esp"));
1729         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S2, S2, in2_y);
1730
1731         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1732         &lea    ("esi",&DWP($in1_z,"esp"));
1733         &lea    ("ebp",&DWP($H,"esp"));
1734         &lea    ("edi",&DWP($res_z,"esp"));
1735         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(res_z, H, in1_z);
1736
1737         &lea    ("esi",&DWP($S2,"esp"));
1738         &lea    ("ebp",&DWP($in1_y,"esp"));
1739         &lea    ("edi",&DWP($R,"esp"));
1740         &call   ("_ecp_nistz256_sub");          # p256_sub(R, S2, in1_y);
1741
1742         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1743         &lea    ("esi",&DWP($H,"esp"));
1744         &lea    ("ebp",&DWP($H,"esp"));
1745         &lea    ("edi",&DWP($Hsqr,"esp"));
1746         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Hsqr, H);
1747
1748         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1749         &lea    ("esi",&DWP($R,"esp"));
1750         &lea    ("ebp",&DWP($R,"esp"));
1751         &lea    ("edi",&DWP($Rsqr,"esp"));
1752         &call   ("_ecp_nistz256_mul_mont");     # p256_sqr_mont(Rsqr, R);
1753
1754         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1755         &lea    ("esi",&DWP($in1_x,"esp"));
1756         &lea    ("ebp",&DWP($Hsqr,"esp"));
1757         &lea    ("edi",&DWP($U2,"esp"));
1758         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(U2, in1_x, Hsqr);
1759
1760         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1761         &lea    ("esi",&DWP($H,"esp"));
1762         &lea    ("ebp",&DWP($Hsqr,"esp"));
1763         &lea    ("edi",&DWP($Hcub,"esp"));
1764         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(Hcub, Hsqr, H);
1765
1766         &lea    ("esi",&DWP($U2,"esp"));
1767         &lea    ("ebp",&DWP($U2,"esp"));
1768         &lea    ("edi",&DWP($Hsqr,"esp"));
1769         &call   ("_ecp_nistz256_add");          # p256_mul_by_2(Hsqr, U2);
1770
1771         &lea    ("esi",&DWP($Rsqr,"esp"));
1772         &lea    ("ebp",&DWP($Hsqr,"esp"));
1773         &lea    ("edi",&DWP($res_x,"esp"));
1774         &call   ("_ecp_nistz256_sub");          # p256_sub(res_x, Rsqr, Hsqr);
1775
1776         &lea    ("esi",&DWP($res_x,"esp"));
1777         &lea    ("ebp",&DWP($Hcub,"esp"));
1778         &lea    ("edi",&DWP($res_x,"esp"));
1779         &call   ("_ecp_nistz256_sub");          # p256_sub(res_x, res_x, Hcub);
1780
1781         &lea    ("esi",&DWP($U2,"esp"));
1782         &lea    ("ebp",&DWP($res_x,"esp"));
1783         &lea    ("edi",&DWP($res_y,"esp"));
1784         &call   ("_ecp_nistz256_sub");          # p256_sub(res_y, U2, res_x);
1785
1786         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1787         &lea    ("esi",&DWP($Hcub,"esp"));
1788         &lea    ("ebp",&DWP($in1_y,"esp"));
1789         &lea    ("edi",&DWP($S2,"esp"));
1790         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(S2, Hcub, in1_y);
1791
1792         &mov    ("eax",&DWP(32*15+8,"esp"));    # OPENSSL_ia32cap_P copy
1793         &lea    ("esi",&DWP($R,"esp"));
1794         &lea    ("ebp",&DWP($res_y,"esp"));
1795         &lea    ("edi",&DWP($res_y,"esp"));
1796         &call   ("_ecp_nistz256_mul_mont");     # p256_mul_mont(res_y, res_y, R);
1797
1798         &lea    ("esi",&DWP($res_y,"esp"));
1799         &lea    ("ebp",&DWP($S2,"esp"));
1800         &lea    ("edi",&DWP($res_y,"esp"));
1801         &call   ("_ecp_nistz256_sub");          # p256_sub(res_y, res_y, S2);
1802
1803         &mov    ("ebp",&DWP(32*15+0,"esp"));    # !in1infty
1804         &mov    ("esi",&DWP(32*15+4,"esp"));    # !in2infty
1805         &mov    ("edi",&wparam(0));
1806         &mov    ("edx","ebp");
1807         &not    ("ebp");
1808         &and    ("edx","esi");
1809         &and    ("ebp","esi");
1810         &not    ("esi");
1811
1812         ########################################
1813         # conditional moves
1814     for($i=64;$i<96;$i+=4) {
1815         my $one=@ONE_mont[($i-64)/4];
1816
1817         &mov    ("eax","edx");
1818         &and    ("eax",&DWP($res_x+$i,"esp"));
1819         &mov    ("ebx","ebp")                   if ($one && $one!=-1);
1820         &and    ("ebx",$one)                    if ($one && $one!=-1);
1821         &mov    ("ecx","esi");
1822         &and    ("ecx",&DWP($in1_x+$i,"esp"));
1823         &or     ("eax",$one==-1?"ebp":"ebx")    if ($one);
1824         &or     ("eax","ecx");
1825         &mov    (&DWP($i,"edi"),"eax");
1826     }
1827     for($i=0;$i<64;$i+=4) {
1828         &mov    ("eax","edx");
1829         &and    ("eax",&DWP($res_x+$i,"esp"));
1830         &mov    ("ebx","ebp");
1831         &and    ("ebx",&DWP($in2_x+$i,"esp"));
1832         &mov    ("ecx","esi");
1833         &and    ("ecx",&DWP($in1_x+$i,"esp"));
1834         &or     ("eax","ebx");
1835         &or     ("eax","ecx");
1836         &mov    (&DWP($i,"edi"),"eax");
1837     }
1838         &stack_pop(8*15+3);
1839 } &function_end("ecp_nistz256_point_add_affine");
1840
1841 &asm_finish();
1842
1843 close STDOUT;