engines/asm/e_padlock-x86.pl

   1 #!/usr/bin/env perl
   2
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9
  10 # September 2011
  11 #
  12 # Assembler helpers for Padlock engine. Compared to original engine
  13 # version relying on inline assembler and compiled with gcc 3.4.6 it
  14 # was measured to provide ~100% improvement on misaligned data in ECB
  15 # mode and ~75% in CBC mode. For aligned data improvement can be
  16 # observed for short inputs only, e.g. 45% for 64-byte messages in
  17 # ECB mode, 20% in CBC. Difference in performance for aligned vs.
  18 # misaligned data depends on misalignment and is either ~1.8x or 2.9x.
  19 # These are approximately same factors as for hardware support, so
  20 # there is little reason to rely on the latter. On the contrary, it
  21 # might actually hurt performance in mixture of aligned and misaligned
  22 # buffers, because a) if you choose to flip 'align' flag in control
  23 # word on per-buffer basis, then you'd have to reload key context,
  24 # which incurs penalty; b) if you choose to set 'align' flag
  25 # permanently, it limits performance even for aligned data to ~1/2.
  26 # All above mentioned results were collected on 1.5GHz C7. Nano on the
  27 # other hand handles unaligned data more gracefully. Depending on
  28 # algorithm and how unaligned data is, hardware can be up to 70% more
  29 # efficient than below software alignment procedures, nor does 'align'
  30 # flag have affect on aligned performance [if has any meaning at all].
  31 # Therefore suggestion is to unconditionally set 'align' flag on Nano
  32 # for optimal performance.
  33
  34 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  35 push(@INC,"${dir}","${dir}../../crypto/perlasm");
  36 require "x86asm.pl";
  37
  38 &asm_init($ARGV[0],$0);
  39
  40 %PADLOCK_PREFETCH=(ecb=>128, cbc=>64);  # prefetch errata
  41 $PADLOCK_CHUNK=512;     # Must be a power of 2 larger than 16
  42
  43 $ctx="edx";
  44 $out="edi";
  45 $inp="esi";
  46 $len="ecx";
  47 $chunk="ebx";
  48
  49 &function_begin_B("padlock_capability");
  50         &push   ("ebx");
  51         &pushf  ();
  52         &pop    ("eax");
  53         &mov    ("ecx","eax");
  54         &xor    ("eax",1<<21);
  55         &push   ("eax");
  56         &popf   ();
  57         &pushf  ();
  58         &pop    ("eax");
  59         &xor    ("ecx","eax");
  60         &xor    ("eax","eax");
  61         &bt     ("ecx",21);
  62         &jnc    (&label("noluck"));
  63         &cpuid  ();
  64         &xor    ("eax","eax");
  65         &cmp    ("ebx","0x".unpack("H*",'tneC'));
  66         &jne    (&label("noluck"));
  67         &cmp    ("edx","0x".unpack("H*",'Hrua'));
  68         &jne    (&label("noluck"));
  69         &cmp    ("ecx","0x".unpack("H*",'slua'));
  70         &jne    (&label("noluck"));
  71         &mov    ("eax",0xC0000000);
  72         &cpuid  ();
  73         &mov    ("edx","eax");
  74         &xor    ("eax","eax");
  75         &cmp    ("edx",0xC0000001);
  76         &jb     (&label("noluck"));
  77         &mov    ("eax",1);
  78         &cpuid  ();
  79         &or     ("eax",0x0f);
  80         &xor    ("ebx","ebx");
  81         &and    ("eax",0x0fff);
  82         &cmp    ("eax",0x06ff);         # check for Nano
  83         &sete   ("bl");
  84         &mov    ("eax",0xC0000001);
  85         &push   ("ebx");
  86         &cpuid  ();
  87         &pop    ("ebx");
  88         &mov    ("eax","edx");
  89         &shl    ("ebx",4);              # bit#4 denotes Nano
  90         &and    ("eax",0xffffffef);
  91         &or     ("eax","ebx")
  92 &set_label("noluck");
  93         &pop    ("ebx");
  94         &ret    ();
  95 &function_end_B("padlock_capability")
  96
  97 &function_begin_B("padlock_key_bswap");
  98         &mov    ("edx",&wparam(0));
  99         &mov    ("ecx",&DWP(240,"edx"));
 100 &set_label("bswap_loop");
 101         &mov    ("eax",&DWP(0,"edx"));
 102         &bswap  ("eax");
 103         &mov    (&DWP(0,"edx"),"eax");
 104         &lea    ("edx",&DWP(4,"edx"));
 105         &sub    ("ecx",1);
 106         &jnz    (&label("bswap_loop"));
 107         &ret    ();
 108 &function_end_B("padlock_key_bswap");
 109
 110 # This is heuristic key context tracing. At first one
 111 # believes that one should use atomic swap instructions,
 112 # but it's not actually necessary. Point is that if
 113 # padlock_saved_context was changed by another thread
 114 # after we've read it and before we compare it with ctx,
 115 # our key *shall* be reloaded upon thread context switch
 116 # and we are therefore set in either case...
 117 &static_label("padlock_saved_context");
 118
 119 &function_begin_B("padlock_verify_context");
 120         &mov    ($ctx,&wparam(0));
 121         &lea    ("eax",($::win32 or $::coff) ? &DWP(&label("padlock_saved_context")) :
 122                        &DWP(&label("padlock_saved_context")."-".&label("verify_pic_point")));
 123         &pushf  ();
 124         &call   ("_padlock_verify_ctx");
 125 &set_label("verify_pic_point");
 126         &lea    ("esp",&DWP(4,"esp"));
 127         &ret    ();
 128 &function_end_B("padlock_verify_context");
 129
 130 &function_begin_B("_padlock_verify_ctx");
 131         &add    ("eax",&DWP(0,"esp")) if(!($::win32 or $::coff));# &padlock_saved_context
 132         &bt     (&DWP(4,"esp"),30);             # eflags
 133         &jnc    (&label("verified"));
 134         &cmp    ($ctx,&DWP(0,"eax"));
 135         &je     (&label("verified"));
 136         &pushf  ();
 137         &popf   ();
 138 &set_label("verified");
 139         &mov    (&DWP(0,"eax"),$ctx);
 140         &ret    ();
 141 &function_end_B("_padlock_verify_ctx");
 142
 143 &function_begin_B("padlock_reload_key");
 144         &pushf  ();
 145         &popf   ();
 146         &ret    ();
 147 &function_end_B("padlock_reload_key");
 148
 149 &function_begin_B("padlock_aes_block");
 150         &push   ("edi");
 151         &push   ("esi");
 152         &push   ("ebx");
 153         &mov    ($out,&wparam(0));              # must be 16-byte aligned
 154         &mov    ($inp,&wparam(1));              # must be 16-byte aligned
 155         &mov    ($ctx,&wparam(2));
 156         &mov    ($len,1);
 157         &lea    ("ebx",&DWP(32,$ctx));          # key
 158         &lea    ($ctx,&DWP(16,$ctx));           # control word
 159         &data_byte(0xf3,0x0f,0xa7,0xc8);        # rep xcryptecb
 160         &pop    ("ebx");
 161         &pop    ("esi");
 162         &pop    ("edi");
 163         &ret    ();
 164 &function_end_B("padlock_aes_block");
 165
 166 sub generate_mode {
 167 my ($mode,$opcode) = @_;
 168 # int padlock_$mode_encrypt(void *out, const void *inp,
 169 #               struct padlock_cipher_data *ctx, size_t len);
 170 &function_begin("padlock_${mode}_encrypt");
 171         &mov    ($out,&wparam(0));
 172         &mov    ($inp,&wparam(1));
 173         &mov    ($ctx,&wparam(2));
 174         &mov    ($len,&wparam(3));
 175         &test   ($ctx,15);
 176         &jnz    (&label("${mode}_abort"));
 177         &test   ($len,15);
 178         &jnz    (&label("${mode}_abort"));
 179         &lea    ("eax",($::win32 or $::coff) ? &DWP(&label("padlock_saved_context")) :
 180                        &DWP(&label("padlock_saved_context")."-".&label("${mode}_pic_point")));
 181         &pushf  ();
 182         &cld    ();
 183         &call   ("_padlock_verify_ctx");
 184 &set_label("${mode}_pic_point");
 185         &lea    ($ctx,&DWP(16,$ctx));   # control word
 186         &xor    ("eax","eax");
 187                                         if ($mode eq "ctr32") {
 188         &movq   ("mm0",&QWP(-16,$ctx)); # load [upper part of] counter
 189                                         } else {
 190         &xor    ("ebx","ebx");
 191         &test   (&DWP(0,$ctx),1<<5);    # align bit in control word
 192         &jnz    (&label("${mode}_aligned"));
 193         &test   ($out,0x0f);
 194         &setz   ("al");                 # !out_misaligned
 195         &test   ($inp,0x0f);
 196         &setz   ("bl");                 # !inp_misaligned
 197         &test   ("eax","ebx");
 198         &jnz    (&label("${mode}_aligned"));
 199         &neg    ("eax");
 200                                         }
 201         &mov    ($chunk,$PADLOCK_CHUNK);
 202         &not    ("eax");                # out_misaligned?-1:0
 203         &lea    ("ebp",&DWP(-24,"esp"));
 204         &cmp    ($len,$chunk);
 205         &cmovc  ($chunk,$len);          # chunk=len>PADLOCK_CHUNK?PADLOCK_CHUNK:len
 206         &and    ("eax",$chunk);         # out_misaligned?chunk:0
 207         &mov    ($chunk,$len);
 208         &neg    ("eax");
 209         &and    ($chunk,$PADLOCK_CHUNK-1);      # chunk=len%PADLOCK_CHUNK
 210         &lea    ("esp",&DWP(0,"eax","ebp"));    # alloca
 211         &mov    ("eax",$PADLOCK_CHUNK);
 212         &cmovz  ($chunk,"eax");                 # chunk=chunk?:PADLOCK_CHUNK
 213         &mov    ("eax","ebp");
 214         &and    ("ebp",-16);
 215         &and    ("esp",-16);
 216         &mov    (&DWP(16,"ebp"),"eax");
 217     if ($PADLOCK_PREFETCH{$mode}) {
 218         &cmp    ($len,$chunk);
 219         &ja     (&label("${mode}_loop"));
 220         &mov    ("eax",$inp);           # check if prefetch crosses page
 221         &cmp    ("ebp","esp");
 222         &cmove  ("eax",$out);
 223         &add    ("eax",$len);
 224         &neg    ("eax");
 225         &and    ("eax",0xfff);          # distance to page boundary
 226         &cmp    ("eax",$PADLOCK_PREFETCH{$mode});
 227         &mov    ("eax",-$PADLOCK_PREFETCH{$mode});
 228         &cmovae ("eax",$chunk);         # mask=distance<prefetch?-prefetch:-1
 229         &and    ($chunk,"eax");
 230         &jz     (&label("${mode}_unaligned_tail"));
 231     }
 232         &jmp    (&label("${mode}_loop"));
 233
 234 &set_label("${mode}_loop",16);
 235         &mov    (&DWP(0,"ebp"),$out);           # save parameters
 236         &mov    (&DWP(4,"ebp"),$inp);
 237         &mov    (&DWP(8,"ebp"),$len);
 238         &mov    ($len,$chunk);
 239         &mov    (&DWP(12,"ebp"),$chunk);        # chunk
 240                                                 if ($mode eq "ctr32") {
 241         &mov    ("ecx",&DWP(-4,$ctx));
 242         &xor    ($out,$out);
 243         &mov    ("eax",&DWP(-8,$ctx));          # borrow $len
 244 &set_label("${mode}_prepare");
 245         &mov    (&DWP(12,"esp",$out),"ecx");
 246         &bswap  ("ecx");
 247         &movq   (&QWP(0,"esp",$out),"mm0");
 248         &inc    ("ecx");
 249         &mov    (&DWP(8,"esp",$out),"eax");
 250         &bswap  ("ecx");
 251         &lea    ($out,&DWP(16,$out));
 252         &cmp    ($out,$chunk);
 253         &jb     (&label("${mode}_prepare"));
 254
 255         &mov    (&DWP(-4,$ctx),"ecx");
 256         &lea    ($inp,&DWP(0,"esp"));
 257         &lea    ($out,&DWP(0,"esp"));
 258         &mov    ($len,$chunk);
 259                                                 } else {
 260         &test   ($out,0x0f);                    # out_misaligned
 261         &cmovnz ($out,"esp");
 262         &test   ($inp,0x0f);                    # inp_misaligned
 263         &jz     (&label("${mode}_inp_aligned"));
 264         &shr    ($len,2);
 265         &data_byte(0xf3,0xa5);                  # rep movsl
 266         &sub    ($out,$chunk);
 267         &mov    ($len,$chunk);
 268         &mov    ($inp,$out);
 269 &set_label("${mode}_inp_aligned");
 270                                                 }
 271         &lea    ("eax",&DWP(-16,$ctx));         # ivp
 272         &lea    ("ebx",&DWP(16,$ctx));          # key
 273         &shr    ($len,4);                       # len/=AES_BLOCK_SIZE
 274         &data_byte(0xf3,0x0f,0xa7,$opcode);     # rep xcrypt*
 275                                                 if ($mode !~ /ecb|ctr/) {
 276         &movaps ("xmm0",&QWP(0,"eax"));
 277         &movaps (&QWP(-16,$ctx),"xmm0");        # copy [or refresh] iv
 278                                                 }
 279         &mov    ($out,&DWP(0,"ebp"));           # restore parameters
 280         &mov    ($chunk,&DWP(12,"ebp"));
 281                                                 if ($mode eq "ctr32") {
 282         &mov    ($inp,&DWP(4,"ebp"));
 283         &xor    ($len,$len);
 284 &set_label("${mode}_xor");
 285         &movups ("xmm1",&QWP(0,$inp,$len));
 286         &lea    ($len,&DWP(16,$len));
 287         &pxor   ("xmm1",&QWP(-16,"esp",$len));
 288         &movups (&QWP(-16,$out,$len),"xmm1");
 289         &cmp    ($len,$chunk);
 290         &jb     (&label("${mode}_xor"));
 291                                                 } else {
 292         &test   ($out,0x0f);
 293         &jz     (&label("${mode}_out_aligned"));
 294         &mov    ($len,$chunk);
 295         &lea    ($inp,&DWP(0,"esp"));
 296         &shr    ($len,2);
 297         &data_byte(0xf3,0xa5);                  # rep movsl
 298         &sub    ($out,$chunk);
 299 &set_label("${mode}_out_aligned");
 300         &mov    ($inp,&DWP(4,"ebp"));
 301                                                 }
 302         &mov    ($len,&DWP(8,"ebp"));
 303         &add    ($out,$chunk);
 304         &add    ($inp,$chunk);
 305         &sub    ($len,$chunk);
 306         &mov    ($chunk,$PADLOCK_CHUNK);
 307     if (!$PADLOCK_PREFETCH{$mode}) {
 308         &jnz    (&label("${mode}_loop"));
 309     } else {
 310         &jz     (&label("${mode}_break"));
 311         &cmp    ($len,$chunk);
 312         &jae    (&label("${mode}_loop"));
 313
 314 &set_label("${mode}_unaligned_tail");
 315         &xor    ("eax","eax");
 316         &cmp    ("esp","ebp");
 317         &cmove  ("eax",$len);
 318         &sub    ("esp","eax");                  # alloca
 319         &mov    ("eax", $out);                  # save parameters
 320         &mov    ($chunk,$len);
 321         &shr    ($len,2);
 322         &lea    ($out,&DWP(0,"esp"));
 323         &data_byte(0xf3,0xa5);                  # rep movsl
 324         &mov    ($inp,"esp");
 325         &mov    ($out,"eax");                   # restore parameters
 326         &mov    ($len,$chunk);
 327         &jmp    (&label("${mode}_loop"));
 328
 329 &set_label("${mode}_break",16);
 330     }
 331                                                 if ($mode ne "ctr32") {
 332         &cmp    ("esp","ebp");
 333         &je     (&label("${mode}_done"));
 334                                                 }
 335         &pxor   ("xmm0","xmm0");
 336         &lea    ("eax",&DWP(0,"esp"));
 337 &set_label("${mode}_bzero");
 338         &movaps (&QWP(0,"eax"),"xmm0");
 339         &lea    ("eax",&DWP(16,"eax"));
 340         &cmp    ("ebp","eax");
 341         &ja     (&label("${mode}_bzero"));
 342
 343 &set_label("${mode}_done");
 344         &mov    ("ebp",&DWP(16,"ebp"));
 345         &lea    ("esp",&DWP(24,"ebp"));
 346                                                 if ($mode ne "ctr32") {
 347         &jmp    (&label("${mode}_exit"));
 348
 349 &set_label("${mode}_aligned",16);
 350     if ($PADLOCK_PREFETCH{$mode}) {
 351         &lea    ("ebp",&DWP(0,$inp,$len));
 352         &neg    ("ebp");
 353         &and    ("ebp",0xfff);                  # distance to page boundary
 354         &xor    ("eax","eax");
 355         &cmp    ("ebp",$PADLOCK_PREFETCH{$mode});
 356         &mov    ("ebp",$PADLOCK_PREFETCH{$mode}-1);
 357         &cmovae ("ebp","eax");
 358         &and    ("ebp",$len);                   # remainder
 359         &sub    ($len,"ebp");
 360         &jz     (&label("${mode}_aligned_tail"));
 361     }
 362         &lea    ("eax",&DWP(-16,$ctx));         # ivp
 363         &lea    ("ebx",&DWP(16,$ctx));          # key
 364         &shr    ($len,4);                       # len/=AES_BLOCK_SIZE
 365         &data_byte(0xf3,0x0f,0xa7,$opcode);     # rep xcrypt*
 366                                                 if ($mode ne "ecb") {
 367         &movaps ("xmm0",&QWP(0,"eax"));
 368         &movaps (&QWP(-16,$ctx),"xmm0");        # copy [or refresh] iv
 369                                                 }
 370     if ($PADLOCK_PREFETCH{$mode}) {
 371         &test   ("ebp","ebp");
 372         &jz     (&label("${mode}_exit"));
 373
 374 &set_label("${mode}_aligned_tail");
 375         &mov    ($len,"ebp");
 376         &lea    ("ebp",&DWP(-24,"esp"));
 377         &mov    ("esp","ebp");
 378         &mov    ("eax","ebp");
 379         &sub    ("esp",$len);
 380         &and    ("ebp",-16);
 381         &and    ("esp",-16);
 382         &mov    (&DWP(16,"ebp"),"eax");
 383         &mov    ("eax", $out);                  # save parameters
 384         &mov    ($chunk,$len);
 385         &shr    ($len,2);
 386         &lea    ($out,&DWP(0,"esp"));
 387         &data_byte(0xf3,0xa5);                  # rep movsl
 388         &mov    ($inp,"esp");
 389         &mov    ($out,"eax");                   # restore parameters
 390         &mov    ($len,$chunk);
 391         &jmp    (&label("${mode}_loop"));
 392     }
 393 &set_label("${mode}_exit");                     }
 394         &mov    ("eax",1);
 395         &lea    ("esp",&DWP(4,"esp"));          # popf
 396         &emms   ()                              if ($mode eq "ctr32");
 397 &set_label("${mode}_abort");
 398 &function_end("padlock_${mode}_encrypt");
 399 }
 400
 401 &generate_mode("ecb",0xc8);
 402 &generate_mode("cbc",0xd0);
 403 &generate_mode("cfb",0xe0);
 404 &generate_mode("ofb",0xe8);
 405 &generate_mode("ctr32",0xc8);   # yes, it implements own CTR with ECB opcode,
 406                                 # because hardware CTR was introduced later
 407                                 # and even has errata on certain C7 stepping.
 408                                 # own implementation *always* works, though
 409                                 # ~15% slower than dedicated hardware...
 410
 411 &function_begin_B("padlock_xstore");
 412         &push   ("edi");
 413         &mov    ("edi",&wparam(0));
 414         &mov    ("edx",&wparam(1));
 415         &data_byte(0x0f,0xa7,0xc0);             # xstore
 416         &pop    ("edi");
 417         &ret    ();
 418 &function_end_B("padlock_xstore");
 419
 420 &function_begin_B("_win32_segv_handler");
 421         &mov    ("eax",1);                      # ExceptionContinueSearch
 422         &mov    ("edx",&wparam(0));             # *ExceptionRecord
 423         &mov    ("ecx",&wparam(2));             # *ContextRecord
 424         &cmp    (&DWP(0,"edx"),0xC0000005)      # ExceptionRecord->ExceptionCode == STATUS_ACCESS_VIOLATION
 425         &jne    (&label("ret"));
 426         &add    (&DWP(184,"ecx"),4);            # skip over rep sha*
 427         &mov    ("eax",0);                      # ExceptionContinueExecution
 428 &set_label("ret");
 429         &ret    ();
 430 &function_end_B("_win32_segv_handler");
 431 &safeseh("_win32_segv_handler")                 if ($::win32);
 432
 433 &function_begin_B("padlock_sha1_oneshot");
 434         &push   ("edi");
 435         &push   ("esi");
 436         &xor    ("eax","eax");
 437         &mov    ("edi",&wparam(0));
 438         &mov    ("esi",&wparam(1));
 439         &mov    ("ecx",&wparam(2));
 440     if ($::win32 or $::coff) {
 441         &push   (&::islabel("_win32_segv_handler"));
 442         &data_byte(0x64,0xff,0x30);             # push  %fs:(%eax)
 443         &data_byte(0x64,0x89,0x20);             # mov   %esp,%fs:(%eax)
 444     }
 445         &mov    ("edx","esp");                  # put aside %esp
 446         &add    ("esp",-128);                   # 32 is enough but spec says 128
 447         &movups ("xmm0",&QWP(0,"edi"));         # copy-in context
 448         &and    ("esp",-16);
 449         &mov    ("eax",&DWP(16,"edi"));
 450         &movaps (&QWP(0,"esp"),"xmm0");
 451         &mov    ("edi","esp");
 452         &mov    (&DWP(16,"esp"),"eax");
 453         &xor    ("eax","eax");
 454         &data_byte(0xf3,0x0f,0xa6,0xc8);        # rep xsha1
 455         &movaps ("xmm0",&QWP(0,"esp"));
 456         &mov    ("eax",&DWP(16,"esp"));
 457         &mov    ("esp","edx");                  # restore %esp
 458     if ($::win32 or $::coff) {
 459         &data_byte(0x64,0x8f,0x05,0,0,0,0);     # pop   %fs:0
 460         &lea    ("esp",&DWP(4,"esp"));
 461     }
 462         &mov    ("edi",&wparam(0));
 463         &movups (&QWP(0,"edi"),"xmm0");         # copy-out context
 464         &mov    (&DWP(16,"edi"),"eax");
 465         &pop    ("esi");
 466         &pop    ("edi");
 467         &ret    ();
 468 &function_end_B("padlock_sha1_oneshot");
 469
 470 &function_begin_B("padlock_sha1_blocks");
 471         &push   ("edi");
 472         &push   ("esi");
 473         &mov    ("edi",&wparam(0));
 474         &mov    ("esi",&wparam(1));
 475         &mov    ("edx","esp");                  # put aside %esp
 476         &mov    ("ecx",&wparam(2));
 477         &add    ("esp",-128);
 478         &movups ("xmm0",&QWP(0,"edi"));         # copy-in context
 479         &and    ("esp",-16);
 480         &mov    ("eax",&DWP(16,"edi"));
 481         &movaps (&QWP(0,"esp"),"xmm0");
 482         &mov    ("edi","esp");
 483         &mov    (&DWP(16,"esp"),"eax");
 484         &mov    ("eax",-1);
 485         &data_byte(0xf3,0x0f,0xa6,0xc8);        # rep xsha1
 486         &movaps ("xmm0",&QWP(0,"esp"));
 487         &mov    ("eax",&DWP(16,"esp"));
 488         &mov    ("esp","edx");                  # restore %esp
 489         &mov    ("edi",&wparam(0));
 490         &movups (&QWP(0,"edi"),"xmm0");         # copy-out context
 491         &mov    (&DWP(16,"edi"),"eax");
 492         &pop    ("esi");
 493         &pop    ("edi");
 494         &ret    ();
 495 &function_end_B("padlock_sha1_blocks");
 496
 497 &function_begin_B("padlock_sha256_oneshot");
 498         &push   ("edi");
 499         &push   ("esi");
 500         &xor    ("eax","eax");
 501         &mov    ("edi",&wparam(0));
 502         &mov    ("esi",&wparam(1));
 503         &mov    ("ecx",&wparam(2));
 504     if ($::win32 or $::coff) {
 505         &push   (&::islabel("_win32_segv_handler"));
 506         &data_byte(0x64,0xff,0x30);             # push  %fs:(%eax)
 507         &data_byte(0x64,0x89,0x20);             # mov   %esp,%fs:(%eax)
 508     }
 509         &mov    ("edx","esp");                  # put aside %esp
 510         &add    ("esp",-128);
 511         &movups ("xmm0",&QWP(0,"edi"));         # copy-in context
 512         &and    ("esp",-16);
 513         &movups ("xmm1",&QWP(16,"edi"));
 514         &movaps (&QWP(0,"esp"),"xmm0");
 515         &mov    ("edi","esp");
 516         &movaps (&QWP(16,"esp"),"xmm1");
 517         &xor    ("eax","eax");
 518         &data_byte(0xf3,0x0f,0xa6,0xd0);        # rep xsha256
 519         &movaps ("xmm0",&QWP(0,"esp"));
 520         &movaps ("xmm1",&QWP(16,"esp"));
 521         &mov    ("esp","edx");                  # restore %esp
 522     if ($::win32 or $::coff) {
 523         &data_byte(0x64,0x8f,0x05,0,0,0,0);     # pop   %fs:0
 524         &lea    ("esp",&DWP(4,"esp"));
 525     }
 526         &mov    ("edi",&wparam(0));
 527         &movups (&QWP(0,"edi"),"xmm0");         # copy-out context
 528         &movups (&QWP(16,"edi"),"xmm1");
 529         &pop    ("esi");
 530         &pop    ("edi");
 531         &ret    ();
 532 &function_end_B("padlock_sha256_oneshot");
 533
 534 &function_begin_B("padlock_sha256_blocks");
 535         &push   ("edi");
 536         &push   ("esi");
 537         &mov    ("edi",&wparam(0));
 538         &mov    ("esi",&wparam(1));
 539         &mov    ("ecx",&wparam(2));
 540         &mov    ("edx","esp");                  # put aside %esp
 541         &add    ("esp",-128);
 542         &movups ("xmm0",&QWP(0,"edi"));         # copy-in context
 543         &and    ("esp",-16);
 544         &movups ("xmm1",&QWP(16,"edi"));
 545         &movaps (&QWP(0,"esp"),"xmm0");
 546         &mov    ("edi","esp");
 547         &movaps (&QWP(16,"esp"),"xmm1");
 548         &mov    ("eax",-1);
 549         &data_byte(0xf3,0x0f,0xa6,0xd0);        # rep xsha256
 550         &movaps ("xmm0",&QWP(0,"esp"));
 551         &movaps ("xmm1",&QWP(16,"esp"));
 552         &mov    ("esp","edx");                  # restore %esp
 553         &mov    ("edi",&wparam(0));
 554         &movups (&QWP(0,"edi"),"xmm0");         # copy-out context
 555         &movups (&QWP(16,"edi"),"xmm1");
 556         &pop    ("esi");
 557         &pop    ("edi");
 558         &ret    ();
 559 &function_end_B("padlock_sha256_blocks");
 560
 561 &function_begin_B("padlock_sha512_blocks");
 562         &push   ("edi");
 563         &push   ("esi");
 564         &mov    ("edi",&wparam(0));
 565         &mov    ("esi",&wparam(1));
 566         &mov    ("ecx",&wparam(2));
 567         &mov    ("edx","esp");                  # put aside %esp
 568         &add    ("esp",-128);
 569         &movups ("xmm0",&QWP(0,"edi"));         # copy-in context
 570         &and    ("esp",-16);
 571         &movups ("xmm1",&QWP(16,"edi"));
 572         &movups ("xmm2",&QWP(32,"edi"));
 573         &movups ("xmm3",&QWP(48,"edi"));
 574         &movaps (&QWP(0,"esp"),"xmm0");
 575         &mov    ("edi","esp");
 576         &movaps (&QWP(16,"esp"),"xmm1");
 577         &movaps (&QWP(32,"esp"),"xmm2");
 578         &movaps (&QWP(48,"esp"),"xmm3");
 579         &data_byte(0xf3,0x0f,0xa6,0xe0);        # rep xsha512
 580         &movaps ("xmm0",&QWP(0,"esp"));
 581         &movaps ("xmm1",&QWP(16,"esp"));
 582         &movaps ("xmm2",&QWP(32,"esp"));
 583         &movaps ("xmm3",&QWP(48,"esp"));
 584         &mov    ("esp","edx");                  # restore %esp
 585         &mov    ("edi",&wparam(0));
 586         &movups (&QWP(0,"edi"),"xmm0");         # copy-out context
 587         &movups (&QWP(16,"edi"),"xmm1");
 588         &movups (&QWP(32,"edi"),"xmm2");
 589         &movups (&QWP(48,"edi"),"xmm3");
 590         &pop    ("esi");
 591         &pop    ("edi");
 592         &ret    ();
 593 &function_end_B("padlock_sha512_blocks");
 594
 595 &asciz  ("VIA Padlock x86 module, CRYPTOGAMS by <appro\@openssl.org>");
 596 &align  (16);
 597
 598 &dataseg();
 599 # Essentially this variable belongs in thread local storage.
 600 # Having this variable global on the other hand can only cause
 601 # few bogus key reloads [if any at all on signle-CPU system],
 602 # so we accept the penalty...
 603 &set_label("padlock_saved_context",4);
 604 &data_word(0);
 605
 606 &asm_finish();