fix error discrepancy
[openssl.git] / engines / asm / e_padlock-x86.pl
1 #!/usr/bin/env perl
2
3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9
10 # September 2011
11 #
12 # Assembler helpers for Padlock engine. Compared to original engine
13 # version relying on inline assembler and compiled with gcc 3.4.6 it
14 # was measured to provide ~100% improvement on misaligned data in ECB
15 # mode and ~75% in CBC mode. For aligned data improvement can be
16 # observed for short inputs only, e.g. 45% for 64-byte messages in
17 # ECB mode, 20% in CBC. Difference in performance for aligned vs.
18 # misaligned data depends on misalignment and is either ~1.8x or 2.9x.
19 # These are approximately same factors as for hardware support, so
20 # there is little reason to rely on the latter. On the contrary, it
21 # might actually hurt performance in mixture of aligned and misaligned
22 # buffers, because a) if you choose to flip 'align' flag in control
23 # word on per-buffer basis, then you'd have to reload key context,
24 # which incurs penalty; b) if you choose to set 'align' flag
25 # permanently, it limits performance even for aligned data to ~1/2.
26 # All above mentioned results were collected on 1.5GHz C7. Nano on the
27 # other hand handles unaligned data more gracefully. Depending on
28 # algorithm and how unaligned data is, hardware can be up to 70% more
29 # efficient than below software alignment procedures, nor does 'align'
30 # flag have affect on aligned performance [if has any meaning at all].
31 # Therefore suggestion is to unconditionally set 'align' flag on Nano
32 # for optimal performance.
33
34 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
35 push(@INC,"${dir}","${dir}../../crypto/perlasm");
36 require "x86asm.pl";
37
38 &asm_init($ARGV[0],$0);
39
40 %PADLOCK_PREFETCH=(ecb=>128, cbc=>64);  # prefetch errata
41 $PADLOCK_CHUNK=512;     # Must be a power of 2 larger than 16
42
43 $ctx="edx";
44 $out="edi";
45 $inp="esi";
46 $len="ecx";
47 $chunk="ebx";
48
49 &function_begin_B("padlock_capability");
50         &push   ("ebx");
51         &pushf  ();
52         &pop    ("eax");
53         &mov    ("ecx","eax");
54         &xor    ("eax",1<<21);
55         &push   ("eax");
56         &popf   ();
57         &pushf  ();
58         &pop    ("eax");
59         &xor    ("ecx","eax");
60         &xor    ("eax","eax");
61         &bt     ("ecx",21);
62         &jnc    (&label("noluck"));
63         &cpuid  ();
64         &xor    ("eax","eax");
65         &cmp    ("ebx","0x".unpack("H*",'tneC'));
66         &jne    (&label("noluck"));
67         &cmp    ("edx","0x".unpack("H*",'Hrua'));
68         &jne    (&label("noluck"));
69         &cmp    ("ecx","0x".unpack("H*",'slua'));
70         &jne    (&label("noluck"));
71         &mov    ("eax",0xC0000000);
72         &cpuid  ();
73         &mov    ("edx","eax");
74         &xor    ("eax","eax");
75         &cmp    ("edx",0xC0000001);
76         &jb     (&label("noluck"));
77         &mov    ("eax",1);
78         &cpuid  ();
79         &or     ("eax",0x0f);
80         &xor    ("ebx","ebx");
81         &and    ("eax",0x0fff);
82         &cmp    ("eax",0x06ff);         # check for Nano
83         &sete   ("bl");
84         &mov    ("eax",0xC0000001);
85         &push   ("ebx");
86         &cpuid  ();
87         &pop    ("ebx");
88         &mov    ("eax","edx");
89         &shl    ("ebx",4);              # bit#4 denotes Nano
90         &and    ("eax",0xffffffef);
91         &or     ("eax","ebx")
92 &set_label("noluck");
93         &pop    ("ebx");
94         &ret    ();
95 &function_end_B("padlock_capability")
96
97 &function_begin_B("padlock_key_bswap");
98         &mov    ("edx",&wparam(0));
99         &mov    ("ecx",&DWP(240,"edx"));
100 &set_label("bswap_loop");
101         &mov    ("eax",&DWP(0,"edx"));
102         &bswap  ("eax");
103         &mov    (&DWP(0,"edx"),"eax");
104         &lea    ("edx",&DWP(4,"edx"));
105         &sub    ("ecx",1);
106         &jnz    (&label("bswap_loop"));
107         &ret    ();
108 &function_end_B("padlock_key_bswap");
109
110 # This is heuristic key context tracing. At first one
111 # believes that one should use atomic swap instructions,
112 # but it's not actually necessary. Point is that if
113 # padlock_saved_context was changed by another thread
114 # after we've read it and before we compare it with ctx,
115 # our key *shall* be reloaded upon thread context switch
116 # and we are therefore set in either case...
117 &static_label("padlock_saved_context");
118
119 &function_begin_B("padlock_verify_context");
120         &mov    ($ctx,&wparam(0));
121         &lea    ("eax",($::win32 or $::coff) ? &DWP(&label("padlock_saved_context")) :
122                        &DWP(&label("padlock_saved_context")."-".&label("verify_pic_point")));
123         &pushf  ();
124         &call   ("_padlock_verify_ctx");
125 &set_label("verify_pic_point");
126         &lea    ("esp",&DWP(4,"esp"));
127         &ret    ();
128 &function_end_B("padlock_verify_context");
129
130 &function_begin_B("_padlock_verify_ctx");
131         &add    ("eax",&DWP(0,"esp")) if(!($::win32 or $::coff));# &padlock_saved_context
132         &bt     (&DWP(4,"esp"),30);             # eflags
133         &jnc    (&label("verified"));
134         &cmp    ($ctx,&DWP(0,"eax"));
135         &je     (&label("verified"));
136         &pushf  ();
137         &popf   ();
138 &set_label("verified");
139         &mov    (&DWP(0,"eax"),$ctx);
140         &ret    ();
141 &function_end_B("_padlock_verify_ctx");
142
143 &function_begin_B("padlock_reload_key");
144         &pushf  ();
145         &popf   ();
146         &ret    ();
147 &function_end_B("padlock_reload_key");
148
149 &function_begin_B("padlock_aes_block");
150         &push   ("edi");
151         &push   ("esi");
152         &push   ("ebx");
153         &mov    ($out,&wparam(0));              # must be 16-byte aligned
154         &mov    ($inp,&wparam(1));              # must be 16-byte aligned
155         &mov    ($ctx,&wparam(2));
156         &mov    ($len,1);
157         &lea    ("ebx",&DWP(32,$ctx));          # key
158         &lea    ($ctx,&DWP(16,$ctx));           # control word
159         &data_byte(0xf3,0x0f,0xa7,0xc8);        # rep xcryptecb
160         &pop    ("ebx");
161         &pop    ("esi");
162         &pop    ("edi");
163         &ret    ();
164 &function_end_B("padlock_aes_block");
165
166 sub generate_mode {
167 my ($mode,$opcode) = @_;
168 # int padlock_$mode_encrypt(void *out, const void *inp,
169 #               struct padlock_cipher_data *ctx, size_t len);
170 &function_begin("padlock_${mode}_encrypt");
171         &mov    ($out,&wparam(0));
172         &mov    ($inp,&wparam(1));
173         &mov    ($ctx,&wparam(2));
174         &mov    ($len,&wparam(3));
175         &test   ($ctx,15);
176         &jnz    (&label("${mode}_abort"));
177         &test   ($len,15);
178         &jnz    (&label("${mode}_abort"));
179         &lea    ("eax",($::win32 or $::coff) ? &DWP(&label("padlock_saved_context")) :
180                        &DWP(&label("padlock_saved_context")."-".&label("${mode}_pic_point")));
181         &pushf  ();
182         &cld    ();
183         &call   ("_padlock_verify_ctx");
184 &set_label("${mode}_pic_point");
185         &lea    ($ctx,&DWP(16,$ctx));   # control word
186         &xor    ("eax","eax");
187                                         if ($mode eq "ctr32") {
188         &movq   ("mm0",&QWP(-16,$ctx)); # load [upper part of] counter
189                                         } else {
190         &xor    ("ebx","ebx");
191         &test   (&DWP(0,$ctx),1<<5);    # align bit in control word
192         &jnz    (&label("${mode}_aligned"));
193         &test   ($out,0x0f);
194         &setz   ("al");                 # !out_misaligned
195         &test   ($inp,0x0f);
196         &setz   ("bl");                 # !inp_misaligned
197         &test   ("eax","ebx");
198         &jnz    (&label("${mode}_aligned"));
199         &neg    ("eax");
200                                         }
201         &mov    ($chunk,$PADLOCK_CHUNK);
202         &not    ("eax");                # out_misaligned?-1:0
203         &lea    ("ebp",&DWP(-24,"esp"));
204         &cmp    ($len,$chunk);
205         &cmovc  ($chunk,$len);          # chunk=len>PADLOCK_CHUNK?PADLOCK_CHUNK:len
206         &and    ("eax",$chunk);         # out_misaligned?chunk:0
207         &mov    ($chunk,$len);
208         &neg    ("eax");
209         &and    ($chunk,$PADLOCK_CHUNK-1);      # chunk=len%PADLOCK_CHUNK
210         &lea    ("esp",&DWP(0,"eax","ebp"));    # alloca
211         &mov    ("eax",$PADLOCK_CHUNK);
212         &cmovz  ($chunk,"eax");                 # chunk=chunk?:PADLOCK_CHUNK
213         &mov    ("eax","ebp");
214         &and    ("ebp",-16);
215         &and    ("esp",-16);
216         &mov    (&DWP(16,"ebp"),"eax");
217     if ($PADLOCK_PREFETCH{$mode}) {
218         &cmp    ($len,$chunk);
219         &ja     (&label("${mode}_loop"));
220         &mov    ("eax",$inp);           # check if prefetch crosses page
221         &cmp    ("ebp","esp");
222         &cmove  ("eax",$out);
223         &add    ("eax",$len);
224         &neg    ("eax");
225         &and    ("eax",0xfff);          # distance to page boundary
226         &cmp    ("eax",$PADLOCK_PREFETCH{$mode});
227         &mov    ("eax",-$PADLOCK_PREFETCH{$mode});
228         &cmovae ("eax",$chunk);         # mask=distance<prefetch?-prefetch:-1
229         &and    ($chunk,"eax");
230         &jz     (&label("${mode}_unaligned_tail"));
231     }
232         &jmp    (&label("${mode}_loop"));
233
234 &set_label("${mode}_loop",16);
235         &mov    (&DWP(0,"ebp"),$out);           # save parameters
236         &mov    (&DWP(4,"ebp"),$inp);
237         &mov    (&DWP(8,"ebp"),$len);
238         &mov    ($len,$chunk);
239         &mov    (&DWP(12,"ebp"),$chunk);        # chunk
240                                                 if ($mode eq "ctr32") {
241         &mov    ("ecx",&DWP(-4,$ctx));
242         &xor    ($out,$out);
243         &mov    ("eax",&DWP(-8,$ctx));          # borrow $len
244 &set_label("${mode}_prepare");
245         &mov    (&DWP(12,"esp",$out),"ecx");
246         &bswap  ("ecx");
247         &movq   (&QWP(0,"esp",$out),"mm0");
248         &inc    ("ecx");
249         &mov    (&DWP(8,"esp",$out),"eax");
250         &bswap  ("ecx");
251         &lea    ($out,&DWP(16,$out));
252         &cmp    ($out,$chunk);
253         &jb     (&label("${mode}_prepare"));
254
255         &mov    (&DWP(-4,$ctx),"ecx");
256         &lea    ($inp,&DWP(0,"esp"));
257         &lea    ($out,&DWP(0,"esp"));
258         &mov    ($len,$chunk);
259                                                 } else {
260         &test   ($out,0x0f);                    # out_misaligned
261         &cmovnz ($out,"esp");
262         &test   ($inp,0x0f);                    # inp_misaligned
263         &jz     (&label("${mode}_inp_aligned"));
264         &shr    ($len,2);
265         &data_byte(0xf3,0xa5);                  # rep movsl
266         &sub    ($out,$chunk);
267         &mov    ($len,$chunk);
268         &mov    ($inp,$out);
269 &set_label("${mode}_inp_aligned");
270                                                 }
271         &lea    ("eax",&DWP(-16,$ctx));         # ivp
272         &lea    ("ebx",&DWP(16,$ctx));          # key
273         &shr    ($len,4);                       # len/=AES_BLOCK_SIZE
274         &data_byte(0xf3,0x0f,0xa7,$opcode);     # rep xcrypt*
275                                                 if ($mode !~ /ecb|ctr/) {
276         &movaps ("xmm0",&QWP(0,"eax"));
277         &movaps (&QWP(-16,$ctx),"xmm0");        # copy [or refresh] iv
278                                                 }
279         &mov    ($out,&DWP(0,"ebp"));           # restore parameters
280         &mov    ($chunk,&DWP(12,"ebp"));
281                                                 if ($mode eq "ctr32") {
282         &mov    ($inp,&DWP(4,"ebp"));
283         &xor    ($len,$len);
284 &set_label("${mode}_xor");
285         &movups ("xmm1",&QWP(0,$inp,$len));
286         &lea    ($len,&DWP(16,$len));
287         &pxor   ("xmm1",&QWP(-16,"esp",$len));
288         &movups (&QWP(-16,$out,$len),"xmm1");
289         &cmp    ($len,$chunk);
290         &jb     (&label("${mode}_xor"));
291                                                 } else {
292         &test   ($out,0x0f);
293         &jz     (&label("${mode}_out_aligned"));
294         &mov    ($len,$chunk);
295         &lea    ($inp,&DWP(0,"esp"));
296         &shr    ($len,2);
297         &data_byte(0xf3,0xa5);                  # rep movsl
298         &sub    ($out,$chunk);
299 &set_label("${mode}_out_aligned");
300         &mov    ($inp,&DWP(4,"ebp"));
301                                                 }
302         &mov    ($len,&DWP(8,"ebp"));
303         &add    ($out,$chunk);
304         &add    ($inp,$chunk);
305         &sub    ($len,$chunk);
306         &mov    ($chunk,$PADLOCK_CHUNK);
307     if (!$PADLOCK_PREFETCH{$mode}) {
308         &jnz    (&label("${mode}_loop"));
309     } else {
310         &jz     (&label("${mode}_break"));
311         &cmp    ($len,$chunk);
312         &jae    (&label("${mode}_loop"));
313
314 &set_label("${mode}_unaligned_tail");
315         &xor    ("eax","eax");
316         &cmp    ("esp","ebp");
317         &cmove  ("eax",$len);
318         &sub    ("esp","eax");                  # alloca
319         &mov    ("eax", $out);                  # save parameters
320         &mov    ($chunk,$len);
321         &shr    ($len,2);
322         &lea    ($out,&DWP(0,"esp"));
323         &data_byte(0xf3,0xa5);                  # rep movsl
324         &mov    ($inp,"esp");
325         &mov    ($out,"eax");                   # restore parameters
326         &mov    ($len,$chunk);
327         &jmp    (&label("${mode}_loop"));
328
329 &set_label("${mode}_break",16);
330     }
331                                                 if ($mode ne "ctr32") {
332         &cmp    ("esp","ebp");
333         &je     (&label("${mode}_done"));
334                                                 }
335         &pxor   ("xmm0","xmm0");
336         &lea    ("eax",&DWP(0,"esp"));
337 &set_label("${mode}_bzero");
338         &movaps (&QWP(0,"eax"),"xmm0");
339         &lea    ("eax",&DWP(16,"eax"));
340         &cmp    ("ebp","eax");
341         &ja     (&label("${mode}_bzero"));
342
343 &set_label("${mode}_done");
344         &mov    ("ebp",&DWP(16,"ebp"));
345         &lea    ("esp",&DWP(24,"ebp"));
346                                                 if ($mode ne "ctr32") {
347         &jmp    (&label("${mode}_exit"));
348
349 &set_label("${mode}_aligned",16);
350     if ($PADLOCK_PREFETCH{$mode}) {
351         &lea    ("ebp",&DWP(0,$inp,$len));
352         &neg    ("ebp");
353         &and    ("ebp",0xfff);                  # distance to page boundary
354         &xor    ("eax","eax");
355         &cmp    ("ebp",$PADLOCK_PREFETCH{$mode});
356         &mov    ("ebp",$PADLOCK_PREFETCH{$mode}-1);
357         &cmovae ("ebp","eax");
358         &and    ("ebp",$len);                   # remainder
359         &sub    ($len,"ebp");
360         &jz     (&label("${mode}_aligned_tail"));
361     }
362         &lea    ("eax",&DWP(-16,$ctx));         # ivp
363         &lea    ("ebx",&DWP(16,$ctx));          # key
364         &shr    ($len,4);                       # len/=AES_BLOCK_SIZE
365         &data_byte(0xf3,0x0f,0xa7,$opcode);     # rep xcrypt*
366                                                 if ($mode ne "ecb") {
367         &movaps ("xmm0",&QWP(0,"eax"));
368         &movaps (&QWP(-16,$ctx),"xmm0");        # copy [or refresh] iv
369                                                 }
370     if ($PADLOCK_PREFETCH{$mode}) {
371         &test   ("ebp","ebp");
372         &jz     (&label("${mode}_exit"));
373
374 &set_label("${mode}_aligned_tail");
375         &mov    ($len,"ebp");
376         &lea    ("ebp",&DWP(-24,"esp"));
377         &mov    ("esp","ebp");
378         &mov    ("eax","ebp");
379         &sub    ("esp",$len);
380         &and    ("ebp",-16);
381         &and    ("esp",-16);
382         &mov    (&DWP(16,"ebp"),"eax");
383         &mov    ("eax", $out);                  # save parameters
384         &mov    ($chunk,$len);
385         &shr    ($len,2);
386         &lea    ($out,&DWP(0,"esp"));
387         &data_byte(0xf3,0xa5);                  # rep movsl
388         &mov    ($inp,"esp");
389         &mov    ($out,"eax");                   # restore parameters
390         &mov    ($len,$chunk);
391         &jmp    (&label("${mode}_loop"));
392     }
393 &set_label("${mode}_exit");                     }
394         &mov    ("eax",1);
395         &lea    ("esp",&DWP(4,"esp"));          # popf
396         &emms   ()                              if ($mode eq "ctr32");
397 &set_label("${mode}_abort");
398 &function_end("padlock_${mode}_encrypt");
399 }
400
401 &generate_mode("ecb",0xc8);
402 &generate_mode("cbc",0xd0);
403 &generate_mode("cfb",0xe0);
404 &generate_mode("ofb",0xe8);
405 &generate_mode("ctr32",0xc8);   # yes, it implements own CTR with ECB opcode,
406                                 # because hardware CTR was introduced later
407                                 # and even has errata on certain C7 stepping.
408                                 # own implementation *always* works, though
409                                 # ~15% slower than dedicated hardware...
410
411 &function_begin_B("padlock_xstore");
412         &push   ("edi");
413         &mov    ("edi",&wparam(0));
414         &mov    ("edx",&wparam(1));
415         &data_byte(0x0f,0xa7,0xc0);             # xstore
416         &pop    ("edi");
417         &ret    ();
418 &function_end_B("padlock_xstore");
419
420 &function_begin_B("_win32_segv_handler");
421         &mov    ("eax",1);                      # ExceptionContinueSearch
422         &mov    ("edx",&wparam(0));             # *ExceptionRecord
423         &mov    ("ecx",&wparam(2));             # *ContextRecord
424         &cmp    (&DWP(0,"edx"),0xC0000005)      # ExceptionRecord->ExceptionCode == STATUS_ACCESS_VIOLATION
425         &jne    (&label("ret"));
426         &add    (&DWP(184,"ecx"),4);            # skip over rep sha*
427         &mov    ("eax",0);                      # ExceptionContinueExecution
428 &set_label("ret");
429         &ret    ();
430 &function_end_B("_win32_segv_handler");
431 &safeseh("_win32_segv_handler")                 if ($::win32);
432
433 &function_begin_B("padlock_sha1_oneshot");
434         &push   ("edi");
435         &push   ("esi");
436         &xor    ("eax","eax");
437         &mov    ("edi",&wparam(0));
438         &mov    ("esi",&wparam(1));
439         &mov    ("ecx",&wparam(2));
440     if ($::win32 or $::coff) {
441         &push   (&::islabel("_win32_segv_handler"));
442         &data_byte(0x64,0xff,0x30);             # push  %fs:(%eax)
443         &data_byte(0x64,0x89,0x20);             # mov   %esp,%fs:(%eax)
444     }
445         &mov    ("edx","esp");                  # put aside %esp
446         &add    ("esp",-128);                   # 32 is enough but spec says 128
447         &movups ("xmm0",&QWP(0,"edi"));         # copy-in context
448         &and    ("esp",-16);
449         &mov    ("eax",&DWP(16,"edi"));
450         &movaps (&QWP(0,"esp"),"xmm0");
451         &mov    ("edi","esp");
452         &mov    (&DWP(16,"esp"),"eax");
453         &xor    ("eax","eax");
454         &data_byte(0xf3,0x0f,0xa6,0xc8);        # rep xsha1
455         &movaps ("xmm0",&QWP(0,"esp"));
456         &mov    ("eax",&DWP(16,"esp"));
457         &mov    ("esp","edx");                  # restore %esp
458     if ($::win32 or $::coff) {
459         &data_byte(0x64,0x8f,0x05,0,0,0,0);     # pop   %fs:0
460         &lea    ("esp",&DWP(4,"esp"));
461     }
462         &mov    ("edi",&wparam(0));
463         &movups (&QWP(0,"edi"),"xmm0");         # copy-out context
464         &mov    (&DWP(16,"edi"),"eax");
465         &pop    ("esi");
466         &pop    ("edi");
467         &ret    ();
468 &function_end_B("padlock_sha1_oneshot");
469
470 &function_begin_B("padlock_sha1_blocks");
471         &push   ("edi");
472         &push   ("esi");
473         &mov    ("edi",&wparam(0));
474         &mov    ("esi",&wparam(1));
475         &mov    ("edx","esp");                  # put aside %esp
476         &mov    ("ecx",&wparam(2));
477         &add    ("esp",-128);
478         &movups ("xmm0",&QWP(0,"edi"));         # copy-in context
479         &and    ("esp",-16);
480         &mov    ("eax",&DWP(16,"edi"));
481         &movaps (&QWP(0,"esp"),"xmm0");
482         &mov    ("edi","esp");
483         &mov    (&DWP(16,"esp"),"eax");
484         &mov    ("eax",-1);
485         &data_byte(0xf3,0x0f,0xa6,0xc8);        # rep xsha1
486         &movaps ("xmm0",&QWP(0,"esp"));
487         &mov    ("eax",&DWP(16,"esp"));
488         &mov    ("esp","edx");                  # restore %esp
489         &mov    ("edi",&wparam(0));
490         &movups (&QWP(0,"edi"),"xmm0");         # copy-out context
491         &mov    (&DWP(16,"edi"),"eax");
492         &pop    ("esi");
493         &pop    ("edi");
494         &ret    ();
495 &function_end_B("padlock_sha1_blocks");
496
497 &function_begin_B("padlock_sha256_oneshot");
498         &push   ("edi");
499         &push   ("esi");
500         &xor    ("eax","eax");
501         &mov    ("edi",&wparam(0));
502         &mov    ("esi",&wparam(1));
503         &mov    ("ecx",&wparam(2));
504     if ($::win32 or $::coff) {
505         &push   (&::islabel("_win32_segv_handler"));
506         &data_byte(0x64,0xff,0x30);             # push  %fs:(%eax)
507         &data_byte(0x64,0x89,0x20);             # mov   %esp,%fs:(%eax)
508     }
509         &mov    ("edx","esp");                  # put aside %esp
510         &add    ("esp",-128);
511         &movups ("xmm0",&QWP(0,"edi"));         # copy-in context
512         &and    ("esp",-16);
513         &movups ("xmm1",&QWP(16,"edi"));
514         &movaps (&QWP(0,"esp"),"xmm0");
515         &mov    ("edi","esp");
516         &movaps (&QWP(16,"esp"),"xmm1");
517         &xor    ("eax","eax");
518         &data_byte(0xf3,0x0f,0xa6,0xd0);        # rep xsha256
519         &movaps ("xmm0",&QWP(0,"esp"));
520         &movaps ("xmm1",&QWP(16,"esp"));
521         &mov    ("esp","edx");                  # restore %esp
522     if ($::win32 or $::coff) {
523         &data_byte(0x64,0x8f,0x05,0,0,0,0);     # pop   %fs:0
524         &lea    ("esp",&DWP(4,"esp"));
525     }
526         &mov    ("edi",&wparam(0));
527         &movups (&QWP(0,"edi"),"xmm0");         # copy-out context
528         &movups (&QWP(16,"edi"),"xmm1");
529         &pop    ("esi");
530         &pop    ("edi");
531         &ret    ();
532 &function_end_B("padlock_sha256_oneshot");
533
534 &function_begin_B("padlock_sha256_blocks");
535         &push   ("edi");
536         &push   ("esi");
537         &mov    ("edi",&wparam(0));
538         &mov    ("esi",&wparam(1));
539         &mov    ("ecx",&wparam(2));
540         &mov    ("edx","esp");                  # put aside %esp
541         &add    ("esp",-128);
542         &movups ("xmm0",&QWP(0,"edi"));         # copy-in context
543         &and    ("esp",-16);
544         &movups ("xmm1",&QWP(16,"edi"));
545         &movaps (&QWP(0,"esp"),"xmm0");
546         &mov    ("edi","esp");
547         &movaps (&QWP(16,"esp"),"xmm1");
548         &mov    ("eax",-1);
549         &data_byte(0xf3,0x0f,0xa6,0xd0);        # rep xsha256
550         &movaps ("xmm0",&QWP(0,"esp"));
551         &movaps ("xmm1",&QWP(16,"esp"));
552         &mov    ("esp","edx");                  # restore %esp
553         &mov    ("edi",&wparam(0));
554         &movups (&QWP(0,"edi"),"xmm0");         # copy-out context
555         &movups (&QWP(16,"edi"),"xmm1");
556         &pop    ("esi");
557         &pop    ("edi");
558         &ret    ();
559 &function_end_B("padlock_sha256_blocks");
560
561 &function_begin_B("padlock_sha512_blocks");
562         &push   ("edi");
563         &push   ("esi");
564         &mov    ("edi",&wparam(0));
565         &mov    ("esi",&wparam(1));
566         &mov    ("ecx",&wparam(2));
567         &mov    ("edx","esp");                  # put aside %esp
568         &add    ("esp",-128);
569         &movups ("xmm0",&QWP(0,"edi"));         # copy-in context
570         &and    ("esp",-16);
571         &movups ("xmm1",&QWP(16,"edi"));
572         &movups ("xmm2",&QWP(32,"edi"));
573         &movups ("xmm3",&QWP(48,"edi"));
574         &movaps (&QWP(0,"esp"),"xmm0");
575         &mov    ("edi","esp");
576         &movaps (&QWP(16,"esp"),"xmm1");
577         &movaps (&QWP(32,"esp"),"xmm2");
578         &movaps (&QWP(48,"esp"),"xmm3");
579         &data_byte(0xf3,0x0f,0xa6,0xe0);        # rep xsha512
580         &movaps ("xmm0",&QWP(0,"esp"));
581         &movaps ("xmm1",&QWP(16,"esp"));
582         &movaps ("xmm2",&QWP(32,"esp"));
583         &movaps ("xmm3",&QWP(48,"esp"));
584         &mov    ("esp","edx");                  # restore %esp
585         &mov    ("edi",&wparam(0));
586         &movups (&QWP(0,"edi"),"xmm0");         # copy-out context
587         &movups (&QWP(16,"edi"),"xmm1");
588         &movups (&QWP(32,"edi"),"xmm2");
589         &movups (&QWP(48,"edi"),"xmm3");
590         &pop    ("esi");
591         &pop    ("edi");
592         &ret    ();
593 &function_end_B("padlock_sha512_blocks");
594
595 &asciz  ("VIA Padlock x86 module, CRYPTOGAMS by <appro\@openssl.org>");
596 &align  (16);
597
598 &dataseg();
599 # Essentially this variable belongs in thread local storage.
600 # Having this variable global on the other hand can only cause
601 # few bogus key reloads [if any at all on signle-CPU system],
602 # so we accept the penalty...
603 &set_label("padlock_saved_context",4);
604 &data_word(0);
605
606 &asm_finish();