Exported gnutls_cpuid() and gnutls_have_cpuid().
[gnutls:gnutls.git] / devel / perlasm / e_padlock-x86.pl
1 #!/usr/bin/env perl
2
3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9
10 # September 2011
11 #
12 # Assembler helpers for Padlock engine. Compared to original engine
13 # version relying on inline assembler and compiled with gcc 3.4.6 it
14 # was measured to provide ~100% improvement on misaligned data in ECB
15 # mode and ~75% in CBC mode. For aligned data improvement can be
16 # observed for short inputs only, e.g. 45% for 64-byte messages in
17 # ECB mode, 20% in CBC. Difference in performance for aligned vs.
18 # misaligned data depends on misalignment and is either ~1.8x or 2.9x.
19 # These are approximately same factors as for hardware support, so
20 # there is little reason to rely on the latter. On the contrary, it
21 # might actually hurt performance in mixture of aligned and misaligned
22 # buffers, because a) if you choose to flip 'align' flag in control
23 # word on per-buffer basis, then you'd have to reload key context,
24 # which incurs penalty; b) if you choose to set 'align' flag
25 # permanently, it limits performance even for aligned data to ~1/2.
26 # All above mentioned results were collected on 1.5GHz C7. Nano on the
27 # other hand handles unaligned data more gracefully. Depending on
28 # algorithm and how unaligned data is, hardware can be up to 70% more
29 # efficient than below software alignment procedures, nor does 'align'
30 # flag have affect on aligned performance [if has any meaning at all].
31 # Therefore suggestion is to unconditionally set 'align' flag on Nano
32 # for optimal performance.
33
34 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
35 push(@INC,"${dir}","${dir}../../crypto/perlasm");
36 require "x86asm.pl";
37
38 &asm_init($ARGV[0],$0);
39
40 %PADLOCK_MARGIN=(ecb=>128, cbc=>64); # prefetch errata
41 $PADLOCK_CHUNK=512;     # Must be a power of 2 larger than 16
42
43 $ctx="edx";
44 $out="edi";
45 $inp="esi";
46 $len="ecx";
47 $chunk="ebx";
48
49 &function_begin_B("padlock_capability");
50         &push   ("ebx");
51         &pushf  ();
52         &pop    ("eax");
53         &mov    ("ecx","eax");
54         &xor    ("eax",1<<21);
55         &push   ("eax");
56         &popf   ();
57         &pushf  ();
58         &pop    ("eax");
59         &xor    ("ecx","eax");
60         &xor    ("eax","eax");
61         &bt     ("ecx",21);
62         &jnc    (&label("noluck"));
63         &cpuid  ();
64         &xor    ("eax","eax");
65         &cmp    ("ebx","0x".unpack("H*",'tneC'));
66         &jne    (&label("noluck"));
67         &cmp    ("edx","0x".unpack("H*",'Hrua'));
68         &jne    (&label("noluck"));
69         &cmp    ("ecx","0x".unpack("H*",'slua'));
70         &jne    (&label("noluck"));
71         &mov    ("eax",0xC0000000);
72         &cpuid  ();
73         &mov    ("edx","eax");
74         &xor    ("eax","eax");
75         &cmp    ("edx",0xC0000001);
76         &jb     (&label("noluck"));
77         &mov    ("eax",1);
78         &cpuid  ();
79         &or     ("eax",0x0f);
80         &xor    ("ebx","ebx");
81         &and    ("eax",0x0fff);
82         &cmp    ("eax",0x06ff);         # check for Nano
83         &sete   ("bl");
84         &mov    ("eax",0xC0000001);
85         &push   ("ebx");
86         &cpuid  ();
87         &pop    ("ebx");
88         &mov    ("eax","edx");
89         &shl    ("ebx",4);              # bit#4 denotes Nano
90         &and    ("eax",0xffffffef);
91         &or     ("eax","ebx")
92 &set_label("noluck");
93         &pop    ("ebx");
94         &ret    ();
95 &function_end_B("padlock_capability")
96
97 &function_begin_B("padlock_key_bswap");
98         &mov    ("edx",&wparam(0));
99         &mov    ("ecx",&DWP(240,"edx"));
100 &set_label("bswap_loop");
101         &mov    ("eax",&DWP(0,"edx"));
102         &bswap  ("eax");
103         &mov    (&DWP(0,"edx"),"eax");
104         &lea    ("edx",&DWP(4,"edx"));
105         &sub    ("ecx",1);
106         &jnz    (&label("bswap_loop"));
107         &ret    ();
108 &function_end_B("padlock_key_bswap");
109
110 # This is heuristic key context tracing. At first one
111 # believes that one should use atomic swap instructions,
112 # but it's not actually necessary. Point is that if
113 # padlock_saved_context was changed by another thread
114 # after we've read it and before we compare it with ctx,
115 # our key *shall* be reloaded upon thread context switch
116 # and we are therefore set in either case...
117 &static_label("padlock_saved_context");
118
119 &function_begin_B("padlock_verify_context");
120         &mov    ($ctx,&wparam(0));
121         &lea    ("eax",($::win32 or $::coff) ? &DWP(&label("padlock_saved_context")) :
122                        &DWP(&label("padlock_saved_context")."-".&label("verify_pic_point")));
123         &pushf  ();
124         &call   ("_padlock_verify_ctx");
125 &set_label("verify_pic_point");
126         &lea    ("esp",&DWP(4,"esp"));
127         &ret    ();
128 &function_end_B("padlock_verify_context");
129
130 &function_begin_B("_padlock_verify_ctx");
131         &add    ("eax",&DWP(0,"esp")) if(!($::win32 or $::coff));# &padlock_saved_context
132         &bt     (&DWP(4,"esp"),30);             # eflags
133         &jnc    (&label("verified"));
134         &cmp    ($ctx,&DWP(0,"eax"));
135         &je     (&label("verified"));
136         &pushf  ();
137         &popf   ();
138 &set_label("verified");
139         &mov    (&DWP(0,"eax"),$ctx);
140         &ret    ();
141 &function_end_B("_padlock_verify_ctx");
142
143 &function_begin_B("padlock_reload_key");
144         &pushf  ();
145         &popf   ();
146         &ret    ();
147 &function_end_B("padlock_reload_key");
148
149 &function_begin_B("padlock_aes_block");
150         &push   ("edi");
151         &push   ("esi");
152         &push   ("ebx");
153         &mov    ($out,&wparam(0));              # must be 16-byte aligned
154         &mov    ($inp,&wparam(1));              # must be 16-byte aligned
155         &mov    ($ctx,&wparam(2));
156         &mov    ($len,1);
157         &lea    ("ebx",&DWP(32,$ctx));          # key
158         &lea    ($ctx,&DWP(16,$ctx));           # control word
159         &data_byte(0xf3,0x0f,0xa7,0xc8);        # rep xcryptecb
160         &pop    ("ebx");
161         &pop    ("esi");
162         &pop    ("edi");
163         &ret    ();
164 &function_end_B("padlock_aes_block");
165
166 sub generate_mode {
167 my ($mode,$opcode) = @_;
168 # int padlock_$mode_encrypt(void *out, const void *inp,
169 #               struct padlock_cipher_data *ctx, size_t len);
170 &function_begin("padlock_${mode}_encrypt");
171         &mov    ($out,&wparam(0));
172         &mov    ($inp,&wparam(1));
173         &mov    ($ctx,&wparam(2));
174         &mov    ($len,&wparam(3));
175         &test   ($ctx,15);
176         &jnz    (&label("${mode}_abort"));
177         &test   ($len,15);
178         &jnz    (&label("${mode}_abort"));
179         &lea    ("eax",($::win32 or $::coff) ? &DWP(&label("padlock_saved_context")) :
180                        &DWP(&label("padlock_saved_context")."-".&label("${mode}_pic_point")));
181         &pushf  ();
182         &cld    ();
183         &call   ("_padlock_verify_ctx");
184 &set_label("${mode}_pic_point");
185         &lea    ($ctx,&DWP(16,$ctx));   # control word
186         &xor    ("eax","eax");
187                                         if ($mode eq "ctr32") {
188         &movq   ("mm0",&QWP(-16,$ctx)); # load [upper part of] counter
189                                         } else {
190         &xor    ("ebx","ebx");
191     if ($PADLOCK_MARGIN{$mode}) {
192         &cmp    ($len,$PADLOCK_MARGIN{$mode});
193         &jbe    (&label("${mode}_short"));
194     }
195         &test   (&DWP(0,$ctx),1<<5);    # align bit in control word
196         &jnz    (&label("${mode}_aligned"));
197         &test   ($out,0x0f);
198         &setz   ("al");                 # !out_misaligned
199         &test   ($inp,0x0f);
200         &setz   ("bl");                 # !inp_misaligned
201         &test   ("eax","ebx");
202         &jnz    (&label("${mode}_aligned"));
203         &neg    ("eax");
204                                         }
205         &mov    ($chunk,$PADLOCK_CHUNK);
206         &not    ("eax");                # out_misaligned?-1:0
207         &lea    ("ebp",&DWP(-24,"esp"));
208         &cmp    ($len,$chunk);
209         &cmovc  ($chunk,$len);          # chunk=len>PADLOCK_CHUNK?PADLOCK_CHUNK:len
210         &and    ("eax",$chunk);         # out_misaligned?chunk:0
211         &mov    ($chunk,$len);
212         &neg    ("eax");
213         &and    ($chunk,$PADLOCK_CHUNK-1);      # chunk=len%PADLOCK_CHUNK
214         &lea    ("esp",&DWP(0,"eax","ebp"));    # alloca
215         &and    ("esp",-16);
216         &jmp    (&label("${mode}_loop"));
217
218 &set_label("${mode}_loop",16);
219         &mov    (&DWP(0,"ebp"),$out);           # save parameters
220         &mov    (&DWP(4,"ebp"),$inp);
221         &mov    (&DWP(8,"ebp"),$len);
222         &mov    ($len,$chunk);
223         &mov    (&DWP(12,"ebp"),$chunk);        # chunk
224                                                 if ($mode eq "ctr32") {
225         &mov    ("ecx",&DWP(-4,$ctx));
226         &xor    ($out,$out);
227         &mov    ("eax",&DWP(-8,$ctx));          # borrow $len
228 &set_label("${mode}_prepare");
229         &mov    (&DWP(12,"esp",$out),"ecx");
230         &bswap  ("ecx");
231         &movq   (&QWP(0,"esp",$out),"mm0");
232         &inc    ("ecx");
233         &mov    (&DWP(8,"esp",$out),"eax");
234         &bswap  ("ecx");
235         &lea    ($out,&DWP(16,$out));
236         &cmp    ($out,$chunk);
237         &jb     (&label("${mode}_prepare"));
238
239         &mov    (&DWP(-4,$ctx),"ecx");
240         &lea    ($inp,&DWP(0,"esp"));
241         &lea    ($out,&DWP(0,"esp"));
242         &mov    ($len,$chunk);
243                                                 } else {
244         &test   ($out,0x0f);                    # out_misaligned
245         &cmovnz ($out,"esp");
246         &test   ($inp,0x0f);                    # inp_misaligned
247         &jz     (&label("${mode}_inp_aligned"));
248         &shr    ($len,2);
249         &data_byte(0xf3,0xa5);                  # rep movsl
250         &sub    ($out,$chunk);
251         &mov    ($len,$chunk);
252         &mov    ($inp,$out);
253 &set_label("${mode}_inp_aligned");
254                                                 }
255         &lea    ("eax",&DWP(-16,$ctx));         # ivp
256         &lea    ("ebx",&DWP(16,$ctx));          # key
257         &shr    ($len,4);                       # len/=AES_BLOCK_SIZE
258         &data_byte(0xf3,0x0f,0xa7,$opcode);     # rep xcrypt*
259                                                 if ($mode !~ /ecb|ctr/) {
260         &movaps ("xmm0",&QWP(0,"eax"));
261         &movaps (&QWP(-16,$ctx),"xmm0");        # copy [or refresh] iv
262                                                 }
263         &mov    ($out,&DWP(0,"ebp"));           # restore parameters
264         &mov    ($chunk,&DWP(12,"ebp"));
265                                                 if ($mode eq "ctr32") {
266         &mov    ($inp,&DWP(4,"ebp"));
267         &xor    ($len,$len);
268 &set_label("${mode}_xor");
269         &movups ("xmm1",&QWP(0,$inp,$len));
270         &lea    ($len,&DWP(16,$len));
271         &pxor   ("xmm1",&QWP(-16,"esp",$len));
272         &movups (&QWP(-16,$out,$len),"xmm1");
273         &cmp    ($len,$chunk);
274         &jb     (&label("${mode}_xor"));
275                                                 } else {
276         &test   ($out,0x0f);
277         &jz     (&label("${mode}_out_aligned"));
278         &mov    ($len,$chunk);
279         &shr    ($len,2);
280         &lea    ($inp,&DWP(0,"esp"));
281         &data_byte(0xf3,0xa5);                  # rep movsl
282         &sub    ($out,$chunk);
283 &set_label("${mode}_out_aligned");
284         &mov    ($inp,&DWP(4,"ebp"));
285                                                 }
286         &mov    ($len,&DWP(8,"ebp"));
287         &add    ($out,$chunk);
288         &add    ($inp,$chunk);
289         &sub    ($len,$chunk);
290         &mov    ($chunk,$PADLOCK_CHUNK);
291         &jnz    (&label("${mode}_loop"));
292                                                 if ($mode ne "ctr32") {
293         &cmp    ("esp","ebp");
294         &je     (&label("${mode}_done"));
295                                                 }
296         &pxor   ("xmm0","xmm0");
297         &lea    ("eax",&DWP(0,"esp"));
298 &set_label("${mode}_bzero");
299         &movaps (&QWP(0,"eax"),"xmm0");
300         &lea    ("eax",&DWP(16,"eax"));
301         &cmp    ("ebp","eax");
302         &ja     (&label("${mode}_bzero"));
303
304 &set_label("${mode}_done");
305         &lea    ("esp",&DWP(24,"ebp"));
306                                                 if ($mode ne "ctr32") {
307         &jmp    (&label("${mode}_exit"));
308
309 &set_label("${mode}_short",16);
310         &xor    ("eax","eax");
311         &lea    ("ebp",&DWP(-24,"esp"));
312         &sub    ("eax",$len);
313         &lea    ("esp",&DWP(0,"eax","ebp"));
314         &and    ("esp",-16);
315         &xor    ($chunk,$chunk);
316 &set_label("${mode}_short_copy");
317         &movups ("xmm0",&QWP(0,$inp,$chunk));
318         &lea    ($chunk,&DWP(16,$chunk));
319         &cmp    ($len,$chunk);
320         &movaps (&QWP(-16,"esp",$chunk),"xmm0");
321         &ja     (&label("${mode}_short_copy"));
322         &mov    ($inp,"esp");
323         &mov    ($chunk,$len);
324         &jmp    (&label("${mode}_loop"));
325
326 &set_label("${mode}_aligned",16);
327         &lea    ("eax",&DWP(-16,$ctx));         # ivp
328         &lea    ("ebx",&DWP(16,$ctx));          # key
329         &shr    ($len,4);                       # len/=AES_BLOCK_SIZE
330         &data_byte(0xf3,0x0f,0xa7,$opcode);     # rep xcrypt*
331                                                 if ($mode ne "ecb") {
332         &movaps ("xmm0",&QWP(0,"eax"));
333         &movaps (&QWP(-16,$ctx),"xmm0");        # copy [or refresh] iv
334                                                 }
335 &set_label("${mode}_exit");                     }
336         &mov    ("eax",1);
337         &lea    ("esp",&DWP(4,"esp"));          # popf
338         &emms   ()                              if ($mode eq "ctr32");
339 &set_label("${mode}_abort");
340 &function_end("padlock_${mode}_encrypt");
341 }
342
343 &generate_mode("ecb",0xc8);
344 &generate_mode("cbc",0xd0);
345 #&generate_mode("cfb",0xe0);
346 #&generate_mode("ofb",0xe8);
347 #&generate_mode("ctr32",0xc8);  # yes, it implements own CTR with ECB opcode,
348                                 # because hardware CTR was introduced later
349                                 # and even has errata on certain C7 stepping.
350                                 # own implementation *always* works, though
351                                 # ~15% slower than dedicated hardware...
352
353 &function_begin_B("padlock_xstore");
354         &push   ("edi");
355         &mov    ("edi",&wparam(0));
356         &mov    ("edx",&wparam(1));
357         &data_byte(0x0f,0xa7,0xc0);             # xstore
358         &pop    ("edi");
359         &ret    ();
360 &function_end_B("padlock_xstore");
361
362 &function_begin_B("_win32_segv_handler");
363         &mov    ("eax",1);                      # ExceptionContinueSearch
364         &mov    ("edx",&wparam(0));             # *ExceptionRecord
365         &mov    ("ecx",&wparam(2));             # *ContextRecord
366         &cmp    (&DWP(0,"edx"),0xC0000005)      # ExceptionRecord->ExceptionCode == STATUS_ACCESS_VIOLATION
367         &jne    (&label("ret"));
368         &add    (&DWP(184,"ecx"),4);            # skip over rep sha*
369         &mov    ("eax",0);                      # ExceptionContinueExecution
370 &set_label("ret");
371         &ret    ();
372 &function_end_B("_win32_segv_handler");
373 &safeseh("_win32_segv_handler")                 if ($::win32);
374
375 &function_begin_B("padlock_sha1_oneshot");
376         &push   ("edi");
377         &push   ("esi");
378         &xor    ("eax","eax");
379         &mov    ("edi",&wparam(0));
380         &mov    ("esi",&wparam(1));
381         &mov    ("ecx",&wparam(2));
382     if ($::win32 or $::coff) {
383         &push   (&::islabel("_win32_segv_handler"));
384         &data_byte(0x64,0xff,0x30);             # push  %fs:(%eax)
385         &data_byte(0x64,0x89,0x20);             # mov   %esp,%fs:(%eax)
386     }
387         &mov    ("edx","esp");                  # put aside %esp
388         &add    ("esp",-128);                   # 32 is enough but spec says 128
389         &movups ("xmm0",&QWP(0,"edi"));         # copy-in context
390         &and    ("esp",-16);
391         &mov    ("eax",&DWP(16,"edi"));
392         &movaps (&QWP(0,"esp"),"xmm0");
393         &mov    ("edi","esp");
394         &mov    (&DWP(16,"esp"),"eax");
395         &xor    ("eax","eax");
396         &data_byte(0xf3,0x0f,0xa6,0xc8);        # rep xsha1
397         &movaps ("xmm0",&QWP(0,"esp"));
398         &mov    ("eax",&DWP(16,"esp"));
399         &mov    ("esp","edx");                  # restore %esp
400     if ($::win32 or $::coff) {
401         &data_byte(0x64,0x8f,0x05,0,0,0,0);     # pop   %fs:0
402         &lea    ("esp",&DWP(4,"esp"));
403     }
404         &mov    ("edi",&wparam(0));
405         &movups (&QWP(0,"edi"),"xmm0");         # copy-out context
406         &mov    (&DWP(16,"edi"),"eax");
407         &pop    ("esi");
408         &pop    ("edi");
409         &ret    ();
410 &function_end_B("padlock_sha1_oneshot");
411
412 &function_begin_B("padlock_sha1_blocks");
413         &push   ("edi");
414         &push   ("esi");
415         &mov    ("edi",&wparam(0));
416         &mov    ("esi",&wparam(1));
417         &mov    ("edx","esp");                  # put aside %esp
418         &mov    ("ecx",&wparam(2));
419         &add    ("esp",-128);
420         &movups ("xmm0",&QWP(0,"edi"));         # copy-in context
421         &and    ("esp",-16);
422         &mov    ("eax",&DWP(16,"edi"));
423         &movaps (&QWP(0,"esp"),"xmm0");
424         &mov    ("edi","esp");
425         &mov    (&DWP(16,"esp"),"eax");
426         &mov    ("eax",-1);
427         &data_byte(0xf3,0x0f,0xa6,0xc8);        # rep xsha1
428         &movaps ("xmm0",&QWP(0,"esp"));
429         &mov    ("eax",&DWP(16,"esp"));
430         &mov    ("esp","edx");                  # restore %esp
431         &mov    ("edi",&wparam(0));
432         &movups (&QWP(0,"edi"),"xmm0");         # copy-out context
433         &mov    (&DWP(16,"edi"),"eax");
434         &pop    ("esi");
435         &pop    ("edi");
436         &ret    ();
437 &function_end_B("padlock_sha1_blocks");
438
439 &function_begin_B("padlock_sha256_oneshot");
440         &push   ("edi");
441         &push   ("esi");
442         &xor    ("eax","eax");
443         &mov    ("edi",&wparam(0));
444         &mov    ("esi",&wparam(1));
445         &mov    ("ecx",&wparam(2));
446     if ($::win32 or $::coff) {
447         &push   (&::islabel("_win32_segv_handler"));
448         &data_byte(0x64,0xff,0x30);             # push  %fs:(%eax)
449         &data_byte(0x64,0x89,0x20);             # mov   %esp,%fs:(%eax)
450     }
451         &mov    ("edx","esp");                  # put aside %esp
452         &add    ("esp",-128);
453         &movups ("xmm0",&QWP(0,"edi"));         # copy-in context
454         &and    ("esp",-16);
455         &movups ("xmm1",&QWP(16,"edi"));
456         &movaps (&QWP(0,"esp"),"xmm0");
457         &mov    ("edi","esp");
458         &movaps (&QWP(16,"esp"),"xmm1");
459         &xor    ("eax","eax");
460         &data_byte(0xf3,0x0f,0xa6,0xd0);        # rep xsha256
461         &movaps ("xmm0",&QWP(0,"esp"));
462         &movaps ("xmm1",&QWP(16,"esp"));
463         &mov    ("esp","edx");                  # restore %esp
464     if ($::win32 or $::coff) {
465         &data_byte(0x64,0x8f,0x05,0,0,0,0);     # pop   %fs:0
466         &lea    ("esp",&DWP(4,"esp"));
467     }
468         &mov    ("edi",&wparam(0));
469         &movups (&QWP(0,"edi"),"xmm0");         # copy-out context
470         &movups (&QWP(16,"edi"),"xmm1");
471         &pop    ("esi");
472         &pop    ("edi");
473         &ret    ();
474 &function_end_B("padlock_sha256_oneshot");
475
476 &function_begin_B("padlock_sha256_blocks");
477         &push   ("edi");
478         &push   ("esi");
479         &mov    ("edi",&wparam(0));
480         &mov    ("esi",&wparam(1));
481         &mov    ("ecx",&wparam(2));
482         &mov    ("edx","esp");                  # put aside %esp
483         &add    ("esp",-128);
484         &movups ("xmm0",&QWP(0,"edi"));         # copy-in context
485         &and    ("esp",-16);
486         &movups ("xmm1",&QWP(16,"edi"));
487         &movaps (&QWP(0,"esp"),"xmm0");
488         &mov    ("edi","esp");
489         &movaps (&QWP(16,"esp"),"xmm1");
490         &mov    ("eax",-1);
491         &data_byte(0xf3,0x0f,0xa6,0xd0);        # rep xsha256
492         &movaps ("xmm0",&QWP(0,"esp"));
493         &movaps ("xmm1",&QWP(16,"esp"));
494         &mov    ("esp","edx");                  # restore %esp
495         &mov    ("edi",&wparam(0));
496         &movups (&QWP(0,"edi"),"xmm0");         # copy-out context
497         &movups (&QWP(16,"edi"),"xmm1");
498         &pop    ("esi");
499         &pop    ("edi");
500         &ret    ();
501 &function_end_B("padlock_sha256_blocks");
502
503 &function_begin_B("padlock_sha512_blocks");
504         &push   ("edi");
505         &push   ("esi");
506         &mov    ("edi",&wparam(0));
507         &mov    ("esi",&wparam(1));
508         &mov    ("ecx",&wparam(2));
509         &mov    ("edx","esp");                  # put aside %esp
510         &add    ("esp",-128);
511         &movups ("xmm0",&QWP(0,"edi"));         # copy-in context
512         &and    ("esp",-16);
513         &movups ("xmm1",&QWP(16,"edi"));
514         &movups ("xmm2",&QWP(32,"edi"));
515         &movups ("xmm3",&QWP(48,"edi"));
516         &movaps (&QWP(0,"esp"),"xmm0");
517         &mov    ("edi","esp");
518         &movaps (&QWP(16,"esp"),"xmm1");
519         &movaps (&QWP(32,"esp"),"xmm2");
520         &movaps (&QWP(48,"esp"),"xmm3");
521         &data_byte(0xf3,0x0f,0xa6,0xe0);        # rep xsha512
522         &movaps ("xmm0",&QWP(0,"esp"));
523         &movaps ("xmm1",&QWP(16,"esp"));
524         &movaps ("xmm2",&QWP(32,"esp"));
525         &movaps ("xmm3",&QWP(48,"esp"));
526         &mov    ("esp","edx");                  # restore %esp
527         &mov    ("edi",&wparam(0));
528         &movups (&QWP(0,"edi"),"xmm0");         # copy-out context
529         &movups (&QWP(16,"edi"),"xmm1");
530         &movups (&QWP(32,"edi"),"xmm2");
531         &movups (&QWP(48,"edi"),"xmm3");
532         &pop    ("esi");
533         &pop    ("edi");
534         &ret    ();
535 &function_end_B("padlock_sha512_blocks");
536
537 &asciz  ("VIA Padlock x86 module, CRYPTOGAMS by <appro\@openssl.org>");
538 &align  (16);
539
540 &dataseg();
541 # Essentially this variable belongs in thread local storage.
542 # Having this variable global on the other hand can only cause
543 # few bogus key reloads [if any at all on signle-CPU system],
544 # so we accept the penalty...
545 &set_label("padlock_saved_context",4);
546 &data_word(0);
547
548 &asm_finish();