Fix macosx-ppc build (and typos in unwind info).
[openssl.git] / crypto / aes / asm / aesp8-ppc.pl
1 #!/usr/bin/env perl
2 #
3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9 #
10 # This module implements support for AES instructions as per PowerISA
11 # specification version 2.07, first implemented by POWER8 processor.
12 # The module is endian-agnostic in sense that it supports both big-
13 # and little-endian cases. Data alignment in parallelizable modes is
14 # handled with VSX loads and stores, which implies MSR.VSX flag being
15 # set. It should also be noted that ISA specification doesn't prohibit
16 # alignment exceptions for these instructions on page boundaries.
17 # Initially alignment was handled in pure AltiVec/VMX way [when data
18 # is aligned programmatically, which in turn guarantees exception-
19 # free execution], but it turned to hamper performance when vcipher
20 # instructions are interleaved. It's reckoned that eventual
21 # misalignment penalties at page boundaries are in average lower
22 # than additional overhead in pure AltiVec approach.
23
24 $flavour = shift;
25
26 if ($flavour =~ /64/) {
27         $SIZE_T =8;
28         $LRSAVE =2*$SIZE_T;
29         $STU    ="stdu";
30         $POP    ="ld";
31         $PUSH   ="std";
32         $UCMP   ="cmpld";
33         $SHL    ="sldi";
34 } elsif ($flavour =~ /32/) {
35         $SIZE_T =4;
36         $LRSAVE =$SIZE_T;
37         $STU    ="stwu";
38         $POP    ="lwz";
39         $PUSH   ="stw";
40         $UCMP   ="cmplw";
41         $SHL    ="slwi";
42 } else { die "nonsense $flavour"; }
43
44 $LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;
45
46 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
47 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
48 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
49 die "can't locate ppc-xlate.pl";
50
51 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
52
53 $FRAME=8*$SIZE_T;
54 $prefix="aes_p8";
55
56 $sp="r1";
57 $vrsave="r12";
58
59 #########################################################################
60 {{{     # Key setup procedures                                          #
61 my ($inp,$bits,$out,$ptr,$cnt,$rounds)=map("r$_",(3..8));
62 my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6));
63 my ($stage,$outperm,$outmask,$outhead,$outtail)=map("v$_",(7..11));
64
65 $code.=<<___;
66 .machine        "any"
67
68 .text
69
70 .align  7
71 rcon:
72 .long   0x01000000, 0x01000000, 0x01000000, 0x01000000  ?rev
73 .long   0x1b000000, 0x1b000000, 0x1b000000, 0x1b000000  ?rev
74 .long   0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c  ?rev
75 .long   0,0,0,0                                         ?asis
76 Lconsts:
77         mflr    r0
78         bcl     20,31,\$+4
79         mflr    $ptr     #vvvvv "distance between . and rcon
80         addi    $ptr,$ptr,-0x48
81         mtlr    r0
82         blr
83         .long   0
84         .byte   0,12,0x14,0,0,0,0,0
85 .asciz  "AES for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
86
87 .globl  .${prefix}_set_encrypt_key
88 .align  5
89 .${prefix}_set_encrypt_key:
90 Lset_encrypt_key:
91         mflr            r11
92         $PUSH           r11,$LRSAVE($sp)
93
94         li              $ptr,-1
95         ${UCMP}i        $inp,0
96         beq-            Lenc_key_abort          # if ($inp==0) return -1;
97         ${UCMP}i        $out,0
98         beq-            Lenc_key_abort          # if ($out==0) return -1;
99         li              $ptr,-2
100         cmpwi           $bits,128
101         blt-            Lenc_key_abort
102         cmpwi           $bits,256
103         bgt-            Lenc_key_abort
104         andi.           r0,$bits,0x3f
105         bne-            Lenc_key_abort
106
107         lis             r0,0xfff0
108         mfspr           $vrsave,256
109         mtspr           256,r0
110
111         bl              Lconsts
112         mtlr            r11
113
114         neg             r9,$inp
115         lvx             $in0,0,$inp
116         addi            $inp,$inp,15            # 15 is not typo
117         lvsr            $key,0,r9               # borrow $key
118         li              r8,0x20
119         cmpwi           $bits,192
120         lvx             $in1,0,$inp
121         le?vspltisb     $mask,0x0f              # borrow $mask
122         lvx             $rcon,0,$ptr
123         le?vxor         $key,$key,$mask         # adjust for byte swap
124         lvx             $mask,r8,$ptr
125         addi            $ptr,$ptr,0x10
126         vperm           $in0,$in0,$in1,$key     # align [and byte swap in LE]
127         li              $cnt,8
128         vxor            $zero,$zero,$zero
129         mtctr           $cnt
130
131         ?lvsr           $outperm,0,$out
132         vspltisb        $outmask,-1
133         lvx             $outhead,0,$out
134         ?vperm          $outmask,$zero,$outmask,$outperm
135
136         blt             Loop128
137         addi            $inp,$inp,8
138         beq             L192
139         addi            $inp,$inp,8
140         b               L256
141
142 .align  4
143 Loop128:
144         vperm           $key,$in0,$in0,$mask    # rotate-n-splat
145         vsldoi          $tmp,$zero,$in0,12      # >>32
146          vperm          $outtail,$in0,$in0,$outperm     # rotate
147          vsel           $stage,$outhead,$outtail,$outmask
148          vmr            $outhead,$outtail
149         vcipherlast     $key,$key,$rcon
150          stvx           $stage,0,$out
151          addi           $out,$out,16
152
153         vxor            $in0,$in0,$tmp
154         vsldoi          $tmp,$zero,$tmp,12      # >>32
155         vxor            $in0,$in0,$tmp
156         vsldoi          $tmp,$zero,$tmp,12      # >>32
157         vxor            $in0,$in0,$tmp
158          vadduwm        $rcon,$rcon,$rcon
159         vxor            $in0,$in0,$key
160         bdnz            Loop128
161
162         lvx             $rcon,0,$ptr            # last two round keys
163
164         vperm           $key,$in0,$in0,$mask    # rotate-n-splat
165         vsldoi          $tmp,$zero,$in0,12      # >>32
166          vperm          $outtail,$in0,$in0,$outperm     # rotate
167          vsel           $stage,$outhead,$outtail,$outmask
168          vmr            $outhead,$outtail
169         vcipherlast     $key,$key,$rcon
170          stvx           $stage,0,$out
171          addi           $out,$out,16
172
173         vxor            $in0,$in0,$tmp
174         vsldoi          $tmp,$zero,$tmp,12      # >>32
175         vxor            $in0,$in0,$tmp
176         vsldoi          $tmp,$zero,$tmp,12      # >>32
177         vxor            $in0,$in0,$tmp
178          vadduwm        $rcon,$rcon,$rcon
179         vxor            $in0,$in0,$key
180
181         vperm           $key,$in0,$in0,$mask    # rotate-n-splat
182         vsldoi          $tmp,$zero,$in0,12      # >>32
183          vperm          $outtail,$in0,$in0,$outperm     # rotate
184          vsel           $stage,$outhead,$outtail,$outmask
185          vmr            $outhead,$outtail
186         vcipherlast     $key,$key,$rcon
187          stvx           $stage,0,$out
188          addi           $out,$out,16
189
190         vxor            $in0,$in0,$tmp
191         vsldoi          $tmp,$zero,$tmp,12      # >>32
192         vxor            $in0,$in0,$tmp
193         vsldoi          $tmp,$zero,$tmp,12      # >>32
194         vxor            $in0,$in0,$tmp
195         vxor            $in0,$in0,$key
196          vperm          $outtail,$in0,$in0,$outperm     # rotate
197          vsel           $stage,$outhead,$outtail,$outmask
198          vmr            $outhead,$outtail
199          stvx           $stage,0,$out
200
201         addi            $inp,$out,15            # 15 is not typo
202         addi            $out,$out,0x50
203
204         li              $rounds,10
205         b               Ldone
206
207 .align  4
208 L192:
209         lvx             $tmp,0,$inp
210         li              $cnt,4
211          vperm          $outtail,$in0,$in0,$outperm     # rotate
212          vsel           $stage,$outhead,$outtail,$outmask
213          vmr            $outhead,$outtail
214          stvx           $stage,0,$out
215          addi           $out,$out,16
216         vperm           $in1,$in1,$tmp,$key     # align [and byte swap in LE]
217         vspltisb        $key,8                  # borrow $key
218         mtctr           $cnt
219         vsububm         $mask,$mask,$key        # adjust the mask
220
221 Loop192:
222         vperm           $key,$in1,$in1,$mask    # roate-n-splat
223         vsldoi          $tmp,$zero,$in0,12      # >>32
224         vcipherlast     $key,$key,$rcon
225
226         vxor            $in0,$in0,$tmp
227         vsldoi          $tmp,$zero,$tmp,12      # >>32
228         vxor            $in0,$in0,$tmp
229         vsldoi          $tmp,$zero,$tmp,12      # >>32
230         vxor            $in0,$in0,$tmp
231
232          vsldoi         $stage,$zero,$in1,8
233         vspltw          $tmp,$in0,3
234         vxor            $tmp,$tmp,$in1
235         vsldoi          $in1,$zero,$in1,12      # >>32
236          vadduwm        $rcon,$rcon,$rcon
237         vxor            $in1,$in1,$tmp
238         vxor            $in0,$in0,$key
239         vxor            $in1,$in1,$key
240          vsldoi         $stage,$stage,$in0,8
241
242         vperm           $key,$in1,$in1,$mask    # rotate-n-splat
243         vsldoi          $tmp,$zero,$in0,12      # >>32
244          vperm          $outtail,$stage,$stage,$outperm # rotate
245          vsel           $stage,$outhead,$outtail,$outmask
246          vmr            $outhead,$outtail
247         vcipherlast     $key,$key,$rcon
248          stvx           $stage,0,$out
249          addi           $out,$out,16
250
251          vsldoi         $stage,$in0,$in1,8
252         vxor            $in0,$in0,$tmp
253         vsldoi          $tmp,$zero,$tmp,12      # >>32
254          vperm          $outtail,$stage,$stage,$outperm # rotate
255          vsel           $stage,$outhead,$outtail,$outmask
256          vmr            $outhead,$outtail
257         vxor            $in0,$in0,$tmp
258         vsldoi          $tmp,$zero,$tmp,12      # >>32
259         vxor            $in0,$in0,$tmp
260          stvx           $stage,0,$out
261          addi           $out,$out,16
262
263         vspltw          $tmp,$in0,3
264         vxor            $tmp,$tmp,$in1
265         vsldoi          $in1,$zero,$in1,12      # >>32
266          vadduwm        $rcon,$rcon,$rcon
267         vxor            $in1,$in1,$tmp
268         vxor            $in0,$in0,$key
269         vxor            $in1,$in1,$key
270          vperm          $outtail,$in0,$in0,$outperm     # rotate
271          vsel           $stage,$outhead,$outtail,$outmask
272          vmr            $outhead,$outtail
273          stvx           $stage,0,$out
274          addi           $inp,$out,15            # 15 is not typo
275          addi           $out,$out,16
276         bdnz            Loop192
277
278         li              $rounds,12
279         addi            $out,$out,0x20
280         b               Ldone
281
282 .align  4
283 L256:
284         lvx             $tmp,0,$inp
285         li              $cnt,7
286         li              $rounds,14
287          vperm          $outtail,$in0,$in0,$outperm     # rotate
288          vsel           $stage,$outhead,$outtail,$outmask
289          vmr            $outhead,$outtail
290          stvx           $stage,0,$out
291          addi           $out,$out,16
292         vperm           $in1,$in1,$tmp,$key     # align [and byte swap in LE]
293         mtctr           $cnt
294
295 Loop256:
296         vperm           $key,$in1,$in1,$mask    # rotate-n-splat
297         vsldoi          $tmp,$zero,$in0,12      # >>32
298          vperm          $outtail,$in1,$in1,$outperm     # rotate
299          vsel           $stage,$outhead,$outtail,$outmask
300          vmr            $outhead,$outtail
301         vcipherlast     $key,$key,$rcon
302          stvx           $stage,0,$out
303          addi           $out,$out,16
304
305         vxor            $in0,$in0,$tmp
306         vsldoi          $tmp,$zero,$tmp,12      # >>32
307         vxor            $in0,$in0,$tmp
308         vsldoi          $tmp,$zero,$tmp,12      # >>32
309         vxor            $in0,$in0,$tmp
310          vadduwm        $rcon,$rcon,$rcon
311         vxor            $in0,$in0,$key
312          vperm          $outtail,$in0,$in0,$outperm     # rotate
313          vsel           $stage,$outhead,$outtail,$outmask
314          vmr            $outhead,$outtail
315          stvx           $stage,0,$out
316          addi           $inp,$out,15            # 15 is not typo
317          addi           $out,$out,16
318         bdz             Ldone
319
320         vspltw          $key,$in0,3             # just splat
321         vsldoi          $tmp,$zero,$in1,12      # >>32
322         vsbox           $key,$key
323
324         vxor            $in1,$in1,$tmp
325         vsldoi          $tmp,$zero,$tmp,12      # >>32
326         vxor            $in1,$in1,$tmp
327         vsldoi          $tmp,$zero,$tmp,12      # >>32
328         vxor            $in1,$in1,$tmp
329
330         vxor            $in1,$in1,$key
331         b               Loop256
332
333 .align  4
334 Ldone:
335         lvx             $in1,0,$inp             # redundant in aligned case
336         vsel            $in1,$outhead,$in1,$outmask
337         stvx            $in1,0,$inp
338         li              $ptr,0
339         mtspr           256,$vrsave
340         stw             $rounds,0($out)
341
342 Lenc_key_abort:
343         mr              r3,$ptr
344         blr
345         .long           0
346         .byte           0,12,0x14,1,0,0,3,0
347         .long           0
348 .size   .${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key
349
350 .globl  .${prefix}_set_decrypt_key
351 .align  5
352 .${prefix}_set_decrypt_key:
353         $STU            $sp,-$FRAME($sp)
354         mflr            r10
355         $PUSH           r10,$FRAME+$LRSAVE($sp)
356         bl              Lset_encrypt_key
357         mtlr            r10
358
359         cmpwi           r3,0
360         bne-            Ldec_key_abort
361
362         slwi            $cnt,$rounds,4
363         subi            $inp,$out,240           # first round key
364         srwi            $rounds,$rounds,1
365         add             $out,$inp,$cnt          # last round key
366         mtctr           $rounds
367
368 Ldeckey:
369         lwz             r0, 0($inp)
370         lwz             r6, 4($inp)
371         lwz             r7, 8($inp)
372         lwz             r8, 12($inp)
373         addi            $inp,$inp,16
374         lwz             r9, 0($out)
375         lwz             r10,4($out)
376         lwz             r11,8($out)
377         lwz             r12,12($out)
378         stw             r0, 0($out)
379         stw             r6, 4($out)
380         stw             r7, 8($out)
381         stw             r8, 12($out)
382         subi            $out,$out,16
383         stw             r9, -16($inp)
384         stw             r10,-12($inp)
385         stw             r11,-8($inp)
386         stw             r12,-4($inp)
387         bdnz            Ldeckey
388
389         xor             r3,r3,r3                # return value
390 Ldec_key_abort:
391         addi            $sp,$sp,$FRAME
392         blr
393         .long           0
394         .byte           0,12,4,1,0x80,0,3,0
395         .long           0
396 .size   .${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key
397 ___
398 }}}
399 #########################################################################
400 {{{     # Single block en- and decrypt procedures                       #
401 sub gen_block () {
402 my $dir = shift;
403 my $n   = $dir eq "de" ? "n" : "";
404 my ($inp,$out,$key,$rounds,$idx)=map("r$_",(3..7));
405
406 $code.=<<___;
407 .globl  .${prefix}_${dir}crypt
408 .align  5
409 .${prefix}_${dir}crypt:
410         lwz             $rounds,240($key)
411         lis             r0,0xfc00
412         mfspr           $vrsave,256
413         li              $idx,15                 # 15 is not typo
414         mtspr           256,r0
415
416         lvx             v0,0,$inp
417         neg             r11,$out
418         lvx             v1,$idx,$inp
419         lvsl            v2,0,$inp               # inpperm
420         le?vspltisb     v4,0x0f
421         ?lvsl           v3,0,r11                # outperm
422         le?vxor         v2,v2,v4
423         li              $idx,16
424         vperm           v0,v0,v1,v2             # align [and byte swap in LE]
425         lvx             v1,0,$key
426         ?lvsl           v5,0,$key               # keyperm
427         srwi            $rounds,$rounds,1
428         lvx             v2,$idx,$key
429         addi            $idx,$idx,16
430         subi            $rounds,$rounds,1
431         ?vperm          v1,v1,v2,v5             # align round key
432
433         vxor            v0,v0,v1
434         lvx             v1,$idx,$key
435         addi            $idx,$idx,16
436         mtctr           $rounds
437
438 Loop_${dir}c:
439         ?vperm          v2,v2,v1,v5
440         v${n}cipher     v0,v0,v2
441         lvx             v2,$idx,$key
442         addi            $idx,$idx,16
443         ?vperm          v1,v1,v2,v5
444         v${n}cipher     v0,v0,v1
445         lvx             v1,$idx,$key
446         addi            $idx,$idx,16
447         bdnz            Loop_${dir}c
448
449         ?vperm          v2,v2,v1,v5
450         v${n}cipher     v0,v0,v2
451         lvx             v2,$idx,$key
452         ?vperm          v1,v1,v2,v5
453         v${n}cipherlast v0,v0,v1
454
455         vspltisb        v2,-1
456         vxor            v1,v1,v1
457         li              $idx,15                 # 15 is not typo
458         ?vperm          v2,v1,v2,v3             # outmask
459         le?vxor         v3,v3,v4
460         lvx             v1,0,$out               # outhead
461         vperm           v0,v0,v0,v3             # rotate [and byte swap in LE]
462         vsel            v1,v1,v0,v2
463         lvx             v4,$idx,$out
464         stvx            v1,0,$out
465         vsel            v0,v0,v4,v2
466         stvx            v0,$idx,$out
467
468         mtspr           256,$vrsave
469         blr
470         .long           0
471         .byte           0,12,0x14,0,0,0,3,0
472         .long           0
473 .size   .${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt
474 ___
475 }
476 &gen_block("en");
477 &gen_block("de");
478 }}}
479 #########################################################################
480 {{{     # CBC en- and decrypt procedures                                #
481 my ($inp,$out,$len,$key,$ivp,$enc,$rounds,$idx)=map("r$_",(3..10));
482 my ($rndkey0,$rndkey1,$inout,$tmp)=             map("v$_",(0..3));
483 my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)=
484                                                 map("v$_",(4..10));
485 $code.=<<___;
486 .globl  .${prefix}_cbc_encrypt
487 .align  5
488 .${prefix}_cbc_encrypt:
489         ${UCMP}i        $len,16
490         bltlr-
491
492         cmpwi           $enc,0                  # test direction
493         lis             r0,0xffe0
494         mfspr           $vrsave,256
495         mtspr           256,r0
496
497         li              $idx,15
498         vxor            $rndkey0,$rndkey0,$rndkey0
499         le?vspltisb     $tmp,0x0f
500
501         lvx             $ivec,0,$ivp            # load [unaligned] iv
502         lvsl            $inpperm,0,$ivp
503         lvx             $inptail,$idx,$ivp
504         le?vxor         $inpperm,$inpperm,$tmp
505         vperm           $ivec,$ivec,$inptail,$inpperm
506
507         neg             r11,$inp
508         ?lvsl           $keyperm,0,$key         # prepare for unaligned key
509         lwz             $rounds,240($key)
510
511         lvsr            $inpperm,0,r11          # prepare for unaligned load
512         lvx             $inptail,0,$inp
513         addi            $inp,$inp,15            # 15 is not typo
514         le?vxor         $inpperm,$inpperm,$tmp
515
516         ?lvsr           $outperm,0,$out         # prepare for unaligned store
517         vspltisb        $outmask,-1
518         lvx             $outhead,0,$out
519         ?vperm          $outmask,$rndkey0,$outmask,$outperm
520         le?vxor         $outperm,$outperm,$tmp
521
522         srwi            $rounds,$rounds,1
523         li              $idx,16
524         subi            $rounds,$rounds,1
525         beq             Lcbc_dec
526
527 Lcbc_enc:
528         vmr             $inout,$inptail
529         lvx             $inptail,0,$inp
530         addi            $inp,$inp,16
531         mtctr           $rounds
532         subi            $len,$len,16            # len-=16
533
534         lvx             $rndkey0,0,$key
535          vperm          $inout,$inout,$inptail,$inpperm
536         lvx             $rndkey1,$idx,$key
537         addi            $idx,$idx,16
538         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
539         vxor            $inout,$inout,$rndkey0
540         lvx             $rndkey0,$idx,$key
541         addi            $idx,$idx,16
542         vxor            $inout,$inout,$ivec
543
544 Loop_cbc_enc:
545         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
546         vcipher         $inout,$inout,$rndkey1
547         lvx             $rndkey1,$idx,$key
548         addi            $idx,$idx,16
549         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
550         vcipher         $inout,$inout,$rndkey0
551         lvx             $rndkey0,$idx,$key
552         addi            $idx,$idx,16
553         bdnz            Loop_cbc_enc
554
555         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
556         vcipher         $inout,$inout,$rndkey1
557         lvx             $rndkey1,$idx,$key
558         li              $idx,16
559         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
560         vcipherlast     $ivec,$inout,$rndkey0
561         ${UCMP}i        $len,16
562
563         vperm           $tmp,$ivec,$ivec,$outperm
564         vsel            $inout,$outhead,$tmp,$outmask
565         vmr             $outhead,$tmp
566         stvx            $inout,0,$out
567         addi            $out,$out,16
568         bge             Lcbc_enc
569
570         b               Lcbc_done
571
572 .align  4
573 Lcbc_dec:
574         ${UCMP}i        $len,128
575         bge             _aesp8_cbc_decrypt8x
576         vmr             $tmp,$inptail
577         lvx             $inptail,0,$inp
578         addi            $inp,$inp,16
579         mtctr           $rounds
580         subi            $len,$len,16            # len-=16
581
582         lvx             $rndkey0,0,$key
583          vperm          $tmp,$tmp,$inptail,$inpperm
584         lvx             $rndkey1,$idx,$key
585         addi            $idx,$idx,16
586         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
587         vxor            $inout,$tmp,$rndkey0
588         lvx             $rndkey0,$idx,$key
589         addi            $idx,$idx,16
590
591 Loop_cbc_dec:
592         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
593         vncipher        $inout,$inout,$rndkey1
594         lvx             $rndkey1,$idx,$key
595         addi            $idx,$idx,16
596         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
597         vncipher        $inout,$inout,$rndkey0
598         lvx             $rndkey0,$idx,$key
599         addi            $idx,$idx,16
600         bdnz            Loop_cbc_dec
601
602         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
603         vncipher        $inout,$inout,$rndkey1
604         lvx             $rndkey1,$idx,$key
605         li              $idx,16
606         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
607         vncipherlast    $inout,$inout,$rndkey0
608         ${UCMP}i        $len,16
609
610         vxor            $inout,$inout,$ivec
611         vmr             $ivec,$tmp
612         vperm           $tmp,$inout,$inout,$outperm
613         vsel            $inout,$outhead,$tmp,$outmask
614         vmr             $outhead,$tmp
615         stvx            $inout,0,$out
616         addi            $out,$out,16
617         bge             Lcbc_dec
618
619 Lcbc_done:
620         addi            $out,$out,-1
621         lvx             $inout,0,$out           # redundant in aligned case
622         vsel            $inout,$outhead,$inout,$outmask
623         stvx            $inout,0,$out
624
625         neg             $enc,$ivp               # write [unaligned] iv
626         li              $idx,15                 # 15 is not typo
627         vxor            $rndkey0,$rndkey0,$rndkey0
628         vspltisb        $outmask,-1
629         le?vspltisb     $tmp,0x0f
630         ?lvsl           $outperm,0,$enc
631         ?vperm          $outmask,$rndkey0,$outmask,$outperm
632         le?vxor         $outperm,$outperm,$tmp
633         lvx             $outhead,0,$ivp
634         vperm           $ivec,$ivec,$ivec,$outperm
635         vsel            $inout,$outhead,$ivec,$outmask
636         lvx             $inptail,$idx,$ivp
637         stvx            $inout,0,$ivp
638         vsel            $inout,$ivec,$inptail,$outmask
639         stvx            $inout,$idx,$ivp
640
641         mtspr           256,$vrsave
642         blr
643         .long           0
644         .byte           0,12,0x14,0,0,0,6,0
645         .long           0
646 ___
647 #########################################################################
648 {{      # Optimized CBC decrypt procedure                               #
649 my $key_="r11";
650 my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
651     $x00=0 if ($flavour =~ /osx/);
652 my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13));
653 my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(14..21));
654 my $rndkey0="v23";      # v24-v25 rotating buffer for first found keys
655                         # v26-v31 last 6 round keys
656 my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment
657
658 $code.=<<___;
659 .align  5
660 _aesp8_cbc_decrypt8x:
661         $STU            $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
662         li              r10,`$FRAME+8*16+15`
663         li              r11,`$FRAME+8*16+31`
664         stvx            v20,r10,$sp             # ABI says so
665         addi            r10,r10,32
666         stvx            v21,r11,$sp
667         addi            r11,r11,32
668         stvx            v22,r10,$sp
669         addi            r10,r10,32
670         stvx            v23,r11,$sp
671         addi            r11,r11,32
672         stvx            v24,r10,$sp
673         addi            r10,r10,32
674         stvx            v25,r11,$sp
675         addi            r11,r11,32
676         stvx            v26,r10,$sp
677         addi            r10,r10,32
678         stvx            v27,r11,$sp
679         addi            r11,r11,32
680         stvx            v28,r10,$sp
681         addi            r10,r10,32
682         stvx            v29,r11,$sp
683         addi            r11,r11,32
684         stvx            v30,r10,$sp
685         stvx            v31,r11,$sp
686         li              r0,-1
687         stw             $vrsave,`$FRAME+21*16-4`($sp)   # save vrsave
688         li              $x10,0x10
689         $PUSH           r26,`$FRAME+21*16+0*$SIZE_T`($sp)
690         li              $x20,0x20
691         $PUSH           r27,`$FRAME+21*16+1*$SIZE_T`($sp)
692         li              $x30,0x30
693         $PUSH           r28,`$FRAME+21*16+2*$SIZE_T`($sp)
694         li              $x40,0x40
695         $PUSH           r29,`$FRAME+21*16+3*$SIZE_T`($sp)
696         li              $x50,0x50
697         $PUSH           r30,`$FRAME+21*16+4*$SIZE_T`($sp)
698         li              $x60,0x60
699         $PUSH           r31,`$FRAME+21*16+5*$SIZE_T`($sp)
700         li              $x70,0x70
701         mtspr           256,r0
702
703         subi            $rounds,$rounds,3       # -4 in total
704         subi            $len,$len,128           # bias
705
706         lvx             $rndkey0,$x00,$key      # load key schedule
707         lvx             v30,$x10,$key
708         addi            $key,$key,0x20
709         lvx             v31,$x00,$key
710         ?vperm          $rndkey0,$rndkey0,v30,$keyperm
711         addi            $key_,$sp,$FRAME+15
712         mtctr           $rounds
713
714 Load_cbc_dec_key:
715         ?vperm          v24,v30,v31,$keyperm
716         lvx             v30,$x10,$key
717         addi            $key,$key,0x20
718         stvx            v24,$x00,$key_          # off-load round[1]
719         ?vperm          v25,v31,v30,$keyperm
720         lvx             v31,$x00,$key
721         stvx            v25,$x10,$key_          # off-load round[2]
722         addi            $key_,$key_,0x20
723         bdnz            Load_cbc_dec_key
724
725         lvx             v26,$x10,$key
726         ?vperm          v24,v30,v31,$keyperm
727         lvx             v27,$x20,$key
728         stvx            v24,$x00,$key_          # off-load round[3]
729         ?vperm          v25,v31,v26,$keyperm
730         lvx             v28,$x30,$key
731         stvx            v25,$x10,$key_          # off-load round[4]
732         addi            $key_,$sp,$FRAME+15     # rewind $key_
733         ?vperm          v26,v26,v27,$keyperm
734         lvx             v29,$x40,$key
735         ?vperm          v27,v27,v28,$keyperm
736         lvx             v30,$x50,$key
737         ?vperm          v28,v28,v29,$keyperm
738         lvx             v31,$x60,$key
739         ?vperm          v29,v29,v30,$keyperm
740         lvx             $out0,$x70,$key         # borrow $out0
741         ?vperm          v30,v30,v31,$keyperm
742         lvx             v24,$x00,$key_          # pre-load round[1]
743         ?vperm          v31,v31,$out0,$keyperm
744         lvx             v25,$x10,$key_          # pre-load round[2]
745
746         #lvx            $inptail,0,$inp         # "caller" already did this
747         #addi           $inp,$inp,15            # 15 is not typo
748         subi            $inp,$inp,15            # undo "caller"
749
750          le?li          $idx,8
751         lvx_u           $in0,$x00,$inp          # load first 8 "words"
752          le?lvsl        $inpperm,0,$idx
753          le?vspltisb    $tmp,0x0f
754         lvx_u           $in1,$x10,$inp
755          le?vxor        $inpperm,$inpperm,$tmp  # transform for lvx_u/stvx_u
756         lvx_u           $in2,$x20,$inp
757          le?vperm       $in0,$in0,$in0,$inpperm
758         lvx_u           $in3,$x30,$inp
759          le?vperm       $in1,$in1,$in1,$inpperm
760         lvx_u           $in4,$x40,$inp
761          le?vperm       $in2,$in2,$in2,$inpperm
762         vxor            $out0,$in0,$rndkey0
763         lvx_u           $in5,$x50,$inp
764          le?vperm       $in3,$in3,$in3,$inpperm
765         vxor            $out1,$in1,$rndkey0
766         lvx_u           $in6,$x60,$inp
767          le?vperm       $in4,$in4,$in4,$inpperm
768         vxor            $out2,$in2,$rndkey0
769         lvx_u           $in7,$x70,$inp
770         addi            $inp,$inp,0x80
771          le?vperm       $in5,$in5,$in5,$inpperm
772         vxor            $out3,$in3,$rndkey0
773          le?vperm       $in6,$in6,$in6,$inpperm
774         vxor            $out4,$in4,$rndkey0
775          le?vperm       $in7,$in7,$in7,$inpperm
776         vxor            $out5,$in5,$rndkey0
777         vxor            $out6,$in6,$rndkey0
778         vxor            $out7,$in7,$rndkey0
779
780         mtctr           $rounds
781         b               Loop_cbc_dec8x
782 .align  5
783 Loop_cbc_dec8x:
784         vncipher        $out0,$out0,v24
785         vncipher        $out1,$out1,v24
786         vncipher        $out2,$out2,v24
787         vncipher        $out3,$out3,v24
788         vncipher        $out4,$out4,v24
789         vncipher        $out5,$out5,v24
790         vncipher        $out6,$out6,v24
791         vncipher        $out7,$out7,v24
792         lvx             v24,$x20,$key_          # round[3]
793         addi            $key_,$key_,0x20
794
795         vncipher        $out0,$out0,v25
796         vncipher        $out1,$out1,v25
797         vncipher        $out2,$out2,v25
798         vncipher        $out3,$out3,v25
799         vncipher        $out4,$out4,v25
800         vncipher        $out5,$out5,v25
801         vncipher        $out6,$out6,v25
802         vncipher        $out7,$out7,v25
803         lvx             v25,$x10,$key_          # round[4]
804         bdnz            Loop_cbc_dec8x
805
806         subic           $len,$len,128           # $len-=128
807         vncipher        $out0,$out0,v24
808         vncipher        $out1,$out1,v24
809         vncipher        $out2,$out2,v24
810         vncipher        $out3,$out3,v24
811         vncipher        $out4,$out4,v24
812         vncipher        $out5,$out5,v24
813         vncipher        $out6,$out6,v24
814         vncipher        $out7,$out7,v24
815
816         subfe.          r0,r0,r0                # borrow?-1:0
817         vncipher        $out0,$out0,v25
818         vncipher        $out1,$out1,v25
819         vncipher        $out2,$out2,v25
820         vncipher        $out3,$out3,v25
821         vncipher        $out4,$out4,v25
822         vncipher        $out5,$out5,v25
823         vncipher        $out6,$out6,v25
824         vncipher        $out7,$out7,v25
825
826         and             r0,r0,$len
827         vncipher        $out0,$out0,v26
828         vncipher        $out1,$out1,v26
829         vncipher        $out2,$out2,v26
830         vncipher        $out3,$out3,v26
831         vncipher        $out4,$out4,v26
832         vncipher        $out5,$out5,v26
833         vncipher        $out6,$out6,v26
834         vncipher        $out7,$out7,v26
835
836         add             $inp,$inp,r0            # $inp is adjusted in such
837                                                 # way that at exit from the
838                                                 # loop inX-in7 are loaded
839                                                 # with last "words"
840         vncipher        $out0,$out0,v27
841         vncipher        $out1,$out1,v27
842         vncipher        $out2,$out2,v27
843         vncipher        $out3,$out3,v27
844         vncipher        $out4,$out4,v27
845         vncipher        $out5,$out5,v27
846         vncipher        $out6,$out6,v27
847         vncipher        $out7,$out7,v27
848
849         addi            $key_,$sp,$FRAME+15     # rewind $key_
850         vncipher        $out0,$out0,v28
851         vncipher        $out1,$out1,v28
852         vncipher        $out2,$out2,v28
853         vncipher        $out3,$out3,v28
854         vncipher        $out4,$out4,v28
855         vncipher        $out5,$out5,v28
856         vncipher        $out6,$out6,v28
857         vncipher        $out7,$out7,v28
858         lvx             v24,$x00,$key_          # re-pre-load round[1]
859
860         vncipher        $out0,$out0,v29
861         vncipher        $out1,$out1,v29
862         vncipher        $out2,$out2,v29
863         vncipher        $out3,$out3,v29
864         vncipher        $out4,$out4,v29
865         vncipher        $out5,$out5,v29
866         vncipher        $out6,$out6,v29
867         vncipher        $out7,$out7,v29
868         lvx             v25,$x10,$key_          # re-pre-load round[2]
869
870         vncipher        $out0,$out0,v30
871          vxor           $ivec,$ivec,v31         # xor with last round key
872         vncipher        $out1,$out1,v30
873          vxor           $in0,$in0,v31
874         vncipher        $out2,$out2,v30
875          vxor           $in1,$in1,v31
876         vncipher        $out3,$out3,v30
877          vxor           $in2,$in2,v31
878         vncipher        $out4,$out4,v30
879          vxor           $in3,$in3,v31
880         vncipher        $out5,$out5,v30
881          vxor           $in4,$in4,v31
882         vncipher        $out6,$out6,v30
883          vxor           $in5,$in5,v31
884         vncipher        $out7,$out7,v30
885          vxor           $in6,$in6,v31
886
887         vncipherlast    $out0,$out0,$ivec
888         vncipherlast    $out1,$out1,$in0
889          lvx_u          $in0,$x00,$inp          # load next input block
890         vncipherlast    $out2,$out2,$in1
891          lvx_u          $in1,$x10,$inp
892         vncipherlast    $out3,$out3,$in2
893          le?vperm       $in0,$in0,$in0,$inpperm
894          lvx_u          $in2,$x20,$inp
895         vncipherlast    $out4,$out4,$in3
896          le?vperm       $in1,$in1,$in1,$inpperm
897          lvx_u          $in3,$x30,$inp
898         vncipherlast    $out5,$out5,$in4
899          le?vperm       $in2,$in2,$in2,$inpperm
900          lvx_u          $in4,$x40,$inp
901         vncipherlast    $out6,$out6,$in5
902          le?vperm       $in3,$in3,$in3,$inpperm
903          lvx_u          $in5,$x50,$inp
904         vncipherlast    $out7,$out7,$in6
905          le?vperm       $in4,$in4,$in4,$inpperm
906          lvx_u          $in6,$x60,$inp
907         vmr             $ivec,$in7
908          le?vperm       $in5,$in5,$in5,$inpperm
909          lvx_u          $in7,$x70,$inp
910          addi           $inp,$inp,0x80
911
912         le?vperm        $out0,$out0,$out0,$inpperm
913         le?vperm        $out1,$out1,$out1,$inpperm
914         stvx_u          $out0,$x00,$out
915          le?vperm       $in6,$in6,$in6,$inpperm
916          vxor           $out0,$in0,$rndkey0
917         le?vperm        $out2,$out2,$out2,$inpperm
918         stvx_u          $out1,$x10,$out
919          le?vperm       $in7,$in7,$in7,$inpperm
920          vxor           $out1,$in1,$rndkey0
921         le?vperm        $out3,$out3,$out3,$inpperm
922         stvx_u          $out2,$x20,$out
923          vxor           $out2,$in2,$rndkey0
924         le?vperm        $out4,$out4,$out4,$inpperm
925         stvx_u          $out3,$x30,$out
926          vxor           $out3,$in3,$rndkey0
927         le?vperm        $out5,$out5,$out5,$inpperm
928         stvx_u          $out4,$x40,$out
929          vxor           $out4,$in4,$rndkey0
930         le?vperm        $out6,$out6,$out6,$inpperm
931         stvx_u          $out5,$x50,$out
932          vxor           $out5,$in5,$rndkey0
933         le?vperm        $out7,$out7,$out7,$inpperm
934         stvx_u          $out6,$x60,$out
935          vxor           $out6,$in6,$rndkey0
936         stvx_u          $out7,$x70,$out
937         addi            $out,$out,0x80
938          vxor           $out7,$in7,$rndkey0
939
940         mtctr           $rounds
941         beq             Loop_cbc_dec8x          # did $len-=128 borrow?
942
943         addic.          $len,$len,128
944         beq             Lcbc_dec8x_done
945         nop
946         nop
947
948 Loop_cbc_dec8x_tail:                            # up to 7 "words" tail...
949         vncipher        $out1,$out1,v24
950         vncipher        $out2,$out2,v24
951         vncipher        $out3,$out3,v24
952         vncipher        $out4,$out4,v24
953         vncipher        $out5,$out5,v24
954         vncipher        $out6,$out6,v24
955         vncipher        $out7,$out7,v24
956         lvx             v24,$x20,$key_          # round[3]
957         addi            $key_,$key_,0x20
958
959         vncipher        $out1,$out1,v25
960         vncipher        $out2,$out2,v25
961         vncipher        $out3,$out3,v25
962         vncipher        $out4,$out4,v25
963         vncipher        $out5,$out5,v25
964         vncipher        $out6,$out6,v25
965         vncipher        $out7,$out7,v25
966         lvx             v25,$x10,$key_          # round[4]
967         bdnz            Loop_cbc_dec8x_tail
968
969         vncipher        $out1,$out1,v24
970         vncipher        $out2,$out2,v24
971         vncipher        $out3,$out3,v24
972         vncipher        $out4,$out4,v24
973         vncipher        $out5,$out5,v24
974         vncipher        $out6,$out6,v24
975         vncipher        $out7,$out7,v24
976
977         vncipher        $out1,$out1,v25
978         vncipher        $out2,$out2,v25
979         vncipher        $out3,$out3,v25
980         vncipher        $out4,$out4,v25
981         vncipher        $out5,$out5,v25
982         vncipher        $out6,$out6,v25
983         vncipher        $out7,$out7,v25
984
985         vncipher        $out1,$out1,v26
986         vncipher        $out2,$out2,v26
987         vncipher        $out3,$out3,v26
988         vncipher        $out4,$out4,v26
989         vncipher        $out5,$out5,v26
990         vncipher        $out6,$out6,v26
991         vncipher        $out7,$out7,v26
992
993         vncipher        $out1,$out1,v27
994         vncipher        $out2,$out2,v27
995         vncipher        $out3,$out3,v27
996         vncipher        $out4,$out4,v27
997         vncipher        $out5,$out5,v27
998         vncipher        $out6,$out6,v27
999         vncipher        $out7,$out7,v27
1000
1001         vncipher        $out1,$out1,v28
1002         vncipher        $out2,$out2,v28
1003         vncipher        $out3,$out3,v28
1004         vncipher        $out4,$out4,v28
1005         vncipher        $out5,$out5,v28
1006         vncipher        $out6,$out6,v28
1007         vncipher        $out7,$out7,v28
1008
1009         vncipher        $out1,$out1,v29
1010         vncipher        $out2,$out2,v29
1011         vncipher        $out3,$out3,v29
1012         vncipher        $out4,$out4,v29
1013         vncipher        $out5,$out5,v29
1014         vncipher        $out6,$out6,v29
1015         vncipher        $out7,$out7,v29
1016
1017         vncipher        $out1,$out1,v30
1018          vxor           $ivec,$ivec,v31         # last round key
1019         vncipher        $out2,$out2,v30
1020          vxor           $in1,$in1,v31
1021         vncipher        $out3,$out3,v30
1022          vxor           $in2,$in2,v31
1023         vncipher        $out4,$out4,v30
1024          vxor           $in3,$in3,v31
1025         vncipher        $out5,$out5,v30
1026          vxor           $in4,$in4,v31
1027         vncipher        $out6,$out6,v30
1028          vxor           $in5,$in5,v31
1029         vncipher        $out7,$out7,v30
1030          vxor           $in6,$in6,v31
1031
1032         cmplwi          $len,32                 # switch($len)
1033         blt             Lcbc_dec8x_one
1034         nop
1035         beq             Lcbc_dec8x_two
1036         cmplwi          $len,64
1037         blt             Lcbc_dec8x_three
1038         nop
1039         beq             Lcbc_dec8x_four
1040         cmplwi          $len,96
1041         blt             Lcbc_dec8x_five
1042         nop
1043         beq             Lcbc_dec8x_six
1044
1045 Lcbc_dec8x_seven:
1046         vncipherlast    $out1,$out1,$ivec
1047         vncipherlast    $out2,$out2,$in1
1048         vncipherlast    $out3,$out3,$in2
1049         vncipherlast    $out4,$out4,$in3
1050         vncipherlast    $out5,$out5,$in4
1051         vncipherlast    $out6,$out6,$in5
1052         vncipherlast    $out7,$out7,$in6
1053         vmr             $ivec,$in7
1054
1055         le?vperm        $out1,$out1,$out1,$inpperm
1056         le?vperm        $out2,$out2,$out2,$inpperm
1057         stvx_u          $out1,$x00,$out
1058         le?vperm        $out3,$out3,$out3,$inpperm
1059         stvx_u          $out2,$x10,$out
1060         le?vperm        $out4,$out4,$out4,$inpperm
1061         stvx_u          $out3,$x20,$out
1062         le?vperm        $out5,$out5,$out5,$inpperm
1063         stvx_u          $out4,$x30,$out
1064         le?vperm        $out6,$out6,$out6,$inpperm
1065         stvx_u          $out5,$x40,$out
1066         le?vperm        $out7,$out7,$out7,$inpperm
1067         stvx_u          $out6,$x50,$out
1068         stvx_u          $out7,$x60,$out
1069         addi            $out,$out,0x70
1070         b               Lcbc_dec8x_done
1071
1072 .align  5
1073 Lcbc_dec8x_six:
1074         vncipherlast    $out2,$out2,$ivec
1075         vncipherlast    $out3,$out3,$in2
1076         vncipherlast    $out4,$out4,$in3
1077         vncipherlast    $out5,$out5,$in4
1078         vncipherlast    $out6,$out6,$in5
1079         vncipherlast    $out7,$out7,$in6
1080         vmr             $ivec,$in7
1081
1082         le?vperm        $out2,$out2,$out2,$inpperm
1083         le?vperm        $out3,$out3,$out3,$inpperm
1084         stvx_u          $out2,$x00,$out
1085         le?vperm        $out4,$out4,$out4,$inpperm
1086         stvx_u          $out3,$x10,$out
1087         le?vperm        $out5,$out5,$out5,$inpperm
1088         stvx_u          $out4,$x20,$out
1089         le?vperm        $out6,$out6,$out6,$inpperm
1090         stvx_u          $out5,$x30,$out
1091         le?vperm        $out7,$out7,$out7,$inpperm
1092         stvx_u          $out6,$x40,$out
1093         stvx_u          $out7,$x50,$out
1094         addi            $out,$out,0x60
1095         b               Lcbc_dec8x_done
1096
1097 .align  5
1098 Lcbc_dec8x_five:
1099         vncipherlast    $out3,$out3,$ivec
1100         vncipherlast    $out4,$out4,$in3
1101         vncipherlast    $out5,$out5,$in4
1102         vncipherlast    $out6,$out6,$in5
1103         vncipherlast    $out7,$out7,$in6
1104         vmr             $ivec,$in7
1105
1106         le?vperm        $out3,$out3,$out3,$inpperm
1107         le?vperm        $out4,$out4,$out4,$inpperm
1108         stvx_u          $out3,$x00,$out
1109         le?vperm        $out5,$out5,$out5,$inpperm
1110         stvx_u          $out4,$x10,$out
1111         le?vperm        $out6,$out6,$out6,$inpperm
1112         stvx_u          $out5,$x20,$out
1113         le?vperm        $out7,$out7,$out7,$inpperm
1114         stvx_u          $out6,$x30,$out
1115         stvx_u          $out7,$x40,$out
1116         addi            $out,$out,0x50
1117         b               Lcbc_dec8x_done
1118
1119 .align  5
1120 Lcbc_dec8x_four:
1121         vncipherlast    $out4,$out4,$ivec
1122         vncipherlast    $out5,$out5,$in4
1123         vncipherlast    $out6,$out6,$in5
1124         vncipherlast    $out7,$out7,$in6
1125         vmr             $ivec,$in7
1126
1127         le?vperm        $out4,$out4,$out4,$inpperm
1128         le?vperm        $out5,$out5,$out5,$inpperm
1129         stvx_u          $out4,$x00,$out
1130         le?vperm        $out6,$out6,$out6,$inpperm
1131         stvx_u          $out5,$x10,$out
1132         le?vperm        $out7,$out7,$out7,$inpperm
1133         stvx_u          $out6,$x20,$out
1134         stvx_u          $out7,$x30,$out
1135         addi            $out,$out,0x40
1136         b               Lcbc_dec8x_done
1137
1138 .align  5
1139 Lcbc_dec8x_three:
1140         vncipherlast    $out5,$out5,$ivec
1141         vncipherlast    $out6,$out6,$in5
1142         vncipherlast    $out7,$out7,$in6
1143         vmr             $ivec,$in7
1144
1145         le?vperm        $out5,$out5,$out5,$inpperm
1146         le?vperm        $out6,$out6,$out6,$inpperm
1147         stvx_u          $out5,$x00,$out
1148         le?vperm        $out7,$out7,$out7,$inpperm
1149         stvx_u          $out6,$x10,$out
1150         stvx_u          $out7,$x20,$out
1151         addi            $out,$out,0x30
1152         b               Lcbc_dec8x_done
1153
1154 .align  5
1155 Lcbc_dec8x_two:
1156         vncipherlast    $out6,$out6,$ivec
1157         vncipherlast    $out7,$out7,$in6
1158         vmr             $ivec,$in7
1159
1160         le?vperm        $out6,$out6,$out6,$inpperm
1161         le?vperm        $out7,$out7,$out7,$inpperm
1162         stvx_u          $out6,$x00,$out
1163         stvx_u          $out7,$x10,$out
1164         addi            $out,$out,0x20
1165         b               Lcbc_dec8x_done
1166
1167 .align  5
1168 Lcbc_dec8x_one:
1169         vncipherlast    $out7,$out7,$ivec
1170         vmr             $ivec,$in7
1171
1172         le?vperm        $out7,$out7,$out7,$inpperm
1173         stvx_u          $out7,0,$out
1174         addi            $out,$out,0x10
1175
1176 Lcbc_dec8x_done:
1177         le?vperm        $ivec,$ivec,$ivec,$inpperm
1178         stvx_u          $ivec,0,$ivp            # write [unaligned] iv
1179
1180         li              r10,`$FRAME+15`
1181         li              r11,`$FRAME+31`
1182         stvx            $inpperm,r10,$sp        # wipe copies of round keys
1183         addi            r10,r10,32
1184         stvx            $inpperm,r11,$sp
1185         addi            r11,r11,32
1186         stvx            $inpperm,r10,$sp
1187         addi            r10,r10,32
1188         stvx            $inpperm,r11,$sp
1189         addi            r11,r11,32
1190         stvx            $inpperm,r10,$sp
1191         addi            r10,r10,32
1192         stvx            $inpperm,r11,$sp
1193         addi            r11,r11,32
1194         stvx            $inpperm,r10,$sp
1195         addi            r10,r10,32
1196         stvx            $inpperm,r11,$sp
1197         addi            r11,r11,32
1198
1199         mtspr           256,$vrsave
1200         lvx             v20,r10,$sp             # ABI says so
1201         addi            r10,r10,32
1202         lvx             v21,r11,$sp
1203         addi            r11,r11,32
1204         lvx             v22,r10,$sp
1205         addi            r10,r10,32
1206         lvx             v23,r11,$sp
1207         addi            r11,r11,32
1208         lvx             v24,r10,$sp
1209         addi            r10,r10,32
1210         lvx             v25,r11,$sp
1211         addi            r11,r11,32
1212         lvx             v26,r10,$sp
1213         addi            r10,r10,32
1214         lvx             v27,r11,$sp
1215         addi            r11,r11,32
1216         lvx             v28,r10,$sp
1217         addi            r10,r10,32
1218         lvx             v29,r11,$sp
1219         addi            r11,r11,32
1220         lvx             v30,r10,$sp
1221         lvx             v31,r11,$sp
1222         $POP            r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1223         $POP            r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1224         $POP            r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1225         $POP            r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1226         $POP            r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1227         $POP            r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1228         addi            $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
1229         blr
1230         .long           0
1231         .byte           0,12,0x04,0,0x80,6,6,0
1232         .long           0
1233 .size   .${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt
1234 ___
1235 }}      }}}
1236
1237 #########################################################################
1238 {{{     # CTR procedure[s]                                              #
1239 my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10));
1240 my ($rndkey0,$rndkey1,$inout,$tmp)=             map("v$_",(0..3));
1241 my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)=
1242                                                 map("v$_",(4..11));
1243 my $dat=$tmp;
1244
1245 $code.=<<___;
1246 .globl  .${prefix}_ctr32_encrypt_blocks
1247 .align  5
1248 .${prefix}_ctr32_encrypt_blocks:
1249         ${UCMP}i        $len,1
1250         bltlr-
1251
1252         lis             r0,0xfff0
1253         mfspr           $vrsave,256
1254         mtspr           256,r0
1255
1256         li              $idx,15
1257         vxor            $rndkey0,$rndkey0,$rndkey0
1258         le?vspltisb     $tmp,0x0f
1259
1260         lvx             $ivec,0,$ivp            # load [unaligned] iv
1261         lvsl            $inpperm,0,$ivp
1262         lvx             $inptail,$idx,$ivp
1263          vspltisb       $one,1
1264         le?vxor         $inpperm,$inpperm,$tmp
1265         vperm           $ivec,$ivec,$inptail,$inpperm
1266          vsldoi         $one,$rndkey0,$one,1
1267
1268         neg             r11,$inp
1269         ?lvsl           $keyperm,0,$key         # prepare for unaligned key
1270         lwz             $rounds,240($key)
1271
1272         lvsr            $inpperm,0,r11          # prepare for unaligned load
1273         lvx             $inptail,0,$inp
1274         addi            $inp,$inp,15            # 15 is not typo
1275         le?vxor         $inpperm,$inpperm,$tmp
1276
1277         srwi            $rounds,$rounds,1
1278         li              $idx,16
1279         subi            $rounds,$rounds,1
1280
1281         ${UCMP}i        $len,8
1282         bge             _aesp8_ctr32_encrypt8x
1283
1284         ?lvsr           $outperm,0,$out         # prepare for unaligned store
1285         vspltisb        $outmask,-1
1286         lvx             $outhead,0,$out
1287         ?vperm          $outmask,$rndkey0,$outmask,$outperm
1288         le?vxor         $outperm,$outperm,$tmp
1289
1290         lvx             $rndkey0,0,$key
1291         mtctr           $rounds
1292         lvx             $rndkey1,$idx,$key
1293         addi            $idx,$idx,16
1294         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
1295         vxor            $inout,$ivec,$rndkey0
1296         lvx             $rndkey0,$idx,$key
1297         addi            $idx,$idx,16
1298         b               Loop_ctr32_enc
1299
1300 .align  5
1301 Loop_ctr32_enc:
1302         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
1303         vcipher         $inout,$inout,$rndkey1
1304         lvx             $rndkey1,$idx,$key
1305         addi            $idx,$idx,16
1306         ?vperm          $rndkey0,$rndkey0,$rndkey1,$keyperm
1307         vcipher         $inout,$inout,$rndkey0
1308         lvx             $rndkey0,$idx,$key
1309         addi            $idx,$idx,16
1310         bdnz            Loop_ctr32_enc
1311
1312         vadduwm         $ivec,$ivec,$one
1313          vmr            $dat,$inptail
1314          lvx            $inptail,0,$inp
1315          addi           $inp,$inp,16
1316          subic.         $len,$len,1             # blocks--
1317
1318         ?vperm          $rndkey1,$rndkey1,$rndkey0,$keyperm
1319         vcipher         $inout,$inout,$rndkey1
1320         lvx             $rndkey1,$idx,$key
1321          vperm          $dat,$dat,$inptail,$inpperm
1322          li             $idx,16
1323         ?vperm          $rndkey1,$rndkey0,$rndkey1,$keyperm
1324          lvx            $rndkey0,0,$key
1325         vxor            $dat,$dat,$rndkey1      # last round key
1326         vcipherlast     $inout,$inout,$dat
1327
1328          lvx            $rndkey1,$idx,$key
1329          addi           $idx,$idx,16
1330         vperm           $inout,$inout,$inout,$outperm
1331         vsel            $dat,$outhead,$inout,$outmask
1332          mtctr          $rounds
1333          ?vperm         $rndkey0,$rndkey0,$rndkey1,$keyperm
1334         vmr             $outhead,$inout
1335          vxor           $inout,$ivec,$rndkey0
1336          lvx            $rndkey0,$idx,$key
1337          addi           $idx,$idx,16
1338         stvx            $dat,0,$out
1339         addi            $out,$out,16
1340         bne             Loop_ctr32_enc
1341
1342         addi            $out,$out,-1
1343         lvx             $inout,0,$out           # redundant in aligned case
1344         vsel            $inout,$outhead,$inout,$outmask
1345         stvx            $inout,0,$out
1346
1347         mtspr           256,$vrsave
1348         blr
1349         .long           0
1350         .byte           0,12,0x14,0,0,0,6,0
1351         .long           0
1352 ___
1353 #########################################################################
1354 {{      # Optimized CTR procedure                                       #
1355 my $key_="r11";
1356 my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
1357     $x00=0 if ($flavour =~ /osx/);
1358 my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14));
1359 my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(15..22));
1360 my $rndkey0="v23";      # v24-v25 rotating buffer for first found keys
1361                         # v26-v31 last 6 round keys
1362 my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment
1363 my ($two,$three,$four)=($outhead,$outperm,$outmask);
1364
1365 $code.=<<___;
1366 .align  5
1367 _aesp8_ctr32_encrypt8x:
1368         $STU            $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
1369         li              r10,`$FRAME+8*16+15`
1370         li              r11,`$FRAME+8*16+31`
1371         stvx            v20,r10,$sp             # ABI says so
1372         addi            r10,r10,32
1373         stvx            v21,r11,$sp
1374         addi            r11,r11,32
1375         stvx            v22,r10,$sp
1376         addi            r10,r10,32
1377         stvx            v23,r11,$sp
1378         addi            r11,r11,32
1379         stvx            v24,r10,$sp
1380         addi            r10,r10,32
1381         stvx            v25,r11,$sp
1382         addi            r11,r11,32
1383         stvx            v26,r10,$sp
1384         addi            r10,r10,32
1385         stvx            v27,r11,$sp
1386         addi            r11,r11,32
1387         stvx            v28,r10,$sp
1388         addi            r10,r10,32
1389         stvx            v29,r11,$sp
1390         addi            r11,r11,32
1391         stvx            v30,r10,$sp
1392         stvx            v31,r11,$sp
1393         li              r0,-1
1394         stw             $vrsave,`$FRAME+21*16-4`($sp)   # save vrsave
1395         li              $x10,0x10
1396         $PUSH           r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1397         li              $x20,0x20
1398         $PUSH           r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1399         li              $x30,0x30
1400         $PUSH           r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1401         li              $x40,0x40
1402         $PUSH           r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1403         li              $x50,0x50
1404         $PUSH           r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1405         li              $x60,0x60
1406         $PUSH           r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1407         li              $x70,0x70
1408         mtspr           256,r0
1409
1410         subi            $rounds,$rounds,3       # -4 in total
1411
1412         lvx             $rndkey0,$x00,$key      # load key schedule
1413         lvx             v30,$x10,$key
1414         addi            $key,$key,0x20
1415         lvx             v31,$x00,$key
1416         ?vperm          $rndkey0,$rndkey0,v30,$keyperm
1417         addi            $key_,$sp,$FRAME+15
1418         mtctr           $rounds
1419
1420 Load_ctr32_enc_key:
1421         ?vperm          v24,v30,v31,$keyperm
1422         lvx             v30,$x10,$key
1423         addi            $key,$key,0x20
1424         stvx            v24,$x00,$key_          # off-load round[1]
1425         ?vperm          v25,v31,v30,$keyperm
1426         lvx             v31,$x00,$key
1427         stvx            v25,$x10,$key_          # off-load round[2]
1428         addi            $key_,$key_,0x20
1429         bdnz            Load_ctr32_enc_key
1430
1431         lvx             v26,$x10,$key
1432         ?vperm          v24,v30,v31,$keyperm
1433         lvx             v27,$x20,$key
1434         stvx            v24,$x00,$key_          # off-load round[3]
1435         ?vperm          v25,v31,v26,$keyperm
1436         lvx             v28,$x30,$key
1437         stvx            v25,$x10,$key_          # off-load round[4]
1438         addi            $key_,$sp,$FRAME+15     # rewind $key_
1439         ?vperm          v26,v26,v27,$keyperm
1440         lvx             v29,$x40,$key
1441         ?vperm          v27,v27,v28,$keyperm
1442         lvx             v30,$x50,$key
1443         ?vperm          v28,v28,v29,$keyperm
1444         lvx             v31,$x60,$key
1445         ?vperm          v29,v29,v30,$keyperm
1446         lvx             $out0,$x70,$key         # borrow $out0
1447         ?vperm          v30,v30,v31,$keyperm
1448         lvx             v24,$x00,$key_          # pre-load round[1]
1449         ?vperm          v31,v31,$out0,$keyperm
1450         lvx             v25,$x10,$key_          # pre-load round[2]
1451
1452         vadduwm         $two,$one,$one
1453         subi            $inp,$inp,15            # undo "caller"
1454         $SHL            $len,$len,4
1455
1456         vadduwm         $out1,$ivec,$one        # counter values ...
1457         vadduwm         $out2,$ivec,$two
1458         vxor            $out0,$ivec,$rndkey0    # ... xored with rndkey[0]
1459          le?li          $idx,8
1460         vadduwm         $out3,$out1,$two
1461         vxor            $out1,$out1,$rndkey0
1462          le?lvsl        $inpperm,0,$idx
1463         vadduwm         $out4,$out2,$two
1464         vxor            $out2,$out2,$rndkey0
1465          le?vspltisb    $tmp,0x0f
1466         vadduwm         $out5,$out3,$two
1467         vxor            $out3,$out3,$rndkey0
1468          le?vxor        $inpperm,$inpperm,$tmp  # transform for lvx_u/stvx_u
1469         vadduwm         $out6,$out4,$two
1470         vxor            $out4,$out4,$rndkey0
1471         vadduwm         $out7,$out5,$two
1472         vxor            $out5,$out5,$rndkey0
1473         vadduwm         $ivec,$out6,$two        # next counter value
1474         vxor            $out6,$out6,$rndkey0
1475         vxor            $out7,$out7,$rndkey0
1476
1477         mtctr           $rounds
1478         b               Loop_ctr32_enc8x
1479 .align  5
1480 Loop_ctr32_enc8x:
1481         vcipher         $out0,$out0,v24
1482         vcipher         $out1,$out1,v24
1483         vcipher         $out2,$out2,v24
1484         vcipher         $out3,$out3,v24
1485         vcipher         $out4,$out4,v24
1486         vcipher         $out5,$out5,v24
1487         vcipher         $out6,$out6,v24
1488         vcipher         $out7,$out7,v24
1489 Loop_ctr32_enc8x_middle:
1490         lvx             v24,$x20,$key_          # round[3]
1491         addi            $key_,$key_,0x20
1492
1493         vcipher         $out0,$out0,v25
1494         vcipher         $out1,$out1,v25
1495         vcipher         $out2,$out2,v25
1496         vcipher         $out3,$out3,v25
1497         vcipher         $out4,$out4,v25
1498         vcipher         $out5,$out5,v25
1499         vcipher         $out6,$out6,v25
1500         vcipher         $out7,$out7,v25
1501         lvx             v25,$x10,$key_          # round[4]
1502         bdnz            Loop_ctr32_enc8x
1503
1504         subic           r11,$len,256            # $len-256, borrow $key_
1505         vcipher         $out0,$out0,v24
1506         vcipher         $out1,$out1,v24
1507         vcipher         $out2,$out2,v24
1508         vcipher         $out3,$out3,v24
1509         vcipher         $out4,$out4,v24
1510         vcipher         $out5,$out5,v24
1511         vcipher         $out6,$out6,v24
1512         vcipher         $out7,$out7,v24
1513
1514         subfe           r0,r0,r0                # borrow?-1:0
1515         vcipher         $out0,$out0,v25
1516         vcipher         $out1,$out1,v25
1517         vcipher         $out2,$out2,v25
1518         vcipher         $out3,$out3,v25
1519         vcipher         $out4,$out4,v25
1520         vcipher         $out5,$out5,v25
1521         vcipher         $out6,$out6,v25
1522         vcipher         $out7,$out7,v25
1523
1524         and             r0,r0,r11
1525         addi            $key_,$sp,$FRAME+15     # rewind $key_
1526         vcipher         $out0,$out0,v26
1527         vcipher         $out1,$out1,v26
1528         vcipher         $out2,$out2,v26
1529         vcipher         $out3,$out3,v26
1530         vcipher         $out4,$out4,v26
1531         vcipher         $out5,$out5,v26
1532         vcipher         $out6,$out6,v26
1533         vcipher         $out7,$out7,v26
1534         lvx             v24,$x00,$key_          # re-pre-load round[1]
1535
1536         subic           $len,$len,129           # $len-=129
1537         vcipher         $out0,$out0,v27
1538         addi            $len,$len,1             # $len-=128 really
1539         vcipher         $out1,$out1,v27
1540         vcipher         $out2,$out2,v27
1541         vcipher         $out3,$out3,v27
1542         vcipher         $out4,$out4,v27
1543         vcipher         $out5,$out5,v27
1544         vcipher         $out6,$out6,v27
1545         vcipher         $out7,$out7,v27
1546         lvx             v25,$x10,$key_          # re-pre-load round[2]
1547
1548         vcipher         $out0,$out0,v28
1549          lvx_u          $in0,$x00,$inp          # load input
1550         vcipher         $out1,$out1,v28
1551          lvx_u          $in1,$x10,$inp
1552         vcipher         $out2,$out2,v28
1553          lvx_u          $in2,$x20,$inp
1554         vcipher         $out3,$out3,v28
1555          lvx_u          $in3,$x30,$inp
1556         vcipher         $out4,$out4,v28
1557          lvx_u          $in4,$x40,$inp
1558         vcipher         $out5,$out5,v28
1559          lvx_u          $in5,$x50,$inp
1560         vcipher         $out6,$out6,v28
1561          lvx_u          $in6,$x60,$inp
1562         vcipher         $out7,$out7,v28
1563          lvx_u          $in7,$x70,$inp
1564          addi           $inp,$inp,0x80
1565
1566         vcipher         $out0,$out0,v29
1567          le?vperm       $in0,$in0,$in0,$inpperm
1568         vcipher         $out1,$out1,v29
1569          le?vperm       $in1,$in1,$in1,$inpperm
1570         vcipher         $out2,$out2,v29
1571          le?vperm       $in2,$in2,$in2,$inpperm
1572         vcipher         $out3,$out3,v29
1573          le?vperm       $in3,$in3,$in3,$inpperm
1574         vcipher         $out4,$out4,v29
1575          le?vperm       $in4,$in4,$in4,$inpperm
1576         vcipher         $out5,$out5,v29
1577          le?vperm       $in5,$in5,$in5,$inpperm
1578         vcipher         $out6,$out6,v29
1579          le?vperm       $in6,$in6,$in6,$inpperm
1580         vcipher         $out7,$out7,v29
1581          le?vperm       $in7,$in7,$in7,$inpperm
1582
1583         add             $inp,$inp,r0            # $inp is adjusted in such
1584                                                 # way that at exit from the
1585                                                 # loop inX-in7 are loaded
1586                                                 # with last "words"
1587         subfe.          r0,r0,r0                # borrow?-1:0
1588         vcipher         $out0,$out0,v30
1589          vxor           $in0,$in0,v31           # xor with last round key
1590         vcipher         $out1,$out1,v30
1591          vxor           $in1,$in1,v31
1592         vcipher         $out2,$out2,v30
1593          vxor           $in2,$in2,v31
1594         vcipher         $out3,$out3,v30
1595          vxor           $in3,$in3,v31
1596         vcipher         $out4,$out4,v30
1597          vxor           $in4,$in4,v31
1598         vcipher         $out5,$out5,v30
1599          vxor           $in5,$in5,v31
1600         vcipher         $out6,$out6,v30
1601          vxor           $in6,$in6,v31
1602         vcipher         $out7,$out7,v30
1603          vxor           $in7,$in7,v31
1604
1605         bne             Lctr32_enc8x_break      # did $len-129 borrow?
1606
1607         vcipherlast     $in0,$out0,$in0
1608         vcipherlast     $in1,$out1,$in1
1609          vadduwm        $out1,$ivec,$one        # counter values ...
1610         vcipherlast     $in2,$out2,$in2
1611          vadduwm        $out2,$ivec,$two
1612          vxor           $out0,$ivec,$rndkey0    # ... xored with rndkey[0]
1613         vcipherlast     $in3,$out3,$in3
1614          vadduwm        $out3,$out1,$two
1615          vxor           $out1,$out1,$rndkey0
1616         vcipherlast     $in4,$out4,$in4
1617          vadduwm        $out4,$out2,$two
1618          vxor           $out2,$out2,$rndkey0
1619         vcipherlast     $in5,$out5,$in5
1620          vadduwm        $out5,$out3,$two
1621          vxor           $out3,$out3,$rndkey0
1622         vcipherlast     $in6,$out6,$in6
1623          vadduwm        $out6,$out4,$two
1624          vxor           $out4,$out4,$rndkey0
1625         vcipherlast     $in7,$out7,$in7
1626          vadduwm        $out7,$out5,$two
1627          vxor           $out5,$out5,$rndkey0
1628         le?vperm        $in0,$in0,$in0,$inpperm
1629          vadduwm        $ivec,$out6,$two        # next counter value
1630          vxor           $out6,$out6,$rndkey0
1631         le?vperm        $in1,$in1,$in1,$inpperm
1632          vxor           $out7,$out7,$rndkey0
1633         mtctr           $rounds
1634
1635          vcipher        $out0,$out0,v24
1636         stvx_u          $in0,$x00,$out
1637         le?vperm        $in2,$in2,$in2,$inpperm
1638          vcipher        $out1,$out1,v24
1639         stvx_u          $in1,$x10,$out
1640         le?vperm        $in3,$in3,$in3,$inpperm
1641          vcipher        $out2,$out2,v24
1642         stvx_u          $in2,$x20,$out
1643         le?vperm        $in4,$in4,$in4,$inpperm
1644          vcipher        $out3,$out3,v24
1645         stvx_u          $in3,$x30,$out
1646         le?vperm        $in5,$in5,$in5,$inpperm
1647          vcipher        $out4,$out4,v24
1648         stvx_u          $in4,$x40,$out
1649         le?vperm        $in6,$in6,$in6,$inpperm
1650          vcipher        $out5,$out5,v24
1651         stvx_u          $in5,$x50,$out
1652         le?vperm        $in7,$in7,$in7,$inpperm
1653          vcipher        $out6,$out6,v24
1654         stvx_u          $in6,$x60,$out
1655          vcipher        $out7,$out7,v24
1656         stvx_u          $in7,$x70,$out
1657         addi            $out,$out,0x80
1658
1659         b               Loop_ctr32_enc8x_middle
1660
1661 .align  5
1662 Lctr32_enc8x_break:
1663         cmpwi           $len,-0x60
1664         blt             Lctr32_enc8x_one
1665         nop
1666         beq             Lctr32_enc8x_two
1667         cmpwi           $len,-0x40
1668         blt             Lctr32_enc8x_three
1669         nop
1670         beq             Lctr32_enc8x_four
1671         cmpwi           $len,-0x20
1672         blt             Lctr32_enc8x_five
1673         nop
1674         beq             Lctr32_enc8x_six
1675         cmpwi           $len,0x00
1676         blt             Lctr32_enc8x_seven
1677
1678 Lctr32_enc8x_eight:
1679         vcipherlast     $out0,$out0,$in0
1680         vcipherlast     $out1,$out1,$in1
1681         vcipherlast     $out2,$out2,$in2
1682         vcipherlast     $out3,$out3,$in3
1683         vcipherlast     $out4,$out4,$in4
1684         vcipherlast     $out5,$out5,$in5
1685         vcipherlast     $out6,$out6,$in6
1686         vcipherlast     $out7,$out7,$in7
1687
1688         le?vperm        $out0,$out0,$out0,$inpperm
1689         le?vperm        $out1,$out1,$out1,$inpperm
1690         stvx_u          $out0,$x00,$out
1691         le?vperm        $out2,$out2,$out2,$inpperm
1692         stvx_u          $out1,$x10,$out
1693         le?vperm        $out3,$out3,$out3,$inpperm
1694         stvx_u          $out2,$x20,$out
1695         le?vperm        $out4,$out4,$out4,$inpperm
1696         stvx_u          $out3,$x30,$out
1697         le?vperm        $out5,$out5,$out5,$inpperm
1698         stvx_u          $out4,$x40,$out
1699         le?vperm        $out6,$out6,$out6,$inpperm
1700         stvx_u          $out5,$x50,$out
1701         le?vperm        $out7,$out7,$out7,$inpperm
1702         stvx_u          $out6,$x60,$out
1703         stvx_u          $out7,$x70,$out
1704         addi            $out,$out,0x80
1705         b               Lctr32_enc8x_done
1706
1707 .align  5
1708 Lctr32_enc8x_seven:
1709         vcipherlast     $out0,$out0,$in1
1710         vcipherlast     $out1,$out1,$in2
1711         vcipherlast     $out2,$out2,$in3
1712         vcipherlast     $out3,$out3,$in4
1713         vcipherlast     $out4,$out4,$in5
1714         vcipherlast     $out5,$out5,$in6
1715         vcipherlast     $out6,$out6,$in7
1716
1717         le?vperm        $out0,$out0,$out0,$inpperm
1718         le?vperm        $out1,$out1,$out1,$inpperm
1719         stvx_u          $out0,$x00,$out
1720         le?vperm        $out2,$out2,$out2,$inpperm
1721         stvx_u          $out1,$x10,$out
1722         le?vperm        $out3,$out3,$out3,$inpperm
1723         stvx_u          $out2,$x20,$out
1724         le?vperm        $out4,$out4,$out4,$inpperm
1725         stvx_u          $out3,$x30,$out
1726         le?vperm        $out5,$out5,$out5,$inpperm
1727         stvx_u          $out4,$x40,$out
1728         le?vperm        $out6,$out6,$out6,$inpperm
1729         stvx_u          $out5,$x50,$out
1730         stvx_u          $out6,$x60,$out
1731         addi            $out,$out,0x70
1732         b               Lctr32_enc8x_done
1733
1734 .align  5
1735 Lctr32_enc8x_six:
1736         vcipherlast     $out0,$out0,$in2
1737         vcipherlast     $out1,$out1,$in3
1738         vcipherlast     $out2,$out2,$in4
1739         vcipherlast     $out3,$out3,$in5
1740         vcipherlast     $out4,$out4,$in6
1741         vcipherlast     $out5,$out5,$in7
1742
1743         le?vperm        $out0,$out0,$out0,$inpperm
1744         le?vperm        $out1,$out1,$out1,$inpperm
1745         stvx_u          $out0,$x00,$out
1746         le?vperm        $out2,$out2,$out2,$inpperm
1747         stvx_u          $out1,$x10,$out
1748         le?vperm        $out3,$out3,$out3,$inpperm
1749         stvx_u          $out2,$x20,$out
1750         le?vperm        $out4,$out4,$out4,$inpperm
1751         stvx_u          $out3,$x30,$out
1752         le?vperm        $out5,$out5,$out5,$inpperm
1753         stvx_u          $out4,$x40,$out
1754         stvx_u          $out5,$x50,$out
1755         addi            $out,$out,0x60
1756         b               Lctr32_enc8x_done
1757
1758 .align  5
1759 Lctr32_enc8x_five:
1760         vcipherlast     $out0,$out0,$in3
1761         vcipherlast     $out1,$out1,$in4
1762         vcipherlast     $out2,$out2,$in5
1763         vcipherlast     $out3,$out3,$in6
1764         vcipherlast     $out4,$out4,$in7
1765
1766         le?vperm        $out0,$out0,$out0,$inpperm
1767         le?vperm        $out1,$out1,$out1,$inpperm
1768         stvx_u          $out0,$x00,$out
1769         le?vperm        $out2,$out2,$out2,$inpperm
1770         stvx_u          $out1,$x10,$out
1771         le?vperm        $out3,$out3,$out3,$inpperm
1772         stvx_u          $out2,$x20,$out
1773         le?vperm        $out4,$out4,$out4,$inpperm
1774         stvx_u          $out3,$x30,$out
1775         stvx_u          $out4,$x40,$out
1776         addi            $out,$out,0x50
1777         b               Lctr32_enc8x_done
1778
1779 .align  5
1780 Lctr32_enc8x_four:
1781         vcipherlast     $out0,$out0,$in4
1782         vcipherlast     $out1,$out1,$in5
1783         vcipherlast     $out2,$out2,$in6
1784         vcipherlast     $out3,$out3,$in7
1785
1786         le?vperm        $out0,$out0,$out0,$inpperm
1787         le?vperm        $out1,$out1,$out1,$inpperm
1788         stvx_u          $out0,$x00,$out
1789         le?vperm        $out2,$out2,$out2,$inpperm
1790         stvx_u          $out1,$x10,$out
1791         le?vperm        $out3,$out3,$out3,$inpperm
1792         stvx_u          $out2,$x20,$out
1793         stvx_u          $out3,$x30,$out
1794         addi            $out,$out,0x40
1795         b               Lctr32_enc8x_done
1796
1797 .align  5
1798 Lctr32_enc8x_three:
1799         vcipherlast     $out0,$out0,$in5
1800         vcipherlast     $out1,$out1,$in6
1801         vcipherlast     $out2,$out2,$in7
1802
1803         le?vperm        $out0,$out0,$out0,$inpperm
1804         le?vperm        $out1,$out1,$out1,$inpperm
1805         stvx_u          $out0,$x00,$out
1806         le?vperm        $out2,$out2,$out2,$inpperm
1807         stvx_u          $out1,$x10,$out
1808         stvx_u          $out2,$x20,$out
1809         addi            $out,$out,0x30
1810         b               Lcbc_dec8x_done
1811
1812 .align  5
1813 Lctr32_enc8x_two:
1814         vcipherlast     $out0,$out0,$in6
1815         vcipherlast     $out1,$out1,$in7
1816
1817         le?vperm        $out0,$out0,$out0,$inpperm
1818         le?vperm        $out1,$out1,$out1,$inpperm
1819         stvx_u          $out0,$x00,$out
1820         stvx_u          $out1,$x10,$out
1821         addi            $out,$out,0x20
1822         b               Lcbc_dec8x_done
1823
1824 .align  5
1825 Lctr32_enc8x_one:
1826         vcipherlast     $out0,$out0,$in7
1827
1828         le?vperm        $out0,$out0,$out0,$inpperm
1829         stvx_u          $out0,0,$out
1830         addi            $out,$out,0x10
1831
1832 Lctr32_enc8x_done:
1833         li              r10,`$FRAME+15`
1834         li              r11,`$FRAME+31`
1835         stvx            $inpperm,r10,$sp        # wipe copies of round keys
1836         addi            r10,r10,32
1837         stvx            $inpperm,r11,$sp
1838         addi            r11,r11,32
1839         stvx            $inpperm,r10,$sp
1840         addi            r10,r10,32
1841         stvx            $inpperm,r11,$sp
1842         addi            r11,r11,32
1843         stvx            $inpperm,r10,$sp
1844         addi            r10,r10,32
1845         stvx            $inpperm,r11,$sp
1846         addi            r11,r11,32
1847         stvx            $inpperm,r10,$sp
1848         addi            r10,r10,32
1849         stvx            $inpperm,r11,$sp
1850         addi            r11,r11,32
1851
1852         mtspr           256,$vrsave
1853         lvx             v20,r10,$sp             # ABI says so
1854         addi            r10,r10,32
1855         lvx             v21,r11,$sp
1856         addi            r11,r11,32
1857         lvx             v22,r10,$sp
1858         addi            r10,r10,32
1859         lvx             v23,r11,$sp
1860         addi            r11,r11,32
1861         lvx             v24,r10,$sp
1862         addi            r10,r10,32
1863         lvx             v25,r11,$sp
1864         addi            r11,r11,32
1865         lvx             v26,r10,$sp
1866         addi            r10,r10,32
1867         lvx             v27,r11,$sp
1868         addi            r11,r11,32
1869         lvx             v28,r10,$sp
1870         addi            r10,r10,32
1871         lvx             v29,r11,$sp
1872         addi            r11,r11,32
1873         lvx             v30,r10,$sp
1874         lvx             v31,r11,$sp
1875         $POP            r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1876         $POP            r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1877         $POP            r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1878         $POP            r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1879         $POP            r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1880         $POP            r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1881         addi            $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
1882         blr
1883         .long           0
1884         .byte           0,12,0x04,0,0x80,6,6,0
1885         .long           0
1886 .size   .${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks
1887 ___
1888 }}      }}}
1889
1890 my $consts=1;
1891 foreach(split("\n",$code)) {
1892         s/\`([^\`]*)\`/eval($1)/geo;
1893
1894         # constants table endian-specific conversion
1895         if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) {
1896             my $conv=$3;
1897             my @bytes=();
1898
1899             # convert to endian-agnostic format
1900             if ($1 eq "long") {
1901               foreach (split(/,\s*/,$2)) {
1902                 my $l = /^0/?oct:int;
1903                 push @bytes,($l>>24)&0xff,($l>>16)&0xff,($l>>8)&0xff,$l&0xff;
1904               }
1905             } else {
1906                 @bytes = map(/^0/?oct:int,split(/,\s*/,$2));
1907             }
1908
1909             # little-endian conversion
1910             if ($flavour =~ /le$/o) {
1911                 SWITCH: for($conv)  {
1912                     /\?inv/ && do   { @bytes=map($_^0xf,@bytes); last; };
1913                     /\?rev/ && do   { @bytes=reverse(@bytes);    last; }; 
1914                 }
1915             }
1916
1917             #emit
1918             print ".byte\t",join(',',map (sprintf("0x%02x",$_),@bytes)),"\n";
1919             next;
1920         }
1921         $consts=0 if (m/Lconsts:/o);    # end of table
1922
1923         # instructions prefixed with '?' are endian-specific and need
1924         # to be adjusted accordingly...
1925         if ($flavour =~ /le$/o) {       # little-endian
1926             s/le\?//o           or
1927             s/be\?/#be#/o       or
1928             s/\?lvsr/lvsl/o     or
1929             s/\?lvsl/lvsr/o     or
1930             s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or
1931             s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or
1932             s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o;
1933         } else {                        # big-endian
1934             s/le\?/#le#/o       or
1935             s/be\?//o           or
1936             s/\?([a-z]+)/$1/o;
1937         }
1938
1939         print $_,"\n";
1940 }
1941
1942 close STDOUT;