chacha/asm/chacha-ppc.pl: add missing .text directive.
[openssl.git] / crypto / chacha / asm / chacha-ppc.pl
1 #! /usr/bin/env perl
2 # Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
3 #
4 # Licensed under the OpenSSL license (the "License").  You may not use
5 # this file except in compliance with the License.  You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
8
9 #
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
16 #
17 # October 2015
18
19 # ChaCha20 for PowerPC/AltiVec.
20 #
21 # Performance in cycles per byte out of large buffer.
22 #
23 #                       IALU/gcc-4.x    3xAltiVec+1xIALU
24 #
25 # Freescale e300        13.6/+115%      -
26 # PPC74x0               6.81/+310%      4.66
27 # POWER7                8.62/+61%       4.27
28 # POWER8                8.70/+51%       3.96
29
30 $flavour = shift;
31
32 if ($flavour =~ /64/) {
33         $SIZE_T =8;
34         $LRSAVE =2*$SIZE_T;
35         $STU    ="stdu";
36         $POP    ="ld";
37         $PUSH   ="std";
38         $UCMP   ="cmpld";
39 } elsif ($flavour =~ /32/) {
40         $SIZE_T =4;
41         $LRSAVE =$SIZE_T;
42         $STU    ="stwu";
43         $POP    ="lwz";
44         $PUSH   ="stw";
45         $UCMP   ="cmplw";
46 } else { die "nonsense $flavour"; }
47
48 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
49
50 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
51 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
52 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
53 die "can't locate ppc-xlate.pl";
54
55 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
56
57 $LOCALS=6*$SIZE_T;
58 $FRAME=$LOCALS+64+18*$SIZE_T;   # 64 is for local variables
59
60 sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
61 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
62     $code .= "\t$opcode\t".join(',',@_)."\n";
63 }
64
65 my $sp = "r1";
66
67 my ($out,$inp,$len,$key,$ctr) = map("r$_",(3..7));
68
69 my @x=map("r$_",(16..31));
70 my @d=map("r$_",(11,12,14,15));
71 my @t=map("r$_",(7..10));
72
73 sub ROUND {
74 my ($a0,$b0,$c0,$d0)=@_;
75 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
76 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
77 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
78
79     (
80         "&add           (@x[$a0],@x[$a0],@x[$b0])",
81          "&add          (@x[$a1],@x[$a1],@x[$b1])",
82           "&add         (@x[$a2],@x[$a2],@x[$b2])",
83            "&add        (@x[$a3],@x[$a3],@x[$b3])",
84         "&xor           (@x[$d0],@x[$d0],@x[$a0])",
85          "&xor          (@x[$d1],@x[$d1],@x[$a1])",
86           "&xor         (@x[$d2],@x[$d2],@x[$a2])",
87            "&xor        (@x[$d3],@x[$d3],@x[$a3])",
88         "&rotlwi        (@x[$d0],@x[$d0],16)",
89          "&rotlwi       (@x[$d1],@x[$d1],16)",
90           "&rotlwi      (@x[$d2],@x[$d2],16)",
91            "&rotlwi     (@x[$d3],@x[$d3],16)",
92
93         "&add           (@x[$c0],@x[$c0],@x[$d0])",
94          "&add          (@x[$c1],@x[$c1],@x[$d1])",
95           "&add         (@x[$c2],@x[$c2],@x[$d2])",
96            "&add        (@x[$c3],@x[$c3],@x[$d3])",
97         "&xor           (@x[$b0],@x[$b0],@x[$c0])",
98          "&xor          (@x[$b1],@x[$b1],@x[$c1])",
99           "&xor         (@x[$b2],@x[$b2],@x[$c2])",
100            "&xor        (@x[$b3],@x[$b3],@x[$c3])",
101         "&rotlwi        (@x[$b0],@x[$b0],12)",
102          "&rotlwi       (@x[$b1],@x[$b1],12)",
103           "&rotlwi      (@x[$b2],@x[$b2],12)",
104            "&rotlwi     (@x[$b3],@x[$b3],12)",
105
106         "&add           (@x[$a0],@x[$a0],@x[$b0])",
107          "&add          (@x[$a1],@x[$a1],@x[$b1])",
108           "&add         (@x[$a2],@x[$a2],@x[$b2])",
109            "&add        (@x[$a3],@x[$a3],@x[$b3])",
110         "&xor           (@x[$d0],@x[$d0],@x[$a0])",
111          "&xor          (@x[$d1],@x[$d1],@x[$a1])",
112           "&xor         (@x[$d2],@x[$d2],@x[$a2])",
113            "&xor        (@x[$d3],@x[$d3],@x[$a3])",
114         "&rotlwi        (@x[$d0],@x[$d0],8)",
115          "&rotlwi       (@x[$d1],@x[$d1],8)",
116           "&rotlwi      (@x[$d2],@x[$d2],8)",
117            "&rotlwi     (@x[$d3],@x[$d3],8)",
118
119         "&add           (@x[$c0],@x[$c0],@x[$d0])",
120          "&add          (@x[$c1],@x[$c1],@x[$d1])",
121           "&add         (@x[$c2],@x[$c2],@x[$d2])",
122            "&add        (@x[$c3],@x[$c3],@x[$d3])",
123         "&xor           (@x[$b0],@x[$b0],@x[$c0])",
124          "&xor          (@x[$b1],@x[$b1],@x[$c1])",
125           "&xor         (@x[$b2],@x[$b2],@x[$c2])",
126            "&xor        (@x[$b3],@x[$b3],@x[$c3])",
127         "&rotlwi        (@x[$b0],@x[$b0],7)",
128          "&rotlwi       (@x[$b1],@x[$b1],7)",
129           "&rotlwi      (@x[$b2],@x[$b2],7)",
130            "&rotlwi     (@x[$b3],@x[$b3],7)"
131     );
132 }
133
134 $code.=<<___;
135 .machine        "any"
136 .text
137
138 .globl  .ChaCha20_ctr32_int
139 .align  5
140 .ChaCha20_ctr32_int:
141 __ChaCha20_ctr32_int:
142         ${UCMP}i $len,0
143         beqlr-
144
145         $STU    $sp,-$FRAME($sp)
146         mflr    r0
147
148         $PUSH   r14,`$FRAME-$SIZE_T*18`($sp)
149         $PUSH   r15,`$FRAME-$SIZE_T*17`($sp)
150         $PUSH   r16,`$FRAME-$SIZE_T*16`($sp)
151         $PUSH   r17,`$FRAME-$SIZE_T*15`($sp)
152         $PUSH   r18,`$FRAME-$SIZE_T*14`($sp)
153         $PUSH   r19,`$FRAME-$SIZE_T*13`($sp)
154         $PUSH   r20,`$FRAME-$SIZE_T*12`($sp)
155         $PUSH   r21,`$FRAME-$SIZE_T*11`($sp)
156         $PUSH   r22,`$FRAME-$SIZE_T*10`($sp)
157         $PUSH   r23,`$FRAME-$SIZE_T*9`($sp)
158         $PUSH   r24,`$FRAME-$SIZE_T*8`($sp)
159         $PUSH   r25,`$FRAME-$SIZE_T*7`($sp)
160         $PUSH   r26,`$FRAME-$SIZE_T*6`($sp)
161         $PUSH   r27,`$FRAME-$SIZE_T*5`($sp)
162         $PUSH   r28,`$FRAME-$SIZE_T*4`($sp)
163         $PUSH   r29,`$FRAME-$SIZE_T*3`($sp)
164         $PUSH   r30,`$FRAME-$SIZE_T*2`($sp)
165         $PUSH   r31,`$FRAME-$SIZE_T*1`($sp)
166         $PUSH   r0,`$FRAME+$LRSAVE`($sp)
167
168         lwz     @d[0],0($ctr)                   # load counter
169         lwz     @d[1],4($ctr)
170         lwz     @d[2],8($ctr)
171         lwz     @d[3],12($ctr)
172
173         bl      __ChaCha20_1x
174
175         $POP    r0,`$FRAME+$LRSAVE`($sp)
176         $POP    r14,`$FRAME-$SIZE_T*18`($sp)
177         $POP    r15,`$FRAME-$SIZE_T*17`($sp)
178         $POP    r16,`$FRAME-$SIZE_T*16`($sp)
179         $POP    r17,`$FRAME-$SIZE_T*15`($sp)
180         $POP    r18,`$FRAME-$SIZE_T*14`($sp)
181         $POP    r19,`$FRAME-$SIZE_T*13`($sp)
182         $POP    r20,`$FRAME-$SIZE_T*12`($sp)
183         $POP    r21,`$FRAME-$SIZE_T*11`($sp)
184         $POP    r22,`$FRAME-$SIZE_T*10`($sp)
185         $POP    r23,`$FRAME-$SIZE_T*9`($sp)
186         $POP    r24,`$FRAME-$SIZE_T*8`($sp)
187         $POP    r25,`$FRAME-$SIZE_T*7`($sp)
188         $POP    r26,`$FRAME-$SIZE_T*6`($sp)
189         $POP    r27,`$FRAME-$SIZE_T*5`($sp)
190         $POP    r28,`$FRAME-$SIZE_T*4`($sp)
191         $POP    r29,`$FRAME-$SIZE_T*3`($sp)
192         $POP    r30,`$FRAME-$SIZE_T*2`($sp)
193         $POP    r31,`$FRAME-$SIZE_T*1`($sp)
194         mtlr    r0
195         addi    $sp,$sp,$FRAME
196         blr
197         .long   0
198         .byte   0,12,4,1,0x80,18,5,0
199         .long   0
200 .size   .ChaCha20_ctr32_int,.-.ChaCha20_ctr32_int
201
202 .align  5
203 __ChaCha20_1x:
204 Loop_outer:
205         lis     @x[0],0x6170                    # synthesize sigma
206         lis     @x[1],0x3320
207         lis     @x[2],0x7962
208         lis     @x[3],0x6b20
209         ori     @x[0],@x[0],0x7865
210         ori     @x[1],@x[1],0x646e
211         ori     @x[2],@x[2],0x2d32
212         ori     @x[3],@x[3],0x6574
213
214         li      r0,10                           # inner loop counter
215         lwz     @x[4],0($key)                   # load key
216         lwz     @x[5],4($key)
217         lwz     @x[6],8($key)
218         lwz     @x[7],12($key)
219         lwz     @x[8],16($key)
220         mr      @x[12],@d[0]                    # copy counter
221         lwz     @x[9],20($key)
222         mr      @x[13],@d[1]
223         lwz     @x[10],24($key)
224         mr      @x[14],@d[2]
225         lwz     @x[11],28($key)
226         mr      @x[15],@d[3]
227
228         mr      @t[0],@x[4]
229         mr      @t[1],@x[5]
230         mr      @t[2],@x[6]
231         mr      @t[3],@x[7]
232
233         mtctr   r0
234 Loop:
235 ___
236         foreach (&ROUND(0, 4, 8,12)) { eval; }
237         foreach (&ROUND(0, 5,10,15)) { eval; }
238 $code.=<<___;
239         bdnz    Loop
240
241         subic   $len,$len,64                    # $len-=64
242         addi    @x[0],@x[0],0x7865              # accumulate key block
243         addi    @x[1],@x[1],0x646e
244         addi    @x[2],@x[2],0x2d32
245         addi    @x[3],@x[3],0x6574
246         addis   @x[0],@x[0],0x6170
247         addis   @x[1],@x[1],0x3320
248         addis   @x[2],@x[2],0x7962
249         addis   @x[3],@x[3],0x6b20
250
251         subfe.  r0,r0,r0                        # borrow?-1:0
252         add     @x[4],@x[4],@t[0]
253         lwz     @t[0],16($key)
254         add     @x[5],@x[5],@t[1]
255         lwz     @t[1],20($key)
256         add     @x[6],@x[6],@t[2]
257         lwz     @t[2],24($key)
258         add     @x[7],@x[7],@t[3]
259         lwz     @t[3],28($key)
260         add     @x[8],@x[8],@t[0]
261         add     @x[9],@x[9],@t[1]
262         add     @x[10],@x[10],@t[2]
263         add     @x[11],@x[11],@t[3]
264
265         add     @x[12],@x[12],@d[0]
266         add     @x[13],@x[13],@d[1]
267         add     @x[14],@x[14],@d[2]
268         add     @x[15],@x[15],@d[3]
269         addi    @d[0],@d[0],1                   # increment counter
270 ___
271 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {   # flip byte order
272 $code.=<<___;
273         mr      @t[$i&3],@x[$i]
274         rotlwi  @x[$i],@x[$i],8
275         rlwimi  @x[$i],@t[$i&3],24,0,7
276         rlwimi  @x[$i],@t[$i&3],24,16,23
277 ___
278 } }
279 $code.=<<___;
280         bne     Ltail                           # $len-=64 borrowed
281
282         lwz     @t[0],0($inp)                   # load input, aligned or not
283         lwz     @t[1],4($inp)
284         ${UCMP}i $len,0                         # done already?
285         lwz     @t[2],8($inp)
286         lwz     @t[3],12($inp)
287         xor     @x[0],@x[0],@t[0]               # xor with input
288         lwz     @t[0],16($inp)
289         xor     @x[1],@x[1],@t[1]
290         lwz     @t[1],20($inp)
291         xor     @x[2],@x[2],@t[2]
292         lwz     @t[2],24($inp)
293         xor     @x[3],@x[3],@t[3]
294         lwz     @t[3],28($inp)
295         xor     @x[4],@x[4],@t[0]
296         lwz     @t[0],32($inp)
297         xor     @x[5],@x[5],@t[1]
298         lwz     @t[1],36($inp)
299         xor     @x[6],@x[6],@t[2]
300         lwz     @t[2],40($inp)
301         xor     @x[7],@x[7],@t[3]
302         lwz     @t[3],44($inp)
303         xor     @x[8],@x[8],@t[0]
304         lwz     @t[0],48($inp)
305         xor     @x[9],@x[9],@t[1]
306         lwz     @t[1],52($inp)
307         xor     @x[10],@x[10],@t[2]
308         lwz     @t[2],56($inp)
309         xor     @x[11],@x[11],@t[3]
310         lwz     @t[3],60($inp)
311         xor     @x[12],@x[12],@t[0]
312         stw     @x[0],0($out)                   # store output, aligned or not
313         xor     @x[13],@x[13],@t[1]
314         stw     @x[1],4($out)
315         xor     @x[14],@x[14],@t[2]
316         stw     @x[2],8($out)
317         xor     @x[15],@x[15],@t[3]
318         stw     @x[3],12($out)
319         stw     @x[4],16($out)
320         stw     @x[5],20($out)
321         stw     @x[6],24($out)
322         stw     @x[7],28($out)
323         stw     @x[8],32($out)
324         stw     @x[9],36($out)
325         stw     @x[10],40($out)
326         stw     @x[11],44($out)
327         stw     @x[12],48($out)
328         stw     @x[13],52($out)
329         stw     @x[14],56($out)
330         addi    $inp,$inp,64
331         stw     @x[15],60($out)
332         addi    $out,$out,64
333
334         bne     Loop_outer
335
336         blr
337
338 .align  4
339 Ltail:
340         addi    $len,$len,64                    # restore tail length
341         subi    $inp,$inp,1                     # prepare for *++ptr
342         subi    $out,$out,1
343         addi    @t[0],$sp,$LOCALS-1
344         mtctr   $len
345
346         stw     @x[0],`$LOCALS+0`($sp)          # save whole block to stack
347         stw     @x[1],`$LOCALS+4`($sp)
348         stw     @x[2],`$LOCALS+8`($sp)
349         stw     @x[3],`$LOCALS+12`($sp)
350         stw     @x[4],`$LOCALS+16`($sp)
351         stw     @x[5],`$LOCALS+20`($sp)
352         stw     @x[6],`$LOCALS+24`($sp)
353         stw     @x[7],`$LOCALS+28`($sp)
354         stw     @x[8],`$LOCALS+32`($sp)
355         stw     @x[9],`$LOCALS+36`($sp)
356         stw     @x[10],`$LOCALS+40`($sp)
357         stw     @x[11],`$LOCALS+44`($sp)
358         stw     @x[12],`$LOCALS+48`($sp)
359         stw     @x[13],`$LOCALS+52`($sp)
360         stw     @x[14],`$LOCALS+56`($sp)
361         stw     @x[15],`$LOCALS+60`($sp)
362
363 Loop_tail:                                      # byte-by-byte loop
364         lbzu    @d[0],1($inp)
365         lbzu    @x[0],1(@t[0])
366         xor     @d[1],@d[0],@x[0]
367         stbu    @d[1],1($out)
368         bdnz    Loop_tail
369
370         stw     $sp,`$LOCALS+0`($sp)            # wipe block on stack
371         stw     $sp,`$LOCALS+4`($sp)
372         stw     $sp,`$LOCALS+8`($sp)
373         stw     $sp,`$LOCALS+12`($sp)
374         stw     $sp,`$LOCALS+16`($sp)
375         stw     $sp,`$LOCALS+20`($sp)
376         stw     $sp,`$LOCALS+24`($sp)
377         stw     $sp,`$LOCALS+28`($sp)
378         stw     $sp,`$LOCALS+32`($sp)
379         stw     $sp,`$LOCALS+36`($sp)
380         stw     $sp,`$LOCALS+40`($sp)
381         stw     $sp,`$LOCALS+44`($sp)
382         stw     $sp,`$LOCALS+48`($sp)
383         stw     $sp,`$LOCALS+52`($sp)
384         stw     $sp,`$LOCALS+56`($sp)
385         stw     $sp,`$LOCALS+60`($sp)
386
387         blr
388         .long   0
389         .byte   0,12,0x14,0,0,0,0,0
390 ___
391
392 {{{
393 my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,$T0,$T1,$T2) =
394     map("v$_",(0..14));
395 my (@K)=map("v$_",(15..20));
396 my ($FOUR,$sixteen,$twenty4,$twenty,$twelve,$twenty5,$seven) =
397     map("v$_",(21..27));
398 my ($inpperm,$outperm,$outmask) = map("v$_",(28..30));
399 my @D=("v31",$seven,$T0,$T1,$T2);
400
401 my $FRAME=$LOCALS+64+13*16+18*$SIZE_T;  # 13*16 is for v20-v31 offload
402
403 sub VMXROUND {
404 my $odd = pop;
405 my ($a,$b,$c,$d,$t)=@_;
406
407         (
408         "&vadduwm       ('$a','$a','$b')",
409         "&vxor          ('$d','$d','$a')",
410         "&vperm         ('$d','$d','$d','$sixteen')",
411
412         "&vadduwm       ('$c','$c','$d')",
413         "&vxor          ('$t','$b','$c')",
414         "&vsrw          ('$b','$t','$twenty')",
415         "&vslw          ('$t','$t','$twelve')",
416         "&vor           ('$b','$b','$t')",
417
418         "&vadduwm       ('$a','$a','$b')",
419         "&vxor          ('$d','$d','$a')",
420         "&vperm         ('$d','$d','$d','$twenty4')",
421
422         "&vadduwm       ('$c','$c','$d')",
423         "&vxor          ('$t','$b','$c')",
424         "&vsrw          ('$b','$t','$twenty5')",
425         "&vslw          ('$t','$t','$seven')",
426         "&vor           ('$b','$b','$t')",
427
428         "&vsldoi        ('$c','$c','$c',8)",
429         "&vsldoi        ('$b','$b','$b',$odd?4:12)",
430         "&vsldoi        ('$d','$d','$d',$odd?12:4)"
431         );
432 }
433
434 $code.=<<___;
435
436 .globl  .ChaCha20_ctr32_vmx
437 .align  5
438 .ChaCha20_ctr32_vmx:
439         ${UCMP}i $len,256
440         blt     __ChaCha20_ctr32_int
441
442         $STU    $sp,-$FRAME($sp)
443         mflr    r0
444         li      r10,`15+$LOCALS+64`
445         li      r11,`31+$LOCALS+64`
446         mfspr   r12,256
447         stvx    v20,r10,$sp
448         addi    r10,r10,32
449         stvx    v21,r11,$sp
450         addi    r11,r11,32
451         stvx    v22,r10,$sp
452         addi    r10,r10,32
453         stvx    v23,r11,$sp
454         addi    r11,r11,32
455         stvx    v24,r10,$sp
456         addi    r10,r10,32
457         stvx    v25,r11,$sp
458         addi    r11,r11,32
459         stvx    v26,r10,$sp
460         addi    r10,r10,32
461         stvx    v27,r11,$sp
462         addi    r11,r11,32
463         stvx    v28,r10,$sp
464         addi    r10,r10,32
465         stvx    v29,r11,$sp
466         addi    r11,r11,32
467         stvx    v30,r10,$sp
468         stvx    v31,r11,$sp
469         stw     r12,`$FRAME-$SIZE_T*18-4`($sp)  # save vrsave
470         $PUSH   r14,`$FRAME-$SIZE_T*18`($sp)
471         $PUSH   r15,`$FRAME-$SIZE_T*17`($sp)
472         $PUSH   r16,`$FRAME-$SIZE_T*16`($sp)
473         $PUSH   r17,`$FRAME-$SIZE_T*15`($sp)
474         $PUSH   r18,`$FRAME-$SIZE_T*14`($sp)
475         $PUSH   r19,`$FRAME-$SIZE_T*13`($sp)
476         $PUSH   r20,`$FRAME-$SIZE_T*12`($sp)
477         $PUSH   r21,`$FRAME-$SIZE_T*11`($sp)
478         $PUSH   r22,`$FRAME-$SIZE_T*10`($sp)
479         $PUSH   r23,`$FRAME-$SIZE_T*9`($sp)
480         $PUSH   r24,`$FRAME-$SIZE_T*8`($sp)
481         $PUSH   r25,`$FRAME-$SIZE_T*7`($sp)
482         $PUSH   r26,`$FRAME-$SIZE_T*6`($sp)
483         $PUSH   r27,`$FRAME-$SIZE_T*5`($sp)
484         $PUSH   r28,`$FRAME-$SIZE_T*4`($sp)
485         $PUSH   r29,`$FRAME-$SIZE_T*3`($sp)
486         $PUSH   r30,`$FRAME-$SIZE_T*2`($sp)
487         $PUSH   r31,`$FRAME-$SIZE_T*1`($sp)
488         li      r12,-1
489         $PUSH   r0, `$FRAME+$LRSAVE`($sp)
490         mtspr   256,r12                         # preserve all AltiVec registers
491
492         bl      Lconsts                         # returns pointer Lsigma in r12
493         li      @x[0],16
494         li      @x[1],32
495         li      @x[2],48
496         li      @x[3],64
497         li      @x[4],31                        # 31 is not a typo
498         li      @x[5],15                        # nor is 15
499
500         lvx     @K[1],0,$key                    # load key
501         ?lvsr   $T0,0,$key                      # prepare unaligned load
502         lvx     @K[2],@x[0],$key
503         lvx     @D[0],@x[4],$key
504
505         lvx     @K[3],0,$ctr                    # load counter
506         ?lvsr   $T1,0,$ctr                      # prepare unaligned load
507         lvx     @D[1],@x[5],$ctr
508
509         lvx     @K[0],0,r12                     # load constants
510         lvx     @K[5],@x[0],r12                 # one
511         lvx     $FOUR,@x[1],r12
512         lvx     $sixteen,@x[2],r12
513         lvx     $twenty4,@x[3],r12
514
515         ?vperm  @K[1],@K[2],@K[1],$T0           # align key
516         ?vperm  @K[2],@D[0],@K[2],$T0
517         ?vperm  @K[3],@D[1],@K[3],$T1           # align counter
518
519         lwz     @d[0],0($ctr)                   # load counter to GPR
520         lwz     @d[1],4($ctr)
521         vadduwm @K[3],@K[3],@K[5]               # adjust AltiVec counter
522         lwz     @d[2],8($ctr)
523         vadduwm @K[4],@K[3],@K[5]
524         lwz     @d[3],12($ctr)
525         vadduwm @K[5],@K[4],@K[5]
526
527         vspltisw $twenty,-12                    # synthesize constants 
528         vspltisw $twelve,12
529         vspltisw $twenty5,-7
530         #vspltisw $seven,7                      # synthesized in the loop
531
532         vxor    $T0,$T0,$T0                     # 0x00..00
533         vspltisw $outmask,-1                    # 0xff..ff
534         ?lvsr   $inpperm,0,$inp                 # prepare for unaligned load
535         ?lvsl   $outperm,0,$out                 # prepare for unaligned store
536         ?vperm  $outmask,$outmask,$T0,$outperm
537
538         be?lvsl $T0,0,@x[0]                     # 0x00..0f
539         be?vspltisb $T1,3                       # 0x03..03
540         be?vxor $T0,$T0,$T1                     # swap bytes within words
541         be?vxor $outperm,$outperm,$T1
542         be?vperm $inpperm,$inpperm,$inpperm,$T0
543
544         b       Loop_outer_vmx
545
546 .align  4
547 Loop_outer_vmx:
548         lis     @x[0],0x6170                    # synthesize sigma
549         lis     @x[1],0x3320
550          vmr    $A0,@K[0]
551         lis     @x[2],0x7962
552         lis     @x[3],0x6b20
553          vmr    $A1,@K[0]
554         ori     @x[0],@x[0],0x7865
555         ori     @x[1],@x[1],0x646e
556          vmr    $A2,@K[0]
557         ori     @x[2],@x[2],0x2d32
558         ori     @x[3],@x[3],0x6574
559          vmr    $B0,@K[1]
560
561         li      r0,10                           # inner loop counter
562         lwz     @x[4],0($key)                   # load key to GPR
563          vmr    $B1,@K[1]
564         lwz     @x[5],4($key)
565          vmr    $B2,@K[1]
566         lwz     @x[6],8($key)
567          vmr    $C0,@K[2]
568         lwz     @x[7],12($key)
569          vmr    $C1,@K[2]
570         lwz     @x[8],16($key)
571          vmr    $C2,@K[2]
572         mr      @x[12],@d[0]                    # copy GPR counter
573         lwz     @x[9],20($key)
574          vmr    $D0,@K[3]
575         mr      @x[13],@d[1]
576         lwz     @x[10],24($key)
577          vmr    $D1,@K[4]
578         mr      @x[14],@d[2]
579         lwz     @x[11],28($key)
580          vmr    $D2,@K[5]
581         mr      @x[15],@d[3]
582
583         mr      @t[0],@x[4]
584         mr      @t[1],@x[5]
585         mr      @t[2],@x[6]
586         mr      @t[3],@x[7]
587         vspltisw $seven,7
588
589         mtctr   r0
590         nop
591 Loop_vmx:
592 ___
593         my @thread0=&VMXROUND($A0,$B0,$C0,$D0,$T0,0);
594         my @thread1=&VMXROUND($A1,$B1,$C1,$D1,$T1,0);
595         my @thread2=&VMXROUND($A2,$B2,$C2,$D2,$T2,0);
596         my @thread3=&ROUND(0,4,8,12);
597
598         foreach (@thread0) {
599                 eval;                   eval(shift(@thread3));
600                 eval(shift(@thread1));  eval(shift(@thread3));
601                 eval(shift(@thread2));  eval(shift(@thread3));
602         }
603
604         @thread0=&VMXROUND($A0,$B0,$C0,$D0,$T0,1);
605         @thread1=&VMXROUND($A1,$B1,$C1,$D1,$T1,1);
606         @thread2=&VMXROUND($A2,$B2,$C2,$D2,$T2,1);
607         @thread3=&ROUND(0,5,10,15);
608
609         foreach (@thread0) {
610                 eval;                   eval(shift(@thread3));
611                 eval(shift(@thread1));  eval(shift(@thread3));
612                 eval(shift(@thread2));  eval(shift(@thread3));
613         }
614 $code.=<<___;
615         bdnz    Loop_vmx
616
617         subi    $len,$len,256                   # $len-=256
618         addi    @x[0],@x[0],0x7865              # accumulate key block
619         addi    @x[1],@x[1],0x646e
620         addi    @x[2],@x[2],0x2d32
621         addi    @x[3],@x[3],0x6574
622         addis   @x[0],@x[0],0x6170
623         addis   @x[1],@x[1],0x3320
624         addis   @x[2],@x[2],0x7962
625         addis   @x[3],@x[3],0x6b20
626         add     @x[4],@x[4],@t[0]
627         lwz     @t[0],16($key)
628         add     @x[5],@x[5],@t[1]
629         lwz     @t[1],20($key)
630         add     @x[6],@x[6],@t[2]
631         lwz     @t[2],24($key)
632         add     @x[7],@x[7],@t[3]
633         lwz     @t[3],28($key)
634         add     @x[8],@x[8],@t[0]
635         add     @x[9],@x[9],@t[1]
636         add     @x[10],@x[10],@t[2]
637         add     @x[11],@x[11],@t[3]
638         add     @x[12],@x[12],@d[0]
639         add     @x[13],@x[13],@d[1]
640         add     @x[14],@x[14],@d[2]
641         add     @x[15],@x[15],@d[3]
642
643         vadduwm $A0,$A0,@K[0]                   # accumulate key block
644         vadduwm $A1,$A1,@K[0]
645         vadduwm $A2,$A2,@K[0]
646         vadduwm $B0,$B0,@K[1]
647         vadduwm $B1,$B1,@K[1]
648         vadduwm $B2,$B2,@K[1]
649         vadduwm $C0,$C0,@K[2]
650         vadduwm $C1,$C1,@K[2]
651         vadduwm $C2,$C2,@K[2]
652         vadduwm $D0,$D0,@K[3]
653         vadduwm $D1,$D1,@K[4]
654         vadduwm $D2,$D2,@K[5]
655
656         addi    @d[0],@d[0],4                   # increment counter
657         vadduwm @K[3],@K[3],$FOUR
658         vadduwm @K[4],@K[4],$FOUR
659         vadduwm @K[5],@K[5],$FOUR
660
661 ___
662 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) {   # flip byte order
663 $code.=<<___;
664         mr      @t[$i&3],@x[$i]
665         rotlwi  @x[$i],@x[$i],8
666         rlwimi  @x[$i],@t[$i&3],24,0,7
667         rlwimi  @x[$i],@t[$i&3],24,16,23
668 ___
669 } }
670 $code.=<<___;
671         lwz     @t[0],0($inp)                   # load input, aligned or not
672         lwz     @t[1],4($inp)
673         lwz     @t[2],8($inp)
674         lwz     @t[3],12($inp)
675         xor     @x[0],@x[0],@t[0]               # xor with input
676         lwz     @t[0],16($inp)
677         xor     @x[1],@x[1],@t[1]
678         lwz     @t[1],20($inp)
679         xor     @x[2],@x[2],@t[2]
680         lwz     @t[2],24($inp)
681         xor     @x[3],@x[3],@t[3]
682         lwz     @t[3],28($inp)
683         xor     @x[4],@x[4],@t[0]
684         lwz     @t[0],32($inp)
685         xor     @x[5],@x[5],@t[1]
686         lwz     @t[1],36($inp)
687         xor     @x[6],@x[6],@t[2]
688         lwz     @t[2],40($inp)
689         xor     @x[7],@x[7],@t[3]
690         lwz     @t[3],44($inp)
691         xor     @x[8],@x[8],@t[0]
692         lwz     @t[0],48($inp)
693         xor     @x[9],@x[9],@t[1]
694         lwz     @t[1],52($inp)
695         xor     @x[10],@x[10],@t[2]
696         lwz     @t[2],56($inp)
697         xor     @x[11],@x[11],@t[3]
698         lwz     @t[3],60($inp)
699         xor     @x[12],@x[12],@t[0]
700         stw     @x[0],0($out)                   # store output, aligned or not
701         xor     @x[13],@x[13],@t[1]
702         stw     @x[1],4($out)
703         xor     @x[14],@x[14],@t[2]
704         stw     @x[2],8($out)
705         xor     @x[15],@x[15],@t[3]
706         stw     @x[3],12($out)
707         addi    $inp,$inp,64
708         stw     @x[4],16($out)
709         li      @t[0],16
710         stw     @x[5],20($out)
711         li      @t[1],32
712         stw     @x[6],24($out)
713         li      @t[2],48
714         stw     @x[7],28($out)
715         li      @t[3],64
716         stw     @x[8],32($out)
717         stw     @x[9],36($out)
718         stw     @x[10],40($out)
719         stw     @x[11],44($out)
720         stw     @x[12],48($out)
721         stw     @x[13],52($out)
722         stw     @x[14],56($out)
723         stw     @x[15],60($out)
724         addi    $out,$out,64
725
726         lvx     @D[0],0,$inp                    # load input
727         lvx     @D[1],@t[0],$inp
728         lvx     @D[2],@t[1],$inp
729         lvx     @D[3],@t[2],$inp
730         lvx     @D[4],@t[3],$inp
731         addi    $inp,$inp,64
732
733         ?vperm  @D[0],@D[1],@D[0],$inpperm      # align input
734         ?vperm  @D[1],@D[2],@D[1],$inpperm
735         ?vperm  @D[2],@D[3],@D[2],$inpperm
736         ?vperm  @D[3],@D[4],@D[3],$inpperm
737         vxor    $A0,$A0,@D[0]                   # xor with input
738         vxor    $B0,$B0,@D[1]
739         lvx     @D[1],@t[0],$inp                # keep loading input
740         vxor    $C0,$C0,@D[2]
741         lvx     @D[2],@t[1],$inp
742         vxor    $D0,$D0,@D[3]
743         lvx     @D[3],@t[2],$inp
744         lvx     @D[0],@t[3],$inp
745         addi    $inp,$inp,64
746         li      @t[3],63                        # 63 is not a typo
747         vperm   $A0,$A0,$A0,$outperm            # pre-misalign output
748         vperm   $B0,$B0,$B0,$outperm
749         vperm   $C0,$C0,$C0,$outperm
750         vperm   $D0,$D0,$D0,$outperm
751
752         ?vperm  @D[4],@D[1],@D[4],$inpperm      # align input
753         ?vperm  @D[1],@D[2],@D[1],$inpperm
754         ?vperm  @D[2],@D[3],@D[2],$inpperm
755         ?vperm  @D[3],@D[0],@D[3],$inpperm
756         vxor    $A1,$A1,@D[4]
757         vxor    $B1,$B1,@D[1]
758         lvx     @D[1],@t[0],$inp                # keep loading input
759         vxor    $C1,$C1,@D[2]
760         lvx     @D[2],@t[1],$inp
761         vxor    $D1,$D1,@D[3]
762         lvx     @D[3],@t[2],$inp
763         lvx     @D[4],@t[3],$inp                # redundant in aligned case
764         addi    $inp,$inp,64
765         vperm   $A1,$A1,$A1,$outperm            # pre-misalign output
766         vperm   $B1,$B1,$B1,$outperm
767         vperm   $C1,$C1,$C1,$outperm
768         vperm   $D1,$D1,$D1,$outperm
769
770         ?vperm  @D[0],@D[1],@D[0],$inpperm      # align input
771         ?vperm  @D[1],@D[2],@D[1],$inpperm
772         ?vperm  @D[2],@D[3],@D[2],$inpperm
773         ?vperm  @D[3],@D[4],@D[3],$inpperm
774         vxor    $A2,$A2,@D[0]
775         vxor    $B2,$B2,@D[1]
776         vxor    $C2,$C2,@D[2]
777         vxor    $D2,$D2,@D[3]
778         vperm   $A2,$A2,$A2,$outperm            # pre-misalign output
779         vperm   $B2,$B2,$B2,$outperm
780         vperm   $C2,$C2,$C2,$outperm
781         vperm   $D2,$D2,$D2,$outperm
782
783         andi.   @x[1],$out,15                   # is $out aligned?
784         mr      @x[0],$out
785
786         vsel    @D[0],$A0,$B0,$outmask          # collect pre-misaligned output
787         vsel    @D[1],$B0,$C0,$outmask
788         vsel    @D[2],$C0,$D0,$outmask
789         vsel    @D[3],$D0,$A1,$outmask
790         vsel    $B0,$A1,$B1,$outmask
791         vsel    $C0,$B1,$C1,$outmask
792         vsel    $D0,$C1,$D1,$outmask
793         vsel    $A1,$D1,$A2,$outmask
794         vsel    $B1,$A2,$B2,$outmask
795         vsel    $C1,$B2,$C2,$outmask
796         vsel    $D1,$C2,$D2,$outmask
797
798         #stvx   $A0,0,$out                      # take it easy on the edges
799         stvx    @D[0],@t[0],$out                # store output
800         stvx    @D[1],@t[1],$out
801         stvx    @D[2],@t[2],$out
802         addi    $out,$out,64
803         stvx    @D[3],0,$out
804         stvx    $B0,@t[0],$out
805         stvx    $C0,@t[1],$out
806         stvx    $D0,@t[2],$out
807         addi    $out,$out,64
808         stvx    $A1,0,$out
809         stvx    $B1,@t[0],$out
810         stvx    $C1,@t[1],$out
811         stvx    $D1,@t[2],$out
812         addi    $out,$out,64
813
814         beq     Laligned_vmx
815
816         sub     @x[2],$out,@x[1]                # in misaligned case edges
817         li      @x[3],0                         # are written byte-by-byte
818 Lunaligned_tail_vmx:
819         stvebx  $D2,@x[3],@x[2]
820         addi    @x[3],@x[3],1
821         cmpw    @x[3],@x[1]
822         bne     Lunaligned_tail_vmx
823
824         sub     @x[2],@x[0],@x[1]
825 Lunaligned_head_vmx:
826         stvebx  $A0,@x[1],@x[2]
827         cmpwi   @x[1],15
828         addi    @x[1],@x[1],1
829         bne     Lunaligned_head_vmx
830
831         ${UCMP}i $len,255                       # done with 256-byte blocks yet?
832         bgt     Loop_outer_vmx
833
834         b       Ldone_vmx
835
836 .align  4
837 Laligned_vmx:
838         stvx    $A0,0,@x[0]                     # head hexaword was not stored
839
840         ${UCMP}i $len,255                       # done with 256-byte blocks yet?
841         bgt     Loop_outer_vmx
842         nop
843
844 Ldone_vmx:
845         ${UCMP}i $len,0                         # done yet?
846         bnel    __ChaCha20_1x
847
848         lwz     r12,`$FRAME-$SIZE_T*18-4`($sp)  # pull vrsave
849         li      r10,`15+$LOCALS+64`
850         li      r11,`31+$LOCALS+64`
851         mtspr   256,r12                         # restore vrsave
852         lvx     v20,r10,$sp
853         addi    r10,r10,32
854         lvx     v21,r11,$sp
855         addi    r11,r11,32
856         lvx     v22,r10,$sp
857         addi    r10,r10,32
858         lvx     v23,r11,$sp
859         addi    r11,r11,32
860         lvx     v24,r10,$sp
861         addi    r10,r10,32
862         lvx     v25,r11,$sp
863         addi    r11,r11,32
864         lvx     v26,r10,$sp
865         addi    r10,r10,32
866         lvx     v27,r11,$sp
867         addi    r11,r11,32
868         lvx     v28,r10,$sp
869         addi    r10,r10,32
870         lvx     v29,r11,$sp
871         addi    r11,r11,32
872         lvx     v30,r10,$sp
873         lvx     v31,r11,$sp
874         $POP    r0, `$FRAME+$LRSAVE`($sp)
875         $POP    r14,`$FRAME-$SIZE_T*18`($sp)
876         $POP    r15,`$FRAME-$SIZE_T*17`($sp)
877         $POP    r16,`$FRAME-$SIZE_T*16`($sp)
878         $POP    r17,`$FRAME-$SIZE_T*15`($sp)
879         $POP    r18,`$FRAME-$SIZE_T*14`($sp)
880         $POP    r19,`$FRAME-$SIZE_T*13`($sp)
881         $POP    r20,`$FRAME-$SIZE_T*12`($sp)
882         $POP    r21,`$FRAME-$SIZE_T*11`($sp)
883         $POP    r22,`$FRAME-$SIZE_T*10`($sp)
884         $POP    r23,`$FRAME-$SIZE_T*9`($sp)
885         $POP    r24,`$FRAME-$SIZE_T*8`($sp)
886         $POP    r25,`$FRAME-$SIZE_T*7`($sp)
887         $POP    r26,`$FRAME-$SIZE_T*6`($sp)
888         $POP    r27,`$FRAME-$SIZE_T*5`($sp)
889         $POP    r28,`$FRAME-$SIZE_T*4`($sp)
890         $POP    r29,`$FRAME-$SIZE_T*3`($sp)
891         $POP    r30,`$FRAME-$SIZE_T*2`($sp)
892         $POP    r31,`$FRAME-$SIZE_T*1`($sp)
893         mtlr    r0
894         addi    $sp,$sp,$FRAME
895         blr
896         .long   0
897         .byte   0,12,0x04,1,0x80,18,5,0
898         .long   0
899 .size   .ChaCha20_ctr32_vmx,.-.ChaCha20_ctr32_vmx
900
901 .align  5
902 Lconsts:
903         mflr    r0
904         bcl     20,31,\$+4
905         mflr    r12     #vvvvv "distance between . and _vpaes_consts
906         addi    r12,r12,`64-8`
907         mtlr    r0
908         blr
909         .long   0
910         .byte   0,12,0x14,0,0,0,0,0
911         .space  `64-9*4`
912 Lsigma:
913         .long   0x61707865,0x3320646e,0x79622d32,0x6b206574
914         .long   1,0,0,0
915         .long   4,0,0,0
916 ___
917 $code.=<<___    if ($LITTLE_ENDIAN);
918         .long   0x0e0f0c0d,0x0a0b0809,0x06070405,0x02030001
919         .long   0x0d0e0f0c,0x090a0b08,0x05060704,0x01020300
920 ___
921 $code.=<<___    if (!$LITTLE_ENDIAN);   # flipped words
922         .long   0x02030001,0x06070405,0x0a0b0809,0x0e0f0c0d
923         .long   0x01020300,0x05060704,0x090a0b08,0x0d0e0f0c
924 ___
925 $code.=<<___;
926 .asciz  "ChaCha20 for PowerPC/AltiVec, CRYPTOGAMS by <appro\@openssl.org>"
927 .align  2
928 ___
929 }}}
930
931 foreach (split("\n",$code)) {
932         s/\`([^\`]*)\`/eval $1/ge;
933
934         # instructions prefixed with '?' are endian-specific and need
935         # to be adjusted accordingly...
936         if ($flavour !~ /le$/) {        # big-endian
937             s/be\?//            or
938             s/le\?/#le#/        or
939             s/\?lvsr/lvsl/      or
940             s/\?lvsl/lvsr/      or
941             s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/ or
942             s/(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/;
943         } else {                        # little-endian
944             s/le\?//            or
945             s/be\?/#be#/        or
946             s/\?([a-z]+)/$1/;
947         }
948
949         print $_,"\n";
950 }
951
952 close STDOUT;