xref: /openssl/crypto/aes/asm/aesni-x86_64.pl (revision 7ed6de99)
1#! /usr/bin/env perl
2# Copyright 2009-2024 The OpenSSL Project Authors. All Rights Reserved.
3#
4# Licensed under the Apache License 2.0 (the "License").  You may not use
5# this file except in compliance with the License.  You can obtain a copy
6# in the file LICENSE in the source distribution or at
7# https://www.openssl.org/source/license.html
8
9#
10# ====================================================================
11# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12# project. The module is, however, dual licensed under OpenSSL and
13# CRYPTOGAMS licenses depending on where you obtain it. For further
14# details see http://www.openssl.org/~appro/cryptogams/.
15# ====================================================================
16#
17# This module implements support for Intel AES-NI extension. In
18# OpenSSL context it's used with Intel engine, but can also be used as
19# drop-in replacement for crypto/aes/asm/aes-x86_64.pl [see below for
20# details].
21#
22# Performance.
23#
24# Given aes(enc|dec) instructions' latency asymptotic performance for
25# non-parallelizable modes such as CBC encrypt is 3.75 cycles per byte
26# processed with 128-bit key. And given their throughput asymptotic
27# performance for parallelizable modes is 1.25 cycles per byte. Being
28# asymptotic limit it's not something you commonly achieve in reality,
29# but how close does one get? Below are results collected for
30# different modes and block sized. Pairs of numbers are for en-/
31# decryption.
32#
33#	16-byte     64-byte     256-byte    1-KB        8-KB
34# ECB	4.25/4.25   1.38/1.38   1.28/1.28   1.26/1.26	1.26/1.26
35# CTR	5.42/5.42   1.92/1.92   1.44/1.44   1.28/1.28   1.26/1.26
36# CBC	4.38/4.43   4.15/1.43   4.07/1.32   4.07/1.29   4.06/1.28
37# CCM	5.66/9.42   4.42/5.41   4.16/4.40   4.09/4.15   4.06/4.07
38# OFB	5.42/5.42   4.64/4.64   4.44/4.44   4.39/4.39   4.38/4.38
39# CFB	5.73/5.85   5.56/5.62   5.48/5.56   5.47/5.55   5.47/5.55
40#
41# ECB, CTR, CBC and CCM results are free from EVP overhead. This means
42# that otherwise used 'openssl speed -evp aes-128-??? -engine aesni
43# [-decrypt]' will exhibit 10-15% worse results for smaller blocks.
44# The results were collected with specially crafted speed.c benchmark
45# in order to compare them with results reported in "Intel Advanced
46# Encryption Standard (AES) New Instruction Set" White Paper Revision
47# 3.0 dated May 2010. All above results are consistently better. This
48# module also provides better performance for block sizes smaller than
49# 128 bytes in points *not* represented in the above table.
50#
51# Looking at the results for 8-KB buffer.
52#
53# CFB and OFB results are far from the limit, because implementation
54# uses "generic" CRYPTO_[c|o]fb128_encrypt interfaces relying on
55# single-block aesni_encrypt, which is not the most optimal way to go.
56# CBC encrypt result is unexpectedly high and there is no documented
57# explanation for it. Seemingly there is a small penalty for feeding
58# the result back to AES unit the way it's done in CBC mode. There is
59# nothing one can do and the result appears optimal. CCM result is
60# identical to CBC, because CBC-MAC is essentially CBC encrypt without
61# saving output. CCM CTR "stays invisible," because it's neatly
62# interleaved with CBC-MAC. This provides ~30% improvement over
63# "straightforward" CCM implementation with CTR and CBC-MAC performed
64# disjointly. Parallelizable modes practically achieve the theoretical
65# limit.
66#
67# Looking at how results vary with buffer size.
68#
69# Curves are practically saturated at 1-KB buffer size. In most cases
70# "256-byte" performance is >95%, and "64-byte" is ~90% of "8-KB" one.
71# CTR curve doesn't follow this pattern and is "slowest" changing one
72# with "256-byte" result being 87% of "8-KB." This is because overhead
73# in CTR mode is most computationally intensive. Small-block CCM
74# decrypt is slower than encrypt, because first CTR and last CBC-MAC
75# iterations can't be interleaved.
76#
77# Results for 192- and 256-bit keys.
78#
79# EVP-free results were observed to scale perfectly with number of
80# rounds for larger block sizes, i.e. 192-bit result being 10/12 times
81# lower and 256-bit one - 10/14. Well, in CBC encrypt case differences
82# are a tad smaller, because the above mentioned penalty biases all
83# results by same constant value. In similar way function call
84# overhead affects small-block performance, as well as OFB and CFB
85# results. Differences are not large, most common coefficients are
86# 10/11.7 and 10/13.4 (as opposite to 10/12.0 and 10/14.0), but one
87# observe even 10/11.2 and 10/12.4 (CTR, OFB, CFB)...
88
89# January 2011
90#
91# While Westmere processor features 6 cycles latency for aes[enc|dec]
92# instructions, which can be scheduled every second cycle, Sandy
93# Bridge spends 8 cycles per instruction, but it can schedule them
94# every cycle. This means that code targeting Westmere would perform
95# suboptimally on Sandy Bridge. Therefore this update.
96#
97# In addition, non-parallelizable CBC encrypt (as well as CCM) is
98# optimized. Relative improvement might appear modest, 8% on Westmere,
99# but in absolute terms it's 3.77 cycles per byte encrypted with
100# 128-bit key on Westmere, and 5.07 - on Sandy Bridge. These numbers
101# should be compared to asymptotic limits of 3.75 for Westmere and
102# 5.00 for Sandy Bridge. Actually, the fact that they get this close
103# to asymptotic limits is quite amazing. Indeed, the limit is
104# calculated as latency times number of rounds, 10 for 128-bit key,
105# and divided by 16, the number of bytes in block, or in other words
106# it accounts *solely* for aesenc instructions. But there are extra
107# instructions, and numbers so close to the asymptotic limits mean
108# that it's as if it takes as little as *one* additional cycle to
109# execute all of them. How is it possible? It is possible thanks to
110# out-of-order execution logic, which manages to overlap post-
111# processing of previous block, things like saving the output, with
112# actual encryption of current block, as well as pre-processing of
113# current block, things like fetching input and xor-ing it with
114# 0-round element of the key schedule, with actual encryption of
115# previous block. Keep this in mind...
116#
117# For parallelizable modes, such as ECB, CBC decrypt, CTR, higher
118# performance is achieved by interleaving instructions working on
119# independent blocks. In which case asymptotic limit for such modes
120# can be obtained by dividing above mentioned numbers by AES
121# instructions' interleave factor. Westmere can execute at most 3
122# instructions at a time, meaning that optimal interleave factor is 3,
123# and that's where the "magic" number of 1.25 come from. "Optimal
124# interleave factor" means that increase of interleave factor does
125# not improve performance. The formula has proven to reflect reality
126# pretty well on Westmere... Sandy Bridge on the other hand can
127# execute up to 8 AES instructions at a time, so how does varying
128# interleave factor affect the performance? Here is table for ECB
129# (numbers are cycles per byte processed with 128-bit key):
130#
131# instruction interleave factor		3x	6x	8x
132# theoretical asymptotic limit		1.67	0.83	0.625
133# measured performance for 8KB block	1.05	0.86	0.84
134#
135# "as if" interleave factor		4.7x	5.8x	6.0x
136#
137# Further data for other parallelizable modes:
138#
139# CBC decrypt				1.16	0.93	0.74
140# CTR					1.14	0.91	0.74
141#
142# Well, given 3x column it's probably inappropriate to call the limit
143# asymptotic, if it can be surpassed, isn't it? What happens there?
144# Rewind to CBC paragraph for the answer. Yes, out-of-order execution
145# magic is responsible for this. Processor overlaps not only the
146# additional instructions with AES ones, but even AES instructions
147# processing adjacent triplets of independent blocks. In the 6x case
148# additional instructions  still claim disproportionally small amount
149# of additional cycles, but in 8x case number of instructions must be
150# a tad too high for out-of-order logic to cope with, and AES unit
151# remains underutilized... As you can see 8x interleave is hardly
152# justifiable, so there no need to feel bad that 32-bit aesni-x86.pl
153# utilizes 6x interleave because of limited register bank capacity.
154#
155# Higher interleave factors do have negative impact on Westmere
156# performance. While for ECB mode it's negligible ~1.5%, other
157# parallelizables perform ~5% worse, which is outweighed by ~25%
158# improvement on Sandy Bridge. To balance regression on Westmere
159# CTR mode was implemented with 6x aesenc interleave factor.
160
161# April 2011
162#
163# Add aesni_xts_[en|de]crypt. Westmere spends 1.25 cycles processing
164# one byte out of 8KB with 128-bit key, Sandy Bridge - 0.90. Just like
165# in CTR mode AES instruction interleave factor was chosen to be 6x.
166
167# November 2015
168#
169# Add aesni_ocb_[en|de]crypt. AES instruction interleave factor was
170# chosen to be 6x.
171
172######################################################################
173# Current large-block performance in cycles per byte processed with
174# 128-bit key (less is better).
175#
176#		CBC en-/decrypt	CTR	XTS	ECB	OCB
177# Westmere	3.77/1.25	1.25	1.25	1.26
178# * Bridge	5.07/0.74	0.75	0.90	0.85	0.98
179# Haswell	4.44/0.63	0.63	0.73	0.63	0.70
180# Skylake	2.62/0.63	0.63	0.63	0.63
181# Silvermont	5.75/3.54	3.56	4.12	3.87(*)	4.11
182# Knights L	2.54/0.77	0.78	0.85	-	1.50
183# Goldmont	3.82/1.26	1.26	1.29	1.29	1.50
184# Bulldozer	5.77/0.70	0.72	0.90	0.70	0.95
185# Ryzen		2.71/0.35	0.35	0.44	0.38	0.49
186#
187# (*)	Atom Silvermont ECB result is suboptimal because of penalties
188#	incurred by operations on %xmm8-15. As ECB is not considered
189#	critical, nothing was done to mitigate the problem.
190
191$PREFIX="aesni";	# if $PREFIX is set to "AES", the script
192			# generates drop-in replacement for
193			# crypto/aes/asm/aes-x86_64.pl:-)
194
195# $output is the last argument if it looks like a file (it has an extension)
196# $flavour is the first argument if it doesn't look like a file
197$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
198$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
199
200$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
201
202$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
203( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
204( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
205die "can't locate x86_64-xlate.pl";
206
207open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""
208    or die "can't call $xlate: $!";
209*STDOUT=*OUT;
210
211$movkey = $PREFIX eq "aesni" ? "movups" : "movups";
212@_4args=$win64?	("%rcx","%rdx","%r8", "%r9") :	# Win64 order
213		("%rdi","%rsi","%rdx","%rcx");	# Unix order
214
215$code=".text\n";
216$code.=".extern	OPENSSL_ia32cap_P\n";
217
218$rounds="%eax";	# input to and changed by aesni_[en|de]cryptN !!!
219# this is natural Unix argument order for public $PREFIX_[ecb|cbc]_encrypt ...
220$inp="%rdi";
221$out="%rsi";
222$len="%rdx";
223$key="%rcx";	# input to and changed by aesni_[en|de]cryptN !!!
224$ivp="%r8";	# cbc, ctr, ...
225
226$rnds_="%r10d";	# backup copy for $rounds
227$key_="%r11";	# backup copy for $key
228
229# %xmm register layout
230$rndkey0="%xmm0";	$rndkey1="%xmm1";
231$inout0="%xmm2";	$inout1="%xmm3";
232$inout2="%xmm4";	$inout3="%xmm5";
233$inout4="%xmm6";	$inout5="%xmm7";
234$inout6="%xmm8";	$inout7="%xmm9";
235
236$in2="%xmm6";		$in1="%xmm7";	# used in CBC decrypt, CTR, ...
237$in0="%xmm8";		$iv="%xmm9";
238
239# Inline version of internal aesni_[en|de]crypt1.
240#
241# Why folded loop? Because aes[enc|dec] is slow enough to accommodate
242# cycles which take care of loop variables...
243{ my $sn;
244sub aesni_generate1 {
245my ($p,$key,$rounds,$inout,$ivec)=@_;	$inout=$inout0 if (!defined($inout));
246++$sn;
247$code.=<<___;
248	$movkey	($key),$rndkey0
249	$movkey	16($key),$rndkey1
250___
251$code.=<<___ if (defined($ivec));
252	xorps	$rndkey0,$ivec
253	lea	32($key),$key
254	xorps	$ivec,$inout
255___
256$code.=<<___ if (!defined($ivec));
257	lea	32($key),$key
258	xorps	$rndkey0,$inout
259___
260$code.=<<___;
261.Loop_${p}1_$sn:
262	aes${p}	$rndkey1,$inout
263	dec	$rounds
264	$movkey	($key),$rndkey1
265	lea	16($key),$key
266	jnz	.Loop_${p}1_$sn	# loop body is 16 bytes
267	aes${p}last	$rndkey1,$inout
268___
269}}
270# void $PREFIX_[en|de]crypt (const void *inp,void *out,const AES_KEY *key);
271#
272{ my ($inp,$out,$key) = @_4args;
273
274$code.=<<___;
275.globl	${PREFIX}_encrypt
276.type	${PREFIX}_encrypt,\@abi-omnipotent
277.align	16
278${PREFIX}_encrypt:
279.cfi_startproc
280	endbranch
281	movups	($inp),$inout0		# load input
282	mov	240($key),$rounds	# key->rounds
283___
284	&aesni_generate1("enc",$key,$rounds);
285$code.=<<___;
286	 pxor	$rndkey0,$rndkey0	# clear register bank
287	 pxor	$rndkey1,$rndkey1
288	movups	$inout0,($out)		# output
289	 pxor	$inout0,$inout0
290	ret
291.cfi_endproc
292.size	${PREFIX}_encrypt,.-${PREFIX}_encrypt
293
294.globl	${PREFIX}_decrypt
295.type	${PREFIX}_decrypt,\@abi-omnipotent
296.align	16
297${PREFIX}_decrypt:
298.cfi_startproc
299	endbranch
300	movups	($inp),$inout0		# load input
301	mov	240($key),$rounds	# key->rounds
302___
303	&aesni_generate1("dec",$key,$rounds);
304$code.=<<___;
305	 pxor	$rndkey0,$rndkey0	# clear register bank
306	 pxor	$rndkey1,$rndkey1
307	movups	$inout0,($out)		# output
308	 pxor	$inout0,$inout0
309	ret
310.cfi_endproc
311.size	${PREFIX}_decrypt, .-${PREFIX}_decrypt
312___
313}
314
315# _aesni_[en|de]cryptN are private interfaces, N denotes interleave
316# factor. Why 3x subroutine were originally used in loops? Even though
317# aes[enc|dec] latency was originally 6, it could be scheduled only
318# every *2nd* cycle. Thus 3x interleave was the one providing optimal
319# utilization, i.e. when subroutine's throughput is virtually same as
320# of non-interleaved subroutine [for number of input blocks up to 3].
321# This is why it originally made no sense to implement 2x subroutine.
322# But times change and it became appropriate to spend extra 192 bytes
323# on 2x subroutine on Atom Silvermont account. For processors that
324# can schedule aes[enc|dec] every cycle optimal interleave factor
325# equals to corresponding instructions latency. 8x is optimal for
326# * Bridge and "super-optimal" for other Intel CPUs...
327
328sub aesni_generate2 {
329my $dir=shift;
330# As already mentioned it takes in $key and $rounds, which are *not*
331# preserved. $inout[0-1] is cipher/clear text...
332$code.=<<___;
333.type	_aesni_${dir}rypt2,\@abi-omnipotent
334.align	16
335_aesni_${dir}rypt2:
336.cfi_startproc
337	$movkey	($key),$rndkey0
338	shl	\$4,$rounds
339	$movkey	16($key),$rndkey1
340	xorps	$rndkey0,$inout0
341	xorps	$rndkey0,$inout1
342	$movkey	32($key),$rndkey0
343	lea	32($key,$rounds),$key
344	neg	%rax				# $rounds
345	add	\$16,%rax
346
347.L${dir}_loop2:
348	aes${dir}	$rndkey1,$inout0
349	aes${dir}	$rndkey1,$inout1
350	$movkey		($key,%rax),$rndkey1
351	add		\$32,%rax
352	aes${dir}	$rndkey0,$inout0
353	aes${dir}	$rndkey0,$inout1
354	$movkey		-16($key,%rax),$rndkey0
355	jnz		.L${dir}_loop2
356
357	aes${dir}	$rndkey1,$inout0
358	aes${dir}	$rndkey1,$inout1
359	aes${dir}last	$rndkey0,$inout0
360	aes${dir}last	$rndkey0,$inout1
361	ret
362.cfi_endproc
363.size	_aesni_${dir}rypt2,.-_aesni_${dir}rypt2
364___
365}
366sub aesni_generate3 {
367my $dir=shift;
368# As already mentioned it takes in $key and $rounds, which are *not*
369# preserved. $inout[0-2] is cipher/clear text...
370$code.=<<___;
371.type	_aesni_${dir}rypt3,\@abi-omnipotent
372.align	16
373_aesni_${dir}rypt3:
374.cfi_startproc
375	$movkey	($key),$rndkey0
376	shl	\$4,$rounds
377	$movkey	16($key),$rndkey1
378	xorps	$rndkey0,$inout0
379	xorps	$rndkey0,$inout1
380	xorps	$rndkey0,$inout2
381	$movkey	32($key),$rndkey0
382	lea	32($key,$rounds),$key
383	neg	%rax				# $rounds
384	add	\$16,%rax
385
386.L${dir}_loop3:
387	aes${dir}	$rndkey1,$inout0
388	aes${dir}	$rndkey1,$inout1
389	aes${dir}	$rndkey1,$inout2
390	$movkey		($key,%rax),$rndkey1
391	add		\$32,%rax
392	aes${dir}	$rndkey0,$inout0
393	aes${dir}	$rndkey0,$inout1
394	aes${dir}	$rndkey0,$inout2
395	$movkey		-16($key,%rax),$rndkey0
396	jnz		.L${dir}_loop3
397
398	aes${dir}	$rndkey1,$inout0
399	aes${dir}	$rndkey1,$inout1
400	aes${dir}	$rndkey1,$inout2
401	aes${dir}last	$rndkey0,$inout0
402	aes${dir}last	$rndkey0,$inout1
403	aes${dir}last	$rndkey0,$inout2
404	ret
405.cfi_endproc
406.size	_aesni_${dir}rypt3,.-_aesni_${dir}rypt3
407___
408}
409# 4x interleave is implemented to improve small block performance,
410# most notably [and naturally] 4 block by ~30%. One can argue that one
411# should have implemented 5x as well, but improvement would be <20%,
412# so it's not worth it...
413sub aesni_generate4 {
414my $dir=shift;
415# As already mentioned it takes in $key and $rounds, which are *not*
416# preserved. $inout[0-3] is cipher/clear text...
417$code.=<<___;
418.type	_aesni_${dir}rypt4,\@abi-omnipotent
419.align	16
420_aesni_${dir}rypt4:
421.cfi_startproc
422	$movkey	($key),$rndkey0
423	shl	\$4,$rounds
424	$movkey	16($key),$rndkey1
425	xorps	$rndkey0,$inout0
426	xorps	$rndkey0,$inout1
427	xorps	$rndkey0,$inout2
428	xorps	$rndkey0,$inout3
429	$movkey	32($key),$rndkey0
430	lea	32($key,$rounds),$key
431	neg	%rax				# $rounds
432	.byte	0x0f,0x1f,0x00
433	add	\$16,%rax
434
435.L${dir}_loop4:
436	aes${dir}	$rndkey1,$inout0
437	aes${dir}	$rndkey1,$inout1
438	aes${dir}	$rndkey1,$inout2
439	aes${dir}	$rndkey1,$inout3
440	$movkey		($key,%rax),$rndkey1
441	add		\$32,%rax
442	aes${dir}	$rndkey0,$inout0
443	aes${dir}	$rndkey0,$inout1
444	aes${dir}	$rndkey0,$inout2
445	aes${dir}	$rndkey0,$inout3
446	$movkey		-16($key,%rax),$rndkey0
447	jnz		.L${dir}_loop4
448
449	aes${dir}	$rndkey1,$inout0
450	aes${dir}	$rndkey1,$inout1
451	aes${dir}	$rndkey1,$inout2
452	aes${dir}	$rndkey1,$inout3
453	aes${dir}last	$rndkey0,$inout0
454	aes${dir}last	$rndkey0,$inout1
455	aes${dir}last	$rndkey0,$inout2
456	aes${dir}last	$rndkey0,$inout3
457	ret
458.cfi_endproc
459.size	_aesni_${dir}rypt4,.-_aesni_${dir}rypt4
460___
461}
462sub aesni_generate6 {
463my $dir=shift;
464# As already mentioned it takes in $key and $rounds, which are *not*
465# preserved. $inout[0-5] is cipher/clear text...
466$code.=<<___;
467.type	_aesni_${dir}rypt6,\@abi-omnipotent
468.align	16
469_aesni_${dir}rypt6:
470.cfi_startproc
471	$movkey		($key),$rndkey0
472	shl		\$4,$rounds
473	$movkey		16($key),$rndkey1
474	xorps		$rndkey0,$inout0
475	pxor		$rndkey0,$inout1
476	pxor		$rndkey0,$inout2
477	aes${dir}	$rndkey1,$inout0
478	lea		32($key,$rounds),$key
479	neg		%rax			# $rounds
480	aes${dir}	$rndkey1,$inout1
481	pxor		$rndkey0,$inout3
482	pxor		$rndkey0,$inout4
483	aes${dir}	$rndkey1,$inout2
484	pxor		$rndkey0,$inout5
485	$movkey		($key,%rax),$rndkey0
486	add		\$16,%rax
487	jmp		.L${dir}_loop6_enter
488.align	16
489.L${dir}_loop6:
490	aes${dir}	$rndkey1,$inout0
491	aes${dir}	$rndkey1,$inout1
492	aes${dir}	$rndkey1,$inout2
493.L${dir}_loop6_enter:
494	aes${dir}	$rndkey1,$inout3
495	aes${dir}	$rndkey1,$inout4
496	aes${dir}	$rndkey1,$inout5
497	$movkey		($key,%rax),$rndkey1
498	add		\$32,%rax
499	aes${dir}	$rndkey0,$inout0
500	aes${dir}	$rndkey0,$inout1
501	aes${dir}	$rndkey0,$inout2
502	aes${dir}	$rndkey0,$inout3
503	aes${dir}	$rndkey0,$inout4
504	aes${dir}	$rndkey0,$inout5
505	$movkey		-16($key,%rax),$rndkey0
506	jnz		.L${dir}_loop6
507
508	aes${dir}	$rndkey1,$inout0
509	aes${dir}	$rndkey1,$inout1
510	aes${dir}	$rndkey1,$inout2
511	aes${dir}	$rndkey1,$inout3
512	aes${dir}	$rndkey1,$inout4
513	aes${dir}	$rndkey1,$inout5
514	aes${dir}last	$rndkey0,$inout0
515	aes${dir}last	$rndkey0,$inout1
516	aes${dir}last	$rndkey0,$inout2
517	aes${dir}last	$rndkey0,$inout3
518	aes${dir}last	$rndkey0,$inout4
519	aes${dir}last	$rndkey0,$inout5
520	ret
521.cfi_endproc
522.size	_aesni_${dir}rypt6,.-_aesni_${dir}rypt6
523___
524}
525sub aesni_generate8 {
526my $dir=shift;
527# As already mentioned it takes in $key and $rounds, which are *not*
528# preserved. $inout[0-7] is cipher/clear text...
529$code.=<<___;
530.type	_aesni_${dir}rypt8,\@abi-omnipotent
531.align	16
532_aesni_${dir}rypt8:
533.cfi_startproc
534	$movkey		($key),$rndkey0
535	shl		\$4,$rounds
536	$movkey		16($key),$rndkey1
537	xorps		$rndkey0,$inout0
538	xorps		$rndkey0,$inout1
539	pxor		$rndkey0,$inout2
540	pxor		$rndkey0,$inout3
541	pxor		$rndkey0,$inout4
542	lea		32($key,$rounds),$key
543	neg		%rax			# $rounds
544	aes${dir}	$rndkey1,$inout0
545	pxor		$rndkey0,$inout5
546	pxor		$rndkey0,$inout6
547	aes${dir}	$rndkey1,$inout1
548	pxor		$rndkey0,$inout7
549	$movkey		($key,%rax),$rndkey0
550	add		\$16,%rax
551	jmp		.L${dir}_loop8_inner
552.align	16
553.L${dir}_loop8:
554	aes${dir}	$rndkey1,$inout0
555	aes${dir}	$rndkey1,$inout1
556.L${dir}_loop8_inner:
557	aes${dir}	$rndkey1,$inout2
558	aes${dir}	$rndkey1,$inout3
559	aes${dir}	$rndkey1,$inout4
560	aes${dir}	$rndkey1,$inout5
561	aes${dir}	$rndkey1,$inout6
562	aes${dir}	$rndkey1,$inout7
563.L${dir}_loop8_enter:
564	$movkey		($key,%rax),$rndkey1
565	add		\$32,%rax
566	aes${dir}	$rndkey0,$inout0
567	aes${dir}	$rndkey0,$inout1
568	aes${dir}	$rndkey0,$inout2
569	aes${dir}	$rndkey0,$inout3
570	aes${dir}	$rndkey0,$inout4
571	aes${dir}	$rndkey0,$inout5
572	aes${dir}	$rndkey0,$inout6
573	aes${dir}	$rndkey0,$inout7
574	$movkey		-16($key,%rax),$rndkey0
575	jnz		.L${dir}_loop8
576
577	aes${dir}	$rndkey1,$inout0
578	aes${dir}	$rndkey1,$inout1
579	aes${dir}	$rndkey1,$inout2
580	aes${dir}	$rndkey1,$inout3
581	aes${dir}	$rndkey1,$inout4
582	aes${dir}	$rndkey1,$inout5
583	aes${dir}	$rndkey1,$inout6
584	aes${dir}	$rndkey1,$inout7
585	aes${dir}last	$rndkey0,$inout0
586	aes${dir}last	$rndkey0,$inout1
587	aes${dir}last	$rndkey0,$inout2
588	aes${dir}last	$rndkey0,$inout3
589	aes${dir}last	$rndkey0,$inout4
590	aes${dir}last	$rndkey0,$inout5
591	aes${dir}last	$rndkey0,$inout6
592	aes${dir}last	$rndkey0,$inout7
593	ret
594.cfi_endproc
595.size	_aesni_${dir}rypt8,.-_aesni_${dir}rypt8
596___
597}
598&aesni_generate2("enc") if ($PREFIX eq "aesni");
599&aesni_generate2("dec");
600&aesni_generate3("enc") if ($PREFIX eq "aesni");
601&aesni_generate3("dec");
602&aesni_generate4("enc") if ($PREFIX eq "aesni");
603&aesni_generate4("dec");
604&aesni_generate6("enc") if ($PREFIX eq "aesni");
605&aesni_generate6("dec");
606&aesni_generate8("enc") if ($PREFIX eq "aesni");
607&aesni_generate8("dec");
608
609if ($PREFIX eq "aesni") {
610########################################################################
611# void aesni_ecb_encrypt (const void *in, void *out,
612#			  size_t length, const AES_KEY *key,
613#			  int enc);
614$code.=<<___;
615.globl	aesni_ecb_encrypt
616.type	aesni_ecb_encrypt,\@function,5
617.align	16
618aesni_ecb_encrypt:
619.cfi_startproc
620	endbranch
621___
622$code.=<<___ if ($win64);
623	lea	-0x58(%rsp),%rsp
624	movaps	%xmm6,(%rsp)		# offload $inout4..7
625	movaps	%xmm7,0x10(%rsp)
626	movaps	%xmm8,0x20(%rsp)
627	movaps	%xmm9,0x30(%rsp)
628.Lecb_enc_body:
629___
630$code.=<<___;
631	and	\$-16,$len		# if ($len<16)
632	jz	.Lecb_ret		# return
633
634	mov	240($key),$rounds	# key->rounds
635	$movkey	($key),$rndkey0
636	mov	$key,$key_		# backup $key
637	mov	$rounds,$rnds_		# backup $rounds
638	test	%r8d,%r8d		# 5th argument
639	jz	.Lecb_decrypt
640#--------------------------- ECB ENCRYPT ------------------------------#
641	cmp	\$0x80,$len		# if ($len<8*16)
642	jb	.Lecb_enc_tail		# short input
643
644	movdqu	($inp),$inout0		# load 8 input blocks
645	movdqu	0x10($inp),$inout1
646	movdqu	0x20($inp),$inout2
647	movdqu	0x30($inp),$inout3
648	movdqu	0x40($inp),$inout4
649	movdqu	0x50($inp),$inout5
650	movdqu	0x60($inp),$inout6
651	movdqu	0x70($inp),$inout7
652	lea	0x80($inp),$inp		# $inp+=8*16
653	sub	\$0x80,$len		# $len-=8*16 (can be zero)
654	jmp	.Lecb_enc_loop8_enter
655.align 16
656.Lecb_enc_loop8:
657	movups	$inout0,($out)		# store 8 output blocks
658	mov	$key_,$key		# restore $key
659	movdqu	($inp),$inout0		# load 8 input blocks
660	mov	$rnds_,$rounds		# restore $rounds
661	movups	$inout1,0x10($out)
662	movdqu	0x10($inp),$inout1
663	movups	$inout2,0x20($out)
664	movdqu	0x20($inp),$inout2
665	movups	$inout3,0x30($out)
666	movdqu	0x30($inp),$inout3
667	movups	$inout4,0x40($out)
668	movdqu	0x40($inp),$inout4
669	movups	$inout5,0x50($out)
670	movdqu	0x50($inp),$inout5
671	movups	$inout6,0x60($out)
672	movdqu	0x60($inp),$inout6
673	movups	$inout7,0x70($out)
674	lea	0x80($out),$out		# $out+=8*16
675	movdqu	0x70($inp),$inout7
676	lea	0x80($inp),$inp		# $inp+=8*16
677.Lecb_enc_loop8_enter:
678
679	call	_aesni_encrypt8
680
681	sub	\$0x80,$len
682	jnc	.Lecb_enc_loop8		# loop if $len-=8*16 didn't borrow
683
684	movups	$inout0,($out)		# store 8 output blocks
685	mov	$key_,$key		# restore $key
686	movups	$inout1,0x10($out)
687	mov	$rnds_,$rounds		# restore $rounds
688	movups	$inout2,0x20($out)
689	movups	$inout3,0x30($out)
690	movups	$inout4,0x40($out)
691	movups	$inout5,0x50($out)
692	movups	$inout6,0x60($out)
693	movups	$inout7,0x70($out)
694	lea	0x80($out),$out		# $out+=8*16
695	add	\$0x80,$len		# restore real remaining $len
696	jz	.Lecb_ret		# done if ($len==0)
697
698.Lecb_enc_tail:				# $len is less than 8*16
699	movups	($inp),$inout0
700	cmp	\$0x20,$len
701	jb	.Lecb_enc_one
702	movups	0x10($inp),$inout1
703	je	.Lecb_enc_two
704	movups	0x20($inp),$inout2
705	cmp	\$0x40,$len
706	jb	.Lecb_enc_three
707	movups	0x30($inp),$inout3
708	je	.Lecb_enc_four
709	movups	0x40($inp),$inout4
710	cmp	\$0x60,$len
711	jb	.Lecb_enc_five
712	movups	0x50($inp),$inout5
713	je	.Lecb_enc_six
714	movdqu	0x60($inp),$inout6
715	xorps	$inout7,$inout7
716	call	_aesni_encrypt8
717	movups	$inout0,($out)		# store 7 output blocks
718	movups	$inout1,0x10($out)
719	movups	$inout2,0x20($out)
720	movups	$inout3,0x30($out)
721	movups	$inout4,0x40($out)
722	movups	$inout5,0x50($out)
723	movups	$inout6,0x60($out)
724	jmp	.Lecb_ret
725.align	16
726.Lecb_enc_one:
727___
728	&aesni_generate1("enc",$key,$rounds);
729$code.=<<___;
730	movups	$inout0,($out)		# store one output block
731	jmp	.Lecb_ret
732.align	16
733.Lecb_enc_two:
734	call	_aesni_encrypt2
735	movups	$inout0,($out)		# store 2 output blocks
736	movups	$inout1,0x10($out)
737	jmp	.Lecb_ret
738.align	16
739.Lecb_enc_three:
740	call	_aesni_encrypt3
741	movups	$inout0,($out)		# store 3 output blocks
742	movups	$inout1,0x10($out)
743	movups	$inout2,0x20($out)
744	jmp	.Lecb_ret
745.align	16
746.Lecb_enc_four:
747	call	_aesni_encrypt4
748	movups	$inout0,($out)		# store 4 output blocks
749	movups	$inout1,0x10($out)
750	movups	$inout2,0x20($out)
751	movups	$inout3,0x30($out)
752	jmp	.Lecb_ret
753.align	16
754.Lecb_enc_five:
755	xorps	$inout5,$inout5
756	call	_aesni_encrypt6
757	movups	$inout0,($out)		# store 5 output blocks
758	movups	$inout1,0x10($out)
759	movups	$inout2,0x20($out)
760	movups	$inout3,0x30($out)
761	movups	$inout4,0x40($out)
762	jmp	.Lecb_ret
763.align	16
764.Lecb_enc_six:
765	call	_aesni_encrypt6
766	movups	$inout0,($out)		# store 6 output blocks
767	movups	$inout1,0x10($out)
768	movups	$inout2,0x20($out)
769	movups	$inout3,0x30($out)
770	movups	$inout4,0x40($out)
771	movups	$inout5,0x50($out)
772	jmp	.Lecb_ret
773#--------------------------- ECB DECRYPT ------------------------------#
774.align	16
775.Lecb_decrypt:
776	cmp	\$0x80,$len		# if ($len<8*16)
777	jb	.Lecb_dec_tail		# short input
778
779	movdqu	($inp),$inout0		# load 8 input blocks
780	movdqu	0x10($inp),$inout1
781	movdqu	0x20($inp),$inout2
782	movdqu	0x30($inp),$inout3
783	movdqu	0x40($inp),$inout4
784	movdqu	0x50($inp),$inout5
785	movdqu	0x60($inp),$inout6
786	movdqu	0x70($inp),$inout7
787	lea	0x80($inp),$inp		# $inp+=8*16
788	sub	\$0x80,$len		# $len-=8*16 (can be zero)
789	jmp	.Lecb_dec_loop8_enter
790.align 16
791.Lecb_dec_loop8:
792	movups	$inout0,($out)		# store 8 output blocks
793	mov	$key_,$key		# restore $key
794	movdqu	($inp),$inout0		# load 8 input blocks
795	mov	$rnds_,$rounds		# restore $rounds
796	movups	$inout1,0x10($out)
797	movdqu	0x10($inp),$inout1
798	movups	$inout2,0x20($out)
799	movdqu	0x20($inp),$inout2
800	movups	$inout3,0x30($out)
801	movdqu	0x30($inp),$inout3
802	movups	$inout4,0x40($out)
803	movdqu	0x40($inp),$inout4
804	movups	$inout5,0x50($out)
805	movdqu	0x50($inp),$inout5
806	movups	$inout6,0x60($out)
807	movdqu	0x60($inp),$inout6
808	movups	$inout7,0x70($out)
809	lea	0x80($out),$out		# $out+=8*16
810	movdqu	0x70($inp),$inout7
811	lea	0x80($inp),$inp		# $inp+=8*16
812.Lecb_dec_loop8_enter:
813
814	call	_aesni_decrypt8
815
816	$movkey	($key_),$rndkey0
817	sub	\$0x80,$len
818	jnc	.Lecb_dec_loop8		# loop if $len-=8*16 didn't borrow
819
820	movups	$inout0,($out)		# store 8 output blocks
821	 pxor	$inout0,$inout0		# clear register bank
822	mov	$key_,$key		# restore $key
823	movups	$inout1,0x10($out)
824	 pxor	$inout1,$inout1
825	mov	$rnds_,$rounds		# restore $rounds
826	movups	$inout2,0x20($out)
827	 pxor	$inout2,$inout2
828	movups	$inout3,0x30($out)
829	 pxor	$inout3,$inout3
830	movups	$inout4,0x40($out)
831	 pxor	$inout4,$inout4
832	movups	$inout5,0x50($out)
833	 pxor	$inout5,$inout5
834	movups	$inout6,0x60($out)
835	 pxor	$inout6,$inout6
836	movups	$inout7,0x70($out)
837	 pxor	$inout7,$inout7
838	lea	0x80($out),$out		# $out+=8*16
839	add	\$0x80,$len		# restore real remaining $len
840	jz	.Lecb_ret		# done if ($len==0)
841
842.Lecb_dec_tail:
843	movups	($inp),$inout0
844	cmp	\$0x20,$len
845	jb	.Lecb_dec_one
846	movups	0x10($inp),$inout1
847	je	.Lecb_dec_two
848	movups	0x20($inp),$inout2
849	cmp	\$0x40,$len
850	jb	.Lecb_dec_three
851	movups	0x30($inp),$inout3
852	je	.Lecb_dec_four
853	movups	0x40($inp),$inout4
854	cmp	\$0x60,$len
855	jb	.Lecb_dec_five
856	movups	0x50($inp),$inout5
857	je	.Lecb_dec_six
858	movups	0x60($inp),$inout6
859	$movkey	($key),$rndkey0
860	xorps	$inout7,$inout7
861	call	_aesni_decrypt8
862	movups	$inout0,($out)		# store 7 output blocks
863	 pxor	$inout0,$inout0		# clear register bank
864	movups	$inout1,0x10($out)
865	 pxor	$inout1,$inout1
866	movups	$inout2,0x20($out)
867	 pxor	$inout2,$inout2
868	movups	$inout3,0x30($out)
869	 pxor	$inout3,$inout3
870	movups	$inout4,0x40($out)
871	 pxor	$inout4,$inout4
872	movups	$inout5,0x50($out)
873	 pxor	$inout5,$inout5
874	movups	$inout6,0x60($out)
875	 pxor	$inout6,$inout6
876	 pxor	$inout7,$inout7
877	jmp	.Lecb_ret
878.align	16
879.Lecb_dec_one:
880___
881	&aesni_generate1("dec",$key,$rounds);
882$code.=<<___;
883	movups	$inout0,($out)		# store one output block
884	 pxor	$inout0,$inout0		# clear register bank
885	jmp	.Lecb_ret
886.align	16
887.Lecb_dec_two:
888	call	_aesni_decrypt2
889	movups	$inout0,($out)		# store 2 output blocks
890	 pxor	$inout0,$inout0		# clear register bank
891	movups	$inout1,0x10($out)
892	 pxor	$inout1,$inout1
893	jmp	.Lecb_ret
894.align	16
895.Lecb_dec_three:
896	call	_aesni_decrypt3
897	movups	$inout0,($out)		# store 3 output blocks
898	 pxor	$inout0,$inout0		# clear register bank
899	movups	$inout1,0x10($out)
900	 pxor	$inout1,$inout1
901	movups	$inout2,0x20($out)
902	 pxor	$inout2,$inout2
903	jmp	.Lecb_ret
904.align	16
905.Lecb_dec_four:
906	call	_aesni_decrypt4
907	movups	$inout0,($out)		# store 4 output blocks
908	 pxor	$inout0,$inout0		# clear register bank
909	movups	$inout1,0x10($out)
910	 pxor	$inout1,$inout1
911	movups	$inout2,0x20($out)
912	 pxor	$inout2,$inout2
913	movups	$inout3,0x30($out)
914	 pxor	$inout3,$inout3
915	jmp	.Lecb_ret
916.align	16
917.Lecb_dec_five:
918	xorps	$inout5,$inout5
919	call	_aesni_decrypt6
920	movups	$inout0,($out)		# store 5 output blocks
921	 pxor	$inout0,$inout0		# clear register bank
922	movups	$inout1,0x10($out)
923	 pxor	$inout1,$inout1
924	movups	$inout2,0x20($out)
925	 pxor	$inout2,$inout2
926	movups	$inout3,0x30($out)
927	 pxor	$inout3,$inout3
928	movups	$inout4,0x40($out)
929	 pxor	$inout4,$inout4
930	 pxor	$inout5,$inout5
931	jmp	.Lecb_ret
932.align	16
933.Lecb_dec_six:
934	call	_aesni_decrypt6
935	movups	$inout0,($out)		# store 6 output blocks
936	 pxor	$inout0,$inout0		# clear register bank
937	movups	$inout1,0x10($out)
938	 pxor	$inout1,$inout1
939	movups	$inout2,0x20($out)
940	 pxor	$inout2,$inout2
941	movups	$inout3,0x30($out)
942	 pxor	$inout3,$inout3
943	movups	$inout4,0x40($out)
944	 pxor	$inout4,$inout4
945	movups	$inout5,0x50($out)
946	 pxor	$inout5,$inout5
947
948.Lecb_ret:
949	xorps	$rndkey0,$rndkey0	# %xmm0
950	pxor	$rndkey1,$rndkey1
951___
952$code.=<<___ if ($win64);
953	movaps	(%rsp),%xmm6
954	movaps	%xmm0,(%rsp)		# clear stack
955	movaps	0x10(%rsp),%xmm7
956	movaps	%xmm0,0x10(%rsp)
957	movaps	0x20(%rsp),%xmm8
958	movaps	%xmm0,0x20(%rsp)
959	movaps	0x30(%rsp),%xmm9
960	movaps	%xmm0,0x30(%rsp)
961	lea	0x58(%rsp),%rsp
962.Lecb_enc_ret:
963___
964$code.=<<___;
965	ret
966.cfi_endproc
967.size	aesni_ecb_encrypt,.-aesni_ecb_encrypt
968___
969
970{
971######################################################################
972# void aesni_ccm64_[en|de]crypt_blocks (const void *in, void *out,
973#                         size_t blocks, const AES_KEY *key,
974#                         const char *ivec,char *cmac);
975#
976# Handles only complete blocks, operates on 64-bit counter and
977# does not update *ivec! Nor does it finalize CMAC value
978# (see engine/eng_aesni.c for details)
979#
980{
981my $cmac="%r9";	# 6th argument
982
983my $increment="%xmm9";
984my $iv="%xmm6";
985my $bswap_mask="%xmm7";
986
987$code.=<<___;
988.globl	aesni_ccm64_encrypt_blocks
989.type	aesni_ccm64_encrypt_blocks,\@function,6
990.align	16
991aesni_ccm64_encrypt_blocks:
992.cfi_startproc
993	endbranch
994___
995$code.=<<___ if ($win64);
996	lea	-0x58(%rsp),%rsp
997	movaps	%xmm6,(%rsp)		# $iv
998	movaps	%xmm7,0x10(%rsp)	# $bswap_mask
999	movaps	%xmm8,0x20(%rsp)	# $in0
1000	movaps	%xmm9,0x30(%rsp)	# $increment
1001.Lccm64_enc_body:
1002___
1003$code.=<<___;
1004	mov	240($key),$rounds		# key->rounds
1005	movdqu	($ivp),$iv
1006	movdqa	.Lincrement64(%rip),$increment
1007	movdqa	.Lbswap_mask(%rip),$bswap_mask
1008
1009	shl	\$4,$rounds
1010	mov	\$16,$rnds_
1011	lea	0($key),$key_
1012	movdqu	($cmac),$inout1
1013	movdqa	$iv,$inout0
1014	lea	32($key,$rounds),$key		# end of key schedule
1015	pshufb	$bswap_mask,$iv
1016	sub	%rax,%r10			# twisted $rounds
1017	jmp	.Lccm64_enc_outer
1018.align	16
1019.Lccm64_enc_outer:
1020	$movkey	($key_),$rndkey0
1021	mov	%r10,%rax
1022	movups	($inp),$in0			# load inp
1023
1024	xorps	$rndkey0,$inout0		# counter
1025	$movkey	16($key_),$rndkey1
1026	xorps	$in0,$rndkey0
1027	xorps	$rndkey0,$inout1		# cmac^=inp
1028	$movkey	32($key_),$rndkey0
1029
1030.Lccm64_enc2_loop:
1031	aesenc	$rndkey1,$inout0
1032	aesenc	$rndkey1,$inout1
1033	$movkey	($key,%rax),$rndkey1
1034	add	\$32,%rax
1035	aesenc	$rndkey0,$inout0
1036	aesenc	$rndkey0,$inout1
1037	$movkey	-16($key,%rax),$rndkey0
1038	jnz	.Lccm64_enc2_loop
1039	aesenc	$rndkey1,$inout0
1040	aesenc	$rndkey1,$inout1
1041	paddq	$increment,$iv
1042	dec	$len				# $len-- ($len is in blocks)
1043	aesenclast	$rndkey0,$inout0
1044	aesenclast	$rndkey0,$inout1
1045
1046	lea	16($inp),$inp
1047	xorps	$inout0,$in0			# inp ^= E(iv)
1048	movdqa	$iv,$inout0
1049	movups	$in0,($out)			# save output
1050	pshufb	$bswap_mask,$inout0
1051	lea	16($out),$out			# $out+=16
1052	jnz	.Lccm64_enc_outer		# loop if ($len!=0)
1053
1054	 pxor	$rndkey0,$rndkey0		# clear register bank
1055	 pxor	$rndkey1,$rndkey1
1056	 pxor	$inout0,$inout0
1057	movups	$inout1,($cmac)			# store resulting mac
1058	 pxor	$inout1,$inout1
1059	 pxor	$in0,$in0
1060	 pxor	$iv,$iv
1061___
1062$code.=<<___ if ($win64);
1063	movaps	(%rsp),%xmm6
1064	movaps	%xmm0,(%rsp)			# clear stack
1065	movaps	0x10(%rsp),%xmm7
1066	movaps	%xmm0,0x10(%rsp)
1067	movaps	0x20(%rsp),%xmm8
1068	movaps	%xmm0,0x20(%rsp)
1069	movaps	0x30(%rsp),%xmm9
1070	movaps	%xmm0,0x30(%rsp)
1071	lea	0x58(%rsp),%rsp
1072.Lccm64_enc_ret:
1073___
1074$code.=<<___;
1075	ret
1076.cfi_endproc
1077.size	aesni_ccm64_encrypt_blocks,.-aesni_ccm64_encrypt_blocks
1078___
1079######################################################################
1080$code.=<<___;
1081.globl	aesni_ccm64_decrypt_blocks
1082.type	aesni_ccm64_decrypt_blocks,\@function,6
1083.align	16
1084aesni_ccm64_decrypt_blocks:
1085.cfi_startproc
1086	endbranch
1087___
1088$code.=<<___ if ($win64);
1089	lea	-0x58(%rsp),%rsp
1090	movaps	%xmm6,(%rsp)		# $iv
1091	movaps	%xmm7,0x10(%rsp)	# $bswap_mask
1092	movaps	%xmm8,0x20(%rsp)	# $in8
1093	movaps	%xmm9,0x30(%rsp)	# $increment
1094.Lccm64_dec_body:
1095___
1096$code.=<<___;
1097	mov	240($key),$rounds		# key->rounds
1098	movups	($ivp),$iv
1099	movdqu	($cmac),$inout1
1100	movdqa	.Lincrement64(%rip),$increment
1101	movdqa	.Lbswap_mask(%rip),$bswap_mask
1102
1103	movaps	$iv,$inout0
1104	mov	$rounds,$rnds_
1105	mov	$key,$key_
1106	pshufb	$bswap_mask,$iv
1107___
1108	&aesni_generate1("enc",$key,$rounds);
1109$code.=<<___;
1110	shl	\$4,$rnds_
1111	mov	\$16,$rounds
1112	movups	($inp),$in0			# load inp
1113	paddq	$increment,$iv
1114	lea	16($inp),$inp			# $inp+=16
1115	sub	%r10,%rax			# twisted $rounds
1116	lea	32($key_,$rnds_),$key		# end of key schedule
1117	mov	%rax,%r10
1118	jmp	.Lccm64_dec_outer
1119.align	16
1120.Lccm64_dec_outer:
1121	xorps	$inout0,$in0			# inp ^= E(iv)
1122	movdqa	$iv,$inout0
1123	movups	$in0,($out)			# save output
1124	lea	16($out),$out			# $out+=16
1125	pshufb	$bswap_mask,$inout0
1126
1127	sub	\$1,$len			# $len-- ($len is in blocks)
1128	jz	.Lccm64_dec_break		# if ($len==0) break
1129
1130	$movkey	($key_),$rndkey0
1131	mov	%r10,%rax
1132	$movkey	16($key_),$rndkey1
1133	xorps	$rndkey0,$in0
1134	xorps	$rndkey0,$inout0
1135	xorps	$in0,$inout1			# cmac^=out
1136	$movkey	32($key_),$rndkey0
1137	jmp	.Lccm64_dec2_loop
1138.align	16
1139.Lccm64_dec2_loop:
1140	aesenc	$rndkey1,$inout0
1141	aesenc	$rndkey1,$inout1
1142	$movkey	($key,%rax),$rndkey1
1143	add	\$32,%rax
1144	aesenc	$rndkey0,$inout0
1145	aesenc	$rndkey0,$inout1
1146	$movkey	-16($key,%rax),$rndkey0
1147	jnz	.Lccm64_dec2_loop
1148	movups	($inp),$in0			# load input
1149	paddq	$increment,$iv
1150	aesenc	$rndkey1,$inout0
1151	aesenc	$rndkey1,$inout1
1152	aesenclast	$rndkey0,$inout0
1153	aesenclast	$rndkey0,$inout1
1154	lea	16($inp),$inp			# $inp+=16
1155	jmp	.Lccm64_dec_outer
1156
1157.align	16
1158.Lccm64_dec_break:
1159	#xorps	$in0,$inout1			# cmac^=out
1160	mov	240($key_),$rounds
1161___
1162	&aesni_generate1("enc",$key_,$rounds,$inout1,$in0);
1163$code.=<<___;
1164	 pxor	$rndkey0,$rndkey0		# clear register bank
1165	 pxor	$rndkey1,$rndkey1
1166	 pxor	$inout0,$inout0
1167	movups	$inout1,($cmac)			# store resulting mac
1168	 pxor	$inout1,$inout1
1169	 pxor	$in0,$in0
1170	 pxor	$iv,$iv
1171___
1172$code.=<<___ if ($win64);
1173	movaps	(%rsp),%xmm6
1174	movaps	%xmm0,(%rsp)			# clear stack
1175	movaps	0x10(%rsp),%xmm7
1176	movaps	%xmm0,0x10(%rsp)
1177	movaps	0x20(%rsp),%xmm8
1178	movaps	%xmm0,0x20(%rsp)
1179	movaps	0x30(%rsp),%xmm9
1180	movaps	%xmm0,0x30(%rsp)
1181	lea	0x58(%rsp),%rsp
1182.Lccm64_dec_ret:
1183___
1184$code.=<<___;
1185	ret
1186.cfi_endproc
1187.size	aesni_ccm64_decrypt_blocks,.-aesni_ccm64_decrypt_blocks
1188___
1189}
1190######################################################################
1191# void aesni_ctr32_encrypt_blocks (const void *in, void *out,
1192#                         size_t blocks, const AES_KEY *key,
1193#                         const char *ivec);
1194#
1195# Handles only complete blocks, operates on 32-bit counter and
1196# does not update *ivec! (see crypto/modes/ctr128.c for details)
1197#
1198# Overhaul based on suggestions from Shay Gueron and Vlad Krasnov,
1199# http://rt.openssl.org/Ticket/Display.html?id=3021&user=guest&pass=guest.
1200# Keywords are full unroll and modulo-schedule counter calculations
1201# with zero-round key xor.
1202{
1203my ($in0,$in1,$in2,$in3,$in4,$in5)=map("%xmm$_",(10..15));
1204my ($key0,$ctr)=("%ebp","${ivp}d");
1205my $frame_size = 0x80 + ($win64?160:0);
1206
1207$code.=<<___;
1208.globl	aesni_ctr32_encrypt_blocks
1209.type	aesni_ctr32_encrypt_blocks,\@function,5
1210.align	16
1211aesni_ctr32_encrypt_blocks:
1212.cfi_startproc
1213	endbranch
1214	cmp	\$1,$len
1215	jne	.Lctr32_bulk
1216
1217	# handle single block without allocating stack frame,
1218	# useful when handling edges
1219	movups	($ivp),$inout0
1220	movups	($inp),$inout1
1221	mov	240($key),%edx			# key->rounds
1222___
1223	&aesni_generate1("enc",$key,"%edx");
1224$code.=<<___;
1225	 pxor	$rndkey0,$rndkey0		# clear register bank
1226	 pxor	$rndkey1,$rndkey1
1227	xorps	$inout1,$inout0
1228	 pxor	$inout1,$inout1
1229	movups	$inout0,($out)
1230	 xorps	$inout0,$inout0
1231	jmp	.Lctr32_epilogue
1232
1233.align	16
1234.Lctr32_bulk:
1235	lea	(%rsp),$key_			# use $key_ as frame pointer
1236.cfi_def_cfa_register	$key_
1237	push	%rbp
1238.cfi_push	%rbp
1239	sub	\$$frame_size,%rsp
1240	and	\$-16,%rsp	# Linux kernel stack can be incorrectly seeded
1241___
1242$code.=<<___ if ($win64);
1243	movaps	%xmm6,-0xa8($key_)		# offload everything
1244	movaps	%xmm7,-0x98($key_)
1245	movaps	%xmm8,-0x88($key_)
1246	movaps	%xmm9,-0x78($key_)
1247	movaps	%xmm10,-0x68($key_)
1248	movaps	%xmm11,-0x58($key_)
1249	movaps	%xmm12,-0x48($key_)
1250	movaps	%xmm13,-0x38($key_)
1251	movaps	%xmm14,-0x28($key_)
1252	movaps	%xmm15,-0x18($key_)
1253.Lctr32_body:
1254___
1255$code.=<<___;
1256
1257	# 8 16-byte words on top of stack are counter values
1258	# xor-ed with zero-round key
1259
1260	movdqu	($ivp),$inout0
1261	movdqu	($key),$rndkey0
1262	mov	12($ivp),$ctr			# counter LSB
1263	pxor	$rndkey0,$inout0
1264	mov	12($key),$key0			# 0-round key LSB
1265	movdqa	$inout0,0x00(%rsp)		# populate counter block
1266	bswap	$ctr
1267	movdqa	$inout0,$inout1
1268	movdqa	$inout0,$inout2
1269	movdqa	$inout0,$inout3
1270	movdqa	$inout0,0x40(%rsp)
1271	movdqa	$inout0,0x50(%rsp)
1272	movdqa	$inout0,0x60(%rsp)
1273	mov	%rdx,%r10			# about to borrow %rdx
1274	movdqa	$inout0,0x70(%rsp)
1275
1276	lea	1($ctr),%rax
1277	 lea	2($ctr),%rdx
1278	bswap	%eax
1279	 bswap	%edx
1280	xor	$key0,%eax
1281	 xor	$key0,%edx
1282	pinsrd	\$3,%eax,$inout1
1283	lea	3($ctr),%rax
1284	movdqa	$inout1,0x10(%rsp)
1285	 pinsrd	\$3,%edx,$inout2
1286	bswap	%eax
1287	 mov	%r10,%rdx			# restore %rdx
1288	 lea	4($ctr),%r10
1289	 movdqa	$inout2,0x20(%rsp)
1290	xor	$key0,%eax
1291	 bswap	%r10d
1292	pinsrd	\$3,%eax,$inout3
1293	 xor	$key0,%r10d
1294	movdqa	$inout3,0x30(%rsp)
1295	lea	5($ctr),%r9
1296	 mov	%r10d,0x40+12(%rsp)
1297	bswap	%r9d
1298	 lea	6($ctr),%r10
1299	mov	240($key),$rounds		# key->rounds
1300	xor	$key0,%r9d
1301	 bswap	%r10d
1302	mov	%r9d,0x50+12(%rsp)
1303	 xor	$key0,%r10d
1304	lea	7($ctr),%r9
1305	 mov	%r10d,0x60+12(%rsp)
1306	bswap	%r9d
1307	 mov	OPENSSL_ia32cap_P+4(%rip),%r10d
1308	xor	$key0,%r9d
1309	 and	\$`1<<26|1<<22`,%r10d		# isolate XSAVE+MOVBE
1310	mov	%r9d,0x70+12(%rsp)
1311
1312	$movkey	0x10($key),$rndkey1
1313
1314	movdqa	0x40(%rsp),$inout4
1315	movdqa	0x50(%rsp),$inout5
1316
1317	cmp	\$8,$len		# $len is in blocks
1318	jb	.Lctr32_tail		# short input if ($len<8)
1319
1320	sub	\$6,$len		# $len is biased by -6
1321	cmp	\$`1<<22`,%r10d		# check for MOVBE without XSAVE
1322	je	.Lctr32_6x		# [which denotes Atom Silvermont]
1323
1324	lea	0x80($key),$key		# size optimization
1325	sub	\$2,$len		# $len is biased by -8
1326	jmp	.Lctr32_loop8
1327
1328.align	16
1329.Lctr32_6x:
1330	shl	\$4,$rounds
1331	mov	\$48,$rnds_
1332	bswap	$key0
1333	lea	32($key,$rounds),$key	# end of key schedule
1334	sub	%rax,%r10		# twisted $rounds
1335	jmp	.Lctr32_loop6
1336
1337.align	16
1338.Lctr32_loop6:
1339	 add	\$6,$ctr		# next counter value
1340	$movkey	-48($key,$rnds_),$rndkey0
1341	aesenc	$rndkey1,$inout0
1342	 mov	$ctr,%eax
1343	 xor	$key0,%eax
1344	aesenc	$rndkey1,$inout1
1345	 movbe	%eax,`0x00+12`(%rsp)	# store next counter value
1346	 lea	1($ctr),%eax
1347	aesenc	$rndkey1,$inout2
1348	 xor	$key0,%eax
1349	 movbe	%eax,`0x10+12`(%rsp)
1350	aesenc	$rndkey1,$inout3
1351	 lea	2($ctr),%eax
1352	 xor	$key0,%eax
1353	aesenc	$rndkey1,$inout4
1354	 movbe	%eax,`0x20+12`(%rsp)
1355	 lea	3($ctr),%eax
1356	aesenc	$rndkey1,$inout5
1357	$movkey	-32($key,$rnds_),$rndkey1
1358	 xor	$key0,%eax
1359
1360	aesenc	$rndkey0,$inout0
1361	 movbe	%eax,`0x30+12`(%rsp)
1362	 lea	4($ctr),%eax
1363	aesenc	$rndkey0,$inout1
1364	 xor	$key0,%eax
1365	 movbe	%eax,`0x40+12`(%rsp)
1366	aesenc	$rndkey0,$inout2
1367	 lea	5($ctr),%eax
1368	 xor	$key0,%eax
1369	aesenc	$rndkey0,$inout3
1370	 movbe	%eax,`0x50+12`(%rsp)
1371	 mov	%r10,%rax		# mov	$rnds_,$rounds
1372	aesenc	$rndkey0,$inout4
1373	aesenc	$rndkey0,$inout5
1374	$movkey	-16($key,$rnds_),$rndkey0
1375
1376	call	.Lenc_loop6
1377
1378	movdqu	($inp),$inout6		# load 6 input blocks
1379	movdqu	0x10($inp),$inout7
1380	movdqu	0x20($inp),$in0
1381	movdqu	0x30($inp),$in1
1382	movdqu	0x40($inp),$in2
1383	movdqu	0x50($inp),$in3
1384	lea	0x60($inp),$inp		# $inp+=6*16
1385	$movkey	-64($key,$rnds_),$rndkey1
1386	pxor	$inout0,$inout6		# inp^=E(ctr)
1387	movaps	0x00(%rsp),$inout0	# load next counter [xor-ed with 0 round]
1388	pxor	$inout1,$inout7
1389	movaps	0x10(%rsp),$inout1
1390	pxor	$inout2,$in0
1391	movaps	0x20(%rsp),$inout2
1392	pxor	$inout3,$in1
1393	movaps	0x30(%rsp),$inout3
1394	pxor	$inout4,$in2
1395	movaps	0x40(%rsp),$inout4
1396	pxor	$inout5,$in3
1397	movaps	0x50(%rsp),$inout5
1398	movdqu	$inout6,($out)		# store 6 output blocks
1399	movdqu	$inout7,0x10($out)
1400	movdqu	$in0,0x20($out)
1401	movdqu	$in1,0x30($out)
1402	movdqu	$in2,0x40($out)
1403	movdqu	$in3,0x50($out)
1404	lea	0x60($out),$out		# $out+=6*16
1405
1406	sub	\$6,$len
1407	jnc	.Lctr32_loop6		# loop if $len-=6 didn't borrow
1408
1409	add	\$6,$len		# restore real remaining $len
1410	jz	.Lctr32_done		# done if ($len==0)
1411
1412	lea	-48($rnds_),$rounds
1413	lea	-80($key,$rnds_),$key	# restore $key
1414	neg	$rounds
1415	shr	\$4,$rounds		# restore $rounds
1416	jmp	.Lctr32_tail
1417
1418.align	32
1419.Lctr32_loop8:
1420	 add		\$8,$ctr		# next counter value
1421	movdqa		0x60(%rsp),$inout6
1422	aesenc		$rndkey1,$inout0
1423	 mov		$ctr,%r9d
1424	movdqa		0x70(%rsp),$inout7
1425	aesenc		$rndkey1,$inout1
1426	 bswap		%r9d
1427	$movkey		0x20-0x80($key),$rndkey0
1428	aesenc		$rndkey1,$inout2
1429	 xor		$key0,%r9d
1430	 nop
1431	aesenc		$rndkey1,$inout3
1432	 mov		%r9d,0x00+12(%rsp)	# store next counter value
1433	 lea		1($ctr),%r9
1434	aesenc		$rndkey1,$inout4
1435	aesenc		$rndkey1,$inout5
1436	aesenc		$rndkey1,$inout6
1437	aesenc		$rndkey1,$inout7
1438	$movkey		0x30-0x80($key),$rndkey1
1439___
1440for($i=2;$i<8;$i++) {
1441my $rndkeyx = ($i&1)?$rndkey1:$rndkey0;
1442$code.=<<___;
1443	 bswap		%r9d
1444	aesenc		$rndkeyx,$inout0
1445	aesenc		$rndkeyx,$inout1
1446	 xor		$key0,%r9d
1447	 .byte		0x66,0x90
1448	aesenc		$rndkeyx,$inout2
1449	aesenc		$rndkeyx,$inout3
1450	 mov		%r9d,`0x10*($i-1)`+12(%rsp)
1451	 lea		$i($ctr),%r9
1452	aesenc		$rndkeyx,$inout4
1453	aesenc		$rndkeyx,$inout5
1454	aesenc		$rndkeyx,$inout6
1455	aesenc		$rndkeyx,$inout7
1456	$movkey		`0x20+0x10*$i`-0x80($key),$rndkeyx
1457___
1458}
1459$code.=<<___;
1460	 bswap		%r9d
1461	aesenc		$rndkey0,$inout0
1462	aesenc		$rndkey0,$inout1
1463	aesenc		$rndkey0,$inout2
1464	 xor		$key0,%r9d
1465	 movdqu		0x00($inp),$in0		# start loading input
1466	aesenc		$rndkey0,$inout3
1467	 mov		%r9d,0x70+12(%rsp)
1468	 cmp		\$11,$rounds
1469	aesenc		$rndkey0,$inout4
1470	aesenc		$rndkey0,$inout5
1471	aesenc		$rndkey0,$inout6
1472	aesenc		$rndkey0,$inout7
1473	$movkey		0xa0-0x80($key),$rndkey0
1474
1475	jb		.Lctr32_enc_done
1476
1477	aesenc		$rndkey1,$inout0
1478	aesenc		$rndkey1,$inout1
1479	aesenc		$rndkey1,$inout2
1480	aesenc		$rndkey1,$inout3
1481	aesenc		$rndkey1,$inout4
1482	aesenc		$rndkey1,$inout5
1483	aesenc		$rndkey1,$inout6
1484	aesenc		$rndkey1,$inout7
1485	$movkey		0xb0-0x80($key),$rndkey1
1486
1487	aesenc		$rndkey0,$inout0
1488	aesenc		$rndkey0,$inout1
1489	aesenc		$rndkey0,$inout2
1490	aesenc		$rndkey0,$inout3
1491	aesenc		$rndkey0,$inout4
1492	aesenc		$rndkey0,$inout5
1493	aesenc		$rndkey0,$inout6
1494	aesenc		$rndkey0,$inout7
1495	$movkey		0xc0-0x80($key),$rndkey0
1496	je		.Lctr32_enc_done
1497
1498	aesenc		$rndkey1,$inout0
1499	aesenc		$rndkey1,$inout1
1500	aesenc		$rndkey1,$inout2
1501	aesenc		$rndkey1,$inout3
1502	aesenc		$rndkey1,$inout4
1503	aesenc		$rndkey1,$inout5
1504	aesenc		$rndkey1,$inout6
1505	aesenc		$rndkey1,$inout7
1506	$movkey		0xd0-0x80($key),$rndkey1
1507
1508	aesenc		$rndkey0,$inout0
1509	aesenc		$rndkey0,$inout1
1510	aesenc		$rndkey0,$inout2
1511	aesenc		$rndkey0,$inout3
1512	aesenc		$rndkey0,$inout4
1513	aesenc		$rndkey0,$inout5
1514	aesenc		$rndkey0,$inout6
1515	aesenc		$rndkey0,$inout7
1516	$movkey		0xe0-0x80($key),$rndkey0
1517	jmp		.Lctr32_enc_done
1518
1519.align	16
1520.Lctr32_enc_done:
1521	movdqu		0x10($inp),$in1
1522	pxor		$rndkey0,$in0		# input^=round[last]
1523	movdqu		0x20($inp),$in2
1524	pxor		$rndkey0,$in1
1525	movdqu		0x30($inp),$in3
1526	pxor		$rndkey0,$in2
1527	movdqu		0x40($inp),$in4
1528	pxor		$rndkey0,$in3
1529	movdqu		0x50($inp),$in5
1530	pxor		$rndkey0,$in4
1531	pxor		$rndkey0,$in5
1532	aesenc		$rndkey1,$inout0
1533	aesenc		$rndkey1,$inout1
1534	aesenc		$rndkey1,$inout2
1535	aesenc		$rndkey1,$inout3
1536	aesenc		$rndkey1,$inout4
1537	aesenc		$rndkey1,$inout5
1538	aesenc		$rndkey1,$inout6
1539	aesenc		$rndkey1,$inout7
1540	movdqu		0x60($inp),$rndkey1	# borrow $rndkey1 for inp[6]
1541	lea		0x80($inp),$inp		# $inp+=8*16
1542
1543	aesenclast	$in0,$inout0		# $inN is inp[N]^round[last]
1544	pxor		$rndkey0,$rndkey1	# borrowed $rndkey
1545	movdqu		0x70-0x80($inp),$in0
1546	aesenclast	$in1,$inout1
1547	pxor		$rndkey0,$in0
1548	movdqa		0x00(%rsp),$in1		# load next counter block
1549	aesenclast	$in2,$inout2
1550	aesenclast	$in3,$inout3
1551	movdqa		0x10(%rsp),$in2
1552	movdqa		0x20(%rsp),$in3
1553	aesenclast	$in4,$inout4
1554	aesenclast	$in5,$inout5
1555	movdqa		0x30(%rsp),$in4
1556	movdqa		0x40(%rsp),$in5
1557	aesenclast	$rndkey1,$inout6
1558	movdqa		0x50(%rsp),$rndkey0
1559	$movkey		0x10-0x80($key),$rndkey1#real 1st-round key
1560	aesenclast	$in0,$inout7
1561
1562	movups		$inout0,($out)		# store 8 output blocks
1563	movdqa		$in1,$inout0
1564	movups		$inout1,0x10($out)
1565	movdqa		$in2,$inout1
1566	movups		$inout2,0x20($out)
1567	movdqa		$in3,$inout2
1568	movups		$inout3,0x30($out)
1569	movdqa		$in4,$inout3
1570	movups		$inout4,0x40($out)
1571	movdqa		$in5,$inout4
1572	movups		$inout5,0x50($out)
1573	movdqa		$rndkey0,$inout5
1574	movups		$inout6,0x60($out)
1575	movups		$inout7,0x70($out)
1576	lea		0x80($out),$out		# $out+=8*16
1577
1578	sub	\$8,$len
1579	jnc	.Lctr32_loop8			# loop if $len-=8 didn't borrow
1580
1581	add	\$8,$len			# restore real remaining $len
1582	jz	.Lctr32_done			# done if ($len==0)
1583	lea	-0x80($key),$key
1584
1585.Lctr32_tail:
1586	# note that at this point $inout0..5 are populated with
1587	# counter values xor-ed with 0-round key
1588	lea	16($key),$key
1589	cmp	\$4,$len
1590	jb	.Lctr32_loop3
1591	je	.Lctr32_loop4
1592
1593	# if ($len>4) compute 7 E(counter)
1594	shl		\$4,$rounds
1595	movdqa		0x60(%rsp),$inout6
1596	pxor		$inout7,$inout7
1597
1598	$movkey		16($key),$rndkey0
1599	aesenc		$rndkey1,$inout0
1600	aesenc		$rndkey1,$inout1
1601	lea		32-16($key,$rounds),$key# prepare for .Lenc_loop8_enter
1602	neg		%rax
1603	aesenc		$rndkey1,$inout2
1604	add		\$16,%rax		# prepare for .Lenc_loop8_enter
1605	 movups		($inp),$in0
1606	aesenc		$rndkey1,$inout3
1607	aesenc		$rndkey1,$inout4
1608	 movups		0x10($inp),$in1		# pre-load input
1609	 movups		0x20($inp),$in2
1610	aesenc		$rndkey1,$inout5
1611	aesenc		$rndkey1,$inout6
1612
1613	call            .Lenc_loop8_enter
1614
1615	movdqu	0x30($inp),$in3
1616	pxor	$in0,$inout0
1617	movdqu	0x40($inp),$in0
1618	pxor	$in1,$inout1
1619	movdqu	$inout0,($out)			# store output
1620	pxor	$in2,$inout2
1621	movdqu	$inout1,0x10($out)
1622	pxor	$in3,$inout3
1623	movdqu	$inout2,0x20($out)
1624	pxor	$in0,$inout4
1625	movdqu	$inout3,0x30($out)
1626	movdqu	$inout4,0x40($out)
1627	cmp	\$6,$len
1628	jb	.Lctr32_done			# $len was 5, stop store
1629
1630	movups	0x50($inp),$in1
1631	xorps	$in1,$inout5
1632	movups	$inout5,0x50($out)
1633	je	.Lctr32_done			# $len was 6, stop store
1634
1635	movups	0x60($inp),$in2
1636	xorps	$in2,$inout6
1637	movups	$inout6,0x60($out)
1638	jmp	.Lctr32_done			# $len was 7, stop store
1639
1640.align	32
1641.Lctr32_loop4:
1642	aesenc		$rndkey1,$inout0
1643	lea		16($key),$key
1644	dec		$rounds
1645	aesenc		$rndkey1,$inout1
1646	aesenc		$rndkey1,$inout2
1647	aesenc		$rndkey1,$inout3
1648	$movkey		($key),$rndkey1
1649	jnz		.Lctr32_loop4
1650	aesenclast	$rndkey1,$inout0
1651	aesenclast	$rndkey1,$inout1
1652	 movups		($inp),$in0		# load input
1653	 movups		0x10($inp),$in1
1654	aesenclast	$rndkey1,$inout2
1655	aesenclast	$rndkey1,$inout3
1656	 movups		0x20($inp),$in2
1657	 movups		0x30($inp),$in3
1658
1659	xorps	$in0,$inout0
1660	movups	$inout0,($out)			# store output
1661	xorps	$in1,$inout1
1662	movups	$inout1,0x10($out)
1663	pxor	$in2,$inout2
1664	movdqu	$inout2,0x20($out)
1665	pxor	$in3,$inout3
1666	movdqu	$inout3,0x30($out)
1667	jmp	.Lctr32_done			# $len was 4, stop store
1668
1669.align	32
1670.Lctr32_loop3:
1671	aesenc		$rndkey1,$inout0
1672	lea		16($key),$key
1673	dec		$rounds
1674	aesenc		$rndkey1,$inout1
1675	aesenc		$rndkey1,$inout2
1676	$movkey		($key),$rndkey1
1677	jnz		.Lctr32_loop3
1678	aesenclast	$rndkey1,$inout0
1679	aesenclast	$rndkey1,$inout1
1680	aesenclast	$rndkey1,$inout2
1681
1682	movups	($inp),$in0			# load input
1683	xorps	$in0,$inout0
1684	movups	$inout0,($out)			# store output
1685	cmp	\$2,$len
1686	jb	.Lctr32_done			# $len was 1, stop store
1687
1688	movups	0x10($inp),$in1
1689	xorps	$in1,$inout1
1690	movups	$inout1,0x10($out)
1691	je	.Lctr32_done			# $len was 2, stop store
1692
1693	movups	0x20($inp),$in2
1694	xorps	$in2,$inout2
1695	movups	$inout2,0x20($out)		# $len was 3, stop store
1696
1697.Lctr32_done:
1698	xorps	%xmm0,%xmm0			# clear register bank
1699	xor	$key0,$key0
1700	pxor	%xmm1,%xmm1
1701	pxor	%xmm2,%xmm2
1702	pxor	%xmm3,%xmm3
1703	pxor	%xmm4,%xmm4
1704	pxor	%xmm5,%xmm5
1705___
1706$code.=<<___ if (!$win64);
1707	pxor	%xmm6,%xmm6
1708	pxor	%xmm7,%xmm7
1709	movaps	%xmm0,0x00(%rsp)		# clear stack
1710	pxor	%xmm8,%xmm8
1711	movaps	%xmm0,0x10(%rsp)
1712	pxor	%xmm9,%xmm9
1713	movaps	%xmm0,0x20(%rsp)
1714	pxor	%xmm10,%xmm10
1715	movaps	%xmm0,0x30(%rsp)
1716	pxor	%xmm11,%xmm11
1717	movaps	%xmm0,0x40(%rsp)
1718	pxor	%xmm12,%xmm12
1719	movaps	%xmm0,0x50(%rsp)
1720	pxor	%xmm13,%xmm13
1721	movaps	%xmm0,0x60(%rsp)
1722	pxor	%xmm14,%xmm14
1723	movaps	%xmm0,0x70(%rsp)
1724	pxor	%xmm15,%xmm15
1725___
1726$code.=<<___ if ($win64);
1727	movaps	-0xa8($key_),%xmm6
1728	movaps	%xmm0,-0xa8($key_)		# clear stack
1729	movaps	-0x98($key_),%xmm7
1730	movaps	%xmm0,-0x98($key_)
1731	movaps	-0x88($key_),%xmm8
1732	movaps	%xmm0,-0x88($key_)
1733	movaps	-0x78($key_),%xmm9
1734	movaps	%xmm0,-0x78($key_)
1735	movaps	-0x68($key_),%xmm10
1736	movaps	%xmm0,-0x68($key_)
1737	movaps	-0x58($key_),%xmm11
1738	movaps	%xmm0,-0x58($key_)
1739	movaps	-0x48($key_),%xmm12
1740	movaps	%xmm0,-0x48($key_)
1741	movaps	-0x38($key_),%xmm13
1742	movaps	%xmm0,-0x38($key_)
1743	movaps	-0x28($key_),%xmm14
1744	movaps	%xmm0,-0x28($key_)
1745	movaps	-0x18($key_),%xmm15
1746	movaps	%xmm0,-0x18($key_)
1747	movaps	%xmm0,0x00(%rsp)
1748	movaps	%xmm0,0x10(%rsp)
1749	movaps	%xmm0,0x20(%rsp)
1750	movaps	%xmm0,0x30(%rsp)
1751	movaps	%xmm0,0x40(%rsp)
1752	movaps	%xmm0,0x50(%rsp)
1753	movaps	%xmm0,0x60(%rsp)
1754	movaps	%xmm0,0x70(%rsp)
1755___
1756$code.=<<___;
1757	mov	-8($key_),%rbp
1758.cfi_restore	%rbp
1759	lea	($key_),%rsp
1760.cfi_def_cfa_register	%rsp
1761.Lctr32_epilogue:
1762	ret
1763.cfi_endproc
1764.size	aesni_ctr32_encrypt_blocks,.-aesni_ctr32_encrypt_blocks
1765___
1766}
1767
1768######################################################################
1769# void aesni_xts_[en|de]crypt(const char *inp,char *out,size_t len,
1770#	const AES_KEY *key1, const AES_KEY *key2
1771#	const unsigned char iv[16]);
1772#
1773{
1774my @tweak=map("%xmm$_",(10..15));
1775my ($twmask,$twres,$twtmp)=("%xmm8","%xmm9",@tweak[4]);
1776my ($key2,$ivp,$len_)=("%r8","%r9","%r9");
1777my $frame_size = 0x70 + ($win64?160:0);
1778my $key_ = "%rbp";	# override so that we can use %r11 as FP
1779
1780$code.=<<___;
1781.globl	aesni_xts_encrypt
1782.type	aesni_xts_encrypt,\@function,6
1783.align	16
1784aesni_xts_encrypt:
1785.cfi_startproc
1786	endbranch
1787	lea	(%rsp),%r11			# frame pointer
1788.cfi_def_cfa_register	%r11
1789	push	%rbp
1790.cfi_push	%rbp
1791	sub	\$$frame_size,%rsp
1792	and	\$-16,%rsp	# Linux kernel stack can be incorrectly seeded
1793___
1794$code.=<<___ if ($win64);
1795	movaps	%xmm6,-0xa8(%r11)		# offload everything
1796	movaps	%xmm7,-0x98(%r11)
1797	movaps	%xmm8,-0x88(%r11)
1798	movaps	%xmm9,-0x78(%r11)
1799	movaps	%xmm10,-0x68(%r11)
1800	movaps	%xmm11,-0x58(%r11)
1801	movaps	%xmm12,-0x48(%r11)
1802	movaps	%xmm13,-0x38(%r11)
1803	movaps	%xmm14,-0x28(%r11)
1804	movaps	%xmm15,-0x18(%r11)
1805.Lxts_enc_body:
1806___
1807$code.=<<___;
1808	movups	($ivp),$inout0			# load clear-text tweak
1809	mov	240(%r8),$rounds		# key2->rounds
1810	mov	240($key),$rnds_		# key1->rounds
1811___
1812	# generate the tweak
1813	&aesni_generate1("enc",$key2,$rounds,$inout0);
1814$code.=<<___;
1815	$movkey	($key),$rndkey0			# zero round key
1816	mov	$key,$key_			# backup $key
1817	mov	$rnds_,$rounds			# backup $rounds
1818	shl	\$4,$rnds_
1819	mov	$len,$len_			# backup $len
1820	and	\$-16,$len
1821
1822	$movkey	16($key,$rnds_),$rndkey1	# last round key
1823
1824	movdqa	.Lxts_magic(%rip),$twmask
1825	movdqa	$inout0,@tweak[5]
1826	pshufd	\$0x5f,$inout0,$twres
1827	pxor	$rndkey0,$rndkey1
1828___
1829    # alternative tweak calculation algorithm is based on suggestions
1830    # by Shay Gueron. psrad doesn't conflict with AES-NI instructions
1831    # and should help in the future...
1832    for ($i=0;$i<4;$i++) {
1833    $code.=<<___;
1834	movdqa	$twres,$twtmp
1835	paddd	$twres,$twres
1836	movdqa	@tweak[5],@tweak[$i]
1837	psrad	\$31,$twtmp			# broadcast upper bits
1838	paddq	@tweak[5],@tweak[5]
1839	pand	$twmask,$twtmp
1840	pxor	$rndkey0,@tweak[$i]
1841	pxor	$twtmp,@tweak[5]
1842___
1843    }
1844$code.=<<___;
1845	movdqa	@tweak[5],@tweak[4]
1846	psrad	\$31,$twres
1847	paddq	@tweak[5],@tweak[5]
1848	pand	$twmask,$twres
1849	pxor	$rndkey0,@tweak[4]
1850	pxor	$twres,@tweak[5]
1851	movaps	$rndkey1,0x60(%rsp)		# save round[0]^round[last]
1852
1853	sub	\$16*6,$len
1854	jc	.Lxts_enc_short			# if $len-=6*16 borrowed
1855
1856	mov	\$16+96,$rounds
1857	lea	32($key_,$rnds_),$key		# end of key schedule
1858	sub	%r10,%rax			# twisted $rounds
1859	$movkey	16($key_),$rndkey1
1860	mov	%rax,%r10			# backup twisted $rounds
1861	lea	.Lxts_magic(%rip),%r8
1862	jmp	.Lxts_enc_grandloop
1863
1864.align	32
1865.Lxts_enc_grandloop:
1866	movdqu	`16*0`($inp),$inout0		# load input
1867	movdqa	$rndkey0,$twmask
1868	movdqu	`16*1`($inp),$inout1
1869	pxor	@tweak[0],$inout0		# input^=tweak^round[0]
1870	movdqu	`16*2`($inp),$inout2
1871	pxor	@tweak[1],$inout1
1872	 aesenc		$rndkey1,$inout0
1873	movdqu	`16*3`($inp),$inout3
1874	pxor	@tweak[2],$inout2
1875	 aesenc		$rndkey1,$inout1
1876	movdqu	`16*4`($inp),$inout4
1877	pxor	@tweak[3],$inout3
1878	 aesenc		$rndkey1,$inout2
1879	movdqu	`16*5`($inp),$inout5
1880	pxor	@tweak[5],$twmask		# round[0]^=tweak[5]
1881	 movdqa	0x60(%rsp),$twres		# load round[0]^round[last]
1882	pxor	@tweak[4],$inout4
1883	 aesenc		$rndkey1,$inout3
1884	$movkey	32($key_),$rndkey0
1885	lea	`16*6`($inp),$inp
1886	pxor	$twmask,$inout5
1887
1888	 pxor	$twres,@tweak[0]		# calculate tweaks^round[last]
1889	aesenc		$rndkey1,$inout4
1890	 pxor	$twres,@tweak[1]
1891	 movdqa	@tweak[0],`16*0`(%rsp)		# put aside tweaks^round[last]
1892	aesenc		$rndkey1,$inout5
1893	$movkey		48($key_),$rndkey1
1894	 pxor	$twres,@tweak[2]
1895
1896	aesenc		$rndkey0,$inout0
1897	 pxor	$twres,@tweak[3]
1898	 movdqa	@tweak[1],`16*1`(%rsp)
1899	aesenc		$rndkey0,$inout1
1900	 pxor	$twres,@tweak[4]
1901	 movdqa	@tweak[2],`16*2`(%rsp)
1902	aesenc		$rndkey0,$inout2
1903	aesenc		$rndkey0,$inout3
1904	 pxor	$twres,$twmask
1905	 movdqa	@tweak[4],`16*4`(%rsp)
1906	aesenc		$rndkey0,$inout4
1907	aesenc		$rndkey0,$inout5
1908	$movkey		64($key_),$rndkey0
1909	 movdqa	$twmask,`16*5`(%rsp)
1910	pshufd	\$0x5f,@tweak[5],$twres
1911	jmp	.Lxts_enc_loop6
1912.align	32
1913.Lxts_enc_loop6:
1914	aesenc		$rndkey1,$inout0
1915	aesenc		$rndkey1,$inout1
1916	aesenc		$rndkey1,$inout2
1917	aesenc		$rndkey1,$inout3
1918	aesenc		$rndkey1,$inout4
1919	aesenc		$rndkey1,$inout5
1920	$movkey		-64($key,%rax),$rndkey1
1921	add		\$32,%rax
1922
1923	aesenc		$rndkey0,$inout0
1924	aesenc		$rndkey0,$inout1
1925	aesenc		$rndkey0,$inout2
1926	aesenc		$rndkey0,$inout3
1927	aesenc		$rndkey0,$inout4
1928	aesenc		$rndkey0,$inout5
1929	$movkey		-80($key,%rax),$rndkey0
1930	jnz		.Lxts_enc_loop6
1931
1932	movdqa	(%r8),$twmask			# start calculating next tweak
1933	movdqa	$twres,$twtmp
1934	paddd	$twres,$twres
1935	 aesenc		$rndkey1,$inout0
1936	paddq	@tweak[5],@tweak[5]
1937	psrad	\$31,$twtmp
1938	 aesenc		$rndkey1,$inout1
1939	pand	$twmask,$twtmp
1940	$movkey	($key_),@tweak[0]		# load round[0]
1941	 aesenc		$rndkey1,$inout2
1942	 aesenc		$rndkey1,$inout3
1943	 aesenc		$rndkey1,$inout4
1944	pxor	$twtmp,@tweak[5]
1945	movaps	@tweak[0],@tweak[1]		# copy round[0]
1946	 aesenc		$rndkey1,$inout5
1947	 $movkey	-64($key),$rndkey1
1948
1949	movdqa	$twres,$twtmp
1950	 aesenc		$rndkey0,$inout0
1951	paddd	$twres,$twres
1952	pxor	@tweak[5],@tweak[0]
1953	 aesenc		$rndkey0,$inout1
1954	psrad	\$31,$twtmp
1955	paddq	@tweak[5],@tweak[5]
1956	 aesenc		$rndkey0,$inout2
1957	 aesenc		$rndkey0,$inout3
1958	pand	$twmask,$twtmp
1959	movaps	@tweak[1],@tweak[2]
1960	 aesenc		$rndkey0,$inout4
1961	pxor	$twtmp,@tweak[5]
1962	movdqa	$twres,$twtmp
1963	 aesenc		$rndkey0,$inout5
1964	 $movkey	-48($key),$rndkey0
1965
1966	paddd	$twres,$twres
1967	 aesenc		$rndkey1,$inout0
1968	pxor	@tweak[5],@tweak[1]
1969	psrad	\$31,$twtmp
1970	 aesenc		$rndkey1,$inout1
1971	paddq	@tweak[5],@tweak[5]
1972	pand	$twmask,$twtmp
1973	 aesenc		$rndkey1,$inout2
1974	 aesenc		$rndkey1,$inout3
1975	 movdqa	@tweak[3],`16*3`(%rsp)
1976	pxor	$twtmp,@tweak[5]
1977	 aesenc		$rndkey1,$inout4
1978	movaps	@tweak[2],@tweak[3]
1979	movdqa	$twres,$twtmp
1980	 aesenc		$rndkey1,$inout5
1981	 $movkey	-32($key),$rndkey1
1982
1983	paddd	$twres,$twres
1984	 aesenc		$rndkey0,$inout0
1985	pxor	@tweak[5],@tweak[2]
1986	psrad	\$31,$twtmp
1987	 aesenc		$rndkey0,$inout1
1988	paddq	@tweak[5],@tweak[5]
1989	pand	$twmask,$twtmp
1990	 aesenc		$rndkey0,$inout2
1991	 aesenc		$rndkey0,$inout3
1992	 aesenc		$rndkey0,$inout4
1993	pxor	$twtmp,@tweak[5]
1994	movaps	@tweak[3],@tweak[4]
1995	 aesenc		$rndkey0,$inout5
1996
1997	movdqa	$twres,$rndkey0
1998	paddd	$twres,$twres
1999	 aesenc		$rndkey1,$inout0
2000	pxor	@tweak[5],@tweak[3]
2001	psrad	\$31,$rndkey0
2002	 aesenc		$rndkey1,$inout1
2003	paddq	@tweak[5],@tweak[5]
2004	pand	$twmask,$rndkey0
2005	 aesenc		$rndkey1,$inout2
2006	 aesenc		$rndkey1,$inout3
2007	pxor	$rndkey0,@tweak[5]
2008	$movkey		($key_),$rndkey0
2009	 aesenc		$rndkey1,$inout4
2010	 aesenc		$rndkey1,$inout5
2011	$movkey		16($key_),$rndkey1
2012
2013	pxor	@tweak[5],@tweak[4]
2014	 aesenclast	`16*0`(%rsp),$inout0
2015	psrad	\$31,$twres
2016	paddq	@tweak[5],@tweak[5]
2017	 aesenclast	`16*1`(%rsp),$inout1
2018	 aesenclast	`16*2`(%rsp),$inout2
2019	pand	$twmask,$twres
2020	mov	%r10,%rax			# restore $rounds
2021	 aesenclast	`16*3`(%rsp),$inout3
2022	 aesenclast	`16*4`(%rsp),$inout4
2023	 aesenclast	`16*5`(%rsp),$inout5
2024	pxor	$twres,@tweak[5]
2025
2026	lea	`16*6`($out),$out		# $out+=6*16
2027	movups	$inout0,`-16*6`($out)		# store 6 output blocks
2028	movups	$inout1,`-16*5`($out)
2029	movups	$inout2,`-16*4`($out)
2030	movups	$inout3,`-16*3`($out)
2031	movups	$inout4,`-16*2`($out)
2032	movups	$inout5,`-16*1`($out)
2033	sub	\$16*6,$len
2034	jnc	.Lxts_enc_grandloop		# loop if $len-=6*16 didn't borrow
2035
2036	mov	\$16+96,$rounds
2037	sub	$rnds_,$rounds
2038	mov	$key_,$key			# restore $key
2039	shr	\$4,$rounds			# restore original value
2040
2041.Lxts_enc_short:
2042	# at the point @tweak[0..5] are populated with tweak values
2043	mov	$rounds,$rnds_			# backup $rounds
2044	pxor	$rndkey0,@tweak[0]
2045	add	\$16*6,$len			# restore real remaining $len
2046	jz	.Lxts_enc_done			# done if ($len==0)
2047
2048	pxor	$rndkey0,@tweak[1]
2049	cmp	\$0x20,$len
2050	jb	.Lxts_enc_one			# $len is 1*16
2051	pxor	$rndkey0,@tweak[2]
2052	je	.Lxts_enc_two			# $len is 2*16
2053
2054	pxor	$rndkey0,@tweak[3]
2055	cmp	\$0x40,$len
2056	jb	.Lxts_enc_three			# $len is 3*16
2057	pxor	$rndkey0,@tweak[4]
2058	je	.Lxts_enc_four			# $len is 4*16
2059
2060	movdqu	($inp),$inout0			# $len is 5*16
2061	movdqu	16*1($inp),$inout1
2062	movdqu	16*2($inp),$inout2
2063	pxor	@tweak[0],$inout0
2064	movdqu	16*3($inp),$inout3
2065	pxor	@tweak[1],$inout1
2066	movdqu	16*4($inp),$inout4
2067	lea	16*5($inp),$inp			# $inp+=5*16
2068	pxor	@tweak[2],$inout2
2069	pxor	@tweak[3],$inout3
2070	pxor	@tweak[4],$inout4
2071	pxor	$inout5,$inout5
2072
2073	call	_aesni_encrypt6
2074
2075	xorps	@tweak[0],$inout0
2076	movdqa	@tweak[5],@tweak[0]
2077	xorps	@tweak[1],$inout1
2078	xorps	@tweak[2],$inout2
2079	movdqu	$inout0,($out)			# store 5 output blocks
2080	xorps	@tweak[3],$inout3
2081	movdqu	$inout1,16*1($out)
2082	xorps	@tweak[4],$inout4
2083	movdqu	$inout2,16*2($out)
2084	movdqu	$inout3,16*3($out)
2085	movdqu	$inout4,16*4($out)
2086	lea	16*5($out),$out			# $out+=5*16
2087	jmp	.Lxts_enc_done
2088
2089.align	16
2090.Lxts_enc_one:
2091	movups	($inp),$inout0
2092	lea	16*1($inp),$inp			# inp+=1*16
2093	xorps	@tweak[0],$inout0
2094___
2095	&aesni_generate1("enc",$key,$rounds);
2096$code.=<<___;
2097	xorps	@tweak[0],$inout0
2098	movdqa	@tweak[1],@tweak[0]
2099	movups	$inout0,($out)			# store one output block
2100	lea	16*1($out),$out			# $out+=1*16
2101	jmp	.Lxts_enc_done
2102
2103.align	16
2104.Lxts_enc_two:
2105	movups	($inp),$inout0
2106	movups	16($inp),$inout1
2107	lea	32($inp),$inp			# $inp+=2*16
2108	xorps	@tweak[0],$inout0
2109	xorps	@tweak[1],$inout1
2110
2111	call	_aesni_encrypt2
2112
2113	xorps	@tweak[0],$inout0
2114	movdqa	@tweak[2],@tweak[0]
2115	xorps	@tweak[1],$inout1
2116	movups	$inout0,($out)			# store 2 output blocks
2117	movups	$inout1,16*1($out)
2118	lea	16*2($out),$out			# $out+=2*16
2119	jmp	.Lxts_enc_done
2120
2121.align	16
2122.Lxts_enc_three:
2123	movups	($inp),$inout0
2124	movups	16*1($inp),$inout1
2125	movups	16*2($inp),$inout2
2126	lea	16*3($inp),$inp			# $inp+=3*16
2127	xorps	@tweak[0],$inout0
2128	xorps	@tweak[1],$inout1
2129	xorps	@tweak[2],$inout2
2130
2131	call	_aesni_encrypt3
2132
2133	xorps	@tweak[0],$inout0
2134	movdqa	@tweak[3],@tweak[0]
2135	xorps	@tweak[1],$inout1
2136	xorps	@tweak[2],$inout2
2137	movups	$inout0,($out)			# store 3 output blocks
2138	movups	$inout1,16*1($out)
2139	movups	$inout2,16*2($out)
2140	lea	16*3($out),$out			# $out+=3*16
2141	jmp	.Lxts_enc_done
2142
2143.align	16
2144.Lxts_enc_four:
2145	movups	($inp),$inout0
2146	movups	16*1($inp),$inout1
2147	movups	16*2($inp),$inout2
2148	xorps	@tweak[0],$inout0
2149	movups	16*3($inp),$inout3
2150	lea	16*4($inp),$inp			# $inp+=4*16
2151	xorps	@tweak[1],$inout1
2152	xorps	@tweak[2],$inout2
2153	xorps	@tweak[3],$inout3
2154
2155	call	_aesni_encrypt4
2156
2157	pxor	@tweak[0],$inout0
2158	movdqa	@tweak[4],@tweak[0]
2159	pxor	@tweak[1],$inout1
2160	pxor	@tweak[2],$inout2
2161	movdqu	$inout0,($out)			# store 4 output blocks
2162	pxor	@tweak[3],$inout3
2163	movdqu	$inout1,16*1($out)
2164	movdqu	$inout2,16*2($out)
2165	movdqu	$inout3,16*3($out)
2166	lea	16*4($out),$out			# $out+=4*16
2167	jmp	.Lxts_enc_done
2168
2169.align	16
2170.Lxts_enc_done:
2171	and	\$15,$len_			# see if $len%16 is 0
2172	jz	.Lxts_enc_ret
2173	mov	$len_,$len
2174
2175.Lxts_enc_steal:
2176	movzb	($inp),%eax			# borrow $rounds ...
2177	movzb	-16($out),%ecx			# ... and $key
2178	lea	1($inp),$inp
2179	mov	%al,-16($out)
2180	mov	%cl,0($out)
2181	lea	1($out),$out
2182	sub	\$1,$len
2183	jnz	.Lxts_enc_steal
2184
2185	sub	$len_,$out			# rewind $out
2186	mov	$key_,$key			# restore $key
2187	mov	$rnds_,$rounds			# restore $rounds
2188
2189	movups	-16($out),$inout0
2190	xorps	@tweak[0],$inout0
2191___
2192	&aesni_generate1("enc",$key,$rounds);
2193$code.=<<___;
2194	xorps	@tweak[0],$inout0
2195	movups	$inout0,-16($out)
2196
2197.Lxts_enc_ret:
2198	xorps	%xmm0,%xmm0			# clear register bank
2199	pxor	%xmm1,%xmm1
2200	pxor	%xmm2,%xmm2
2201	pxor	%xmm3,%xmm3
2202	pxor	%xmm4,%xmm4
2203	pxor	%xmm5,%xmm5
2204___
2205$code.=<<___ if (!$win64);
2206	pxor	%xmm6,%xmm6
2207	pxor	%xmm7,%xmm7
2208	movaps	%xmm0,0x00(%rsp)		# clear stack
2209	pxor	%xmm8,%xmm8
2210	movaps	%xmm0,0x10(%rsp)
2211	pxor	%xmm9,%xmm9
2212	movaps	%xmm0,0x20(%rsp)
2213	pxor	%xmm10,%xmm10
2214	movaps	%xmm0,0x30(%rsp)
2215	pxor	%xmm11,%xmm11
2216	movaps	%xmm0,0x40(%rsp)
2217	pxor	%xmm12,%xmm12
2218	movaps	%xmm0,0x50(%rsp)
2219	pxor	%xmm13,%xmm13
2220	movaps	%xmm0,0x60(%rsp)
2221	pxor	%xmm14,%xmm14
2222	pxor	%xmm15,%xmm15
2223___
2224$code.=<<___ if ($win64);
2225	movaps	-0xa8(%r11),%xmm6
2226	movaps	%xmm0,-0xa8(%r11)		# clear stack
2227	movaps	-0x98(%r11),%xmm7
2228	movaps	%xmm0,-0x98(%r11)
2229	movaps	-0x88(%r11),%xmm8
2230	movaps	%xmm0,-0x88(%r11)
2231	movaps	-0x78(%r11),%xmm9
2232	movaps	%xmm0,-0x78(%r11)
2233	movaps	-0x68(%r11),%xmm10
2234	movaps	%xmm0,-0x68(%r11)
2235	movaps	-0x58(%r11),%xmm11
2236	movaps	%xmm0,-0x58(%r11)
2237	movaps	-0x48(%r11),%xmm12
2238	movaps	%xmm0,-0x48(%r11)
2239	movaps	-0x38(%r11),%xmm13
2240	movaps	%xmm0,-0x38(%r11)
2241	movaps	-0x28(%r11),%xmm14
2242	movaps	%xmm0,-0x28(%r11)
2243	movaps	-0x18(%r11),%xmm15
2244	movaps	%xmm0,-0x18(%r11)
2245	movaps	%xmm0,0x00(%rsp)
2246	movaps	%xmm0,0x10(%rsp)
2247	movaps	%xmm0,0x20(%rsp)
2248	movaps	%xmm0,0x30(%rsp)
2249	movaps	%xmm0,0x40(%rsp)
2250	movaps	%xmm0,0x50(%rsp)
2251	movaps	%xmm0,0x60(%rsp)
2252___
2253$code.=<<___;
2254	mov	-8(%r11),%rbp
2255.cfi_restore	%rbp
2256	lea	(%r11),%rsp
2257.cfi_def_cfa_register	%rsp
2258.Lxts_enc_epilogue:
2259	ret
2260.cfi_endproc
2261.size	aesni_xts_encrypt,.-aesni_xts_encrypt
2262___
2263
2264$code.=<<___;
2265.globl	aesni_xts_decrypt
2266.type	aesni_xts_decrypt,\@function,6
2267.align	16
2268aesni_xts_decrypt:
2269.cfi_startproc
2270	endbranch
2271	lea	(%rsp),%r11			# frame pointer
2272.cfi_def_cfa_register	%r11
2273	push	%rbp
2274.cfi_push	%rbp
2275	sub	\$$frame_size,%rsp
2276	and	\$-16,%rsp	# Linux kernel stack can be incorrectly seeded
2277___
2278$code.=<<___ if ($win64);
2279	movaps	%xmm6,-0xa8(%r11)		# offload everything
2280	movaps	%xmm7,-0x98(%r11)
2281	movaps	%xmm8,-0x88(%r11)
2282	movaps	%xmm9,-0x78(%r11)
2283	movaps	%xmm10,-0x68(%r11)
2284	movaps	%xmm11,-0x58(%r11)
2285	movaps	%xmm12,-0x48(%r11)
2286	movaps	%xmm13,-0x38(%r11)
2287	movaps	%xmm14,-0x28(%r11)
2288	movaps	%xmm15,-0x18(%r11)
2289.Lxts_dec_body:
2290___
2291$code.=<<___;
2292	movups	($ivp),$inout0			# load clear-text tweak
2293	mov	240($key2),$rounds		# key2->rounds
2294	mov	240($key),$rnds_		# key1->rounds
2295___
2296	# generate the tweak
2297	&aesni_generate1("enc",$key2,$rounds,$inout0);
2298$code.=<<___;
2299	xor	%eax,%eax			# if ($len%16) len-=16;
2300	test	\$15,$len
2301	setnz	%al
2302	shl	\$4,%rax
2303	sub	%rax,$len
2304
2305	$movkey	($key),$rndkey0			# zero round key
2306	mov	$key,$key_			# backup $key
2307	mov	$rnds_,$rounds			# backup $rounds
2308	shl	\$4,$rnds_
2309	mov	$len,$len_			# backup $len
2310	and	\$-16,$len
2311
2312	$movkey	16($key,$rnds_),$rndkey1	# last round key
2313
2314	movdqa	.Lxts_magic(%rip),$twmask
2315	movdqa	$inout0,@tweak[5]
2316	pshufd	\$0x5f,$inout0,$twres
2317	pxor	$rndkey0,$rndkey1
2318___
2319    for ($i=0;$i<4;$i++) {
2320    $code.=<<___;
2321	movdqa	$twres,$twtmp
2322	paddd	$twres,$twres
2323	movdqa	@tweak[5],@tweak[$i]
2324	psrad	\$31,$twtmp			# broadcast upper bits
2325	paddq	@tweak[5],@tweak[5]
2326	pand	$twmask,$twtmp
2327	pxor	$rndkey0,@tweak[$i]
2328	pxor	$twtmp,@tweak[5]
2329___
2330    }
2331$code.=<<___;
2332	movdqa	@tweak[5],@tweak[4]
2333	psrad	\$31,$twres
2334	paddq	@tweak[5],@tweak[5]
2335	pand	$twmask,$twres
2336	pxor	$rndkey0,@tweak[4]
2337	pxor	$twres,@tweak[5]
2338	movaps	$rndkey1,0x60(%rsp)		# save round[0]^round[last]
2339
2340	sub	\$16*6,$len
2341	jc	.Lxts_dec_short			# if $len-=6*16 borrowed
2342
2343	mov	\$16+96,$rounds
2344	lea	32($key_,$rnds_),$key		# end of key schedule
2345	sub	%r10,%rax			# twisted $rounds
2346	$movkey	16($key_),$rndkey1
2347	mov	%rax,%r10			# backup twisted $rounds
2348	lea	.Lxts_magic(%rip),%r8
2349	jmp	.Lxts_dec_grandloop
2350
2351.align	32
2352.Lxts_dec_grandloop:
2353	movdqu	`16*0`($inp),$inout0		# load input
2354	movdqa	$rndkey0,$twmask
2355	movdqu	`16*1`($inp),$inout1
2356	pxor	@tweak[0],$inout0		# input^=tweak^round[0]
2357	movdqu	`16*2`($inp),$inout2
2358	pxor	@tweak[1],$inout1
2359	 aesdec		$rndkey1,$inout0
2360	movdqu	`16*3`($inp),$inout3
2361	pxor	@tweak[2],$inout2
2362	 aesdec		$rndkey1,$inout1
2363	movdqu	`16*4`($inp),$inout4
2364	pxor	@tweak[3],$inout3
2365	 aesdec		$rndkey1,$inout2
2366	movdqu	`16*5`($inp),$inout5
2367	pxor	@tweak[5],$twmask		# round[0]^=tweak[5]
2368	 movdqa	0x60(%rsp),$twres		# load round[0]^round[last]
2369	pxor	@tweak[4],$inout4
2370	 aesdec		$rndkey1,$inout3
2371	$movkey	32($key_),$rndkey0
2372	lea	`16*6`($inp),$inp
2373	pxor	$twmask,$inout5
2374
2375	 pxor	$twres,@tweak[0]		# calculate tweaks^round[last]
2376	aesdec		$rndkey1,$inout4
2377	 pxor	$twres,@tweak[1]
2378	 movdqa	@tweak[0],`16*0`(%rsp)		# put aside tweaks^last round key
2379	aesdec		$rndkey1,$inout5
2380	$movkey		48($key_),$rndkey1
2381	 pxor	$twres,@tweak[2]
2382
2383	aesdec		$rndkey0,$inout0
2384	 pxor	$twres,@tweak[3]
2385	 movdqa	@tweak[1],`16*1`(%rsp)
2386	aesdec		$rndkey0,$inout1
2387	 pxor	$twres,@tweak[4]
2388	 movdqa	@tweak[2],`16*2`(%rsp)
2389	aesdec		$rndkey0,$inout2
2390	aesdec		$rndkey0,$inout3
2391	 pxor	$twres,$twmask
2392	 movdqa	@tweak[4],`16*4`(%rsp)
2393	aesdec		$rndkey0,$inout4
2394	aesdec		$rndkey0,$inout5
2395	$movkey		64($key_),$rndkey0
2396	 movdqa	$twmask,`16*5`(%rsp)
2397	pshufd	\$0x5f,@tweak[5],$twres
2398	jmp	.Lxts_dec_loop6
2399.align	32
2400.Lxts_dec_loop6:
2401	aesdec		$rndkey1,$inout0
2402	aesdec		$rndkey1,$inout1
2403	aesdec		$rndkey1,$inout2
2404	aesdec		$rndkey1,$inout3
2405	aesdec		$rndkey1,$inout4
2406	aesdec		$rndkey1,$inout5
2407	$movkey		-64($key,%rax),$rndkey1
2408	add		\$32,%rax
2409
2410	aesdec		$rndkey0,$inout0
2411	aesdec		$rndkey0,$inout1
2412	aesdec		$rndkey0,$inout2
2413	aesdec		$rndkey0,$inout3
2414	aesdec		$rndkey0,$inout4
2415	aesdec		$rndkey0,$inout5
2416	$movkey		-80($key,%rax),$rndkey0
2417	jnz		.Lxts_dec_loop6
2418
2419	movdqa	(%r8),$twmask			# start calculating next tweak
2420	movdqa	$twres,$twtmp
2421	paddd	$twres,$twres
2422	 aesdec		$rndkey1,$inout0
2423	paddq	@tweak[5],@tweak[5]
2424	psrad	\$31,$twtmp
2425	 aesdec		$rndkey1,$inout1
2426	pand	$twmask,$twtmp
2427	$movkey	($key_),@tweak[0]		# load round[0]
2428	 aesdec		$rndkey1,$inout2
2429	 aesdec		$rndkey1,$inout3
2430	 aesdec		$rndkey1,$inout4
2431	pxor	$twtmp,@tweak[5]
2432	movaps	@tweak[0],@tweak[1]		# copy round[0]
2433	 aesdec		$rndkey1,$inout5
2434	 $movkey	-64($key),$rndkey1
2435
2436	movdqa	$twres,$twtmp
2437	 aesdec		$rndkey0,$inout0
2438	paddd	$twres,$twres
2439	pxor	@tweak[5],@tweak[0]
2440	 aesdec		$rndkey0,$inout1
2441	psrad	\$31,$twtmp
2442	paddq	@tweak[5],@tweak[5]
2443	 aesdec		$rndkey0,$inout2
2444	 aesdec		$rndkey0,$inout3
2445	pand	$twmask,$twtmp
2446	movaps	@tweak[1],@tweak[2]
2447	 aesdec		$rndkey0,$inout4
2448	pxor	$twtmp,@tweak[5]
2449	movdqa	$twres,$twtmp
2450	 aesdec		$rndkey0,$inout5
2451	 $movkey	-48($key),$rndkey0
2452
2453	paddd	$twres,$twres
2454	 aesdec		$rndkey1,$inout0
2455	pxor	@tweak[5],@tweak[1]
2456	psrad	\$31,$twtmp
2457	 aesdec		$rndkey1,$inout1
2458	paddq	@tweak[5],@tweak[5]
2459	pand	$twmask,$twtmp
2460	 aesdec		$rndkey1,$inout2
2461	 aesdec		$rndkey1,$inout3
2462	 movdqa	@tweak[3],`16*3`(%rsp)
2463	pxor	$twtmp,@tweak[5]
2464	 aesdec		$rndkey1,$inout4
2465	movaps	@tweak[2],@tweak[3]
2466	movdqa	$twres,$twtmp
2467	 aesdec		$rndkey1,$inout5
2468	 $movkey	-32($key),$rndkey1
2469
2470	paddd	$twres,$twres
2471	 aesdec		$rndkey0,$inout0
2472	pxor	@tweak[5],@tweak[2]
2473	psrad	\$31,$twtmp
2474	 aesdec		$rndkey0,$inout1
2475	paddq	@tweak[5],@tweak[5]
2476	pand	$twmask,$twtmp
2477	 aesdec		$rndkey0,$inout2
2478	 aesdec		$rndkey0,$inout3
2479	 aesdec		$rndkey0,$inout4
2480	pxor	$twtmp,@tweak[5]
2481	movaps	@tweak[3],@tweak[4]
2482	 aesdec		$rndkey0,$inout5
2483
2484	movdqa	$twres,$rndkey0
2485	paddd	$twres,$twres
2486	 aesdec		$rndkey1,$inout0
2487	pxor	@tweak[5],@tweak[3]
2488	psrad	\$31,$rndkey0
2489	 aesdec		$rndkey1,$inout1
2490	paddq	@tweak[5],@tweak[5]
2491	pand	$twmask,$rndkey0
2492	 aesdec		$rndkey1,$inout2
2493	 aesdec		$rndkey1,$inout3
2494	pxor	$rndkey0,@tweak[5]
2495	$movkey		($key_),$rndkey0
2496	 aesdec		$rndkey1,$inout4
2497	 aesdec		$rndkey1,$inout5
2498	$movkey		16($key_),$rndkey1
2499
2500	pxor	@tweak[5],@tweak[4]
2501	 aesdeclast	`16*0`(%rsp),$inout0
2502	psrad	\$31,$twres
2503	paddq	@tweak[5],@tweak[5]
2504	 aesdeclast	`16*1`(%rsp),$inout1
2505	 aesdeclast	`16*2`(%rsp),$inout2
2506	pand	$twmask,$twres
2507	mov	%r10,%rax			# restore $rounds
2508	 aesdeclast	`16*3`(%rsp),$inout3
2509	 aesdeclast	`16*4`(%rsp),$inout4
2510	 aesdeclast	`16*5`(%rsp),$inout5
2511	pxor	$twres,@tweak[5]
2512
2513	lea	`16*6`($out),$out		# $out+=6*16
2514	movups	$inout0,`-16*6`($out)		# store 6 output blocks
2515	movups	$inout1,`-16*5`($out)
2516	movups	$inout2,`-16*4`($out)
2517	movups	$inout3,`-16*3`($out)
2518	movups	$inout4,`-16*2`($out)
2519	movups	$inout5,`-16*1`($out)
2520	sub	\$16*6,$len
2521	jnc	.Lxts_dec_grandloop		# loop if $len-=6*16 didn't borrow
2522
2523	mov	\$16+96,$rounds
2524	sub	$rnds_,$rounds
2525	mov	$key_,$key			# restore $key
2526	shr	\$4,$rounds			# restore original value
2527
2528.Lxts_dec_short:
2529	# at the point @tweak[0..5] are populated with tweak values
2530	mov	$rounds,$rnds_			# backup $rounds
2531	pxor	$rndkey0,@tweak[0]
2532	pxor	$rndkey0,@tweak[1]
2533	add	\$16*6,$len			# restore real remaining $len
2534	jz	.Lxts_dec_done			# done if ($len==0)
2535
2536	pxor	$rndkey0,@tweak[2]
2537	cmp	\$0x20,$len
2538	jb	.Lxts_dec_one			# $len is 1*16
2539	pxor	$rndkey0,@tweak[3]
2540	je	.Lxts_dec_two			# $len is 2*16
2541
2542	pxor	$rndkey0,@tweak[4]
2543	cmp	\$0x40,$len
2544	jb	.Lxts_dec_three			# $len is 3*16
2545	je	.Lxts_dec_four			# $len is 4*16
2546
2547	movdqu	($inp),$inout0			# $len is 5*16
2548	movdqu	16*1($inp),$inout1
2549	movdqu	16*2($inp),$inout2
2550	pxor	@tweak[0],$inout0
2551	movdqu	16*3($inp),$inout3
2552	pxor	@tweak[1],$inout1
2553	movdqu	16*4($inp),$inout4
2554	lea	16*5($inp),$inp			# $inp+=5*16
2555	pxor	@tweak[2],$inout2
2556	pxor	@tweak[3],$inout3
2557	pxor	@tweak[4],$inout4
2558
2559	call	_aesni_decrypt6
2560
2561	xorps	@tweak[0],$inout0
2562	xorps	@tweak[1],$inout1
2563	xorps	@tweak[2],$inout2
2564	movdqu	$inout0,($out)			# store 5 output blocks
2565	xorps	@tweak[3],$inout3
2566	movdqu	$inout1,16*1($out)
2567	xorps	@tweak[4],$inout4
2568	movdqu	$inout2,16*2($out)
2569	 pxor		$twtmp,$twtmp
2570	movdqu	$inout3,16*3($out)
2571	 pcmpgtd	@tweak[5],$twtmp
2572	movdqu	$inout4,16*4($out)
2573	lea	16*5($out),$out			# $out+=5*16
2574	 pshufd		\$0x13,$twtmp,@tweak[1]	# $twres
2575	and	\$15,$len_
2576	jz	.Lxts_dec_ret
2577
2578	movdqa	@tweak[5],@tweak[0]
2579	paddq	@tweak[5],@tweak[5]		# psllq 1,$tweak
2580	pand	$twmask,@tweak[1]		# isolate carry and residue
2581	pxor	@tweak[5],@tweak[1]
2582	jmp	.Lxts_dec_done2
2583
2584.align	16
2585.Lxts_dec_one:
2586	movups	($inp),$inout0
2587	lea	16*1($inp),$inp			# $inp+=1*16
2588	xorps	@tweak[0],$inout0
2589___
2590	&aesni_generate1("dec",$key,$rounds);
2591$code.=<<___;
2592	xorps	@tweak[0],$inout0
2593	movdqa	@tweak[1],@tweak[0]
2594	movups	$inout0,($out)			# store one output block
2595	movdqa	@tweak[2],@tweak[1]
2596	lea	16*1($out),$out			# $out+=1*16
2597	jmp	.Lxts_dec_done
2598
2599.align	16
2600.Lxts_dec_two:
2601	movups	($inp),$inout0
2602	movups	16($inp),$inout1
2603	lea	32($inp),$inp			# $inp+=2*16
2604	xorps	@tweak[0],$inout0
2605	xorps	@tweak[1],$inout1
2606
2607	call	_aesni_decrypt2
2608
2609	xorps	@tweak[0],$inout0
2610	movdqa	@tweak[2],@tweak[0]
2611	xorps	@tweak[1],$inout1
2612	movdqa	@tweak[3],@tweak[1]
2613	movups	$inout0,($out)			# store 2 output blocks
2614	movups	$inout1,16*1($out)
2615	lea	16*2($out),$out			# $out+=2*16
2616	jmp	.Lxts_dec_done
2617
2618.align	16
2619.Lxts_dec_three:
2620	movups	($inp),$inout0
2621	movups	16*1($inp),$inout1
2622	movups	16*2($inp),$inout2
2623	lea	16*3($inp),$inp			# $inp+=3*16
2624	xorps	@tweak[0],$inout0
2625	xorps	@tweak[1],$inout1
2626	xorps	@tweak[2],$inout2
2627
2628	call	_aesni_decrypt3
2629
2630	xorps	@tweak[0],$inout0
2631	movdqa	@tweak[3],@tweak[0]
2632	xorps	@tweak[1],$inout1
2633	movdqa	@tweak[4],@tweak[1]
2634	xorps	@tweak[2],$inout2
2635	movups	$inout0,($out)			# store 3 output blocks
2636	movups	$inout1,16*1($out)
2637	movups	$inout2,16*2($out)
2638	lea	16*3($out),$out			# $out+=3*16
2639	jmp	.Lxts_dec_done
2640
2641.align	16
2642.Lxts_dec_four:
2643	movups	($inp),$inout0
2644	movups	16*1($inp),$inout1
2645	movups	16*2($inp),$inout2
2646	xorps	@tweak[0],$inout0
2647	movups	16*3($inp),$inout3
2648	lea	16*4($inp),$inp			# $inp+=4*16
2649	xorps	@tweak[1],$inout1
2650	xorps	@tweak[2],$inout2
2651	xorps	@tweak[3],$inout3
2652
2653	call	_aesni_decrypt4
2654
2655	pxor	@tweak[0],$inout0
2656	movdqa	@tweak[4],@tweak[0]
2657	pxor	@tweak[1],$inout1
2658	movdqa	@tweak[5],@tweak[1]
2659	pxor	@tweak[2],$inout2
2660	movdqu	$inout0,($out)			# store 4 output blocks
2661	pxor	@tweak[3],$inout3
2662	movdqu	$inout1,16*1($out)
2663	movdqu	$inout2,16*2($out)
2664	movdqu	$inout3,16*3($out)
2665	lea	16*4($out),$out			# $out+=4*16
2666	jmp	.Lxts_dec_done
2667
2668.align	16
2669.Lxts_dec_done:
2670	and	\$15,$len_			# see if $len%16 is 0
2671	jz	.Lxts_dec_ret
2672.Lxts_dec_done2:
2673	mov	$len_,$len
2674	mov	$key_,$key			# restore $key
2675	mov	$rnds_,$rounds			# restore $rounds
2676
2677	movups	($inp),$inout0
2678	xorps	@tweak[1],$inout0
2679___
2680	&aesni_generate1("dec",$key,$rounds);
2681$code.=<<___;
2682	xorps	@tweak[1],$inout0
2683	movups	$inout0,($out)
2684
2685.Lxts_dec_steal:
2686	movzb	16($inp),%eax			# borrow $rounds ...
2687	movzb	($out),%ecx			# ... and $key
2688	lea	1($inp),$inp
2689	mov	%al,($out)
2690	mov	%cl,16($out)
2691	lea	1($out),$out
2692	sub	\$1,$len
2693	jnz	.Lxts_dec_steal
2694
2695	sub	$len_,$out			# rewind $out
2696	mov	$key_,$key			# restore $key
2697	mov	$rnds_,$rounds			# restore $rounds
2698
2699	movups	($out),$inout0
2700	xorps	@tweak[0],$inout0
2701___
2702	&aesni_generate1("dec",$key,$rounds);
2703$code.=<<___;
2704	xorps	@tweak[0],$inout0
2705	movups	$inout0,($out)
2706
2707.Lxts_dec_ret:
2708	xorps	%xmm0,%xmm0			# clear register bank
2709	pxor	%xmm1,%xmm1
2710	pxor	%xmm2,%xmm2
2711	pxor	%xmm3,%xmm3
2712	pxor	%xmm4,%xmm4
2713	pxor	%xmm5,%xmm5
2714___
2715$code.=<<___ if (!$win64);
2716	pxor	%xmm6,%xmm6
2717	pxor	%xmm7,%xmm7
2718	movaps	%xmm0,0x00(%rsp)		# clear stack
2719	pxor	%xmm8,%xmm8
2720	movaps	%xmm0,0x10(%rsp)
2721	pxor	%xmm9,%xmm9
2722	movaps	%xmm0,0x20(%rsp)
2723	pxor	%xmm10,%xmm10
2724	movaps	%xmm0,0x30(%rsp)
2725	pxor	%xmm11,%xmm11
2726	movaps	%xmm0,0x40(%rsp)
2727	pxor	%xmm12,%xmm12
2728	movaps	%xmm0,0x50(%rsp)
2729	pxor	%xmm13,%xmm13
2730	movaps	%xmm0,0x60(%rsp)
2731	pxor	%xmm14,%xmm14
2732	pxor	%xmm15,%xmm15
2733___
2734$code.=<<___ if ($win64);
2735	movaps	-0xa8(%r11),%xmm6
2736	movaps	%xmm0,-0xa8(%r11)		# clear stack
2737	movaps	-0x98(%r11),%xmm7
2738	movaps	%xmm0,-0x98(%r11)
2739	movaps	-0x88(%r11),%xmm8
2740	movaps	%xmm0,-0x88(%r11)
2741	movaps	-0x78(%r11),%xmm9
2742	movaps	%xmm0,-0x78(%r11)
2743	movaps	-0x68(%r11),%xmm10
2744	movaps	%xmm0,-0x68(%r11)
2745	movaps	-0x58(%r11),%xmm11
2746	movaps	%xmm0,-0x58(%r11)
2747	movaps	-0x48(%r11),%xmm12
2748	movaps	%xmm0,-0x48(%r11)
2749	movaps	-0x38(%r11),%xmm13
2750	movaps	%xmm0,-0x38(%r11)
2751	movaps	-0x28(%r11),%xmm14
2752	movaps	%xmm0,-0x28(%r11)
2753	movaps	-0x18(%r11),%xmm15
2754	movaps	%xmm0,-0x18(%r11)
2755	movaps	%xmm0,0x00(%rsp)
2756	movaps	%xmm0,0x10(%rsp)
2757	movaps	%xmm0,0x20(%rsp)
2758	movaps	%xmm0,0x30(%rsp)
2759	movaps	%xmm0,0x40(%rsp)
2760	movaps	%xmm0,0x50(%rsp)
2761	movaps	%xmm0,0x60(%rsp)
2762___
2763$code.=<<___;
2764	mov	-8(%r11),%rbp
2765.cfi_restore	%rbp
2766	lea	(%r11),%rsp
2767.cfi_def_cfa_register	%rsp
2768.Lxts_dec_epilogue:
2769	ret
2770.cfi_endproc
2771.size	aesni_xts_decrypt,.-aesni_xts_decrypt
2772___
2773}
2774
2775######################################################################
2776# void aesni_ocb_[en|de]crypt(const char *inp, char *out, size_t blocks,
2777#	const AES_KEY *key, unsigned int start_block_num,
2778#	unsigned char offset_i[16], const unsigned char L_[][16],
2779#	unsigned char checksum[16]);
2780#
2781{
2782my @offset=map("%xmm$_",(10..15));
2783my ($checksum,$rndkey0l)=("%xmm8","%xmm9");
2784my ($block_num,$offset_p)=("%r8","%r9");		# 5th and 6th arguments
2785my ($L_p,$checksum_p) = ("%rbx","%rbp");
2786my ($i1,$i3,$i5) = ("%r12","%r13","%r14");
2787my $seventh_arg = $win64 ? 56 : 8;
2788my $blocks = $len;
2789
2790$code.=<<___;
2791.globl	aesni_ocb_encrypt
2792.type	aesni_ocb_encrypt,\@function,6
2793.align	32
2794aesni_ocb_encrypt:
2795.cfi_startproc
2796	endbranch
2797	lea	(%rsp),%rax
2798	push	%rbx
2799.cfi_push	%rbx
2800	push	%rbp
2801.cfi_push	%rbp
2802	push	%r12
2803.cfi_push	%r12
2804	push	%r13
2805.cfi_push	%r13
2806	push	%r14
2807.cfi_push	%r14
2808___
2809$code.=<<___ if ($win64);
2810	lea	-0xa0(%rsp),%rsp
2811	movaps	%xmm6,0x00(%rsp)		# offload everything
2812	movaps	%xmm7,0x10(%rsp)
2813	movaps	%xmm8,0x20(%rsp)
2814	movaps	%xmm9,0x30(%rsp)
2815	movaps	%xmm10,0x40(%rsp)
2816	movaps	%xmm11,0x50(%rsp)
2817	movaps	%xmm12,0x60(%rsp)
2818	movaps	%xmm13,0x70(%rsp)
2819	movaps	%xmm14,0x80(%rsp)
2820	movaps	%xmm15,0x90(%rsp)
2821.Locb_enc_body:
2822___
2823$code.=<<___;
2824	mov	$seventh_arg(%rax),$L_p		# 7th argument
2825	mov	$seventh_arg+8(%rax),$checksum_p# 8th argument
2826
2827	mov	240($key),$rnds_
2828	mov	$key,$key_
2829	shl	\$4,$rnds_
2830	$movkey	($key),$rndkey0l		# round[0]
2831	$movkey	16($key,$rnds_),$rndkey1	# round[last]
2832
2833	movdqu	($offset_p),@offset[5]		# load last offset_i
2834	pxor	$rndkey1,$rndkey0l		# round[0] ^ round[last]
2835	pxor	$rndkey1,@offset[5]		# offset_i ^ round[last]
2836
2837	mov	\$16+32,$rounds
2838	lea	32($key_,$rnds_),$key
2839	$movkey	16($key_),$rndkey1		# round[1]
2840	sub	%r10,%rax			# twisted $rounds
2841	mov	%rax,%r10			# backup twisted $rounds
2842
2843	movdqu	($L_p),@offset[0]		# L_0 for all odd-numbered blocks
2844	movdqu	($checksum_p),$checksum		# load checksum
2845
2846	test	\$1,$block_num			# is first block number odd?
2847	jnz	.Locb_enc_odd
2848
2849	bsf	$block_num,$i1
2850	add	\$1,$block_num
2851	shl	\$4,$i1
2852	movdqu	($L_p,$i1),$inout5		# borrow
2853	movdqu	($inp),$inout0
2854	lea	16($inp),$inp
2855
2856	call	__ocb_encrypt1
2857
2858	movdqa	$inout5,@offset[5]
2859	movups	$inout0,($out)
2860	lea	16($out),$out
2861	sub	\$1,$blocks
2862	jz	.Locb_enc_done
2863
2864.Locb_enc_odd:
2865	lea	1($block_num),$i1		# even-numbered blocks
2866	lea	3($block_num),$i3
2867	lea	5($block_num),$i5
2868	lea	6($block_num),$block_num
2869	bsf	$i1,$i1				# ntz(block)
2870	bsf	$i3,$i3
2871	bsf	$i5,$i5
2872	shl	\$4,$i1				# ntz(block) -> table offset
2873	shl	\$4,$i3
2874	shl	\$4,$i5
2875
2876	sub	\$6,$blocks
2877	jc	.Locb_enc_short
2878	jmp	.Locb_enc_grandloop
2879
2880.align	32
2881.Locb_enc_grandloop:
2882	movdqu	`16*0`($inp),$inout0		# load input
2883	movdqu	`16*1`($inp),$inout1
2884	movdqu	`16*2`($inp),$inout2
2885	movdqu	`16*3`($inp),$inout3
2886	movdqu	`16*4`($inp),$inout4
2887	movdqu	`16*5`($inp),$inout5
2888	lea	`16*6`($inp),$inp
2889
2890	call	__ocb_encrypt6
2891
2892	movups	$inout0,`16*0`($out)		# store output
2893	movups	$inout1,`16*1`($out)
2894	movups	$inout2,`16*2`($out)
2895	movups	$inout3,`16*3`($out)
2896	movups	$inout4,`16*4`($out)
2897	movups	$inout5,`16*5`($out)
2898	lea	`16*6`($out),$out
2899	sub	\$6,$blocks
2900	jnc	.Locb_enc_grandloop
2901
2902.Locb_enc_short:
2903	add	\$6,$blocks
2904	jz	.Locb_enc_done
2905
2906	movdqu	`16*0`($inp),$inout0
2907	cmp	\$2,$blocks
2908	jb	.Locb_enc_one
2909	movdqu	`16*1`($inp),$inout1
2910	je	.Locb_enc_two
2911
2912	movdqu	`16*2`($inp),$inout2
2913	cmp	\$4,$blocks
2914	jb	.Locb_enc_three
2915	movdqu	`16*3`($inp),$inout3
2916	je	.Locb_enc_four
2917
2918	movdqu	`16*4`($inp),$inout4
2919	pxor	$inout5,$inout5
2920
2921	call	__ocb_encrypt6
2922
2923	movdqa	@offset[4],@offset[5]
2924	movups	$inout0,`16*0`($out)
2925	movups	$inout1,`16*1`($out)
2926	movups	$inout2,`16*2`($out)
2927	movups	$inout3,`16*3`($out)
2928	movups	$inout4,`16*4`($out)
2929
2930	jmp	.Locb_enc_done
2931
2932.align	16
2933.Locb_enc_one:
2934	movdqa	@offset[0],$inout5		# borrow
2935
2936	call	__ocb_encrypt1
2937
2938	movdqa	$inout5,@offset[5]
2939	movups	$inout0,`16*0`($out)
2940	jmp	.Locb_enc_done
2941
2942.align	16
2943.Locb_enc_two:
2944	pxor	$inout2,$inout2
2945	pxor	$inout3,$inout3
2946
2947	call	__ocb_encrypt4
2948
2949	movdqa	@offset[1],@offset[5]
2950	movups	$inout0,`16*0`($out)
2951	movups	$inout1,`16*1`($out)
2952
2953	jmp	.Locb_enc_done
2954
2955.align	16
2956.Locb_enc_three:
2957	pxor	$inout3,$inout3
2958
2959	call	__ocb_encrypt4
2960
2961	movdqa	@offset[2],@offset[5]
2962	movups	$inout0,`16*0`($out)
2963	movups	$inout1,`16*1`($out)
2964	movups	$inout2,`16*2`($out)
2965
2966	jmp	.Locb_enc_done
2967
2968.align	16
2969.Locb_enc_four:
2970	call	__ocb_encrypt4
2971
2972	movdqa	@offset[3],@offset[5]
2973	movups	$inout0,`16*0`($out)
2974	movups	$inout1,`16*1`($out)
2975	movups	$inout2,`16*2`($out)
2976	movups	$inout3,`16*3`($out)
2977
2978.Locb_enc_done:
2979	pxor	$rndkey0,@offset[5]		# "remove" round[last]
2980	movdqu	$checksum,($checksum_p)		# store checksum
2981	movdqu	@offset[5],($offset_p)		# store last offset_i
2982
2983	xorps	%xmm0,%xmm0			# clear register bank
2984	pxor	%xmm1,%xmm1
2985	pxor	%xmm2,%xmm2
2986	pxor	%xmm3,%xmm3
2987	pxor	%xmm4,%xmm4
2988	pxor	%xmm5,%xmm5
2989___
2990$code.=<<___ if (!$win64);
2991	pxor	%xmm6,%xmm6
2992	pxor	%xmm7,%xmm7
2993	pxor	%xmm8,%xmm8
2994	pxor	%xmm9,%xmm9
2995	pxor	%xmm10,%xmm10
2996	pxor	%xmm11,%xmm11
2997	pxor	%xmm12,%xmm12
2998	pxor	%xmm13,%xmm13
2999	pxor	%xmm14,%xmm14
3000	pxor	%xmm15,%xmm15
3001	lea	0x28(%rsp),%rax
3002.cfi_def_cfa	%rax,8
3003___
3004$code.=<<___ if ($win64);
3005	movaps	0x00(%rsp),%xmm6
3006	movaps	%xmm0,0x00(%rsp)		# clear stack
3007	movaps	0x10(%rsp),%xmm7
3008	movaps	%xmm0,0x10(%rsp)
3009	movaps	0x20(%rsp),%xmm8
3010	movaps	%xmm0,0x20(%rsp)
3011	movaps	0x30(%rsp),%xmm9
3012	movaps	%xmm0,0x30(%rsp)
3013	movaps	0x40(%rsp),%xmm10
3014	movaps	%xmm0,0x40(%rsp)
3015	movaps	0x50(%rsp),%xmm11
3016	movaps	%xmm0,0x50(%rsp)
3017	movaps	0x60(%rsp),%xmm12
3018	movaps	%xmm0,0x60(%rsp)
3019	movaps	0x70(%rsp),%xmm13
3020	movaps	%xmm0,0x70(%rsp)
3021	movaps	0x80(%rsp),%xmm14
3022	movaps	%xmm0,0x80(%rsp)
3023	movaps	0x90(%rsp),%xmm15
3024	movaps	%xmm0,0x90(%rsp)
3025	lea	0xa0+0x28(%rsp),%rax
3026.Locb_enc_pop:
3027___
3028$code.=<<___;
3029	mov	-40(%rax),%r14
3030.cfi_restore	%r14
3031	mov	-32(%rax),%r13
3032.cfi_restore	%r13
3033	mov	-24(%rax),%r12
3034.cfi_restore	%r12
3035	mov	-16(%rax),%rbp
3036.cfi_restore	%rbp
3037	mov	-8(%rax),%rbx
3038.cfi_restore	%rbx
3039	lea	(%rax),%rsp
3040.cfi_def_cfa_register	%rsp
3041.Locb_enc_epilogue:
3042	ret
3043.cfi_endproc
3044.size	aesni_ocb_encrypt,.-aesni_ocb_encrypt
3045
3046.type	__ocb_encrypt6,\@abi-omnipotent
3047.align	32
3048__ocb_encrypt6:
3049.cfi_startproc
3050	 pxor		$rndkey0l,@offset[5]	# offset_i ^ round[0]
3051	 movdqu		($L_p,$i1),@offset[1]
3052	 movdqa		@offset[0],@offset[2]
3053	 movdqu		($L_p,$i3),@offset[3]
3054	 movdqa		@offset[0],@offset[4]
3055	 pxor		@offset[5],@offset[0]
3056	 movdqu		($L_p,$i5),@offset[5]
3057	 pxor		@offset[0],@offset[1]
3058	pxor		$inout0,$checksum	# accumulate checksum
3059	pxor		@offset[0],$inout0	# input ^ round[0] ^ offset_i
3060	 pxor		@offset[1],@offset[2]
3061	pxor		$inout1,$checksum
3062	pxor		@offset[1],$inout1
3063	 pxor		@offset[2],@offset[3]
3064	pxor		$inout2,$checksum
3065	pxor		@offset[2],$inout2
3066	 pxor		@offset[3],@offset[4]
3067	pxor		$inout3,$checksum
3068	pxor		@offset[3],$inout3
3069	 pxor		@offset[4],@offset[5]
3070	pxor		$inout4,$checksum
3071	pxor		@offset[4],$inout4
3072	pxor		$inout5,$checksum
3073	pxor		@offset[5],$inout5
3074	$movkey		32($key_),$rndkey0
3075
3076	lea		1($block_num),$i1	# even-numbered blocks
3077	lea		3($block_num),$i3
3078	lea		5($block_num),$i5
3079	add		\$6,$block_num
3080	 pxor		$rndkey0l,@offset[0]	# offset_i ^ round[last]
3081	bsf		$i1,$i1			# ntz(block)
3082	bsf		$i3,$i3
3083	bsf		$i5,$i5
3084
3085	aesenc		$rndkey1,$inout0
3086	aesenc		$rndkey1,$inout1
3087	aesenc		$rndkey1,$inout2
3088	aesenc		$rndkey1,$inout3
3089	 pxor		$rndkey0l,@offset[1]
3090	 pxor		$rndkey0l,@offset[2]
3091	aesenc		$rndkey1,$inout4
3092	 pxor		$rndkey0l,@offset[3]
3093	 pxor		$rndkey0l,@offset[4]
3094	aesenc		$rndkey1,$inout5
3095	$movkey		48($key_),$rndkey1
3096	 pxor		$rndkey0l,@offset[5]
3097
3098	aesenc		$rndkey0,$inout0
3099	aesenc		$rndkey0,$inout1
3100	aesenc		$rndkey0,$inout2
3101	aesenc		$rndkey0,$inout3
3102	aesenc		$rndkey0,$inout4
3103	aesenc		$rndkey0,$inout5
3104	$movkey		64($key_),$rndkey0
3105	shl		\$4,$i1			# ntz(block) -> table offset
3106	shl		\$4,$i3
3107	jmp		.Locb_enc_loop6
3108
3109.align	32
3110.Locb_enc_loop6:
3111	aesenc		$rndkey1,$inout0
3112	aesenc		$rndkey1,$inout1
3113	aesenc		$rndkey1,$inout2
3114	aesenc		$rndkey1,$inout3
3115	aesenc		$rndkey1,$inout4
3116	aesenc		$rndkey1,$inout5
3117	$movkey		($key,%rax),$rndkey1
3118	add		\$32,%rax
3119
3120	aesenc		$rndkey0,$inout0
3121	aesenc		$rndkey0,$inout1
3122	aesenc		$rndkey0,$inout2
3123	aesenc		$rndkey0,$inout3
3124	aesenc		$rndkey0,$inout4
3125	aesenc		$rndkey0,$inout5
3126	$movkey		-16($key,%rax),$rndkey0
3127	jnz		.Locb_enc_loop6
3128
3129	aesenc		$rndkey1,$inout0
3130	aesenc		$rndkey1,$inout1
3131	aesenc		$rndkey1,$inout2
3132	aesenc		$rndkey1,$inout3
3133	aesenc		$rndkey1,$inout4
3134	aesenc		$rndkey1,$inout5
3135	$movkey		16($key_),$rndkey1
3136	shl		\$4,$i5
3137
3138	aesenclast	@offset[0],$inout0
3139	movdqu		($L_p),@offset[0]	# L_0 for all odd-numbered blocks
3140	mov		%r10,%rax		# restore twisted rounds
3141	aesenclast	@offset[1],$inout1
3142	aesenclast	@offset[2],$inout2
3143	aesenclast	@offset[3],$inout3
3144	aesenclast	@offset[4],$inout4
3145	aesenclast	@offset[5],$inout5
3146	ret
3147.cfi_endproc
3148.size	__ocb_encrypt6,.-__ocb_encrypt6
3149
3150.type	__ocb_encrypt4,\@abi-omnipotent
3151.align	32
3152__ocb_encrypt4:
3153.cfi_startproc
3154	 pxor		$rndkey0l,@offset[5]	# offset_i ^ round[0]
3155	 movdqu		($L_p,$i1),@offset[1]
3156	 movdqa		@offset[0],@offset[2]
3157	 movdqu		($L_p,$i3),@offset[3]
3158	 pxor		@offset[5],@offset[0]
3159	 pxor		@offset[0],@offset[1]
3160	pxor		$inout0,$checksum	# accumulate checksum
3161	pxor		@offset[0],$inout0	# input ^ round[0] ^ offset_i
3162	 pxor		@offset[1],@offset[2]
3163	pxor		$inout1,$checksum
3164	pxor		@offset[1],$inout1
3165	 pxor		@offset[2],@offset[3]
3166	pxor		$inout2,$checksum
3167	pxor		@offset[2],$inout2
3168	pxor		$inout3,$checksum
3169	pxor		@offset[3],$inout3
3170	$movkey		32($key_),$rndkey0
3171
3172	 pxor		$rndkey0l,@offset[0]	# offset_i ^ round[last]
3173	 pxor		$rndkey0l,@offset[1]
3174	 pxor		$rndkey0l,@offset[2]
3175	 pxor		$rndkey0l,@offset[3]
3176
3177	aesenc		$rndkey1,$inout0
3178	aesenc		$rndkey1,$inout1
3179	aesenc		$rndkey1,$inout2
3180	aesenc		$rndkey1,$inout3
3181	$movkey		48($key_),$rndkey1
3182
3183	aesenc		$rndkey0,$inout0
3184	aesenc		$rndkey0,$inout1
3185	aesenc		$rndkey0,$inout2
3186	aesenc		$rndkey0,$inout3
3187	$movkey		64($key_),$rndkey0
3188	jmp		.Locb_enc_loop4
3189
3190.align	32
3191.Locb_enc_loop4:
3192	aesenc		$rndkey1,$inout0
3193	aesenc		$rndkey1,$inout1
3194	aesenc		$rndkey1,$inout2
3195	aesenc		$rndkey1,$inout3
3196	$movkey		($key,%rax),$rndkey1
3197	add		\$32,%rax
3198
3199	aesenc		$rndkey0,$inout0
3200	aesenc		$rndkey0,$inout1
3201	aesenc		$rndkey0,$inout2
3202	aesenc		$rndkey0,$inout3
3203	$movkey		-16($key,%rax),$rndkey0
3204	jnz		.Locb_enc_loop4
3205
3206	aesenc		$rndkey1,$inout0
3207	aesenc		$rndkey1,$inout1
3208	aesenc		$rndkey1,$inout2
3209	aesenc		$rndkey1,$inout3
3210	$movkey		16($key_),$rndkey1
3211	mov		%r10,%rax		# restore twisted rounds
3212
3213	aesenclast	@offset[0],$inout0
3214	aesenclast	@offset[1],$inout1
3215	aesenclast	@offset[2],$inout2
3216	aesenclast	@offset[3],$inout3
3217	ret
3218.cfi_endproc
3219.size	__ocb_encrypt4,.-__ocb_encrypt4
3220
3221.type	__ocb_encrypt1,\@abi-omnipotent
3222.align	32
3223__ocb_encrypt1:
3224.cfi_startproc
3225	 pxor		@offset[5],$inout5	# offset_i
3226	 pxor		$rndkey0l,$inout5	# offset_i ^ round[0]
3227	pxor		$inout0,$checksum	# accumulate checksum
3228	pxor		$inout5,$inout0		# input ^ round[0] ^ offset_i
3229	$movkey		32($key_),$rndkey0
3230
3231	aesenc		$rndkey1,$inout0
3232	$movkey		48($key_),$rndkey1
3233	pxor		$rndkey0l,$inout5	# offset_i ^ round[last]
3234
3235	aesenc		$rndkey0,$inout0
3236	$movkey		64($key_),$rndkey0
3237	jmp		.Locb_enc_loop1
3238
3239.align	32
3240.Locb_enc_loop1:
3241	aesenc		$rndkey1,$inout0
3242	$movkey		($key,%rax),$rndkey1
3243	add		\$32,%rax
3244
3245	aesenc		$rndkey0,$inout0
3246	$movkey		-16($key,%rax),$rndkey0
3247	jnz		.Locb_enc_loop1
3248
3249	aesenc		$rndkey1,$inout0
3250	$movkey		16($key_),$rndkey1	# redundant in tail
3251	mov		%r10,%rax		# restore twisted rounds
3252
3253	aesenclast	$inout5,$inout0
3254	ret
3255.cfi_endproc
3256.size	__ocb_encrypt1,.-__ocb_encrypt1
3257
3258.globl	aesni_ocb_decrypt
3259.type	aesni_ocb_decrypt,\@function,6
3260.align	32
3261aesni_ocb_decrypt:
3262.cfi_startproc
3263	endbranch
3264	lea	(%rsp),%rax
3265	push	%rbx
3266.cfi_push	%rbx
3267	push	%rbp
3268.cfi_push	%rbp
3269	push	%r12
3270.cfi_push	%r12
3271	push	%r13
3272.cfi_push	%r13
3273	push	%r14
3274.cfi_push	%r14
3275___
3276$code.=<<___ if ($win64);
3277	lea	-0xa0(%rsp),%rsp
3278	movaps	%xmm6,0x00(%rsp)		# offload everything
3279	movaps	%xmm7,0x10(%rsp)
3280	movaps	%xmm8,0x20(%rsp)
3281	movaps	%xmm9,0x30(%rsp)
3282	movaps	%xmm10,0x40(%rsp)
3283	movaps	%xmm11,0x50(%rsp)
3284	movaps	%xmm12,0x60(%rsp)
3285	movaps	%xmm13,0x70(%rsp)
3286	movaps	%xmm14,0x80(%rsp)
3287	movaps	%xmm15,0x90(%rsp)
3288.Locb_dec_body:
3289___
3290$code.=<<___;
3291	mov	$seventh_arg(%rax),$L_p		# 7th argument
3292	mov	$seventh_arg+8(%rax),$checksum_p# 8th argument
3293
3294	mov	240($key),$rnds_
3295	mov	$key,$key_
3296	shl	\$4,$rnds_
3297	$movkey	($key),$rndkey0l		# round[0]
3298	$movkey	16($key,$rnds_),$rndkey1	# round[last]
3299
3300	movdqu	($offset_p),@offset[5]		# load last offset_i
3301	pxor	$rndkey1,$rndkey0l		# round[0] ^ round[last]
3302	pxor	$rndkey1,@offset[5]		# offset_i ^ round[last]
3303
3304	mov	\$16+32,$rounds
3305	lea	32($key_,$rnds_),$key
3306	$movkey	16($key_),$rndkey1		# round[1]
3307	sub	%r10,%rax			# twisted $rounds
3308	mov	%rax,%r10			# backup twisted $rounds
3309
3310	movdqu	($L_p),@offset[0]		# L_0 for all odd-numbered blocks
3311	movdqu	($checksum_p),$checksum		# load checksum
3312
3313	test	\$1,$block_num			# is first block number odd?
3314	jnz	.Locb_dec_odd
3315
3316	bsf	$block_num,$i1
3317	add	\$1,$block_num
3318	shl	\$4,$i1
3319	movdqu	($L_p,$i1),$inout5		# borrow
3320	movdqu	($inp),$inout0
3321	lea	16($inp),$inp
3322
3323	call	__ocb_decrypt1
3324
3325	movdqa	$inout5,@offset[5]
3326	movups	$inout0,($out)
3327	xorps	$inout0,$checksum		# accumulate checksum
3328	lea	16($out),$out
3329	sub	\$1,$blocks
3330	jz	.Locb_dec_done
3331
3332.Locb_dec_odd:
3333	lea	1($block_num),$i1		# even-numbered blocks
3334	lea	3($block_num),$i3
3335	lea	5($block_num),$i5
3336	lea	6($block_num),$block_num
3337	bsf	$i1,$i1				# ntz(block)
3338	bsf	$i3,$i3
3339	bsf	$i5,$i5
3340	shl	\$4,$i1				# ntz(block) -> table offset
3341	shl	\$4,$i3
3342	shl	\$4,$i5
3343
3344	sub	\$6,$blocks
3345	jc	.Locb_dec_short
3346	jmp	.Locb_dec_grandloop
3347
3348.align	32
3349.Locb_dec_grandloop:
3350	movdqu	`16*0`($inp),$inout0		# load input
3351	movdqu	`16*1`($inp),$inout1
3352	movdqu	`16*2`($inp),$inout2
3353	movdqu	`16*3`($inp),$inout3
3354	movdqu	`16*4`($inp),$inout4
3355	movdqu	`16*5`($inp),$inout5
3356	lea	`16*6`($inp),$inp
3357
3358	call	__ocb_decrypt6
3359
3360	movups	$inout0,`16*0`($out)		# store output
3361	pxor	$inout0,$checksum		# accumulate checksum
3362	movups	$inout1,`16*1`($out)
3363	pxor	$inout1,$checksum
3364	movups	$inout2,`16*2`($out)
3365	pxor	$inout2,$checksum
3366	movups	$inout3,`16*3`($out)
3367	pxor	$inout3,$checksum
3368	movups	$inout4,`16*4`($out)
3369	pxor	$inout4,$checksum
3370	movups	$inout5,`16*5`($out)
3371	pxor	$inout5,$checksum
3372	lea	`16*6`($out),$out
3373	sub	\$6,$blocks
3374	jnc	.Locb_dec_grandloop
3375
3376.Locb_dec_short:
3377	add	\$6,$blocks
3378	jz	.Locb_dec_done
3379
3380	movdqu	`16*0`($inp),$inout0
3381	cmp	\$2,$blocks
3382	jb	.Locb_dec_one
3383	movdqu	`16*1`($inp),$inout1
3384	je	.Locb_dec_two
3385
3386	movdqu	`16*2`($inp),$inout2
3387	cmp	\$4,$blocks
3388	jb	.Locb_dec_three
3389	movdqu	`16*3`($inp),$inout3
3390	je	.Locb_dec_four
3391
3392	movdqu	`16*4`($inp),$inout4
3393	pxor	$inout5,$inout5
3394
3395	call	__ocb_decrypt6
3396
3397	movdqa	@offset[4],@offset[5]
3398	movups	$inout0,`16*0`($out)		# store output
3399	pxor	$inout0,$checksum		# accumulate checksum
3400	movups	$inout1,`16*1`($out)
3401	pxor	$inout1,$checksum
3402	movups	$inout2,`16*2`($out)
3403	pxor	$inout2,$checksum
3404	movups	$inout3,`16*3`($out)
3405	pxor	$inout3,$checksum
3406	movups	$inout4,`16*4`($out)
3407	pxor	$inout4,$checksum
3408
3409	jmp	.Locb_dec_done
3410
3411.align	16
3412.Locb_dec_one:
3413	movdqa	@offset[0],$inout5		# borrow
3414
3415	call	__ocb_decrypt1
3416
3417	movdqa	$inout5,@offset[5]
3418	movups	$inout0,`16*0`($out)		# store output
3419	xorps	$inout0,$checksum		# accumulate checksum
3420	jmp	.Locb_dec_done
3421
3422.align	16
3423.Locb_dec_two:
3424	pxor	$inout2,$inout2
3425	pxor	$inout3,$inout3
3426
3427	call	__ocb_decrypt4
3428
3429	movdqa	@offset[1],@offset[5]
3430	movups	$inout0,`16*0`($out)		# store output
3431	xorps	$inout0,$checksum		# accumulate checksum
3432	movups	$inout1,`16*1`($out)
3433	xorps	$inout1,$checksum
3434
3435	jmp	.Locb_dec_done
3436
3437.align	16
3438.Locb_dec_three:
3439	pxor	$inout3,$inout3
3440
3441	call	__ocb_decrypt4
3442
3443	movdqa	@offset[2],@offset[5]
3444	movups	$inout0,`16*0`($out)		# store output
3445	xorps	$inout0,$checksum		# accumulate checksum
3446	movups	$inout1,`16*1`($out)
3447	xorps	$inout1,$checksum
3448	movups	$inout2,`16*2`($out)
3449	xorps	$inout2,$checksum
3450
3451	jmp	.Locb_dec_done
3452
3453.align	16
3454.Locb_dec_four:
3455	call	__ocb_decrypt4
3456
3457	movdqa	@offset[3],@offset[5]
3458	movups	$inout0,`16*0`($out)		# store output
3459	pxor	$inout0,$checksum		# accumulate checksum
3460	movups	$inout1,`16*1`($out)
3461	pxor	$inout1,$checksum
3462	movups	$inout2,`16*2`($out)
3463	pxor	$inout2,$checksum
3464	movups	$inout3,`16*3`($out)
3465	pxor	$inout3,$checksum
3466
3467.Locb_dec_done:
3468	pxor	$rndkey0,@offset[5]		# "remove" round[last]
3469	movdqu	$checksum,($checksum_p)		# store checksum
3470	movdqu	@offset[5],($offset_p)		# store last offset_i
3471
3472	xorps	%xmm0,%xmm0			# clear register bank
3473	pxor	%xmm1,%xmm1
3474	pxor	%xmm2,%xmm2
3475	pxor	%xmm3,%xmm3
3476	pxor	%xmm4,%xmm4
3477	pxor	%xmm5,%xmm5
3478___
3479$code.=<<___ if (!$win64);
3480	pxor	%xmm6,%xmm6
3481	pxor	%xmm7,%xmm7
3482	pxor	%xmm8,%xmm8
3483	pxor	%xmm9,%xmm9
3484	pxor	%xmm10,%xmm10
3485	pxor	%xmm11,%xmm11
3486	pxor	%xmm12,%xmm12
3487	pxor	%xmm13,%xmm13
3488	pxor	%xmm14,%xmm14
3489	pxor	%xmm15,%xmm15
3490	lea	0x28(%rsp),%rax
3491.cfi_def_cfa	%rax,8
3492___
3493$code.=<<___ if ($win64);
3494	movaps	0x00(%rsp),%xmm6
3495	movaps	%xmm0,0x00(%rsp)		# clear stack
3496	movaps	0x10(%rsp),%xmm7
3497	movaps	%xmm0,0x10(%rsp)
3498	movaps	0x20(%rsp),%xmm8
3499	movaps	%xmm0,0x20(%rsp)
3500	movaps	0x30(%rsp),%xmm9
3501	movaps	%xmm0,0x30(%rsp)
3502	movaps	0x40(%rsp),%xmm10
3503	movaps	%xmm0,0x40(%rsp)
3504	movaps	0x50(%rsp),%xmm11
3505	movaps	%xmm0,0x50(%rsp)
3506	movaps	0x60(%rsp),%xmm12
3507	movaps	%xmm0,0x60(%rsp)
3508	movaps	0x70(%rsp),%xmm13
3509	movaps	%xmm0,0x70(%rsp)
3510	movaps	0x80(%rsp),%xmm14
3511	movaps	%xmm0,0x80(%rsp)
3512	movaps	0x90(%rsp),%xmm15
3513	movaps	%xmm0,0x90(%rsp)
3514	lea	0xa0+0x28(%rsp),%rax
3515.Locb_dec_pop:
3516___
3517$code.=<<___;
3518	mov	-40(%rax),%r14
3519.cfi_restore	%r14
3520	mov	-32(%rax),%r13
3521.cfi_restore	%r13
3522	mov	-24(%rax),%r12
3523.cfi_restore	%r12
3524	mov	-16(%rax),%rbp
3525.cfi_restore	%rbp
3526	mov	-8(%rax),%rbx
3527.cfi_restore	%rbx
3528	lea	(%rax),%rsp
3529.cfi_def_cfa_register	%rsp
3530.Locb_dec_epilogue:
3531	ret
3532.cfi_endproc
3533.size	aesni_ocb_decrypt,.-aesni_ocb_decrypt
3534
3535.type	__ocb_decrypt6,\@abi-omnipotent
3536.align	32
3537__ocb_decrypt6:
3538.cfi_startproc
3539	 pxor		$rndkey0l,@offset[5]	# offset_i ^ round[0]
3540	 movdqu		($L_p,$i1),@offset[1]
3541	 movdqa		@offset[0],@offset[2]
3542	 movdqu		($L_p,$i3),@offset[3]
3543	 movdqa		@offset[0],@offset[4]
3544	 pxor		@offset[5],@offset[0]
3545	 movdqu		($L_p,$i5),@offset[5]
3546	 pxor		@offset[0],@offset[1]
3547	pxor		@offset[0],$inout0	# input ^ round[0] ^ offset_i
3548	 pxor		@offset[1],@offset[2]
3549	pxor		@offset[1],$inout1
3550	 pxor		@offset[2],@offset[3]
3551	pxor		@offset[2],$inout2
3552	 pxor		@offset[3],@offset[4]
3553	pxor		@offset[3],$inout3
3554	 pxor		@offset[4],@offset[5]
3555	pxor		@offset[4],$inout4
3556	pxor		@offset[5],$inout5
3557	$movkey		32($key_),$rndkey0
3558
3559	lea		1($block_num),$i1	# even-numbered blocks
3560	lea		3($block_num),$i3
3561	lea		5($block_num),$i5
3562	add		\$6,$block_num
3563	 pxor		$rndkey0l,@offset[0]	# offset_i ^ round[last]
3564	bsf		$i1,$i1			# ntz(block)
3565	bsf		$i3,$i3
3566	bsf		$i5,$i5
3567
3568	aesdec		$rndkey1,$inout0
3569	aesdec		$rndkey1,$inout1
3570	aesdec		$rndkey1,$inout2
3571	aesdec		$rndkey1,$inout3
3572	 pxor		$rndkey0l,@offset[1]
3573	 pxor		$rndkey0l,@offset[2]
3574	aesdec		$rndkey1,$inout4
3575	 pxor		$rndkey0l,@offset[3]
3576	 pxor		$rndkey0l,@offset[4]
3577	aesdec		$rndkey1,$inout5
3578	$movkey		48($key_),$rndkey1
3579	 pxor		$rndkey0l,@offset[5]
3580
3581	aesdec		$rndkey0,$inout0
3582	aesdec		$rndkey0,$inout1
3583	aesdec		$rndkey0,$inout2
3584	aesdec		$rndkey0,$inout3
3585	aesdec		$rndkey0,$inout4
3586	aesdec		$rndkey0,$inout5
3587	$movkey		64($key_),$rndkey0
3588	shl		\$4,$i1			# ntz(block) -> table offset
3589	shl		\$4,$i3
3590	jmp		.Locb_dec_loop6
3591
3592.align	32
3593.Locb_dec_loop6:
3594	aesdec		$rndkey1,$inout0
3595	aesdec		$rndkey1,$inout1
3596	aesdec		$rndkey1,$inout2
3597	aesdec		$rndkey1,$inout3
3598	aesdec		$rndkey1,$inout4
3599	aesdec		$rndkey1,$inout5
3600	$movkey		($key,%rax),$rndkey1
3601	add		\$32,%rax
3602
3603	aesdec		$rndkey0,$inout0
3604	aesdec		$rndkey0,$inout1
3605	aesdec		$rndkey0,$inout2
3606	aesdec		$rndkey0,$inout3
3607	aesdec		$rndkey0,$inout4
3608	aesdec		$rndkey0,$inout5
3609	$movkey		-16($key,%rax),$rndkey0
3610	jnz		.Locb_dec_loop6
3611
3612	aesdec		$rndkey1,$inout0
3613	aesdec		$rndkey1,$inout1
3614	aesdec		$rndkey1,$inout2
3615	aesdec		$rndkey1,$inout3
3616	aesdec		$rndkey1,$inout4
3617	aesdec		$rndkey1,$inout5
3618	$movkey		16($key_),$rndkey1
3619	shl		\$4,$i5
3620
3621	aesdeclast	@offset[0],$inout0
3622	movdqu		($L_p),@offset[0]	# L_0 for all odd-numbered blocks
3623	mov		%r10,%rax		# restore twisted rounds
3624	aesdeclast	@offset[1],$inout1
3625	aesdeclast	@offset[2],$inout2
3626	aesdeclast	@offset[3],$inout3
3627	aesdeclast	@offset[4],$inout4
3628	aesdeclast	@offset[5],$inout5
3629	ret
3630.cfi_endproc
3631.size	__ocb_decrypt6,.-__ocb_decrypt6
3632
3633.type	__ocb_decrypt4,\@abi-omnipotent
3634.align	32
3635__ocb_decrypt4:
3636.cfi_startproc
3637	 pxor		$rndkey0l,@offset[5]	# offset_i ^ round[0]
3638	 movdqu		($L_p,$i1),@offset[1]
3639	 movdqa		@offset[0],@offset[2]
3640	 movdqu		($L_p,$i3),@offset[3]
3641	 pxor		@offset[5],@offset[0]
3642	 pxor		@offset[0],@offset[1]
3643	pxor		@offset[0],$inout0	# input ^ round[0] ^ offset_i
3644	 pxor		@offset[1],@offset[2]
3645	pxor		@offset[1],$inout1
3646	 pxor		@offset[2],@offset[3]
3647	pxor		@offset[2],$inout2
3648	pxor		@offset[3],$inout3
3649	$movkey		32($key_),$rndkey0
3650
3651	 pxor		$rndkey0l,@offset[0]	# offset_i ^ round[last]
3652	 pxor		$rndkey0l,@offset[1]
3653	 pxor		$rndkey0l,@offset[2]
3654	 pxor		$rndkey0l,@offset[3]
3655
3656	aesdec		$rndkey1,$inout0
3657	aesdec		$rndkey1,$inout1
3658	aesdec		$rndkey1,$inout2
3659	aesdec		$rndkey1,$inout3
3660	$movkey		48($key_),$rndkey1
3661
3662	aesdec		$rndkey0,$inout0
3663	aesdec		$rndkey0,$inout1
3664	aesdec		$rndkey0,$inout2
3665	aesdec		$rndkey0,$inout3
3666	$movkey		64($key_),$rndkey0
3667	jmp		.Locb_dec_loop4
3668
3669.align	32
3670.Locb_dec_loop4:
3671	aesdec		$rndkey1,$inout0
3672	aesdec		$rndkey1,$inout1
3673	aesdec		$rndkey1,$inout2
3674	aesdec		$rndkey1,$inout3
3675	$movkey		($key,%rax),$rndkey1
3676	add		\$32,%rax
3677
3678	aesdec		$rndkey0,$inout0
3679	aesdec		$rndkey0,$inout1
3680	aesdec		$rndkey0,$inout2
3681	aesdec		$rndkey0,$inout3
3682	$movkey		-16($key,%rax),$rndkey0
3683	jnz		.Locb_dec_loop4
3684
3685	aesdec		$rndkey1,$inout0
3686	aesdec		$rndkey1,$inout1
3687	aesdec		$rndkey1,$inout2
3688	aesdec		$rndkey1,$inout3
3689	$movkey		16($key_),$rndkey1
3690	mov		%r10,%rax		# restore twisted rounds
3691
3692	aesdeclast	@offset[0],$inout0
3693	aesdeclast	@offset[1],$inout1
3694	aesdeclast	@offset[2],$inout2
3695	aesdeclast	@offset[3],$inout3
3696	ret
3697.cfi_endproc
3698.size	__ocb_decrypt4,.-__ocb_decrypt4
3699
3700.type	__ocb_decrypt1,\@abi-omnipotent
3701.align	32
3702__ocb_decrypt1:
3703.cfi_startproc
3704	 pxor		@offset[5],$inout5	# offset_i
3705	 pxor		$rndkey0l,$inout5	# offset_i ^ round[0]
3706	pxor		$inout5,$inout0		# input ^ round[0] ^ offset_i
3707	$movkey		32($key_),$rndkey0
3708
3709	aesdec		$rndkey1,$inout0
3710	$movkey		48($key_),$rndkey1
3711	pxor		$rndkey0l,$inout5	# offset_i ^ round[last]
3712
3713	aesdec		$rndkey0,$inout0
3714	$movkey		64($key_),$rndkey0
3715	jmp		.Locb_dec_loop1
3716
3717.align	32
3718.Locb_dec_loop1:
3719	aesdec		$rndkey1,$inout0
3720	$movkey		($key,%rax),$rndkey1
3721	add		\$32,%rax
3722
3723	aesdec		$rndkey0,$inout0
3724	$movkey		-16($key,%rax),$rndkey0
3725	jnz		.Locb_dec_loop1
3726
3727	aesdec		$rndkey1,$inout0
3728	$movkey		16($key_),$rndkey1	# redundant in tail
3729	mov		%r10,%rax		# restore twisted rounds
3730
3731	aesdeclast	$inout5,$inout0
3732	ret
3733.cfi_endproc
3734.size	__ocb_decrypt1,.-__ocb_decrypt1
3735___
3736} }}
3737
3738########################################################################
3739# void $PREFIX_cbc_encrypt (const void *inp, void *out,
3740#			    size_t length, const AES_KEY *key,
3741#			    unsigned char *ivp,const int enc);
3742{
3743my $frame_size = 0x10 + ($win64?0xa0:0);	# used in decrypt
3744my ($iv,$in0,$in1,$in2,$in3,$in4)=map("%xmm$_",(10..15));
3745
3746$code.=<<___;
3747.globl	${PREFIX}_cbc_encrypt
3748.type	${PREFIX}_cbc_encrypt,\@function,6
3749.align	16
3750${PREFIX}_cbc_encrypt:
3751.cfi_startproc
3752	endbranch
3753	test	$len,$len		# check length
3754	jz	.Lcbc_ret
3755
3756	mov	240($key),$rnds_	# key->rounds
3757	mov	$key,$key_		# backup $key
3758	test	%r9d,%r9d		# 6th argument
3759	jz	.Lcbc_decrypt
3760#--------------------------- CBC ENCRYPT ------------------------------#
3761	movups	($ivp),$inout0		# load iv as initial state
3762	mov	$rnds_,$rounds
3763	cmp	\$16,$len
3764	jb	.Lcbc_enc_tail
3765	sub	\$16,$len
3766	jmp	.Lcbc_enc_loop
3767.align	16
3768.Lcbc_enc_loop:
3769	movups	($inp),$inout1		# load input
3770	lea	16($inp),$inp
3771	#xorps	$inout1,$inout0
3772___
3773	&aesni_generate1("enc",$key,$rounds,$inout0,$inout1);
3774$code.=<<___;
3775	mov	$rnds_,$rounds		# restore $rounds
3776	mov	$key_,$key		# restore $key
3777	movups	$inout0,0($out)		# store output
3778	lea	16($out),$out
3779	sub	\$16,$len
3780	jnc	.Lcbc_enc_loop
3781	add	\$16,$len
3782	jnz	.Lcbc_enc_tail
3783	 pxor	$rndkey0,$rndkey0	# clear register bank
3784	 pxor	$rndkey1,$rndkey1
3785	movups	$inout0,($ivp)
3786	 pxor	$inout0,$inout0
3787	 pxor	$inout1,$inout1
3788	jmp	.Lcbc_ret
3789
3790.Lcbc_enc_tail:
3791	mov	$len,%rcx	# zaps $key
3792	xchg	$inp,$out	# $inp is %rsi and $out is %rdi now
3793	.long	0x9066A4F3	# rep movsb
3794	mov	\$16,%ecx	# zero tail
3795	sub	$len,%rcx
3796	xor	%eax,%eax
3797	.long	0x9066AAF3	# rep stosb
3798	lea	-16(%rdi),%rdi	# rewind $out by 1 block
3799	mov	$rnds_,$rounds	# restore $rounds
3800	mov	%rdi,%rsi	# $inp and $out are the same
3801	mov	$key_,$key	# restore $key
3802	xor	$len,$len	# len=16
3803	jmp	.Lcbc_enc_loop	# one more spin
3804#--------------------------- CBC DECRYPT ------------------------------#
3805.align	16
3806.Lcbc_decrypt:
3807	cmp	\$16,$len
3808	jne	.Lcbc_decrypt_bulk
3809
3810	# handle single block without allocating stack frame,
3811	# useful in ciphertext stealing mode
3812	movdqu	($inp),$inout0		# load input
3813	movdqu	($ivp),$inout1		# load iv
3814	movdqa	$inout0,$inout2		# future iv
3815___
3816	&aesni_generate1("dec",$key,$rnds_);
3817$code.=<<___;
3818	 pxor	$rndkey0,$rndkey0	# clear register bank
3819	 pxor	$rndkey1,$rndkey1
3820	movdqu	$inout2,($ivp)		# store iv
3821	xorps	$inout1,$inout0		# ^=iv
3822	 pxor	$inout1,$inout1
3823	movups	$inout0,($out)		# store output
3824	 pxor	$inout0,$inout0
3825	jmp	.Lcbc_ret
3826.align	16
3827.Lcbc_decrypt_bulk:
3828	lea	(%rsp),%r11		# frame pointer
3829.cfi_def_cfa_register	%r11
3830	push	%rbp
3831.cfi_push	%rbp
3832	sub	\$$frame_size,%rsp
3833	and	\$-16,%rsp	# Linux kernel stack can be incorrectly seeded
3834___
3835$code.=<<___ if ($win64);
3836	movaps	%xmm6,0x10(%rsp)
3837	movaps	%xmm7,0x20(%rsp)
3838	movaps	%xmm8,0x30(%rsp)
3839	movaps	%xmm9,0x40(%rsp)
3840	movaps	%xmm10,0x50(%rsp)
3841	movaps	%xmm11,0x60(%rsp)
3842	movaps	%xmm12,0x70(%rsp)
3843	movaps	%xmm13,0x80(%rsp)
3844	movaps	%xmm14,0x90(%rsp)
3845	movaps	%xmm15,0xa0(%rsp)
3846.Lcbc_decrypt_body:
3847___
3848
3849my $inp_=$key_="%rbp";			# reassign $key_
3850
3851$code.=<<___;
3852	mov	$key,$key_		# [re-]backup $key [after reassignment]
3853	movups	($ivp),$iv
3854	mov	$rnds_,$rounds
3855	cmp	\$0x50,$len
3856	jbe	.Lcbc_dec_tail
3857
3858	$movkey	($key),$rndkey0
3859	movdqu	0x00($inp),$inout0	# load input
3860	movdqu	0x10($inp),$inout1
3861	movdqa	$inout0,$in0
3862	movdqu	0x20($inp),$inout2
3863	movdqa	$inout1,$in1
3864	movdqu	0x30($inp),$inout3
3865	movdqa	$inout2,$in2
3866	movdqu	0x40($inp),$inout4
3867	movdqa	$inout3,$in3
3868	movdqu	0x50($inp),$inout5
3869	movdqa	$inout4,$in4
3870	mov	OPENSSL_ia32cap_P+4(%rip),%r9d
3871	cmp	\$0x70,$len
3872	jbe	.Lcbc_dec_six_or_seven
3873
3874	and	\$`1<<26|1<<22`,%r9d	# isolate XSAVE+MOVBE
3875	sub	\$0x50,$len		# $len is biased by -5*16
3876	cmp	\$`1<<22`,%r9d		# check for MOVBE without XSAVE
3877	je	.Lcbc_dec_loop6_enter	# [which denotes Atom Silvermont]
3878	sub	\$0x20,$len		# $len is biased by -7*16
3879	lea	0x70($key),$key		# size optimization
3880	jmp	.Lcbc_dec_loop8_enter
3881.align	16
3882.Lcbc_dec_loop8:
3883	movups	$inout7,($out)
3884	lea	0x10($out),$out
3885.Lcbc_dec_loop8_enter:
3886	movdqu		0x60($inp),$inout6
3887	pxor		$rndkey0,$inout0
3888	movdqu		0x70($inp),$inout7
3889	pxor		$rndkey0,$inout1
3890	$movkey		0x10-0x70($key),$rndkey1
3891	pxor		$rndkey0,$inout2
3892	mov		\$-1,$inp_
3893	cmp		\$0x70,$len	# is there at least 0x60 bytes ahead?
3894	pxor		$rndkey0,$inout3
3895	pxor		$rndkey0,$inout4
3896	pxor		$rndkey0,$inout5
3897	pxor		$rndkey0,$inout6
3898
3899	aesdec		$rndkey1,$inout0
3900	pxor		$rndkey0,$inout7
3901	$movkey		0x20-0x70($key),$rndkey0
3902	aesdec		$rndkey1,$inout1
3903	aesdec		$rndkey1,$inout2
3904	aesdec		$rndkey1,$inout3
3905	aesdec		$rndkey1,$inout4
3906	aesdec		$rndkey1,$inout5
3907	aesdec		$rndkey1,$inout6
3908	adc		\$0,$inp_
3909	and		\$128,$inp_
3910	aesdec		$rndkey1,$inout7
3911	add		$inp,$inp_
3912	$movkey		0x30-0x70($key),$rndkey1
3913___
3914for($i=1;$i<12;$i++) {
3915my $rndkeyx = ($i&1)?$rndkey0:$rndkey1;
3916$code.=<<___	if ($i==7);
3917	cmp		\$11,$rounds
3918___
3919$code.=<<___;
3920	aesdec		$rndkeyx,$inout0
3921	aesdec		$rndkeyx,$inout1
3922	aesdec		$rndkeyx,$inout2
3923	aesdec		$rndkeyx,$inout3
3924	aesdec		$rndkeyx,$inout4
3925	aesdec		$rndkeyx,$inout5
3926	aesdec		$rndkeyx,$inout6
3927	aesdec		$rndkeyx,$inout7
3928	$movkey		`0x30+0x10*$i`-0x70($key),$rndkeyx
3929___
3930$code.=<<___	if ($i<6 || (!($i&1) && $i>7));
3931	nop
3932___
3933$code.=<<___	if ($i==7);
3934	jb		.Lcbc_dec_done
3935___
3936$code.=<<___	if ($i==9);
3937	je		.Lcbc_dec_done
3938___
3939$code.=<<___	if ($i==11);
3940	jmp		.Lcbc_dec_done
3941___
3942}
3943$code.=<<___;
3944.align	16
3945.Lcbc_dec_done:
3946	aesdec		$rndkey1,$inout0
3947	aesdec		$rndkey1,$inout1
3948	pxor		$rndkey0,$iv
3949	pxor		$rndkey0,$in0
3950	aesdec		$rndkey1,$inout2
3951	aesdec		$rndkey1,$inout3
3952	pxor		$rndkey0,$in1
3953	pxor		$rndkey0,$in2
3954	aesdec		$rndkey1,$inout4
3955	aesdec		$rndkey1,$inout5
3956	pxor		$rndkey0,$in3
3957	pxor		$rndkey0,$in4
3958	aesdec		$rndkey1,$inout6
3959	aesdec		$rndkey1,$inout7
3960	movdqu		0x50($inp),$rndkey1
3961
3962	aesdeclast	$iv,$inout0
3963	movdqu		0x60($inp),$iv		# borrow $iv
3964	pxor		$rndkey0,$rndkey1
3965	aesdeclast	$in0,$inout1
3966	pxor		$rndkey0,$iv
3967	movdqu		0x70($inp),$rndkey0	# next IV
3968	aesdeclast	$in1,$inout2
3969	lea		0x80($inp),$inp
3970	movdqu		0x00($inp_),$in0
3971	aesdeclast	$in2,$inout3
3972	aesdeclast	$in3,$inout4
3973	movdqu		0x10($inp_),$in1
3974	movdqu		0x20($inp_),$in2
3975	aesdeclast	$in4,$inout5
3976	aesdeclast	$rndkey1,$inout6
3977	movdqu		0x30($inp_),$in3
3978	movdqu		0x40($inp_),$in4
3979	aesdeclast	$iv,$inout7
3980	movdqa		$rndkey0,$iv		# return $iv
3981	movdqu		0x50($inp_),$rndkey1
3982	$movkey		-0x70($key),$rndkey0
3983
3984	movups		$inout0,($out)		# store output
3985	movdqa		$in0,$inout0
3986	movups		$inout1,0x10($out)
3987	movdqa		$in1,$inout1
3988	movups		$inout2,0x20($out)
3989	movdqa		$in2,$inout2
3990	movups		$inout3,0x30($out)
3991	movdqa		$in3,$inout3
3992	movups		$inout4,0x40($out)
3993	movdqa		$in4,$inout4
3994	movups		$inout5,0x50($out)
3995	movdqa		$rndkey1,$inout5
3996	movups		$inout6,0x60($out)
3997	lea		0x70($out),$out
3998
3999	sub	\$0x80,$len
4000	ja	.Lcbc_dec_loop8
4001
4002	movaps	$inout7,$inout0
4003	lea	-0x70($key),$key
4004	add	\$0x70,$len
4005	jle	.Lcbc_dec_clear_tail_collected
4006	movups	$inout7,($out)
4007	lea	0x10($out),$out
4008	cmp	\$0x50,$len
4009	jbe	.Lcbc_dec_tail
4010
4011	movaps	$in0,$inout0
4012.Lcbc_dec_six_or_seven:
4013	cmp	\$0x60,$len
4014	ja	.Lcbc_dec_seven
4015
4016	movaps	$inout5,$inout6
4017	call	_aesni_decrypt6
4018	pxor	$iv,$inout0		# ^= IV
4019	movaps	$inout6,$iv
4020	pxor	$in0,$inout1
4021	movdqu	$inout0,($out)
4022	pxor	$in1,$inout2
4023	movdqu	$inout1,0x10($out)
4024	 pxor	$inout1,$inout1		# clear register bank
4025	pxor	$in2,$inout3
4026	movdqu	$inout2,0x20($out)
4027	 pxor	$inout2,$inout2
4028	pxor	$in3,$inout4
4029	movdqu	$inout3,0x30($out)
4030	 pxor	$inout3,$inout3
4031	pxor	$in4,$inout5
4032	movdqu	$inout4,0x40($out)
4033	 pxor	$inout4,$inout4
4034	lea	0x50($out),$out
4035	movdqa	$inout5,$inout0
4036	 pxor	$inout5,$inout5
4037	jmp	.Lcbc_dec_tail_collected
4038
4039.align	16
4040.Lcbc_dec_seven:
4041	movups	0x60($inp),$inout6
4042	xorps	$inout7,$inout7
4043	call	_aesni_decrypt8
4044	movups	0x50($inp),$inout7
4045	pxor	$iv,$inout0		# ^= IV
4046	movups	0x60($inp),$iv
4047	pxor	$in0,$inout1
4048	movdqu	$inout0,($out)
4049	pxor	$in1,$inout2
4050	movdqu	$inout1,0x10($out)
4051	 pxor	$inout1,$inout1		# clear register bank
4052	pxor	$in2,$inout3
4053	movdqu	$inout2,0x20($out)
4054	 pxor	$inout2,$inout2
4055	pxor	$in3,$inout4
4056	movdqu	$inout3,0x30($out)
4057	 pxor	$inout3,$inout3
4058	pxor	$in4,$inout5
4059	movdqu	$inout4,0x40($out)
4060	 pxor	$inout4,$inout4
4061	pxor	$inout7,$inout6
4062	movdqu	$inout5,0x50($out)
4063	 pxor	$inout5,$inout5
4064	lea	0x60($out),$out
4065	movdqa	$inout6,$inout0
4066	 pxor	$inout6,$inout6
4067	 pxor	$inout7,$inout7
4068	jmp	.Lcbc_dec_tail_collected
4069
4070.align	16
4071.Lcbc_dec_loop6:
4072	movups	$inout5,($out)
4073	lea	0x10($out),$out
4074	movdqu	0x00($inp),$inout0	# load input
4075	movdqu	0x10($inp),$inout1
4076	movdqa	$inout0,$in0
4077	movdqu	0x20($inp),$inout2
4078	movdqa	$inout1,$in1
4079	movdqu	0x30($inp),$inout3
4080	movdqa	$inout2,$in2
4081	movdqu	0x40($inp),$inout4
4082	movdqa	$inout3,$in3
4083	movdqu	0x50($inp),$inout5
4084	movdqa	$inout4,$in4
4085.Lcbc_dec_loop6_enter:
4086	lea	0x60($inp),$inp
4087	movdqa	$inout5,$inout6
4088
4089	call	_aesni_decrypt6
4090
4091	pxor	$iv,$inout0		# ^= IV
4092	movdqa	$inout6,$iv
4093	pxor	$in0,$inout1
4094	movdqu	$inout0,($out)
4095	pxor	$in1,$inout2
4096	movdqu	$inout1,0x10($out)
4097	pxor	$in2,$inout3
4098	movdqu	$inout2,0x20($out)
4099	pxor	$in3,$inout4
4100	mov	$key_,$key
4101	movdqu	$inout3,0x30($out)
4102	pxor	$in4,$inout5
4103	mov	$rnds_,$rounds
4104	movdqu	$inout4,0x40($out)
4105	lea	0x50($out),$out
4106	sub	\$0x60,$len
4107	ja	.Lcbc_dec_loop6
4108
4109	movdqa	$inout5,$inout0
4110	add	\$0x50,$len
4111	jle	.Lcbc_dec_clear_tail_collected
4112	movups	$inout5,($out)
4113	lea	0x10($out),$out
4114
4115.Lcbc_dec_tail:
4116	movups	($inp),$inout0
4117	sub	\$0x10,$len
4118	jbe	.Lcbc_dec_one		# $len is 1*16 or less
4119
4120	movups	0x10($inp),$inout1
4121	movaps	$inout0,$in0
4122	sub	\$0x10,$len
4123	jbe	.Lcbc_dec_two		# $len is 2*16 or less
4124
4125	movups	0x20($inp),$inout2
4126	movaps	$inout1,$in1
4127	sub	\$0x10,$len
4128	jbe	.Lcbc_dec_three		# $len is 3*16 or less
4129
4130	movups	0x30($inp),$inout3
4131	movaps	$inout2,$in2
4132	sub	\$0x10,$len
4133	jbe	.Lcbc_dec_four		# $len is 4*16 or less
4134
4135	movups	0x40($inp),$inout4	# $len is 5*16 or less
4136	movaps	$inout3,$in3
4137	movaps	$inout4,$in4
4138	xorps	$inout5,$inout5
4139	call	_aesni_decrypt6
4140	pxor	$iv,$inout0
4141	movaps	$in4,$iv
4142	pxor	$in0,$inout1
4143	movdqu	$inout0,($out)
4144	pxor	$in1,$inout2
4145	movdqu	$inout1,0x10($out)
4146	 pxor	$inout1,$inout1		# clear register bank
4147	pxor	$in2,$inout3
4148	movdqu	$inout2,0x20($out)
4149	 pxor	$inout2,$inout2
4150	pxor	$in3,$inout4
4151	movdqu	$inout3,0x30($out)
4152	 pxor	$inout3,$inout3
4153	lea	0x40($out),$out
4154	movdqa	$inout4,$inout0
4155	 pxor	$inout4,$inout4
4156	 pxor	$inout5,$inout5
4157	sub	\$0x10,$len
4158	jmp	.Lcbc_dec_tail_collected
4159
4160.align	16
4161.Lcbc_dec_one:
4162	movaps	$inout0,$in0
4163___
4164	&aesni_generate1("dec",$key,$rounds);
4165$code.=<<___;
4166	xorps	$iv,$inout0
4167	movaps	$in0,$iv
4168	jmp	.Lcbc_dec_tail_collected
4169.align	16
4170.Lcbc_dec_two:
4171	movaps	$inout1,$in1
4172	call	_aesni_decrypt2
4173	pxor	$iv,$inout0
4174	movaps	$in1,$iv
4175	pxor	$in0,$inout1
4176	movdqu	$inout0,($out)
4177	movdqa	$inout1,$inout0
4178	 pxor	$inout1,$inout1		# clear register bank
4179	lea	0x10($out),$out
4180	jmp	.Lcbc_dec_tail_collected
4181.align	16
4182.Lcbc_dec_three:
4183	movaps	$inout2,$in2
4184	call	_aesni_decrypt3
4185	pxor	$iv,$inout0
4186	movaps	$in2,$iv
4187	pxor	$in0,$inout1
4188	movdqu	$inout0,($out)
4189	pxor	$in1,$inout2
4190	movdqu	$inout1,0x10($out)
4191	 pxor	$inout1,$inout1		# clear register bank
4192	movdqa	$inout2,$inout0
4193	 pxor	$inout2,$inout2
4194	lea	0x20($out),$out
4195	jmp	.Lcbc_dec_tail_collected
4196.align	16
4197.Lcbc_dec_four:
4198	movaps	$inout3,$in3
4199	call	_aesni_decrypt4
4200	pxor	$iv,$inout0
4201	movaps	$in3,$iv
4202	pxor	$in0,$inout1
4203	movdqu	$inout0,($out)
4204	pxor	$in1,$inout2
4205	movdqu	$inout1,0x10($out)
4206	 pxor	$inout1,$inout1		# clear register bank
4207	pxor	$in2,$inout3
4208	movdqu	$inout2,0x20($out)
4209	 pxor	$inout2,$inout2
4210	movdqa	$inout3,$inout0
4211	 pxor	$inout3,$inout3
4212	lea	0x30($out),$out
4213	jmp	.Lcbc_dec_tail_collected
4214
4215.align	16
4216.Lcbc_dec_clear_tail_collected:
4217	pxor	$inout1,$inout1		# clear register bank
4218	pxor	$inout2,$inout2
4219	pxor	$inout3,$inout3
4220___
4221$code.=<<___ if (!$win64);
4222	pxor	$inout4,$inout4		# %xmm6..9
4223	pxor	$inout5,$inout5
4224	pxor	$inout6,$inout6
4225	pxor	$inout7,$inout7
4226___
4227$code.=<<___;
4228.Lcbc_dec_tail_collected:
4229	movups	$iv,($ivp)
4230	and	\$15,$len
4231	jnz	.Lcbc_dec_tail_partial
4232	movups	$inout0,($out)
4233	pxor	$inout0,$inout0
4234	jmp	.Lcbc_dec_ret
4235.align	16
4236.Lcbc_dec_tail_partial:
4237	movaps	$inout0,(%rsp)
4238	pxor	$inout0,$inout0
4239	mov	\$16,%rcx
4240	mov	$out,%rdi
4241	sub	$len,%rcx
4242	lea	(%rsp),%rsi
4243	.long	0x9066A4F3		# rep movsb
4244	movdqa	$inout0,(%rsp)
4245
4246.Lcbc_dec_ret:
4247	xorps	$rndkey0,$rndkey0	# %xmm0
4248	pxor	$rndkey1,$rndkey1
4249___
4250$code.=<<___ if ($win64);
4251	movaps	0x10(%rsp),%xmm6
4252	movaps	%xmm0,0x10(%rsp)	# clear stack
4253	movaps	0x20(%rsp),%xmm7
4254	movaps	%xmm0,0x20(%rsp)
4255	movaps	0x30(%rsp),%xmm8
4256	movaps	%xmm0,0x30(%rsp)
4257	movaps	0x40(%rsp),%xmm9
4258	movaps	%xmm0,0x40(%rsp)
4259	movaps	0x50(%rsp),%xmm10
4260	movaps	%xmm0,0x50(%rsp)
4261	movaps	0x60(%rsp),%xmm11
4262	movaps	%xmm0,0x60(%rsp)
4263	movaps	0x70(%rsp),%xmm12
4264	movaps	%xmm0,0x70(%rsp)
4265	movaps	0x80(%rsp),%xmm13
4266	movaps	%xmm0,0x80(%rsp)
4267	movaps	0x90(%rsp),%xmm14
4268	movaps	%xmm0,0x90(%rsp)
4269	movaps	0xa0(%rsp),%xmm15
4270	movaps	%xmm0,0xa0(%rsp)
4271___
4272$code.=<<___;
4273	mov	-8(%r11),%rbp
4274.cfi_restore	%rbp
4275	lea	(%r11),%rsp
4276.cfi_def_cfa_register	%rsp
4277.Lcbc_ret:
4278	ret
4279.cfi_endproc
4280.size	${PREFIX}_cbc_encrypt,.-${PREFIX}_cbc_encrypt
4281___
4282}
4283# int ${PREFIX}_set_decrypt_key(const unsigned char *inp,
4284#				int bits, AES_KEY *key)
4285#
4286# input:	$inp	user-supplied key
4287#		$bits	$inp length in bits
4288#		$key	pointer to key schedule
4289# output:	%eax	0 denoting success, -1 or -2 - failure (see C)
4290#		*$key	key schedule
4291#
4292{ my ($inp,$bits,$key) = @_4args;
4293  $bits =~ s/%r/%e/;
4294
4295$code.=<<___;
4296.globl	${PREFIX}_set_decrypt_key
4297.type	${PREFIX}_set_decrypt_key,\@abi-omnipotent
4298.align	16
4299${PREFIX}_set_decrypt_key:
4300.cfi_startproc
4301	.byte	0x48,0x83,0xEC,0x08	# sub rsp,8
4302.cfi_adjust_cfa_offset	8
4303	call	__aesni_set_encrypt_key
4304	shl	\$4,$bits		# rounds-1 after _aesni_set_encrypt_key
4305	test	%eax,%eax
4306	jnz	.Ldec_key_ret
4307	lea	16($key,$bits),$inp	# points at the end of key schedule
4308
4309	$movkey	($key),%xmm0		# just swap
4310	$movkey	($inp),%xmm1
4311	$movkey	%xmm0,($inp)
4312	$movkey	%xmm1,($key)
4313	lea	16($key),$key
4314	lea	-16($inp),$inp
4315
4316.Ldec_key_inverse:
4317	$movkey	($key),%xmm0		# swap and inverse
4318	$movkey	($inp),%xmm1
4319	aesimc	%xmm0,%xmm0
4320	aesimc	%xmm1,%xmm1
4321	lea	16($key),$key
4322	lea	-16($inp),$inp
4323	$movkey	%xmm0,16($inp)
4324	$movkey	%xmm1,-16($key)
4325	cmp	$key,$inp
4326	ja	.Ldec_key_inverse
4327
4328	$movkey	($key),%xmm0		# inverse middle
4329	aesimc	%xmm0,%xmm0
4330	pxor	%xmm1,%xmm1
4331	$movkey	%xmm0,($inp)
4332	pxor	%xmm0,%xmm0
4333.Ldec_key_ret:
4334	add	\$8,%rsp
4335.cfi_adjust_cfa_offset	-8
4336	ret
4337.cfi_endproc
4338.LSEH_end_set_decrypt_key:
4339.size	${PREFIX}_set_decrypt_key,.-${PREFIX}_set_decrypt_key
4340___
4341
4342# This is based on submission from Intel by
4343#	Huang Ying
4344#	Vinodh Gopal
4345#	Kahraman Akdemir
4346#
4347# Aggressively optimized in respect to aeskeygenassist's critical path
4348# and is contained in %xmm0-5 to meet Win64 ABI requirement.
4349#
4350# int ${PREFIX}_set_encrypt_key(const unsigned char *inp,
4351#				int bits, AES_KEY * const key);
4352#
4353# input:	$inp	user-supplied key
4354#		$bits	$inp length in bits
4355#		$key	pointer to key schedule
4356# output:	%eax	0 denoting success, -1 or -2 - failure (see C)
4357#		$bits	rounds-1 (used in aesni_set_decrypt_key)
4358#		*$key	key schedule
4359#		$key	pointer to key schedule (used in
4360#			aesni_set_decrypt_key)
4361#
4362# Subroutine is frame-less, which means that only volatile registers
4363# are used. Note that it's declared "abi-omnipotent", which means that
4364# amount of volatile registers is smaller on Windows.
4365#
4366$code.=<<___;
4367.globl	${PREFIX}_set_encrypt_key
4368.type	${PREFIX}_set_encrypt_key,\@abi-omnipotent
4369.align	16
4370${PREFIX}_set_encrypt_key:
4371__aesni_set_encrypt_key:
4372.cfi_startproc
4373	.byte	0x48,0x83,0xEC,0x08	# sub rsp,8
4374.cfi_adjust_cfa_offset	8
4375	mov	\$-1,%rax
4376	test	$inp,$inp
4377	jz	.Lenc_key_ret
4378	test	$key,$key
4379	jz	.Lenc_key_ret
4380
4381	mov	\$`1<<28|1<<11`,%r10d	# AVX and XOP bits
4382	movups	($inp),%xmm0		# pull first 128 bits of *userKey
4383	xorps	%xmm4,%xmm4		# low dword of xmm4 is assumed 0
4384	and	OPENSSL_ia32cap_P+4(%rip),%r10d
4385	lea	16($key),%rax		# %rax is used as modifiable copy of $key
4386	cmp	\$256,$bits
4387	je	.L14rounds
4388	cmp	\$192,$bits
4389	je	.L12rounds
4390	cmp	\$128,$bits
4391	jne	.Lbad_keybits
4392
4393.L10rounds:
4394	mov	\$9,$bits			# 10 rounds for 128-bit key
4395	cmp	\$`1<<28`,%r10d			# AVX, bit no XOP
4396	je	.L10rounds_alt
4397
4398	$movkey	%xmm0,($key)			# round 0
4399	aeskeygenassist	\$0x1,%xmm0,%xmm1	# round 1
4400	call		.Lkey_expansion_128_cold
4401	aeskeygenassist	\$0x2,%xmm0,%xmm1	# round 2
4402	call		.Lkey_expansion_128
4403	aeskeygenassist	\$0x4,%xmm0,%xmm1	# round 3
4404	call		.Lkey_expansion_128
4405	aeskeygenassist	\$0x8,%xmm0,%xmm1	# round 4
4406	call		.Lkey_expansion_128
4407	aeskeygenassist	\$0x10,%xmm0,%xmm1	# round 5
4408	call		.Lkey_expansion_128
4409	aeskeygenassist	\$0x20,%xmm0,%xmm1	# round 6
4410	call		.Lkey_expansion_128
4411	aeskeygenassist	\$0x40,%xmm0,%xmm1	# round 7
4412	call		.Lkey_expansion_128
4413	aeskeygenassist	\$0x80,%xmm0,%xmm1	# round 8
4414	call		.Lkey_expansion_128
4415	aeskeygenassist	\$0x1b,%xmm0,%xmm1	# round 9
4416	call		.Lkey_expansion_128
4417	aeskeygenassist	\$0x36,%xmm0,%xmm1	# round 10
4418	call		.Lkey_expansion_128
4419	$movkey	%xmm0,(%rax)
4420	mov	$bits,80(%rax)	# 240(%rdx)
4421	xor	%eax,%eax
4422	jmp	.Lenc_key_ret
4423
4424.align	16
4425.L10rounds_alt:
4426	movdqa	.Lkey_rotate(%rip),%xmm5
4427	mov	\$8,%r10d
4428	movdqa	.Lkey_rcon1(%rip),%xmm4
4429	movdqa	%xmm0,%xmm2
4430	movdqu	%xmm0,($key)
4431	jmp	.Loop_key128
4432
4433.align	16
4434.Loop_key128:
4435	pshufb		%xmm5,%xmm0
4436	aesenclast	%xmm4,%xmm0
4437	pslld		\$1,%xmm4
4438	lea		16(%rax),%rax
4439
4440	movdqa		%xmm2,%xmm3
4441	pslldq		\$4,%xmm2
4442	pxor		%xmm2,%xmm3
4443	pslldq		\$4,%xmm2
4444	pxor		%xmm2,%xmm3
4445	pslldq		\$4,%xmm2
4446	pxor		%xmm3,%xmm2
4447
4448	pxor		%xmm2,%xmm0
4449	movdqu		%xmm0,-16(%rax)
4450	movdqa		%xmm0,%xmm2
4451
4452	dec	%r10d
4453	jnz	.Loop_key128
4454
4455	movdqa		.Lkey_rcon1b(%rip),%xmm4
4456
4457	pshufb		%xmm5,%xmm0
4458	aesenclast	%xmm4,%xmm0
4459	pslld		\$1,%xmm4
4460
4461	movdqa		%xmm2,%xmm3
4462	pslldq		\$4,%xmm2
4463	pxor		%xmm2,%xmm3
4464	pslldq		\$4,%xmm2
4465	pxor		%xmm2,%xmm3
4466	pslldq		\$4,%xmm2
4467	pxor		%xmm3,%xmm2
4468
4469	pxor		%xmm2,%xmm0
4470	movdqu		%xmm0,(%rax)
4471
4472	movdqa		%xmm0,%xmm2
4473	pshufb		%xmm5,%xmm0
4474	aesenclast	%xmm4,%xmm0
4475
4476	movdqa		%xmm2,%xmm3
4477	pslldq		\$4,%xmm2
4478	pxor		%xmm2,%xmm3
4479	pslldq		\$4,%xmm2
4480	pxor		%xmm2,%xmm3
4481	pslldq		\$4,%xmm2
4482	pxor		%xmm3,%xmm2
4483
4484	pxor		%xmm2,%xmm0
4485	movdqu		%xmm0,16(%rax)
4486
4487	mov	$bits,96(%rax)	# 240($key)
4488	xor	%eax,%eax
4489	jmp	.Lenc_key_ret
4490
4491.align	16
4492.L12rounds:
4493	movq	16($inp),%xmm2			# remaining 1/3 of *userKey
4494	mov	\$11,$bits			# 12 rounds for 192
4495	cmp	\$`1<<28`,%r10d			# AVX, but no XOP
4496	je	.L12rounds_alt
4497
4498	$movkey	%xmm0,($key)			# round 0
4499	aeskeygenassist	\$0x1,%xmm2,%xmm1	# round 1,2
4500	call		.Lkey_expansion_192a_cold
4501	aeskeygenassist	\$0x2,%xmm2,%xmm1	# round 2,3
4502	call		.Lkey_expansion_192b
4503	aeskeygenassist	\$0x4,%xmm2,%xmm1	# round 4,5
4504	call		.Lkey_expansion_192a
4505	aeskeygenassist	\$0x8,%xmm2,%xmm1	# round 5,6
4506	call		.Lkey_expansion_192b
4507	aeskeygenassist	\$0x10,%xmm2,%xmm1	# round 7,8
4508	call		.Lkey_expansion_192a
4509	aeskeygenassist	\$0x20,%xmm2,%xmm1	# round 8,9
4510	call		.Lkey_expansion_192b
4511	aeskeygenassist	\$0x40,%xmm2,%xmm1	# round 10,11
4512	call		.Lkey_expansion_192a
4513	aeskeygenassist	\$0x80,%xmm2,%xmm1	# round 11,12
4514	call		.Lkey_expansion_192b
4515	$movkey	%xmm0,(%rax)
4516	mov	$bits,48(%rax)	# 240(%rdx)
4517	xor	%rax, %rax
4518	jmp	.Lenc_key_ret
4519
4520.align	16
4521.L12rounds_alt:
4522	movdqa	.Lkey_rotate192(%rip),%xmm5
4523	movdqa	.Lkey_rcon1(%rip),%xmm4
4524	mov	\$8,%r10d
4525	movdqu	%xmm0,($key)
4526	jmp	.Loop_key192
4527
4528.align	16
4529.Loop_key192:
4530	movq		%xmm2,0(%rax)
4531	movdqa		%xmm2,%xmm1
4532	pshufb		%xmm5,%xmm2
4533	aesenclast	%xmm4,%xmm2
4534	pslld		\$1, %xmm4
4535	lea		24(%rax),%rax
4536
4537	movdqa		%xmm0,%xmm3
4538	pslldq		\$4,%xmm0
4539	pxor		%xmm0,%xmm3
4540	pslldq		\$4,%xmm0
4541	pxor		%xmm0,%xmm3
4542	pslldq		\$4,%xmm0
4543	pxor		%xmm3,%xmm0
4544
4545	pshufd		\$0xff,%xmm0,%xmm3
4546	pxor		%xmm1,%xmm3
4547	pslldq		\$4,%xmm1
4548	pxor		%xmm1,%xmm3
4549
4550	pxor		%xmm2,%xmm0
4551	pxor		%xmm3,%xmm2
4552	movdqu		%xmm0,-16(%rax)
4553
4554	dec	%r10d
4555	jnz	.Loop_key192
4556
4557	mov	$bits,32(%rax)	# 240($key)
4558	xor	%eax,%eax
4559	jmp	.Lenc_key_ret
4560
4561.align	16
4562.L14rounds:
4563	movups	16($inp),%xmm2			# remaining half of *userKey
4564	mov	\$13,$bits			# 14 rounds for 256
4565	lea	16(%rax),%rax
4566	cmp	\$`1<<28`,%r10d			# AVX, but no XOP
4567	je	.L14rounds_alt
4568
4569	$movkey	%xmm0,($key)			# round 0
4570	$movkey	%xmm2,16($key)			# round 1
4571	aeskeygenassist	\$0x1,%xmm2,%xmm1	# round 2
4572	call		.Lkey_expansion_256a_cold
4573	aeskeygenassist	\$0x1,%xmm0,%xmm1	# round 3
4574	call		.Lkey_expansion_256b
4575	aeskeygenassist	\$0x2,%xmm2,%xmm1	# round 4
4576	call		.Lkey_expansion_256a
4577	aeskeygenassist	\$0x2,%xmm0,%xmm1	# round 5
4578	call		.Lkey_expansion_256b
4579	aeskeygenassist	\$0x4,%xmm2,%xmm1	# round 6
4580	call		.Lkey_expansion_256a
4581	aeskeygenassist	\$0x4,%xmm0,%xmm1	# round 7
4582	call		.Lkey_expansion_256b
4583	aeskeygenassist	\$0x8,%xmm2,%xmm1	# round 8
4584	call		.Lkey_expansion_256a
4585	aeskeygenassist	\$0x8,%xmm0,%xmm1	# round 9
4586	call		.Lkey_expansion_256b
4587	aeskeygenassist	\$0x10,%xmm2,%xmm1	# round 10
4588	call		.Lkey_expansion_256a
4589	aeskeygenassist	\$0x10,%xmm0,%xmm1	# round 11
4590	call		.Lkey_expansion_256b
4591	aeskeygenassist	\$0x20,%xmm2,%xmm1	# round 12
4592	call		.Lkey_expansion_256a
4593	aeskeygenassist	\$0x20,%xmm0,%xmm1	# round 13
4594	call		.Lkey_expansion_256b
4595	aeskeygenassist	\$0x40,%xmm2,%xmm1	# round 14
4596	call		.Lkey_expansion_256a
4597	$movkey	%xmm0,(%rax)
4598	mov	$bits,16(%rax)	# 240(%rdx)
4599	xor	%rax,%rax
4600	jmp	.Lenc_key_ret
4601
4602.align	16
4603.L14rounds_alt:
4604	movdqa	.Lkey_rotate(%rip),%xmm5
4605	movdqa	.Lkey_rcon1(%rip),%xmm4
4606	mov	\$7,%r10d
4607	movdqu	%xmm0,0($key)
4608	movdqa	%xmm2,%xmm1
4609	movdqu	%xmm2,16($key)
4610	jmp	.Loop_key256
4611
4612.align	16
4613.Loop_key256:
4614	pshufb		%xmm5,%xmm2
4615	aesenclast	%xmm4,%xmm2
4616
4617	movdqa		%xmm0,%xmm3
4618	pslldq		\$4,%xmm0
4619	pxor		%xmm0,%xmm3
4620	pslldq		\$4,%xmm0
4621	pxor		%xmm0,%xmm3
4622	pslldq		\$4,%xmm0
4623	pxor		%xmm3,%xmm0
4624	pslld		\$1,%xmm4
4625
4626	pxor		%xmm2,%xmm0
4627	movdqu		%xmm0,(%rax)
4628
4629	dec	%r10d
4630	jz	.Ldone_key256
4631
4632	pshufd		\$0xff,%xmm0,%xmm2
4633	pxor		%xmm3,%xmm3
4634	aesenclast	%xmm3,%xmm2
4635
4636	movdqa		%xmm1,%xmm3
4637	pslldq		\$4,%xmm1
4638	pxor		%xmm1,%xmm3
4639	pslldq		\$4,%xmm1
4640	pxor		%xmm1,%xmm3
4641	pslldq		\$4,%xmm1
4642	pxor		%xmm3,%xmm1
4643
4644	pxor		%xmm1,%xmm2
4645	movdqu		%xmm2,16(%rax)
4646	lea		32(%rax),%rax
4647	movdqa		%xmm2,%xmm1
4648
4649	jmp	.Loop_key256
4650
4651.Ldone_key256:
4652	mov	$bits,16(%rax)	# 240($key)
4653	xor	%eax,%eax
4654	jmp	.Lenc_key_ret
4655
4656.align	16
4657.Lbad_keybits:
4658	mov	\$-2,%rax
4659.Lenc_key_ret:
4660	pxor	%xmm0,%xmm0
4661	pxor	%xmm1,%xmm1
4662	pxor	%xmm2,%xmm2
4663	pxor	%xmm3,%xmm3
4664	pxor	%xmm4,%xmm4
4665	pxor	%xmm5,%xmm5
4666	add	\$8,%rsp
4667.cfi_adjust_cfa_offset	-8
4668	ret
4669.LSEH_end_set_encrypt_key:
4670
4671.align	16
4672.Lkey_expansion_128:
4673	$movkey	%xmm0,(%rax)
4674	lea	16(%rax),%rax
4675.Lkey_expansion_128_cold:
4676	shufps	\$0b00010000,%xmm0,%xmm4
4677	xorps	%xmm4, %xmm0
4678	shufps	\$0b10001100,%xmm0,%xmm4
4679	xorps	%xmm4, %xmm0
4680	shufps	\$0b11111111,%xmm1,%xmm1	# critical path
4681	xorps	%xmm1,%xmm0
4682	ret
4683
4684.align 16
4685.Lkey_expansion_192a:
4686	$movkey	%xmm0,(%rax)
4687	lea	16(%rax),%rax
4688.Lkey_expansion_192a_cold:
4689	movaps	%xmm2, %xmm5
4690.Lkey_expansion_192b_warm:
4691	shufps	\$0b00010000,%xmm0,%xmm4
4692	movdqa	%xmm2,%xmm3
4693	xorps	%xmm4,%xmm0
4694	shufps	\$0b10001100,%xmm0,%xmm4
4695	pslldq	\$4,%xmm3
4696	xorps	%xmm4,%xmm0
4697	pshufd	\$0b01010101,%xmm1,%xmm1	# critical path
4698	pxor	%xmm3,%xmm2
4699	pxor	%xmm1,%xmm0
4700	pshufd	\$0b11111111,%xmm0,%xmm3
4701	pxor	%xmm3,%xmm2
4702	ret
4703
4704.align 16
4705.Lkey_expansion_192b:
4706	movaps	%xmm0,%xmm3
4707	shufps	\$0b01000100,%xmm0,%xmm5
4708	$movkey	%xmm5,(%rax)
4709	shufps	\$0b01001110,%xmm2,%xmm3
4710	$movkey	%xmm3,16(%rax)
4711	lea	32(%rax),%rax
4712	jmp	.Lkey_expansion_192b_warm
4713
4714.align	16
4715.Lkey_expansion_256a:
4716	$movkey	%xmm2,(%rax)
4717	lea	16(%rax),%rax
4718.Lkey_expansion_256a_cold:
4719	shufps	\$0b00010000,%xmm0,%xmm4
4720	xorps	%xmm4,%xmm0
4721	shufps	\$0b10001100,%xmm0,%xmm4
4722	xorps	%xmm4,%xmm0
4723	shufps	\$0b11111111,%xmm1,%xmm1	# critical path
4724	xorps	%xmm1,%xmm0
4725	ret
4726
4727.align 16
4728.Lkey_expansion_256b:
4729	$movkey	%xmm0,(%rax)
4730	lea	16(%rax),%rax
4731
4732	shufps	\$0b00010000,%xmm2,%xmm4
4733	xorps	%xmm4,%xmm2
4734	shufps	\$0b10001100,%xmm2,%xmm4
4735	xorps	%xmm4,%xmm2
4736	shufps	\$0b10101010,%xmm1,%xmm1	# critical path
4737	xorps	%xmm1,%xmm2
4738	ret
4739.cfi_endproc
4740.size	${PREFIX}_set_encrypt_key,.-${PREFIX}_set_encrypt_key
4741.size	__aesni_set_encrypt_key,.-__aesni_set_encrypt_key
4742___
4743}
4744
4745$code.=<<___;
4746.section .rodata align=64
4747.align	64
4748.Lbswap_mask:
4749	.byte	15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0
4750.Lincrement32:
4751	.long	6,6,6,0
4752.Lincrement64:
4753	.long	1,0,0,0
4754.Lxts_magic:
4755	.long	0x87,0,1,0
4756.Lincrement1:
4757	.byte	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
4758.Lkey_rotate:
4759	.long	0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d
4760.Lkey_rotate192:
4761	.long	0x04070605,0x04070605,0x04070605,0x04070605
4762.Lkey_rcon1:
4763	.long	1,1,1,1
4764.Lkey_rcon1b:
4765	.long	0x1b,0x1b,0x1b,0x1b
4766
4767.asciz  "AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>"
4768.align	64
4769.previous
4770___
4771
4772# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
4773#		CONTEXT *context,DISPATCHER_CONTEXT *disp)
4774if ($win64) {
4775$rec="%rcx";
4776$frame="%rdx";
4777$context="%r8";
4778$disp="%r9";
4779
4780$code.=<<___;
4781.extern	__imp_RtlVirtualUnwind
4782___
4783$code.=<<___ if ($PREFIX eq "aesni");
4784.type	ecb_ccm64_se_handler,\@abi-omnipotent
4785.align	16
4786ecb_ccm64_se_handler:
4787	push	%rsi
4788	push	%rdi
4789	push	%rbx
4790	push	%rbp
4791	push	%r12
4792	push	%r13
4793	push	%r14
4794	push	%r15
4795	pushfq
4796	sub	\$64,%rsp
4797
4798	mov	120($context),%rax	# pull context->Rax
4799	mov	248($context),%rbx	# pull context->Rip
4800
4801	mov	8($disp),%rsi		# disp->ImageBase
4802	mov	56($disp),%r11		# disp->HandlerData
4803
4804	mov	0(%r11),%r10d		# HandlerData[0]
4805	lea	(%rsi,%r10),%r10	# prologue label
4806	cmp	%r10,%rbx		# context->Rip<prologue label
4807	jb	.Lcommon_seh_tail
4808
4809	mov	152($context),%rax	# pull context->Rsp
4810
4811	mov	4(%r11),%r10d		# HandlerData[1]
4812	lea	(%rsi,%r10),%r10	# epilogue label
4813	cmp	%r10,%rbx		# context->Rip>=epilogue label
4814	jae	.Lcommon_seh_tail
4815
4816	lea	0(%rax),%rsi		# %xmm save area
4817	lea	512($context),%rdi	# &context.Xmm6
4818	mov	\$8,%ecx		# 4*sizeof(%xmm0)/sizeof(%rax)
4819	.long	0xa548f3fc		# cld; rep movsq
4820	lea	0x58(%rax),%rax		# adjust stack pointer
4821
4822	jmp	.Lcommon_seh_tail
4823.size	ecb_ccm64_se_handler,.-ecb_ccm64_se_handler
4824
4825.type	ctr_xts_se_handler,\@abi-omnipotent
4826.align	16
4827ctr_xts_se_handler:
4828	push	%rsi
4829	push	%rdi
4830	push	%rbx
4831	push	%rbp
4832	push	%r12
4833	push	%r13
4834	push	%r14
4835	push	%r15
4836	pushfq
4837	sub	\$64,%rsp
4838
4839	mov	120($context),%rax	# pull context->Rax
4840	mov	248($context),%rbx	# pull context->Rip
4841
4842	mov	8($disp),%rsi		# disp->ImageBase
4843	mov	56($disp),%r11		# disp->HandlerData
4844
4845	mov	0(%r11),%r10d		# HandlerData[0]
4846	lea	(%rsi,%r10),%r10	# prologue label
4847	cmp	%r10,%rbx		# context->Rip<prologue label
4848	jb	.Lcommon_seh_tail
4849
4850	mov	152($context),%rax	# pull context->Rsp
4851
4852	mov	4(%r11),%r10d		# HandlerData[1]
4853	lea	(%rsi,%r10),%r10	# epilogue label
4854	cmp	%r10,%rbx		# context->Rip>=epilogue label
4855	jae	.Lcommon_seh_tail
4856
4857	mov	208($context),%rax	# pull context->R11
4858
4859	lea	-0xa8(%rax),%rsi	# %xmm save area
4860	lea	512($context),%rdi	# & context.Xmm6
4861	mov	\$20,%ecx		# 10*sizeof(%xmm0)/sizeof(%rax)
4862	.long	0xa548f3fc		# cld; rep movsq
4863
4864	mov	-8(%rax),%rbp		# restore saved %rbp
4865	mov	%rbp,160($context)	# restore context->Rbp
4866	jmp	.Lcommon_seh_tail
4867.size	ctr_xts_se_handler,.-ctr_xts_se_handler
4868
4869.type	ocb_se_handler,\@abi-omnipotent
4870.align	16
4871ocb_se_handler:
4872	push	%rsi
4873	push	%rdi
4874	push	%rbx
4875	push	%rbp
4876	push	%r12
4877	push	%r13
4878	push	%r14
4879	push	%r15
4880	pushfq
4881	sub	\$64,%rsp
4882
4883	mov	120($context),%rax	# pull context->Rax
4884	mov	248($context),%rbx	# pull context->Rip
4885
4886	mov	8($disp),%rsi		# disp->ImageBase
4887	mov	56($disp),%r11		# disp->HandlerData
4888
4889	mov	0(%r11),%r10d		# HandlerData[0]
4890	lea	(%rsi,%r10),%r10	# prologue label
4891	cmp	%r10,%rbx		# context->Rip<prologue label
4892	jb	.Lcommon_seh_tail
4893
4894	mov	4(%r11),%r10d		# HandlerData[1]
4895	lea	(%rsi,%r10),%r10	# epilogue label
4896	cmp	%r10,%rbx		# context->Rip>=epilogue label
4897	jae	.Lcommon_seh_tail
4898
4899	mov	8(%r11),%r10d		# HandlerData[2]
4900	lea	(%rsi,%r10),%r10
4901	cmp	%r10,%rbx		# context->Rip>=pop label
4902	jae	.Locb_no_xmm
4903
4904	mov	152($context),%rax	# pull context->Rsp
4905
4906	lea	(%rax),%rsi		# %xmm save area
4907	lea	512($context),%rdi	# & context.Xmm6
4908	mov	\$20,%ecx		# 10*sizeof(%xmm0)/sizeof(%rax)
4909	.long	0xa548f3fc		# cld; rep movsq
4910	lea	0xa0+0x28(%rax),%rax
4911
4912.Locb_no_xmm:
4913	mov	-8(%rax),%rbx
4914	mov	-16(%rax),%rbp
4915	mov	-24(%rax),%r12
4916	mov	-32(%rax),%r13
4917	mov	-40(%rax),%r14
4918
4919	mov	%rbx,144($context)	# restore context->Rbx
4920	mov	%rbp,160($context)	# restore context->Rbp
4921	mov	%r12,216($context)	# restore context->R12
4922	mov	%r13,224($context)	# restore context->R13
4923	mov	%r14,232($context)	# restore context->R14
4924
4925	jmp	.Lcommon_seh_tail
4926.size	ocb_se_handler,.-ocb_se_handler
4927___
4928$code.=<<___;
4929.type	cbc_se_handler,\@abi-omnipotent
4930.align	16
4931cbc_se_handler:
4932	push	%rsi
4933	push	%rdi
4934	push	%rbx
4935	push	%rbp
4936	push	%r12
4937	push	%r13
4938	push	%r14
4939	push	%r15
4940	pushfq
4941	sub	\$64,%rsp
4942
4943	mov	152($context),%rax	# pull context->Rsp
4944	mov	248($context),%rbx	# pull context->Rip
4945
4946	lea	.Lcbc_decrypt_bulk(%rip),%r10
4947	cmp	%r10,%rbx		# context->Rip<"prologue" label
4948	jb	.Lcommon_seh_tail
4949
4950	mov	120($context),%rax	# pull context->Rax
4951
4952	lea	.Lcbc_decrypt_body(%rip),%r10
4953	cmp	%r10,%rbx		# context->Rip<cbc_decrypt_body
4954	jb	.Lcommon_seh_tail
4955
4956	mov	152($context),%rax	# pull context->Rsp
4957
4958	lea	.Lcbc_ret(%rip),%r10
4959	cmp	%r10,%rbx		# context->Rip>="epilogue" label
4960	jae	.Lcommon_seh_tail
4961
4962	lea	16(%rax),%rsi		# %xmm save area
4963	lea	512($context),%rdi	# &context.Xmm6
4964	mov	\$20,%ecx		# 10*sizeof(%xmm0)/sizeof(%rax)
4965	.long	0xa548f3fc		# cld; rep movsq
4966
4967	mov	208($context),%rax	# pull context->R11
4968
4969	mov	-8(%rax),%rbp		# restore saved %rbp
4970	mov	%rbp,160($context)	# restore context->Rbp
4971
4972.Lcommon_seh_tail:
4973	mov	8(%rax),%rdi
4974	mov	16(%rax),%rsi
4975	mov	%rax,152($context)	# restore context->Rsp
4976	mov	%rsi,168($context)	# restore context->Rsi
4977	mov	%rdi,176($context)	# restore context->Rdi
4978
4979	mov	40($disp),%rdi		# disp->ContextRecord
4980	mov	$context,%rsi		# context
4981	mov	\$154,%ecx		# sizeof(CONTEXT)
4982	.long	0xa548f3fc		# cld; rep movsq
4983
4984	mov	$disp,%rsi
4985	xor	%rcx,%rcx		# arg1, UNW_FLAG_NHANDLER
4986	mov	8(%rsi),%rdx		# arg2, disp->ImageBase
4987	mov	0(%rsi),%r8		# arg3, disp->ControlPc
4988	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry
4989	mov	40(%rsi),%r10		# disp->ContextRecord
4990	lea	56(%rsi),%r11		# &disp->HandlerData
4991	lea	24(%rsi),%r12		# &disp->EstablisherFrame
4992	mov	%r10,32(%rsp)		# arg5
4993	mov	%r11,40(%rsp)		# arg6
4994	mov	%r12,48(%rsp)		# arg7
4995	mov	%rcx,56(%rsp)		# arg8, (NULL)
4996	call	*__imp_RtlVirtualUnwind(%rip)
4997
4998	mov	\$1,%eax		# ExceptionContinueSearch
4999	add	\$64,%rsp
5000	popfq
5001	pop	%r15
5002	pop	%r14
5003	pop	%r13
5004	pop	%r12
5005	pop	%rbp
5006	pop	%rbx
5007	pop	%rdi
5008	pop	%rsi
5009	ret
5010.size	cbc_se_handler,.-cbc_se_handler
5011
5012.section	.pdata
5013.align	4
5014___
5015$code.=<<___ if ($PREFIX eq "aesni");
5016	.rva	.LSEH_begin_aesni_ecb_encrypt
5017	.rva	.LSEH_end_aesni_ecb_encrypt
5018	.rva	.LSEH_info_ecb
5019
5020	.rva	.LSEH_begin_aesni_ccm64_encrypt_blocks
5021	.rva	.LSEH_end_aesni_ccm64_encrypt_blocks
5022	.rva	.LSEH_info_ccm64_enc
5023
5024	.rva	.LSEH_begin_aesni_ccm64_decrypt_blocks
5025	.rva	.LSEH_end_aesni_ccm64_decrypt_blocks
5026	.rva	.LSEH_info_ccm64_dec
5027
5028	.rva	.LSEH_begin_aesni_ctr32_encrypt_blocks
5029	.rva	.LSEH_end_aesni_ctr32_encrypt_blocks
5030	.rva	.LSEH_info_ctr32
5031
5032	.rva	.LSEH_begin_aesni_xts_encrypt
5033	.rva	.LSEH_end_aesni_xts_encrypt
5034	.rva	.LSEH_info_xts_enc
5035
5036	.rva	.LSEH_begin_aesni_xts_decrypt
5037	.rva	.LSEH_end_aesni_xts_decrypt
5038	.rva	.LSEH_info_xts_dec
5039
5040	.rva	.LSEH_begin_aesni_ocb_encrypt
5041	.rva	.LSEH_end_aesni_ocb_encrypt
5042	.rva	.LSEH_info_ocb_enc
5043
5044	.rva	.LSEH_begin_aesni_ocb_decrypt
5045	.rva	.LSEH_end_aesni_ocb_decrypt
5046	.rva	.LSEH_info_ocb_dec
5047___
5048$code.=<<___;
5049	.rva	.LSEH_begin_${PREFIX}_cbc_encrypt
5050	.rva	.LSEH_end_${PREFIX}_cbc_encrypt
5051	.rva	.LSEH_info_cbc
5052
5053	.rva	${PREFIX}_set_decrypt_key
5054	.rva	.LSEH_end_set_decrypt_key
5055	.rva	.LSEH_info_key
5056
5057	.rva	${PREFIX}_set_encrypt_key
5058	.rva	.LSEH_end_set_encrypt_key
5059	.rva	.LSEH_info_key
5060.section	.xdata
5061.align	8
5062___
5063$code.=<<___ if ($PREFIX eq "aesni");
5064.LSEH_info_ecb:
5065	.byte	9,0,0,0
5066	.rva	ecb_ccm64_se_handler
5067	.rva	.Lecb_enc_body,.Lecb_enc_ret		# HandlerData[]
5068.LSEH_info_ccm64_enc:
5069	.byte	9,0,0,0
5070	.rva	ecb_ccm64_se_handler
5071	.rva	.Lccm64_enc_body,.Lccm64_enc_ret	# HandlerData[]
5072.LSEH_info_ccm64_dec:
5073	.byte	9,0,0,0
5074	.rva	ecb_ccm64_se_handler
5075	.rva	.Lccm64_dec_body,.Lccm64_dec_ret	# HandlerData[]
5076.LSEH_info_ctr32:
5077	.byte	9,0,0,0
5078	.rva	ctr_xts_se_handler
5079	.rva	.Lctr32_body,.Lctr32_epilogue		# HandlerData[]
5080.LSEH_info_xts_enc:
5081	.byte	9,0,0,0
5082	.rva	ctr_xts_se_handler
5083	.rva	.Lxts_enc_body,.Lxts_enc_epilogue	# HandlerData[]
5084.LSEH_info_xts_dec:
5085	.byte	9,0,0,0
5086	.rva	ctr_xts_se_handler
5087	.rva	.Lxts_dec_body,.Lxts_dec_epilogue	# HandlerData[]
5088.LSEH_info_ocb_enc:
5089	.byte	9,0,0,0
5090	.rva	ocb_se_handler
5091	.rva	.Locb_enc_body,.Locb_enc_epilogue	# HandlerData[]
5092	.rva	.Locb_enc_pop
5093	.long	0
5094.LSEH_info_ocb_dec:
5095	.byte	9,0,0,0
5096	.rva	ocb_se_handler
5097	.rva	.Locb_dec_body,.Locb_dec_epilogue	# HandlerData[]
5098	.rva	.Locb_dec_pop
5099	.long	0
5100___
5101$code.=<<___;
5102.LSEH_info_cbc:
5103	.byte	9,0,0,0
5104	.rva	cbc_se_handler
5105.LSEH_info_key:
5106	.byte	0x01,0x04,0x01,0x00
5107	.byte	0x04,0x02,0x00,0x00	# sub rsp,8
5108___
5109}
5110
5111sub rex {
5112  local *opcode=shift;
5113  my ($dst,$src)=@_;
5114  my $rex=0;
5115
5116    $rex|=0x04			if($dst>=8);
5117    $rex|=0x01			if($src>=8);
5118    push @opcode,$rex|0x40	if($rex);
5119}
5120
5121sub aesni {
5122  my $line=shift;
5123  my @opcode=(0x66);
5124
5125    if ($line=~/(aeskeygenassist)\s+\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
5126	rex(\@opcode,$4,$3);
5127	push @opcode,0x0f,0x3a,0xdf;
5128	push @opcode,0xc0|($3&7)|(($4&7)<<3);	# ModR/M
5129	my $c=$2;
5130	push @opcode,$c=~/^0/?oct($c):$c;
5131	return ".byte\t".join(',',@opcode);
5132    }
5133    elsif ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) {
5134	my %opcodelet = (
5135		"aesimc" => 0xdb,
5136		"aesenc" => 0xdc,	"aesenclast" => 0xdd,
5137		"aesdec" => 0xde,	"aesdeclast" => 0xdf
5138	);
5139	return undef if (!defined($opcodelet{$1}));
5140	rex(\@opcode,$3,$2);
5141	push @opcode,0x0f,0x38,$opcodelet{$1};
5142	push @opcode,0xc0|($2&7)|(($3&7)<<3);	# ModR/M
5143	return ".byte\t".join(',',@opcode);
5144    }
5145    elsif ($line=~/(aes[a-z]+)\s+([0x1-9a-fA-F]*)\(%rsp\),\s*%xmm([0-9]+)/) {
5146	my %opcodelet = (
5147		"aesenc" => 0xdc,	"aesenclast" => 0xdd,
5148		"aesdec" => 0xde,	"aesdeclast" => 0xdf
5149	);
5150	return undef if (!defined($opcodelet{$1}));
5151	my $off = $2;
5152	push @opcode,0x44 if ($3>=8);
5153	push @opcode,0x0f,0x38,$opcodelet{$1};
5154	push @opcode,0x44|(($3&7)<<3),0x24;	# ModR/M
5155	push @opcode,($off=~/^0/?oct($off):$off)&0xff;
5156	return ".byte\t".join(',',@opcode);
5157    }
5158    return $line;
5159}
5160
5161sub movbe {
5162	".byte	0x0f,0x38,0xf1,0x44,0x24,".shift;
5163}
5164
5165$code =~ s/\`([^\`]*)\`/eval($1)/gem;
5166$code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem;
5167#$code =~ s/\bmovbe\s+%eax/bswap %eax; mov %eax/gm;	# debugging artefact
5168$code =~ s/\bmovbe\s+%eax,\s*([0-9]+)\(%rsp\)/movbe($1)/gem;
5169
5170print $code;
5171
5172close STDOUT or die "error closing STDOUT: $!";
5173