xref: /openssl/crypto/aes/asm/vpaes-x86.pl (revision 33388b44) 
1#! /usr/bin/env perl
2# Copyright 2011-2020 The OpenSSL Project Authors. All Rights Reserved.
3#
4# Licensed under the Apache License 2.0 (the "License").  You may not use
5# this file except in compliance with the License.  You can obtain a copy
6# in the file LICENSE in the source distribution or at
7# https://www.openssl.org/source/license.html
8
9
10######################################################################
11## Constant-time SSSE3 AES core implementation.
12## version 0.1
13##
14## By Mike Hamburg (Stanford University), 2009
15## Public domain.
16##
17## For details see http://shiftleft.org/papers/vector_aes/ and
18## http://crypto.stanford.edu/vpaes/.
19
20######################################################################
21# September 2011.
22#
23# Port vpaes-x86_64.pl as 32-bit "almost" drop-in replacement for
24# aes-586.pl. "Almost" refers to the fact that AES_cbc_encrypt
25# doesn't handle partial vectors (doesn't have to if called from
26# EVP only). "Drop-in" implies that this module doesn't share key
27# schedule structure with the original nor does it make assumption
28# about its alignment...
29#
30# Performance summary. aes-586.pl column lists large-block CBC
31# encrypt/decrypt/with-hyper-threading-off(*) results in cycles per
32# byte processed with 128-bit key, and vpaes-x86.pl column - [also
33# large-block CBC] encrypt/decrypt.
34#
35#		aes-586.pl		vpaes-x86.pl
36#
37# Core 2(**)	28.1/41.4/18.3		21.9/25.2(***)
38# Nehalem	27.9/40.4/18.1		10.2/11.9
39# Atom		70.7/92.1/60.1		61.1/75.4(***)
40# Silvermont	45.4/62.9/24.1		49.2/61.1(***)
41#
42# (*)	"Hyper-threading" in the context refers rather to cache shared
43#	among multiple cores, than to specifically Intel HTT. As vast
44#	majority of contemporary cores share cache, slower code path
45#	is common place. In other words "with-hyper-threading-off"
46#	results are presented mostly for reference purposes.
47#
48# (**)	"Core 2" refers to initial 65nm design, a.k.a. Conroe.
49#
50# (***)	Less impressive improvement on Core 2 and Atom is due to slow
51#	pshufb,	yet it's respectable +28%/64%  improvement on Core 2
52#	and +15% on Atom (as implied, over "hyper-threading-safe"
53#	code path).
54#
55#						<appro@openssl.org>
56
57$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
58push(@INC,"${dir}","${dir}../../perlasm");
59require "x86asm.pl";
60
61$output = pop and open STDOUT,">$output";
62
63&asm_init($ARGV[0],$x86only = $ARGV[$#ARGV] eq "386");
64
65$PREFIX="vpaes";
66
67my  ($round, $base, $magic, $key, $const, $inp, $out)=
68    ("eax",  "ebx", "ecx",  "edx","ebp",  "esi","edi");
69
70&static_label("_vpaes_consts");
71&static_label("_vpaes_schedule_low_round");
72
73&set_label("_vpaes_consts",64);
74$k_inv=-0x30;		# inv, inva
75	&data_word(0x0D080180,0x0E05060F,0x0A0B0C02,0x04070309);
76	&data_word(0x0F0B0780,0x01040A06,0x02050809,0x030D0E0C);
77
78$k_s0F=-0x10;		# s0F
79	&data_word(0x0F0F0F0F,0x0F0F0F0F,0x0F0F0F0F,0x0F0F0F0F);
80
81$k_ipt=0x00;		# input transform (lo, hi)
82	&data_word(0x5A2A7000,0xC2B2E898,0x52227808,0xCABAE090);
83	&data_word(0x317C4D00,0x4C01307D,0xB0FDCC81,0xCD80B1FC);
84
85$k_sb1=0x20;		# sb1u, sb1t
86	&data_word(0xCB503E00,0xB19BE18F,0x142AF544,0xA5DF7A6E);
87	&data_word(0xFAE22300,0x3618D415,0x0D2ED9EF,0x3BF7CCC1);
88$k_sb2=0x40;		# sb2u, sb2t
89	&data_word(0x0B712400,0xE27A93C6,0xBC982FCD,0x5EB7E955);
90	&data_word(0x0AE12900,0x69EB8840,0xAB82234A,0xC2A163C8);
91$k_sbo=0x60;		# sbou, sbot
92	&data_word(0x6FBDC700,0xD0D26D17,0xC502A878,0x15AABF7A);
93	&data_word(0x5FBB6A00,0xCFE474A5,0x412B35FA,0x8E1E90D1);
94
95$k_mc_forward=0x80;	# mc_forward
96	&data_word(0x00030201,0x04070605,0x080B0A09,0x0C0F0E0D);
97	&data_word(0x04070605,0x080B0A09,0x0C0F0E0D,0x00030201);
98	&data_word(0x080B0A09,0x0C0F0E0D,0x00030201,0x04070605);
99	&data_word(0x0C0F0E0D,0x00030201,0x04070605,0x080B0A09);
100
101$k_mc_backward=0xc0;	# mc_backward
102	&data_word(0x02010003,0x06050407,0x0A09080B,0x0E0D0C0F);
103	&data_word(0x0E0D0C0F,0x02010003,0x06050407,0x0A09080B);
104	&data_word(0x0A09080B,0x0E0D0C0F,0x02010003,0x06050407);
105	&data_word(0x06050407,0x0A09080B,0x0E0D0C0F,0x02010003);
106
107$k_sr=0x100;		# sr
108	&data_word(0x03020100,0x07060504,0x0B0A0908,0x0F0E0D0C);
109	&data_word(0x0F0A0500,0x030E0904,0x07020D08,0x0B06010C);
110	&data_word(0x0B020900,0x0F060D04,0x030A0108,0x070E050C);
111	&data_word(0x070A0D00,0x0B0E0104,0x0F020508,0x0306090C);
112
113$k_rcon=0x140;		# rcon
114	&data_word(0xAF9DEEB6,0x1F8391B9,0x4D7C7D81,0x702A9808);
115
116$k_s63=0x150;		# s63: all equal to 0x63 transformed
117	&data_word(0x5B5B5B5B,0x5B5B5B5B,0x5B5B5B5B,0x5B5B5B5B);
118
119$k_opt=0x160;		# output transform
120	&data_word(0xD6B66000,0xFF9F4929,0xDEBE6808,0xF7974121);
121	&data_word(0x50BCEC00,0x01EDBD51,0xB05C0CE0,0xE10D5DB1);
122
123$k_deskew=0x180;	# deskew tables: inverts the sbox's "skew"
124	&data_word(0x47A4E300,0x07E4A340,0x5DBEF91A,0x1DFEB95A);
125	&data_word(0x83EA6900,0x5F36B5DC,0xF49D1E77,0x2841C2AB);
126##
127##  Decryption stuff
128##  Key schedule constants
129##
130$k_dksd=0x1a0;		# decryption key schedule: invskew x*D
131	&data_word(0xA3E44700,0xFEB91A5D,0x5A1DBEF9,0x0740E3A4);
132	&data_word(0xB5368300,0x41C277F4,0xAB289D1E,0x5FDC69EA);
133$k_dksb=0x1c0;		# decryption key schedule: invskew x*B
134	&data_word(0x8550D500,0x9A4FCA1F,0x1CC94C99,0x03D65386);
135	&data_word(0xB6FC4A00,0x115BEDA7,0x7E3482C8,0xD993256F);
136$k_dkse=0x1e0;		# decryption key schedule: invskew x*E + 0x63
137	&data_word(0x1FC9D600,0xD5031CCA,0x994F5086,0x53859A4C);
138	&data_word(0x4FDC7BE8,0xA2319605,0x20B31487,0xCD5EF96A);
139$k_dks9=0x200;		# decryption key schedule: invskew x*9
140	&data_word(0x7ED9A700,0xB6116FC8,0x82255BFC,0x4AED9334);
141	&data_word(0x27143300,0x45765162,0xE9DAFDCE,0x8BB89FAC);
142
143##
144##  Decryption stuff
145##  Round function constants
146##
147$k_dipt=0x220;		# decryption input transform
148	&data_word(0x0B545F00,0x0F505B04,0x114E451A,0x154A411E);
149	&data_word(0x60056500,0x86E383E6,0xF491F194,0x12771772);
150
151$k_dsb9=0x240;		# decryption sbox output *9*u, *9*t
152	&data_word(0x9A86D600,0x851C0353,0x4F994CC9,0xCAD51F50);
153	&data_word(0xECD74900,0xC03B1789,0xB2FBA565,0x725E2C9E);
154$k_dsbd=0x260;		# decryption sbox output *D*u, *D*t
155	&data_word(0xE6B1A200,0x7D57CCDF,0x882A4439,0xF56E9B13);
156	&data_word(0x24C6CB00,0x3CE2FAF7,0x15DEEFD3,0x2931180D);
157$k_dsbb=0x280;		# decryption sbox output *B*u, *B*t
158	&data_word(0x96B44200,0xD0226492,0xB0F2D404,0x602646F6);
159	&data_word(0xCD596700,0xC19498A6,0x3255AA6B,0xF3FF0C3E);
160$k_dsbe=0x2a0;		# decryption sbox output *E*u, *E*t
161	&data_word(0x26D4D000,0x46F29296,0x64B4F6B0,0x22426004);
162	&data_word(0xFFAAC100,0x0C55A6CD,0x98593E32,0x9467F36B);
163$k_dsbo=0x2c0;		# decryption sbox final output
164	&data_word(0x7EF94000,0x1387EA53,0xD4943E2D,0xC7AA6DB9);
165	&data_word(0x93441D00,0x12D7560F,0xD8C58E9C,0xCA4B8159);
166&asciz	("Vector Permutation AES for x86/SSSE3, Mike Hamburg (Stanford University)");
167&align	(64);
168
169&function_begin_B("_vpaes_preheat");
170	&add	($const,&DWP(0,"esp"));
171	&movdqa	("xmm7",&QWP($k_inv,$const));
172	&movdqa	("xmm6",&QWP($k_s0F,$const));
173	&ret	();
174&function_end_B("_vpaes_preheat");
175
176##
177##  _aes_encrypt_core
178##
179##  AES-encrypt %xmm0.
180##
181##  Inputs:
182##     %xmm0 = input
183##     %xmm6-%xmm7 as in _vpaes_preheat
184##    (%edx) = scheduled keys
185##
186##  Output in %xmm0
187##  Clobbers  %xmm1-%xmm5, %eax, %ebx, %ecx, %edx
188##
189##
190&function_begin_B("_vpaes_encrypt_core");
191	&mov	($magic,16);
192	&mov	($round,&DWP(240,$key));
193	&movdqa	("xmm1","xmm6")
194	&movdqa	("xmm2",&QWP($k_ipt,$const));
195	&pandn	("xmm1","xmm0");
196	&pand	("xmm0","xmm6");
197	&movdqu	("xmm5",&QWP(0,$key));
198	&pshufb	("xmm2","xmm0");
199	&movdqa	("xmm0",&QWP($k_ipt+16,$const));
200	&pxor	("xmm2","xmm5");
201	&psrld	("xmm1",4);
202	&add	($key,16);
203	&pshufb	("xmm0","xmm1");
204	&lea	($base,&DWP($k_mc_backward,$const));
205	&pxor	("xmm0","xmm2");
206	&jmp	(&label("enc_entry"));
207
208
209&set_label("enc_loop",16);
210	# middle of middle round
211	&movdqa	("xmm4",&QWP($k_sb1,$const));	# 4 : sb1u
212	&movdqa	("xmm0",&QWP($k_sb1+16,$const));# 0 : sb1t
213	&pshufb	("xmm4","xmm2");		# 4 = sb1u
214	&pshufb	("xmm0","xmm3");		# 0 = sb1t
215	&pxor	("xmm4","xmm5");		# 4 = sb1u + k
216	&movdqa	("xmm5",&QWP($k_sb2,$const));	# 4 : sb2u
217	&pxor	("xmm0","xmm4");		# 0 = A
218	&movdqa	("xmm1",&QWP(-0x40,$base,$magic));# .Lk_mc_forward[]
219	&pshufb	("xmm5","xmm2");		# 4 = sb2u
220	&movdqa	("xmm2",&QWP($k_sb2+16,$const));# 2 : sb2t
221	&movdqa	("xmm4",&QWP(0,$base,$magic));	# .Lk_mc_backward[]
222	&pshufb	("xmm2","xmm3");		# 2 = sb2t
223	&movdqa	("xmm3","xmm0");		# 3 = A
224	&pxor	("xmm2","xmm5");		# 2 = 2A
225	&pshufb	("xmm0","xmm1");		# 0 = B
226	&add	($key,16);			# next key
227	&pxor	("xmm0","xmm2");		# 0 = 2A+B
228	&pshufb	("xmm3","xmm4");		# 3 = D
229	&add	($magic,16);			# next mc
230	&pxor	("xmm3","xmm0");		# 3 = 2A+B+D
231	&pshufb	("xmm0","xmm1");		# 0 = 2B+C
232	&and	($magic,0x30);			# ... mod 4
233	&sub	($round,1);			# nr--
234	&pxor	("xmm0","xmm3");		# 0 = 2A+3B+C+D
235
236&set_label("enc_entry");
237	# top of round
238	&movdqa	("xmm1","xmm6");		# 1 : i
239	&movdqa	("xmm5",&QWP($k_inv+16,$const));# 2 : a/k
240	&pandn	("xmm1","xmm0");		# 1 = i<<4
241	&psrld	("xmm1",4);			# 1 = i
242	&pand	("xmm0","xmm6");		# 0 = k
243	&pshufb	("xmm5","xmm0");		# 2 = a/k
244	&movdqa	("xmm3","xmm7");		# 3 : 1/i
245	&pxor	("xmm0","xmm1");		# 0 = j
246	&pshufb	("xmm3","xmm1");		# 3 = 1/i
247	&movdqa	("xmm4","xmm7");		# 4 : 1/j
248	&pxor	("xmm3","xmm5");		# 3 = iak = 1/i + a/k
249	&pshufb	("xmm4","xmm0");		# 4 = 1/j
250	&movdqa	("xmm2","xmm7");		# 2 : 1/iak
251	&pxor	("xmm4","xmm5");		# 4 = jak = 1/j + a/k
252	&pshufb	("xmm2","xmm3");		# 2 = 1/iak
253	&movdqa	("xmm3","xmm7");		# 3 : 1/jak
254	&pxor	("xmm2","xmm0");		# 2 = io
255	&pshufb	("xmm3","xmm4");		# 3 = 1/jak
256	&movdqu	("xmm5",&QWP(0,$key));
257	&pxor	("xmm3","xmm1");		# 3 = jo
258	&jnz	(&label("enc_loop"));
259
260	# middle of last round
261	&movdqa	("xmm4",&QWP($k_sbo,$const));	# 3 : sbou      .Lk_sbo
262	&movdqa	("xmm0",&QWP($k_sbo+16,$const));# 3 : sbot      .Lk_sbo+16
263	&pshufb	("xmm4","xmm2");		# 4 = sbou
264	&pxor	("xmm4","xmm5");		# 4 = sb1u + k
265	&pshufb	("xmm0","xmm3");		# 0 = sb1t
266	&movdqa	("xmm1",&QWP(0x40,$base,$magic));# .Lk_sr[]
267	&pxor	("xmm0","xmm4");		# 0 = A
268	&pshufb	("xmm0","xmm1");
269	&ret	();
270&function_end_B("_vpaes_encrypt_core");
271
272##
273##  Decryption core
274##
275##  Same API as encryption core.
276##
277&function_begin_B("_vpaes_decrypt_core");
278	&lea	($base,&DWP($k_dsbd,$const));
279	&mov	($round,&DWP(240,$key));
280	&movdqa	("xmm1","xmm6");
281	&movdqa	("xmm2",&QWP($k_dipt-$k_dsbd,$base));
282	&pandn	("xmm1","xmm0");
283	&mov	($magic,$round);
284	&psrld	("xmm1",4)
285	&movdqu	("xmm5",&QWP(0,$key));
286	&shl	($magic,4);
287	&pand	("xmm0","xmm6");
288	&pshufb	("xmm2","xmm0");
289	&movdqa	("xmm0",&QWP($k_dipt-$k_dsbd+16,$base));
290	&xor	($magic,0x30);
291	&pshufb	("xmm0","xmm1");
292	&and	($magic,0x30);
293	&pxor	("xmm2","xmm5");
294	&movdqa	("xmm5",&QWP($k_mc_forward+48,$const));
295	&pxor	("xmm0","xmm2");
296	&add	($key,16);
297	&lea	($magic,&DWP($k_sr-$k_dsbd,$base,$magic));
298	&jmp	(&label("dec_entry"));
299
300&set_label("dec_loop",16);
301##
302##  Inverse mix columns
303##
304	&movdqa	("xmm4",&QWP(-0x20,$base));	# 4 : sb9u
305	&movdqa	("xmm1",&QWP(-0x10,$base));	# 0 : sb9t
306	&pshufb	("xmm4","xmm2");		# 4 = sb9u
307	&pshufb	("xmm1","xmm3");		# 0 = sb9t
308	&pxor	("xmm0","xmm4");
309	&movdqa	("xmm4",&QWP(0,$base));		# 4 : sbdu
310	&pxor	("xmm0","xmm1");		# 0 = ch
311	&movdqa	("xmm1",&QWP(0x10,$base));	# 0 : sbdt
312
313	&pshufb	("xmm4","xmm2");		# 4 = sbdu
314	&pshufb	("xmm0","xmm5");		# MC ch
315	&pshufb	("xmm1","xmm3");		# 0 = sbdt
316	&pxor	("xmm0","xmm4");		# 4 = ch
317	&movdqa	("xmm4",&QWP(0x20,$base));	# 4 : sbbu
318	&pxor	("xmm0","xmm1");		# 0 = ch
319	&movdqa	("xmm1",&QWP(0x30,$base));	# 0 : sbbt
320
321	&pshufb	("xmm4","xmm2");		# 4 = sbbu
322	&pshufb	("xmm0","xmm5");		# MC ch
323	&pshufb	("xmm1","xmm3");		# 0 = sbbt
324	&pxor	("xmm0","xmm4");		# 4 = ch
325	&movdqa	("xmm4",&QWP(0x40,$base));	# 4 : sbeu
326	&pxor	("xmm0","xmm1");		# 0 = ch
327	&movdqa	("xmm1",&QWP(0x50,$base));	# 0 : sbet
328
329	&pshufb	("xmm4","xmm2");		# 4 = sbeu
330	&pshufb	("xmm0","xmm5");		# MC ch
331	&pshufb	("xmm1","xmm3");		# 0 = sbet
332	&pxor	("xmm0","xmm4");		# 4 = ch
333	&add	($key,16);			# next round key
334	&palignr("xmm5","xmm5",12);
335	&pxor	("xmm0","xmm1");		# 0 = ch
336	&sub	($round,1);			# nr--
337
338&set_label("dec_entry");
339	# top of round
340	&movdqa	("xmm1","xmm6");		# 1 : i
341	&movdqa	("xmm2",&QWP($k_inv+16,$const));# 2 : a/k
342	&pandn	("xmm1","xmm0");		# 1 = i<<4
343	&pand	("xmm0","xmm6");		# 0 = k
344	&psrld	("xmm1",4);			# 1 = i
345	&pshufb	("xmm2","xmm0");		# 2 = a/k
346	&movdqa	("xmm3","xmm7");		# 3 : 1/i
347	&pxor	("xmm0","xmm1");		# 0 = j
348	&pshufb	("xmm3","xmm1");		# 3 = 1/i
349	&movdqa	("xmm4","xmm7");		# 4 : 1/j
350	&pxor	("xmm3","xmm2");		# 3 = iak = 1/i + a/k
351	&pshufb	("xmm4","xmm0");		# 4 = 1/j
352	&pxor	("xmm4","xmm2");		# 4 = jak = 1/j + a/k
353	&movdqa	("xmm2","xmm7");		# 2 : 1/iak
354	&pshufb	("xmm2","xmm3");		# 2 = 1/iak
355	&movdqa	("xmm3","xmm7");		# 3 : 1/jak
356	&pxor	("xmm2","xmm0");		# 2 = io
357	&pshufb	("xmm3","xmm4");		# 3 = 1/jak
358	&movdqu	("xmm0",&QWP(0,$key));
359	&pxor	("xmm3","xmm1");		# 3 = jo
360	&jnz	(&label("dec_loop"));
361
362	# middle of last round
363	&movdqa	("xmm4",&QWP(0x60,$base));	# 3 : sbou
364	&pshufb	("xmm4","xmm2");		# 4 = sbou
365	&pxor	("xmm4","xmm0");		# 4 = sb1u + k
366	&movdqa	("xmm0",&QWP(0x70,$base));	# 0 : sbot
367	&movdqa	("xmm2",&QWP(0,$magic));
368	&pshufb	("xmm0","xmm3");		# 0 = sb1t
369	&pxor	("xmm0","xmm4");		# 0 = A
370	&pshufb	("xmm0","xmm2");
371	&ret	();
372&function_end_B("_vpaes_decrypt_core");
373
374########################################################
375##                                                    ##
376##                  AES key schedule                  ##
377##                                                    ##
378########################################################
379&function_begin_B("_vpaes_schedule_core");
380	&add	($const,&DWP(0,"esp"));
381	&movdqu	("xmm0",&QWP(0,$inp));		# load key (unaligned)
382	&movdqa	("xmm2",&QWP($k_rcon,$const));	# load rcon
383
384	# input transform
385	&movdqa	("xmm3","xmm0");
386	&lea	($base,&DWP($k_ipt,$const));
387	&movdqa	(&QWP(4,"esp"),"xmm2");		# xmm8
388	&call	("_vpaes_schedule_transform");
389	&movdqa	("xmm7","xmm0");
390
391	&test	($out,$out);
392	&jnz	(&label("schedule_am_decrypting"));
393
394	# encrypting, output zeroth round key after transform
395	&movdqu	(&QWP(0,$key),"xmm0");
396	&jmp	(&label("schedule_go"));
397
398&set_label("schedule_am_decrypting");
399	# decrypting, output zeroth round key after shiftrows
400	&movdqa	("xmm1",&QWP($k_sr,$const,$magic));
401	&pshufb	("xmm3","xmm1");
402	&movdqu	(&QWP(0,$key),"xmm3");
403	&xor	($magic,0x30);
404
405&set_label("schedule_go");
406	&cmp	($round,192);
407	&ja	(&label("schedule_256"));
408	&je	(&label("schedule_192"));
409	# 128: fall though
410
411##
412##  .schedule_128
413##
414##  128-bit specific part of key schedule.
415##
416##  This schedule is really simple, because all its parts
417##  are accomplished by the subroutines.
418##
419&set_label("schedule_128");
420	&mov	($round,10);
421
422&set_label("loop_schedule_128");
423	&call	("_vpaes_schedule_round");
424	&dec	($round);
425	&jz	(&label("schedule_mangle_last"));
426	&call	("_vpaes_schedule_mangle");	# write output
427	&jmp	(&label("loop_schedule_128"));
428
429##
430##  .aes_schedule_192
431##
432##  192-bit specific part of key schedule.
433##
434##  The main body of this schedule is the same as the 128-bit
435##  schedule, but with more smearing.  The long, high side is
436##  stored in %xmm7 as before, and the short, low side is in
437##  the high bits of %xmm6.
438##
439##  This schedule is somewhat nastier, however, because each
440##  round produces 192 bits of key material, or 1.5 round keys.
441##  Therefore, on each cycle we do 2 rounds and produce 3 round
442##  keys.
443##
444&set_label("schedule_192",16);
445	&movdqu	("xmm0",&QWP(8,$inp));		# load key part 2 (very unaligned)
446	&call	("_vpaes_schedule_transform");	# input transform
447	&movdqa	("xmm6","xmm0");		# save short part
448	&pxor	("xmm4","xmm4");		# clear 4
449	&movhlps("xmm6","xmm4");		# clobber low side with zeros
450	&mov	($round,4);
451
452&set_label("loop_schedule_192");
453	&call	("_vpaes_schedule_round");
454	&palignr("xmm0","xmm6",8);
455	&call	("_vpaes_schedule_mangle");	# save key n
456	&call	("_vpaes_schedule_192_smear");
457	&call	("_vpaes_schedule_mangle");	# save key n+1
458	&call	("_vpaes_schedule_round");
459	&dec	($round);
460	&jz	(&label("schedule_mangle_last"));
461	&call	("_vpaes_schedule_mangle");	# save key n+2
462	&call	("_vpaes_schedule_192_smear");
463	&jmp	(&label("loop_schedule_192"));
464
465##
466##  .aes_schedule_256
467##
468##  256-bit specific part of key schedule.
469##
470##  The structure here is very similar to the 128-bit
471##  schedule, but with an additional "low side" in
472##  %xmm6.  The low side's rounds are the same as the
473##  high side's, except no rcon and no rotation.
474##
475&set_label("schedule_256",16);
476	&movdqu	("xmm0",&QWP(16,$inp));		# load key part 2 (unaligned)
477	&call	("_vpaes_schedule_transform");	# input transform
478	&mov	($round,7);
479
480&set_label("loop_schedule_256");
481	&call	("_vpaes_schedule_mangle");	# output low result
482	&movdqa	("xmm6","xmm0");		# save cur_lo in xmm6
483
484	# high round
485	&call	("_vpaes_schedule_round");
486	&dec	($round);
487	&jz	(&label("schedule_mangle_last"));
488	&call	("_vpaes_schedule_mangle");
489
490	# low round. swap xmm7 and xmm6
491	&pshufd	("xmm0","xmm0",0xFF);
492	&movdqa	(&QWP(20,"esp"),"xmm7");
493	&movdqa	("xmm7","xmm6");
494	&call	("_vpaes_schedule_low_round");
495	&movdqa	("xmm7",&QWP(20,"esp"));
496
497	&jmp	(&label("loop_schedule_256"));
498
499##
500##  .aes_schedule_mangle_last
501##
502##  Mangler for last round of key schedule
503##  Mangles %xmm0
504##    when encrypting, outputs out(%xmm0) ^ 63
505##    when decrypting, outputs unskew(%xmm0)
506##
507##  Always called right before return... jumps to cleanup and exits
508##
509&set_label("schedule_mangle_last",16);
510	# schedule last round key from xmm0
511	&lea	($base,&DWP($k_deskew,$const));
512	&test	($out,$out);
513	&jnz	(&label("schedule_mangle_last_dec"));
514
515	# encrypting
516	&movdqa	("xmm1",&QWP($k_sr,$const,$magic));
517	&pshufb	("xmm0","xmm1");		# output permute
518	&lea	($base,&DWP($k_opt,$const));	# prepare to output transform
519	&add	($key,32);
520
521&set_label("schedule_mangle_last_dec");
522	&add	($key,-16);
523	&pxor	("xmm0",&QWP($k_s63,$const));
524	&call	("_vpaes_schedule_transform");	# output transform
525	&movdqu	(&QWP(0,$key),"xmm0");		# save last key
526
527	# cleanup
528	&pxor	("xmm0","xmm0");
529	&pxor	("xmm1","xmm1");
530	&pxor	("xmm2","xmm2");
531	&pxor	("xmm3","xmm3");
532	&pxor	("xmm4","xmm4");
533	&pxor	("xmm5","xmm5");
534	&pxor	("xmm6","xmm6");
535	&pxor	("xmm7","xmm7");
536	&ret	();
537&function_end_B("_vpaes_schedule_core");
538
539##
540##  .aes_schedule_192_smear
541##
542##  Smear the short, low side in the 192-bit key schedule.
543##
544##  Inputs:
545##    %xmm7: high side, b  a  x  y
546##    %xmm6:  low side, d  c  0  0
547##    %xmm13: 0
548##
549##  Outputs:
550##    %xmm6: b+c+d  b+c  0  0
551##    %xmm0: b+c+d  b+c  b  a
552##
553&function_begin_B("_vpaes_schedule_192_smear");
554	&pshufd	("xmm1","xmm6",0x80);		# d c 0 0 -> c 0 0 0
555	&pshufd	("xmm0","xmm7",0xFE);		# b a _ _ -> b b b a
556	&pxor	("xmm6","xmm1");		# -> c+d c 0 0
557	&pxor	("xmm1","xmm1");
558	&pxor	("xmm6","xmm0");		# -> b+c+d b+c b a
559	&movdqa	("xmm0","xmm6");
560	&movhlps("xmm6","xmm1");		# clobber low side with zeros
561	&ret	();
562&function_end_B("_vpaes_schedule_192_smear");
563
564##
565##  .aes_schedule_round
566##
567##  Runs one main round of the key schedule on %xmm0, %xmm7
568##
569##  Specifically, runs subbytes on the high dword of %xmm0
570##  then rotates it by one byte and xors into the low dword of
571##  %xmm7.
572##
573##  Adds rcon from low byte of %xmm8, then rotates %xmm8 for
574##  next rcon.
575##
576##  Smears the dwords of %xmm7 by xoring the low into the
577##  second low, result into third, result into highest.
578##
579##  Returns results in %xmm7 = %xmm0.
580##  Clobbers %xmm1-%xmm5.
581##
582&function_begin_B("_vpaes_schedule_round");
583	# extract rcon from xmm8
584	&movdqa	("xmm2",&QWP(8,"esp"));		# xmm8
585	&pxor	("xmm1","xmm1");
586	&palignr("xmm1","xmm2",15);
587	&palignr("xmm2","xmm2",15);
588	&pxor	("xmm7","xmm1");
589
590	# rotate
591	&pshufd	("xmm0","xmm0",0xFF);
592	&palignr("xmm0","xmm0",1);
593
594	# fall through...
595	&movdqa	(&QWP(8,"esp"),"xmm2");		# xmm8
596
597	# low round: same as high round, but no rotation and no rcon.
598&set_label("_vpaes_schedule_low_round");
599	# smear xmm7
600	&movdqa	("xmm1","xmm7");
601	&pslldq	("xmm7",4);
602	&pxor	("xmm7","xmm1");
603	&movdqa	("xmm1","xmm7");
604	&pslldq	("xmm7",8);
605	&pxor	("xmm7","xmm1");
606	&pxor	("xmm7",&QWP($k_s63,$const));
607
608	# subbyte
609	&movdqa	("xmm4",&QWP($k_s0F,$const));
610	&movdqa	("xmm5",&QWP($k_inv,$const));	# 4 : 1/j
611	&movdqa	("xmm1","xmm4");
612	&pandn	("xmm1","xmm0");
613	&psrld	("xmm1",4);			# 1 = i
614	&pand	("xmm0","xmm4");		# 0 = k
615	&movdqa	("xmm2",&QWP($k_inv+16,$const));# 2 : a/k
616	&pshufb	("xmm2","xmm0");		# 2 = a/k
617	&pxor	("xmm0","xmm1");		# 0 = j
618	&movdqa	("xmm3","xmm5");		# 3 : 1/i
619	&pshufb	("xmm3","xmm1");		# 3 = 1/i
620	&pxor	("xmm3","xmm2");		# 3 = iak = 1/i + a/k
621	&movdqa	("xmm4","xmm5");		# 4 : 1/j
622	&pshufb	("xmm4","xmm0");		# 4 = 1/j
623	&pxor	("xmm4","xmm2");		# 4 = jak = 1/j + a/k
624	&movdqa	("xmm2","xmm5");		# 2 : 1/iak
625	&pshufb	("xmm2","xmm3");		# 2 = 1/iak
626	&pxor	("xmm2","xmm0");		# 2 = io
627	&movdqa	("xmm3","xmm5");		# 3 : 1/jak
628	&pshufb	("xmm3","xmm4");		# 3 = 1/jak
629	&pxor	("xmm3","xmm1");		# 3 = jo
630	&movdqa	("xmm4",&QWP($k_sb1,$const));	# 4 : sbou
631	&pshufb	("xmm4","xmm2");		# 4 = sbou
632	&movdqa	("xmm0",&QWP($k_sb1+16,$const));# 0 : sbot
633	&pshufb	("xmm0","xmm3");		# 0 = sb1t
634	&pxor	("xmm0","xmm4");		# 0 = sbox output
635
636	# add in smeared stuff
637	&pxor	("xmm0","xmm7");
638	&movdqa	("xmm7","xmm0");
639	&ret	();
640&function_end_B("_vpaes_schedule_round");
641
642##
643##  .aes_schedule_transform
644##
645##  Linear-transform %xmm0 according to tables at (%ebx)
646##
647##  Output in %xmm0
648##  Clobbers %xmm1, %xmm2
649##
650&function_begin_B("_vpaes_schedule_transform");
651	&movdqa	("xmm2",&QWP($k_s0F,$const));
652	&movdqa	("xmm1","xmm2");
653	&pandn	("xmm1","xmm0");
654	&psrld	("xmm1",4);
655	&pand	("xmm0","xmm2");
656	&movdqa	("xmm2",&QWP(0,$base));
657	&pshufb	("xmm2","xmm0");
658	&movdqa	("xmm0",&QWP(16,$base));
659	&pshufb	("xmm0","xmm1");
660	&pxor	("xmm0","xmm2");
661	&ret	();
662&function_end_B("_vpaes_schedule_transform");
663
664##
665##  .aes_schedule_mangle
666##
667##  Mangle xmm0 from (basis-transformed) standard version
668##  to our version.
669##
670##  On encrypt,
671##    xor with 0x63
672##    multiply by circulant 0,1,1,1
673##    apply shiftrows transform
674##
675##  On decrypt,
676##    xor with 0x63
677##    multiply by "inverse mixcolumns" circulant E,B,D,9
678##    deskew
679##    apply shiftrows transform
680##
681##
682##  Writes out to (%edx), and increments or decrements it
683##  Keeps track of round number mod 4 in %ecx
684##  Preserves xmm0
685##  Clobbers xmm1-xmm5
686##
687&function_begin_B("_vpaes_schedule_mangle");
688	&movdqa	("xmm4","xmm0");	# save xmm0 for later
689	&movdqa	("xmm5",&QWP($k_mc_forward,$const));
690	&test	($out,$out);
691	&jnz	(&label("schedule_mangle_dec"));
692
693	# encrypting
694	&add	($key,16);
695	&pxor	("xmm4",&QWP($k_s63,$const));
696	&pshufb	("xmm4","xmm5");
697	&movdqa	("xmm3","xmm4");
698	&pshufb	("xmm4","xmm5");
699	&pxor	("xmm3","xmm4");
700	&pshufb	("xmm4","xmm5");
701	&pxor	("xmm3","xmm4");
702
703	&jmp	(&label("schedule_mangle_both"));
704
705&set_label("schedule_mangle_dec",16);
706	# inverse mix columns
707	&movdqa	("xmm2",&QWP($k_s0F,$const));
708	&lea	($inp,&DWP($k_dksd,$const));
709	&movdqa	("xmm1","xmm2");
710	&pandn	("xmm1","xmm4");
711	&psrld	("xmm1",4);			# 1 = hi
712	&pand	("xmm4","xmm2");		# 4 = lo
713
714	&movdqa	("xmm2",&QWP(0,$inp));
715	&pshufb	("xmm2","xmm4");
716	&movdqa	("xmm3",&QWP(0x10,$inp));
717	&pshufb	("xmm3","xmm1");
718	&pxor	("xmm3","xmm2");
719	&pshufb	("xmm3","xmm5");
720
721	&movdqa	("xmm2",&QWP(0x20,$inp));
722	&pshufb	("xmm2","xmm4");
723	&pxor	("xmm2","xmm3");
724	&movdqa	("xmm3",&QWP(0x30,$inp));
725	&pshufb	("xmm3","xmm1");
726	&pxor	("xmm3","xmm2");
727	&pshufb	("xmm3","xmm5");
728
729	&movdqa	("xmm2",&QWP(0x40,$inp));
730	&pshufb	("xmm2","xmm4");
731	&pxor	("xmm2","xmm3");
732	&movdqa	("xmm3",&QWP(0x50,$inp));
733	&pshufb	("xmm3","xmm1");
734	&pxor	("xmm3","xmm2");
735	&pshufb	("xmm3","xmm5");
736
737	&movdqa	("xmm2",&QWP(0x60,$inp));
738	&pshufb	("xmm2","xmm4");
739	&pxor	("xmm2","xmm3");
740	&movdqa	("xmm3",&QWP(0x70,$inp));
741	&pshufb	("xmm3","xmm1");
742	&pxor	("xmm3","xmm2");
743
744	&add	($key,-16);
745
746&set_label("schedule_mangle_both");
747	&movdqa	("xmm1",&QWP($k_sr,$const,$magic));
748	&pshufb	("xmm3","xmm1");
749	&add	($magic,-16);
750	&and	($magic,0x30);
751	&movdqu	(&QWP(0,$key),"xmm3");
752	&ret	();
753&function_end_B("_vpaes_schedule_mangle");
754
755#
756# Interface to OpenSSL
757#
758&function_begin("${PREFIX}_set_encrypt_key");
759	&mov	($inp,&wparam(0));		# inp
760	&lea	($base,&DWP(-56,"esp"));
761	&mov	($round,&wparam(1));		# bits
762	&and	($base,-16);
763	&mov	($key,&wparam(2));		# key
764	&xchg	($base,"esp");			# alloca
765	&mov	(&DWP(48,"esp"),$base);
766
767	&mov	($base,$round);
768	&shr	($base,5);
769	&add	($base,5);
770	&mov	(&DWP(240,$key),$base);		# AES_KEY->rounds = nbits/32+5;
771	&mov	($magic,0x30);
772	&mov	($out,0);
773
774	&lea	($const,&DWP(&label("_vpaes_consts")."+0x30-".&label("pic_point")));
775	&call	("_vpaes_schedule_core");
776&set_label("pic_point");
777
778	&mov	("esp",&DWP(48,"esp"));
779	&xor	("eax","eax");
780&function_end("${PREFIX}_set_encrypt_key");
781
782&function_begin("${PREFIX}_set_decrypt_key");
783	&mov	($inp,&wparam(0));		# inp
784	&lea	($base,&DWP(-56,"esp"));
785	&mov	($round,&wparam(1));		# bits
786	&and	($base,-16);
787	&mov	($key,&wparam(2));		# key
788	&xchg	($base,"esp");			# alloca
789	&mov	(&DWP(48,"esp"),$base);
790
791	&mov	($base,$round);
792	&shr	($base,5);
793	&add	($base,5);
794	&mov	(&DWP(240,$key),$base);	# AES_KEY->rounds = nbits/32+5;
795	&shl	($base,4);
796	&lea	($key,&DWP(16,$key,$base));
797
798	&mov	($out,1);
799	&mov	($magic,$round);
800	&shr	($magic,1);
801	&and	($magic,32);
802	&xor	($magic,32);			# nbist==192?0:32;
803
804	&lea	($const,&DWP(&label("_vpaes_consts")."+0x30-".&label("pic_point")));
805	&call	("_vpaes_schedule_core");
806&set_label("pic_point");
807
808	&mov	("esp",&DWP(48,"esp"));
809	&xor	("eax","eax");
810&function_end("${PREFIX}_set_decrypt_key");
811
812&function_begin("${PREFIX}_encrypt");
813	&lea	($const,&DWP(&label("_vpaes_consts")."+0x30-".&label("pic_point")));
814	&call	("_vpaes_preheat");
815&set_label("pic_point");
816	&mov	($inp,&wparam(0));		# inp
817	&lea	($base,&DWP(-56,"esp"));
818	&mov	($out,&wparam(1));		# out
819	&and	($base,-16);
820	&mov	($key,&wparam(2));		# key
821	&xchg	($base,"esp");			# alloca
822	&mov	(&DWP(48,"esp"),$base);
823
824	&movdqu	("xmm0",&QWP(0,$inp));
825	&call	("_vpaes_encrypt_core");
826	&movdqu	(&QWP(0,$out),"xmm0");
827
828	&mov	("esp",&DWP(48,"esp"));
829&function_end("${PREFIX}_encrypt");
830
831&function_begin("${PREFIX}_decrypt");
832	&lea	($const,&DWP(&label("_vpaes_consts")."+0x30-".&label("pic_point")));
833	&call	("_vpaes_preheat");
834&set_label("pic_point");
835	&mov	($inp,&wparam(0));		# inp
836	&lea	($base,&DWP(-56,"esp"));
837	&mov	($out,&wparam(1));		# out
838	&and	($base,-16);
839	&mov	($key,&wparam(2));		# key
840	&xchg	($base,"esp");			# alloca
841	&mov	(&DWP(48,"esp"),$base);
842
843	&movdqu	("xmm0",&QWP(0,$inp));
844	&call	("_vpaes_decrypt_core");
845	&movdqu	(&QWP(0,$out),"xmm0");
846
847	&mov	("esp",&DWP(48,"esp"));
848&function_end("${PREFIX}_decrypt");
849
850&function_begin("${PREFIX}_cbc_encrypt");
851	&mov	($inp,&wparam(0));		# inp
852	&mov	($out,&wparam(1));		# out
853	&mov	($round,&wparam(2));		# len
854	&mov	($key,&wparam(3));		# key
855	&sub	($round,16);
856	&jc	(&label("cbc_abort"));
857	&lea	($base,&DWP(-56,"esp"));
858	&mov	($const,&wparam(4));		# ivp
859	&and	($base,-16);
860	&mov	($magic,&wparam(5));		# enc
861	&xchg	($base,"esp");			# alloca
862	&movdqu	("xmm1",&QWP(0,$const));	# load IV
863	&sub	($out,$inp);
864	&mov	(&DWP(48,"esp"),$base);
865
866	&mov	(&DWP(0,"esp"),$out);		# save out
867	&mov	(&DWP(4,"esp"),$key)		# save key
868	&mov	(&DWP(8,"esp"),$const);		# save ivp
869	&mov	($out,$round);			# $out works as $len
870
871	&lea	($const,&DWP(&label("_vpaes_consts")."+0x30-".&label("pic_point")));
872	&call	("_vpaes_preheat");
873&set_label("pic_point");
874	&cmp	($magic,0);
875	&je	(&label("cbc_dec_loop"));
876	&jmp	(&label("cbc_enc_loop"));
877
878&set_label("cbc_enc_loop",16);
879	&movdqu	("xmm0",&QWP(0,$inp));		# load input
880	&pxor	("xmm0","xmm1");		# inp^=iv
881	&call	("_vpaes_encrypt_core");
882	&mov	($base,&DWP(0,"esp"));		# restore out
883	&mov	($key,&DWP(4,"esp"));		# restore key
884	&movdqa	("xmm1","xmm0");
885	&movdqu	(&QWP(0,$base,$inp),"xmm0");	# write output
886	&lea	($inp,&DWP(16,$inp));
887	&sub	($out,16);
888	&jnc	(&label("cbc_enc_loop"));
889	&jmp	(&label("cbc_done"));
890
891&set_label("cbc_dec_loop",16);
892	&movdqu	("xmm0",&QWP(0,$inp));		# load input
893	&movdqa	(&QWP(16,"esp"),"xmm1");	# save IV
894	&movdqa	(&QWP(32,"esp"),"xmm0");	# save future IV
895	&call	("_vpaes_decrypt_core");
896	&mov	($base,&DWP(0,"esp"));		# restore out
897	&mov	($key,&DWP(4,"esp"));		# restore key
898	&pxor	("xmm0",&QWP(16,"esp"));	# out^=iv
899	&movdqa	("xmm1",&QWP(32,"esp"));	# load next IV
900	&movdqu	(&QWP(0,$base,$inp),"xmm0");	# write output
901	&lea	($inp,&DWP(16,$inp));
902	&sub	($out,16);
903	&jnc	(&label("cbc_dec_loop"));
904
905&set_label("cbc_done");
906	&mov	($base,&DWP(8,"esp"));		# restore ivp
907	&mov	("esp",&DWP(48,"esp"));
908	&movdqu	(&QWP(0,$base),"xmm1");		# write IV
909&set_label("cbc_abort");
910&function_end("${PREFIX}_cbc_encrypt");
911
912&asm_finish();
913
914close STDOUT or die "error closing STDOUT: $!";
915