1#! /usr/bin/env perl 2# Copyright 2010-2020 The OpenSSL Project Authors. All Rights Reserved. 3# 4# Licensed under the Apache License 2.0 (the "License"). You may not use 5# this file except in compliance with the License. You can obtain a copy 6# in the file LICENSE in the source distribution or at 7# https://www.openssl.org/source/license.html 8 9# 10# ==================================================================== 11# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL 12# project. The module is, however, dual licensed under OpenSSL and 13# CRYPTOGAMS licenses depending on where you obtain it. For further 14# details see http://www.openssl.org/~appro/cryptogams/. 15# ==================================================================== 16# 17# March, June 2010 18# 19# The module implements "4-bit" GCM GHASH function and underlying 20# single multiplication operation in GF(2^128). "4-bit" means that 21# it uses 256 bytes per-key table [+128 bytes shared table]. GHASH 22# function features so called "528B" variant utilizing additional 23# 256+16 bytes of per-key storage [+512 bytes shared table]. 24# Performance results are for this streamed GHASH subroutine and are 25# expressed in cycles per processed byte, less is better: 26# 27# gcc 3.4.x(*) assembler 28# 29# P4 28.6 14.0 +100% 30# Opteron 19.3 7.7 +150% 31# Core2 17.8 8.1(**) +120% 32# Atom 31.6 16.8 +88% 33# VIA Nano 21.8 10.1 +115% 34# 35# (*) comparison is not completely fair, because C results are 36# for vanilla "256B" implementation, while assembler results 37# are for "528B";-) 38# (**) it's mystery [to me] why Core2 result is not same as for 39# Opteron; 40 41# May 2010 42# 43# Add PCLMULQDQ version performing at 2.02 cycles per processed byte. 44# See ghash-x86.pl for background information and details about coding 45# techniques. 46# 47# Special thanks to David Woodhouse for providing access to a 48# Westmere-based system on behalf of Intel Open Source Technology Centre. 49 50# December 2012 51# 52# Overhaul: aggregate Karatsuba post-processing, improve ILP in 53# reduction_alg9, increase reduction aggregate factor to 4x. As for 54# the latter. ghash-x86.pl discusses that it makes lesser sense to 55# increase aggregate factor. Then why increase here? Critical path 56# consists of 3 independent pclmulqdq instructions, Karatsuba post- 57# processing and reduction. "On top" of this we lay down aggregated 58# multiplication operations, triplets of independent pclmulqdq's. As 59# issue rate for pclmulqdq is limited, it makes lesser sense to 60# aggregate more multiplications than it takes to perform remaining 61# non-multiplication operations. 2x is near-optimal coefficient for 62# contemporary Intel CPUs (therefore modest improvement coefficient), 63# but not for Bulldozer. Latter is because logical SIMD operations 64# are twice as slow in comparison to Intel, so that critical path is 65# longer. A CPU with higher pclmulqdq issue rate would also benefit 66# from higher aggregate factor... 67# 68# Westmere 1.78(+13%) 69# Sandy Bridge 1.80(+8%) 70# Ivy Bridge 1.80(+7%) 71# Haswell 0.55(+93%) (if system doesn't support AVX) 72# Broadwell 0.45(+110%)(if system doesn't support AVX) 73# Skylake 0.44(+110%)(if system doesn't support AVX) 74# Bulldozer 1.49(+27%) 75# Silvermont 2.88(+13%) 76# Knights L 2.12(-) (if system doesn't support AVX) 77# Goldmont 1.08(+24%) 78 79# March 2013 80# 81# ... 8x aggregate factor AVX code path is using reduction algorithm 82# suggested by Shay Gueron[1]. Even though contemporary AVX-capable 83# CPUs such as Sandy and Ivy Bridge can execute it, the code performs 84# sub-optimally in comparison to above mentioned version. But thanks 85# to Ilya Albrekht and Max Locktyukhin of Intel Corp. we knew that 86# it performs in 0.41 cycles per byte on Haswell processor, in 87# 0.29 on Broadwell, and in 0.36 on Skylake. 88# 89# Knights Landing achieves 1.09 cpb. 90# 91# [1] http://rt.openssl.org/Ticket/Display.html?id=2900&user=guest&pass=guest 92 93# $output is the last argument if it looks like a file (it has an extension) 94# $flavour is the first argument if it doesn't look like a file 95$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; 96$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; 97 98$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); 99 100$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; 101( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or 102( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or 103die "can't locate x86_64-xlate.pl"; 104 105if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` 106 =~ /GNU assembler version ([2-9]\.[0-9]+)/) { 107 $avx = ($1>=2.20) + ($1>=2.22); 108} 109 110if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && 111 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) { 112 $avx = ($1>=2.09) + ($1>=2.10); 113} 114 115if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && 116 `ml64 2>&1` =~ /Version ([0-9]+)\./) { 117 $avx = ($1>=10) + ($1>=11); 118} 119 120if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) { 121 $avx = ($2>=3.0) + ($2>3.0); 122} 123 124open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"" 125 or die "can't call $xlate: $!"; 126*STDOUT=*OUT; 127 128$do4xaggr=1; 129 130# common register layout 131$nlo="%rax"; 132$nhi="%rbx"; 133$Zlo="%r8"; 134$Zhi="%r9"; 135$tmp="%r10"; 136$rem_4bit = "%r11"; 137 138$Xi="%rdi"; 139$Htbl="%rsi"; 140 141# per-function register layout 142$cnt="%rcx"; 143$rem="%rdx"; 144 145sub LB() { my $r=shift; $r =~ s/%[er]([a-d])x/%\1l/ or 146 $r =~ s/%[er]([sd]i)/%\1l/ or 147 $r =~ s/%[er](bp)/%\1l/ or 148 $r =~ s/%(r[0-9]+)[d]?/%\1b/; $r; } 149 150sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm 151{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; 152 my $arg = pop; 153 $arg = "\$$arg" if ($arg*1 eq $arg); 154 $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n"; 155} 156 157{ my $N; 158 sub loop() { 159 my $inp = shift; 160 161 $N++; 162$code.=<<___; 163 xor $nlo,$nlo 164 xor $nhi,$nhi 165 mov `&LB("$Zlo")`,`&LB("$nlo")` 166 mov `&LB("$Zlo")`,`&LB("$nhi")` 167 shl \$4,`&LB("$nlo")` 168 mov \$14,$cnt 169 mov 8($Htbl,$nlo),$Zlo 170 mov ($Htbl,$nlo),$Zhi 171 and \$0xf0,`&LB("$nhi")` 172 mov $Zlo,$rem 173 jmp .Loop$N 174 175.align 16 176.Loop$N: 177 shr \$4,$Zlo 178 and \$0xf,$rem 179 mov $Zhi,$tmp 180 mov ($inp,$cnt),`&LB("$nlo")` 181 shr \$4,$Zhi 182 xor 8($Htbl,$nhi),$Zlo 183 shl \$60,$tmp 184 xor ($Htbl,$nhi),$Zhi 185 mov `&LB("$nlo")`,`&LB("$nhi")` 186 xor ($rem_4bit,$rem,8),$Zhi 187 mov $Zlo,$rem 188 shl \$4,`&LB("$nlo")` 189 xor $tmp,$Zlo 190 dec $cnt 191 js .Lbreak$N 192 193 shr \$4,$Zlo 194 and \$0xf,$rem 195 mov $Zhi,$tmp 196 shr \$4,$Zhi 197 xor 8($Htbl,$nlo),$Zlo 198 shl \$60,$tmp 199 xor ($Htbl,$nlo),$Zhi 200 and \$0xf0,`&LB("$nhi")` 201 xor ($rem_4bit,$rem,8),$Zhi 202 mov $Zlo,$rem 203 xor $tmp,$Zlo 204 jmp .Loop$N 205 206.align 16 207.Lbreak$N: 208 shr \$4,$Zlo 209 and \$0xf,$rem 210 mov $Zhi,$tmp 211 shr \$4,$Zhi 212 xor 8($Htbl,$nlo),$Zlo 213 shl \$60,$tmp 214 xor ($Htbl,$nlo),$Zhi 215 and \$0xf0,`&LB("$nhi")` 216 xor ($rem_4bit,$rem,8),$Zhi 217 mov $Zlo,$rem 218 xor $tmp,$Zlo 219 220 shr \$4,$Zlo 221 and \$0xf,$rem 222 mov $Zhi,$tmp 223 shr \$4,$Zhi 224 xor 8($Htbl,$nhi),$Zlo 225 shl \$60,$tmp 226 xor ($Htbl,$nhi),$Zhi 227 xor $tmp,$Zlo 228 xor ($rem_4bit,$rem,8),$Zhi 229 230 bswap $Zlo 231 bswap $Zhi 232___ 233}} 234 235$code=<<___; 236.text 237.extern OPENSSL_ia32cap_P 238 239.globl gcm_gmult_4bit 240.type gcm_gmult_4bit,\@function,2 241.align 16 242gcm_gmult_4bit: 243.cfi_startproc 244 endbranch 245 push %rbx 246.cfi_push %rbx 247 push %rbp # %rbp and others are pushed exclusively in 248.cfi_push %rbp 249 push %r12 # order to reuse Win64 exception handler... 250.cfi_push %r12 251 push %r13 252.cfi_push %r13 253 push %r14 254.cfi_push %r14 255 push %r15 256.cfi_push %r15 257 sub \$280,%rsp 258.cfi_adjust_cfa_offset 280 259.Lgmult_prologue: 260 261 movzb 15($Xi),$Zlo 262 lea .Lrem_4bit(%rip),$rem_4bit 263___ 264 &loop ($Xi); 265$code.=<<___; 266 mov $Zlo,8($Xi) 267 mov $Zhi,($Xi) 268 269 lea 280+48(%rsp),%rsi 270.cfi_def_cfa %rsi,8 271 mov -8(%rsi),%rbx 272.cfi_restore %rbx 273 lea (%rsi),%rsp 274.cfi_def_cfa_register %rsp 275.Lgmult_epilogue: 276 ret 277.cfi_endproc 278.size gcm_gmult_4bit,.-gcm_gmult_4bit 279___ 280 281# per-function register layout 282$inp="%rdx"; 283$len="%rcx"; 284$rem_8bit=$rem_4bit; 285 286$code.=<<___; 287.globl gcm_ghash_4bit 288.type gcm_ghash_4bit,\@function,4 289.align 16 290gcm_ghash_4bit: 291.cfi_startproc 292 endbranch 293 push %rbx 294.cfi_push %rbx 295 push %rbp 296.cfi_push %rbp 297 push %r12 298.cfi_push %r12 299 push %r13 300.cfi_push %r13 301 push %r14 302.cfi_push %r14 303 push %r15 304.cfi_push %r15 305 sub \$280,%rsp 306.cfi_adjust_cfa_offset 280 307.Lghash_prologue: 308 mov $inp,%r14 # reassign couple of args 309 mov $len,%r15 310___ 311{ my $inp="%r14"; 312 my $dat="%edx"; 313 my $len="%r15"; 314 my @nhi=("%ebx","%ecx"); 315 my @rem=("%r12","%r13"); 316 my $Hshr4="%rbp"; 317 318 &sub ($Htbl,-128); # size optimization 319 &lea ($Hshr4,"16+128(%rsp)"); 320 { my @lo =($nlo,$nhi); 321 my @hi =($Zlo,$Zhi); 322 323 &xor ($dat,$dat); 324 for ($i=0,$j=-2;$i<18;$i++,$j++) { 325 &mov ("$j(%rsp)",&LB($dat)) if ($i>1); 326 &or ($lo[0],$tmp) if ($i>1); 327 &mov (&LB($dat),&LB($lo[1])) if ($i>0 && $i<17); 328 &shr ($lo[1],4) if ($i>0 && $i<17); 329 &mov ($tmp,$hi[1]) if ($i>0 && $i<17); 330 &shr ($hi[1],4) if ($i>0 && $i<17); 331 &mov ("8*$j($Hshr4)",$hi[0]) if ($i>1); 332 &mov ($hi[0],"16*$i+0-128($Htbl)") if ($i<16); 333 &shl (&LB($dat),4) if ($i>0 && $i<17); 334 &mov ("8*$j-128($Hshr4)",$lo[0]) if ($i>1); 335 &mov ($lo[0],"16*$i+8-128($Htbl)") if ($i<16); 336 &shl ($tmp,60) if ($i>0 && $i<17); 337 338 push (@lo,shift(@lo)); 339 push (@hi,shift(@hi)); 340 } 341 } 342 &add ($Htbl,-128); 343 &mov ($Zlo,"8($Xi)"); 344 &mov ($Zhi,"0($Xi)"); 345 &add ($len,$inp); # pointer to the end of data 346 &lea ($rem_8bit,".Lrem_8bit(%rip)"); 347 &jmp (".Louter_loop"); 348 349$code.=".align 16\n.Louter_loop:\n"; 350 &xor ($Zhi,"($inp)"); 351 &mov ("%rdx","8($inp)"); 352 &lea ($inp,"16($inp)"); 353 &xor ("%rdx",$Zlo); 354 &mov ("($Xi)",$Zhi); 355 &mov ("8($Xi)","%rdx"); 356 &shr ("%rdx",32); 357 358 &xor ($nlo,$nlo); 359 &rol ($dat,8); 360 &mov (&LB($nlo),&LB($dat)); 361 &movz ($nhi[0],&LB($dat)); 362 &shl (&LB($nlo),4); 363 &shr ($nhi[0],4); 364 365 for ($j=11,$i=0;$i<15;$i++) { 366 &rol ($dat,8); 367 &xor ($Zlo,"8($Htbl,$nlo)") if ($i>0); 368 &xor ($Zhi,"($Htbl,$nlo)") if ($i>0); 369 &mov ($Zlo,"8($Htbl,$nlo)") if ($i==0); 370 &mov ($Zhi,"($Htbl,$nlo)") if ($i==0); 371 372 &mov (&LB($nlo),&LB($dat)); 373 &xor ($Zlo,$tmp) if ($i>0); 374 &movzw ($rem[1],"($rem_8bit,$rem[1],2)") if ($i>0); 375 376 &movz ($nhi[1],&LB($dat)); 377 &shl (&LB($nlo),4); 378 &movzb ($rem[0],"(%rsp,$nhi[0])"); 379 380 &shr ($nhi[1],4) if ($i<14); 381 &and ($nhi[1],0xf0) if ($i==14); 382 &shl ($rem[1],48) if ($i>0); 383 &xor ($rem[0],$Zlo); 384 385 &mov ($tmp,$Zhi); 386 &xor ($Zhi,$rem[1]) if ($i>0); 387 &shr ($Zlo,8); 388 389 &movz ($rem[0],&LB($rem[0])); 390 &mov ($dat,"$j($Xi)") if (--$j%4==0); 391 &shr ($Zhi,8); 392 393 &xor ($Zlo,"-128($Hshr4,$nhi[0],8)"); 394 &shl ($tmp,56); 395 &xor ($Zhi,"($Hshr4,$nhi[0],8)"); 396 397 unshift (@nhi,pop(@nhi)); # "rotate" registers 398 unshift (@rem,pop(@rem)); 399 } 400 &movzw ($rem[1],"($rem_8bit,$rem[1],2)"); 401 &xor ($Zlo,"8($Htbl,$nlo)"); 402 &xor ($Zhi,"($Htbl,$nlo)"); 403 404 &shl ($rem[1],48); 405 &xor ($Zlo,$tmp); 406 407 &xor ($Zhi,$rem[1]); 408 &movz ($rem[0],&LB($Zlo)); 409 &shr ($Zlo,4); 410 411 &mov ($tmp,$Zhi); 412 &shl (&LB($rem[0]),4); 413 &shr ($Zhi,4); 414 415 &xor ($Zlo,"8($Htbl,$nhi[0])"); 416 &movzw ($rem[0],"($rem_8bit,$rem[0],2)"); 417 &shl ($tmp,60); 418 419 &xor ($Zhi,"($Htbl,$nhi[0])"); 420 &xor ($Zlo,$tmp); 421 &shl ($rem[0],48); 422 423 &bswap ($Zlo); 424 &xor ($Zhi,$rem[0]); 425 426 &bswap ($Zhi); 427 &cmp ($inp,$len); 428 &jb (".Louter_loop"); 429} 430$code.=<<___; 431 mov $Zlo,8($Xi) 432 mov $Zhi,($Xi) 433 434 lea 280+48(%rsp),%rsi 435.cfi_def_cfa %rsi,8 436 mov -48(%rsi),%r15 437.cfi_restore %r15 438 mov -40(%rsi),%r14 439.cfi_restore %r14 440 mov -32(%rsi),%r13 441.cfi_restore %r13 442 mov -24(%rsi),%r12 443.cfi_restore %r12 444 mov -16(%rsi),%rbp 445.cfi_restore %rbp 446 mov -8(%rsi),%rbx 447.cfi_restore %rbx 448 lea 0(%rsi),%rsp 449.cfi_def_cfa_register %rsp 450.Lghash_epilogue: 451 ret 452.cfi_endproc 453.size gcm_ghash_4bit,.-gcm_ghash_4bit 454___ 455 456###################################################################### 457# PCLMULQDQ version. 458 459@_4args=$win64? ("%rcx","%rdx","%r8", "%r9") : # Win64 order 460 ("%rdi","%rsi","%rdx","%rcx"); # Unix order 461 462($Xi,$Xhi)=("%xmm0","%xmm1"); $Hkey="%xmm2"; 463($T1,$T2,$T3)=("%xmm3","%xmm4","%xmm5"); 464 465sub clmul64x64_T2 { # minimal register pressure 466my ($Xhi,$Xi,$Hkey,$HK)=@_; 467 468if (!defined($HK)) { $HK = $T2; 469$code.=<<___; 470 movdqa $Xi,$Xhi # 471 pshufd \$0b01001110,$Xi,$T1 472 pshufd \$0b01001110,$Hkey,$T2 473 pxor $Xi,$T1 # 474 pxor $Hkey,$T2 475___ 476} else { 477$code.=<<___; 478 movdqa $Xi,$Xhi # 479 pshufd \$0b01001110,$Xi,$T1 480 pxor $Xi,$T1 # 481___ 482} 483$code.=<<___; 484 pclmulqdq \$0x00,$Hkey,$Xi ####### 485 pclmulqdq \$0x11,$Hkey,$Xhi ####### 486 pclmulqdq \$0x00,$HK,$T1 ####### 487 pxor $Xi,$T1 # 488 pxor $Xhi,$T1 # 489 490 movdqa $T1,$T2 # 491 psrldq \$8,$T1 492 pslldq \$8,$T2 # 493 pxor $T1,$Xhi 494 pxor $T2,$Xi # 495___ 496} 497 498sub reduction_alg9 { # 17/11 times faster than Intel version 499my ($Xhi,$Xi) = @_; 500 501$code.=<<___; 502 # 1st phase 503 movdqa $Xi,$T2 # 504 movdqa $Xi,$T1 505 psllq \$5,$Xi 506 pxor $Xi,$T1 # 507 psllq \$1,$Xi 508 pxor $T1,$Xi # 509 psllq \$57,$Xi # 510 movdqa $Xi,$T1 # 511 pslldq \$8,$Xi 512 psrldq \$8,$T1 # 513 pxor $T2,$Xi 514 pxor $T1,$Xhi # 515 516 # 2nd phase 517 movdqa $Xi,$T2 518 psrlq \$1,$Xi 519 pxor $T2,$Xhi # 520 pxor $Xi,$T2 521 psrlq \$5,$Xi 522 pxor $T2,$Xi # 523 psrlq \$1,$Xi # 524 pxor $Xhi,$Xi # 525___ 526} 527 528{ my ($Htbl,$Xip)=@_4args; 529 my $HK="%xmm6"; 530 531$code.=<<___; 532.globl gcm_init_clmul 533.type gcm_init_clmul,\@abi-omnipotent 534.align 16 535gcm_init_clmul: 536.cfi_startproc 537 endbranch 538.L_init_clmul: 539___ 540$code.=<<___ if ($win64); 541.LSEH_begin_gcm_init_clmul: 542 # I can't trust assembler to use specific encoding:-( 543 .byte 0x48,0x83,0xec,0x18 #sub $0x18,%rsp 544 .byte 0x0f,0x29,0x34,0x24 #movaps %xmm6,(%rsp) 545___ 546$code.=<<___; 547 movdqu ($Xip),$Hkey 548 pshufd \$0b01001110,$Hkey,$Hkey # dword swap 549 550 # <<1 twist 551 pshufd \$0b11111111,$Hkey,$T2 # broadcast uppermost dword 552 movdqa $Hkey,$T1 553 psllq \$1,$Hkey 554 pxor $T3,$T3 # 555 psrlq \$63,$T1 556 pcmpgtd $T2,$T3 # broadcast carry bit 557 pslldq \$8,$T1 558 por $T1,$Hkey # H<<=1 559 560 # magic reduction 561 pand .L0x1c2_polynomial(%rip),$T3 562 pxor $T3,$Hkey # if(carry) H^=0x1c2_polynomial 563 564 # calculate H^2 565 pshufd \$0b01001110,$Hkey,$HK 566 movdqa $Hkey,$Xi 567 pxor $Hkey,$HK 568___ 569 &clmul64x64_T2 ($Xhi,$Xi,$Hkey,$HK); 570 &reduction_alg9 ($Xhi,$Xi); 571$code.=<<___; 572 pshufd \$0b01001110,$Hkey,$T1 573 pshufd \$0b01001110,$Xi,$T2 574 pxor $Hkey,$T1 # Karatsuba pre-processing 575 movdqu $Hkey,0x00($Htbl) # save H 576 pxor $Xi,$T2 # Karatsuba pre-processing 577 movdqu $Xi,0x10($Htbl) # save H^2 578 palignr \$8,$T1,$T2 # low part is H.lo^H.hi... 579 movdqu $T2,0x20($Htbl) # save Karatsuba "salt" 580___ 581if ($do4xaggr) { 582 &clmul64x64_T2 ($Xhi,$Xi,$Hkey,$HK); # H^3 583 &reduction_alg9 ($Xhi,$Xi); 584$code.=<<___; 585 movdqa $Xi,$T3 586___ 587 &clmul64x64_T2 ($Xhi,$Xi,$Hkey,$HK); # H^4 588 &reduction_alg9 ($Xhi,$Xi); 589$code.=<<___; 590 pshufd \$0b01001110,$T3,$T1 591 pshufd \$0b01001110,$Xi,$T2 592 pxor $T3,$T1 # Karatsuba pre-processing 593 movdqu $T3,0x30($Htbl) # save H^3 594 pxor $Xi,$T2 # Karatsuba pre-processing 595 movdqu $Xi,0x40($Htbl) # save H^4 596 palignr \$8,$T1,$T2 # low part is H^3.lo^H^3.hi... 597 movdqu $T2,0x50($Htbl) # save Karatsuba "salt" 598___ 599} 600$code.=<<___ if ($win64); 601 movaps (%rsp),%xmm6 602 lea 0x18(%rsp),%rsp 603.LSEH_end_gcm_init_clmul: 604___ 605$code.=<<___; 606 ret 607.cfi_endproc 608.size gcm_init_clmul,.-gcm_init_clmul 609___ 610} 611 612{ my ($Xip,$Htbl)=@_4args; 613 614$code.=<<___; 615.globl gcm_gmult_clmul 616.type gcm_gmult_clmul,\@abi-omnipotent 617.align 16 618gcm_gmult_clmul: 619.cfi_startproc 620 endbranch 621.L_gmult_clmul: 622 movdqu ($Xip),$Xi 623 movdqa .Lbswap_mask(%rip),$T3 624 movdqu ($Htbl),$Hkey 625 movdqu 0x20($Htbl),$T2 626 pshufb $T3,$Xi 627___ 628 &clmul64x64_T2 ($Xhi,$Xi,$Hkey,$T2); 629$code.=<<___ if (0 || (&reduction_alg9($Xhi,$Xi)&&0)); 630 # experimental alternative. special thing about is that there 631 # no dependency between the two multiplications... 632 mov \$`0xE1<<1`,%eax 633 mov \$0xA040608020C0E000,%r10 # ((7..0)·0xE0)&0xff 634 mov \$0x07,%r11d 635 movq %rax,$T1 636 movq %r10,$T2 637 movq %r11,$T3 # borrow $T3 638 pand $Xi,$T3 639 pshufb $T3,$T2 # ($Xi&7)·0xE0 640 movq %rax,$T3 641 pclmulqdq \$0x00,$Xi,$T1 # ·(0xE1<<1) 642 pxor $Xi,$T2 643 pslldq \$15,$T2 644 paddd $T2,$T2 # <<(64+56+1) 645 pxor $T2,$Xi 646 pclmulqdq \$0x01,$T3,$Xi 647 movdqa .Lbswap_mask(%rip),$T3 # reload $T3 648 psrldq \$1,$T1 649 pxor $T1,$Xhi 650 pslldq \$7,$Xi 651 pxor $Xhi,$Xi 652___ 653$code.=<<___; 654 pshufb $T3,$Xi 655 movdqu $Xi,($Xip) 656 ret 657.cfi_endproc 658.size gcm_gmult_clmul,.-gcm_gmult_clmul 659___ 660} 661 662{ my ($Xip,$Htbl,$inp,$len)=@_4args; 663 my ($Xln,$Xmn,$Xhn,$Hkey2,$HK) = map("%xmm$_",(3..7)); 664 my ($T1,$T2,$T3)=map("%xmm$_",(8..10)); 665 666$code.=<<___; 667.globl gcm_ghash_clmul 668.type gcm_ghash_clmul,\@abi-omnipotent 669.align 32 670gcm_ghash_clmul: 671.cfi_startproc 672 endbranch 673.L_ghash_clmul: 674___ 675$code.=<<___ if ($win64); 676 lea -0x88(%rsp),%rax 677.LSEH_begin_gcm_ghash_clmul: 678 # I can't trust assembler to use specific encoding:-( 679 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax),%rsp 680 .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6,-0x20(%rax) 681 .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7,-0x10(%rax) 682 .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8,0(%rax) 683 .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9,0x10(%rax) 684 .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10,0x20(%rax) 685 .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11,0x30(%rax) 686 .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12,0x40(%rax) 687 .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13,0x50(%rax) 688 .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14,0x60(%rax) 689 .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15,0x70(%rax) 690___ 691$code.=<<___; 692 movdqa .Lbswap_mask(%rip),$T3 693 694 movdqu ($Xip),$Xi 695 movdqu ($Htbl),$Hkey 696 movdqu 0x20($Htbl),$HK 697 pshufb $T3,$Xi 698 699 sub \$0x10,$len 700 jz .Lodd_tail 701 702 movdqu 0x10($Htbl),$Hkey2 703___ 704if ($do4xaggr) { 705my ($Xl,$Xm,$Xh,$Hkey3,$Hkey4)=map("%xmm$_",(11..15)); 706 707$code.=<<___; 708 mov OPENSSL_ia32cap_P+4(%rip),%eax 709 cmp \$0x30,$len 710 jb .Lskip4x 711 712 and \$`1<<26|1<<22`,%eax # isolate MOVBE+XSAVE 713 cmp \$`1<<22`,%eax # check for MOVBE without XSAVE 714 je .Lskip4x 715 716 sub \$0x30,$len 717 mov \$0xA040608020C0E000,%rax # ((7..0)·0xE0)&0xff 718 movdqu 0x30($Htbl),$Hkey3 719 movdqu 0x40($Htbl),$Hkey4 720 721 ####### 722 # Xi+4 =[(H*Ii+3) + (H^2*Ii+2) + (H^3*Ii+1) + H^4*(Ii+Xi)] mod P 723 # 724 movdqu 0x30($inp),$Xln 725 movdqu 0x20($inp),$Xl 726 pshufb $T3,$Xln 727 pshufb $T3,$Xl 728 movdqa $Xln,$Xhn 729 pshufd \$0b01001110,$Xln,$Xmn 730 pxor $Xln,$Xmn 731 pclmulqdq \$0x00,$Hkey,$Xln 732 pclmulqdq \$0x11,$Hkey,$Xhn 733 pclmulqdq \$0x00,$HK,$Xmn 734 735 movdqa $Xl,$Xh 736 pshufd \$0b01001110,$Xl,$Xm 737 pxor $Xl,$Xm 738 pclmulqdq \$0x00,$Hkey2,$Xl 739 pclmulqdq \$0x11,$Hkey2,$Xh 740 pclmulqdq \$0x10,$HK,$Xm 741 xorps $Xl,$Xln 742 xorps $Xh,$Xhn 743 movups 0x50($Htbl),$HK 744 xorps $Xm,$Xmn 745 746 movdqu 0x10($inp),$Xl 747 movdqu 0($inp),$T1 748 pshufb $T3,$Xl 749 pshufb $T3,$T1 750 movdqa $Xl,$Xh 751 pshufd \$0b01001110,$Xl,$Xm 752 pxor $T1,$Xi 753 pxor $Xl,$Xm 754 pclmulqdq \$0x00,$Hkey3,$Xl 755 movdqa $Xi,$Xhi 756 pshufd \$0b01001110,$Xi,$T1 757 pxor $Xi,$T1 758 pclmulqdq \$0x11,$Hkey3,$Xh 759 pclmulqdq \$0x00,$HK,$Xm 760 xorps $Xl,$Xln 761 xorps $Xh,$Xhn 762 763 lea 0x40($inp),$inp 764 sub \$0x40,$len 765 jc .Ltail4x 766 767 jmp .Lmod4_loop 768.align 32 769.Lmod4_loop: 770 pclmulqdq \$0x00,$Hkey4,$Xi 771 xorps $Xm,$Xmn 772 movdqu 0x30($inp),$Xl 773 pshufb $T3,$Xl 774 pclmulqdq \$0x11,$Hkey4,$Xhi 775 xorps $Xln,$Xi 776 movdqu 0x20($inp),$Xln 777 movdqa $Xl,$Xh 778 pclmulqdq \$0x10,$HK,$T1 779 pshufd \$0b01001110,$Xl,$Xm 780 xorps $Xhn,$Xhi 781 pxor $Xl,$Xm 782 pshufb $T3,$Xln 783 movups 0x20($Htbl),$HK 784 xorps $Xmn,$T1 785 pclmulqdq \$0x00,$Hkey,$Xl 786 pshufd \$0b01001110,$Xln,$Xmn 787 788 pxor $Xi,$T1 # aggregated Karatsuba post-processing 789 movdqa $Xln,$Xhn 790 pxor $Xhi,$T1 # 791 pxor $Xln,$Xmn 792 movdqa $T1,$T2 # 793 pclmulqdq \$0x11,$Hkey,$Xh 794 pslldq \$8,$T1 795 psrldq \$8,$T2 # 796 pxor $T1,$Xi 797 movdqa .L7_mask(%rip),$T1 798 pxor $T2,$Xhi # 799 movq %rax,$T2 800 801 pand $Xi,$T1 # 1st phase 802 pshufb $T1,$T2 # 803 pxor $Xi,$T2 # 804 pclmulqdq \$0x00,$HK,$Xm 805 psllq \$57,$T2 # 806 movdqa $T2,$T1 # 807 pslldq \$8,$T2 808 pclmulqdq \$0x00,$Hkey2,$Xln 809 psrldq \$8,$T1 # 810 pxor $T2,$Xi 811 pxor $T1,$Xhi # 812 movdqu 0($inp),$T1 813 814 movdqa $Xi,$T2 # 2nd phase 815 psrlq \$1,$Xi 816 pclmulqdq \$0x11,$Hkey2,$Xhn 817 xorps $Xl,$Xln 818 movdqu 0x10($inp),$Xl 819 pshufb $T3,$Xl 820 pclmulqdq \$0x10,$HK,$Xmn 821 xorps $Xh,$Xhn 822 movups 0x50($Htbl),$HK 823 pshufb $T3,$T1 824 pxor $T2,$Xhi # 825 pxor $Xi,$T2 826 psrlq \$5,$Xi 827 828 movdqa $Xl,$Xh 829 pxor $Xm,$Xmn 830 pshufd \$0b01001110,$Xl,$Xm 831 pxor $T2,$Xi # 832 pxor $T1,$Xhi 833 pxor $Xl,$Xm 834 pclmulqdq \$0x00,$Hkey3,$Xl 835 psrlq \$1,$Xi # 836 pxor $Xhi,$Xi # 837 movdqa $Xi,$Xhi 838 pclmulqdq \$0x11,$Hkey3,$Xh 839 xorps $Xl,$Xln 840 pshufd \$0b01001110,$Xi,$T1 841 pxor $Xi,$T1 842 843 pclmulqdq \$0x00,$HK,$Xm 844 xorps $Xh,$Xhn 845 846 lea 0x40($inp),$inp 847 sub \$0x40,$len 848 jnc .Lmod4_loop 849 850.Ltail4x: 851 pclmulqdq \$0x00,$Hkey4,$Xi 852 pclmulqdq \$0x11,$Hkey4,$Xhi 853 pclmulqdq \$0x10,$HK,$T1 854 xorps $Xm,$Xmn 855 xorps $Xln,$Xi 856 xorps $Xhn,$Xhi 857 pxor $Xi,$Xhi # aggregated Karatsuba post-processing 858 pxor $Xmn,$T1 859 860 pxor $Xhi,$T1 # 861 pxor $Xi,$Xhi 862 863 movdqa $T1,$T2 # 864 psrldq \$8,$T1 865 pslldq \$8,$T2 # 866 pxor $T1,$Xhi 867 pxor $T2,$Xi # 868___ 869 &reduction_alg9($Xhi,$Xi); 870$code.=<<___; 871 add \$0x40,$len 872 jz .Ldone 873 movdqu 0x20($Htbl),$HK 874 sub \$0x10,$len 875 jz .Lodd_tail 876.Lskip4x: 877___ 878} 879$code.=<<___; 880 ####### 881 # Xi+2 =[H*(Ii+1 + Xi+1)] mod P = 882 # [(H*Ii+1) + (H*Xi+1)] mod P = 883 # [(H*Ii+1) + H^2*(Ii+Xi)] mod P 884 # 885 movdqu ($inp),$T1 # Ii 886 movdqu 16($inp),$Xln # Ii+1 887 pshufb $T3,$T1 888 pshufb $T3,$Xln 889 pxor $T1,$Xi # Ii+Xi 890 891 movdqa $Xln,$Xhn 892 pshufd \$0b01001110,$Xln,$Xmn 893 pxor $Xln,$Xmn 894 pclmulqdq \$0x00,$Hkey,$Xln 895 pclmulqdq \$0x11,$Hkey,$Xhn 896 pclmulqdq \$0x00,$HK,$Xmn 897 898 lea 32($inp),$inp # i+=2 899 nop 900 sub \$0x20,$len 901 jbe .Leven_tail 902 nop 903 jmp .Lmod_loop 904 905.align 32 906.Lmod_loop: 907 movdqa $Xi,$Xhi 908 movdqa $Xmn,$T1 909 pshufd \$0b01001110,$Xi,$Xmn # 910 pxor $Xi,$Xmn # 911 912 pclmulqdq \$0x00,$Hkey2,$Xi 913 pclmulqdq \$0x11,$Hkey2,$Xhi 914 pclmulqdq \$0x10,$HK,$Xmn 915 916 pxor $Xln,$Xi # (H*Ii+1) + H^2*(Ii+Xi) 917 pxor $Xhn,$Xhi 918 movdqu ($inp),$T2 # Ii 919 pxor $Xi,$T1 # aggregated Karatsuba post-processing 920 pshufb $T3,$T2 921 movdqu 16($inp),$Xln # Ii+1 922 923 pxor $Xhi,$T1 924 pxor $T2,$Xhi # "Ii+Xi", consume early 925 pxor $T1,$Xmn 926 pshufb $T3,$Xln 927 movdqa $Xmn,$T1 # 928 psrldq \$8,$T1 929 pslldq \$8,$Xmn # 930 pxor $T1,$Xhi 931 pxor $Xmn,$Xi # 932 933 movdqa $Xln,$Xhn # 934 935 movdqa $Xi,$T2 # 1st phase 936 movdqa $Xi,$T1 937 psllq \$5,$Xi 938 pxor $Xi,$T1 # 939 pclmulqdq \$0x00,$Hkey,$Xln ####### 940 psllq \$1,$Xi 941 pxor $T1,$Xi # 942 psllq \$57,$Xi # 943 movdqa $Xi,$T1 # 944 pslldq \$8,$Xi 945 psrldq \$8,$T1 # 946 pxor $T2,$Xi 947 pshufd \$0b01001110,$Xhn,$Xmn 948 pxor $T1,$Xhi # 949 pxor $Xhn,$Xmn # 950 951 movdqa $Xi,$T2 # 2nd phase 952 psrlq \$1,$Xi 953 pclmulqdq \$0x11,$Hkey,$Xhn ####### 954 pxor $T2,$Xhi # 955 pxor $Xi,$T2 956 psrlq \$5,$Xi 957 pxor $T2,$Xi # 958 lea 32($inp),$inp 959 psrlq \$1,$Xi # 960 pclmulqdq \$0x00,$HK,$Xmn ####### 961 pxor $Xhi,$Xi # 962 963 sub \$0x20,$len 964 ja .Lmod_loop 965 966.Leven_tail: 967 movdqa $Xi,$Xhi 968 movdqa $Xmn,$T1 969 pshufd \$0b01001110,$Xi,$Xmn # 970 pxor $Xi,$Xmn # 971 972 pclmulqdq \$0x00,$Hkey2,$Xi 973 pclmulqdq \$0x11,$Hkey2,$Xhi 974 pclmulqdq \$0x10,$HK,$Xmn 975 976 pxor $Xln,$Xi # (H*Ii+1) + H^2*(Ii+Xi) 977 pxor $Xhn,$Xhi 978 pxor $Xi,$T1 979 pxor $Xhi,$T1 980 pxor $T1,$Xmn 981 movdqa $Xmn,$T1 # 982 psrldq \$8,$T1 983 pslldq \$8,$Xmn # 984 pxor $T1,$Xhi 985 pxor $Xmn,$Xi # 986___ 987 &reduction_alg9 ($Xhi,$Xi); 988$code.=<<___; 989 test $len,$len 990 jnz .Ldone 991 992.Lodd_tail: 993 movdqu ($inp),$T1 # Ii 994 pshufb $T3,$T1 995 pxor $T1,$Xi # Ii+Xi 996___ 997 &clmul64x64_T2 ($Xhi,$Xi,$Hkey,$HK); # H*(Ii+Xi) 998 &reduction_alg9 ($Xhi,$Xi); 999$code.=<<___; 1000.Ldone: 1001 pshufb $T3,$Xi 1002 movdqu $Xi,($Xip) 1003___ 1004$code.=<<___ if ($win64); 1005 movaps (%rsp),%xmm6 1006 movaps 0x10(%rsp),%xmm7 1007 movaps 0x20(%rsp),%xmm8 1008 movaps 0x30(%rsp),%xmm9 1009 movaps 0x40(%rsp),%xmm10 1010 movaps 0x50(%rsp),%xmm11 1011 movaps 0x60(%rsp),%xmm12 1012 movaps 0x70(%rsp),%xmm13 1013 movaps 0x80(%rsp),%xmm14 1014 movaps 0x90(%rsp),%xmm15 1015 lea 0xa8(%rsp),%rsp 1016.LSEH_end_gcm_ghash_clmul: 1017___ 1018$code.=<<___; 1019 ret 1020.cfi_endproc 1021.size gcm_ghash_clmul,.-gcm_ghash_clmul 1022___ 1023} 1024 1025$code.=<<___; 1026.globl gcm_init_avx 1027.type gcm_init_avx,\@abi-omnipotent 1028.align 32 1029gcm_init_avx: 1030.cfi_startproc 1031 endbranch 1032___ 1033if ($avx) { 1034my ($Htbl,$Xip)=@_4args; 1035my $HK="%xmm6"; 1036 1037$code.=<<___ if ($win64); 1038.LSEH_begin_gcm_init_avx: 1039 # I can't trust assembler to use specific encoding:-( 1040 .byte 0x48,0x83,0xec,0x18 #sub $0x18,%rsp 1041 .byte 0x0f,0x29,0x34,0x24 #movaps %xmm6,(%rsp) 1042___ 1043$code.=<<___; 1044 vzeroupper 1045 1046 vmovdqu ($Xip),$Hkey 1047 vpshufd \$0b01001110,$Hkey,$Hkey # dword swap 1048 1049 # <<1 twist 1050 vpshufd \$0b11111111,$Hkey,$T2 # broadcast uppermost dword 1051 vpsrlq \$63,$Hkey,$T1 1052 vpsllq \$1,$Hkey,$Hkey 1053 vpxor $T3,$T3,$T3 # 1054 vpcmpgtd $T2,$T3,$T3 # broadcast carry bit 1055 vpslldq \$8,$T1,$T1 1056 vpor $T1,$Hkey,$Hkey # H<<=1 1057 1058 # magic reduction 1059 vpand .L0x1c2_polynomial(%rip),$T3,$T3 1060 vpxor $T3,$Hkey,$Hkey # if(carry) H^=0x1c2_polynomial 1061 1062 vpunpckhqdq $Hkey,$Hkey,$HK 1063 vmovdqa $Hkey,$Xi 1064 vpxor $Hkey,$HK,$HK 1065 mov \$4,%r10 # up to H^8 1066 jmp .Linit_start_avx 1067___ 1068 1069sub clmul64x64_avx { 1070my ($Xhi,$Xi,$Hkey,$HK)=@_; 1071 1072if (!defined($HK)) { $HK = $T2; 1073$code.=<<___; 1074 vpunpckhqdq $Xi,$Xi,$T1 1075 vpunpckhqdq $Hkey,$Hkey,$T2 1076 vpxor $Xi,$T1,$T1 # 1077 vpxor $Hkey,$T2,$T2 1078___ 1079} else { 1080$code.=<<___; 1081 vpunpckhqdq $Xi,$Xi,$T1 1082 vpxor $Xi,$T1,$T1 # 1083___ 1084} 1085$code.=<<___; 1086 vpclmulqdq \$0x11,$Hkey,$Xi,$Xhi ####### 1087 vpclmulqdq \$0x00,$Hkey,$Xi,$Xi ####### 1088 vpclmulqdq \$0x00,$HK,$T1,$T1 ####### 1089 vpxor $Xi,$Xhi,$T2 # 1090 vpxor $T2,$T1,$T1 # 1091 1092 vpslldq \$8,$T1,$T2 # 1093 vpsrldq \$8,$T1,$T1 1094 vpxor $T2,$Xi,$Xi # 1095 vpxor $T1,$Xhi,$Xhi 1096___ 1097} 1098 1099sub reduction_avx { 1100my ($Xhi,$Xi) = @_; 1101 1102$code.=<<___; 1103 vpsllq \$57,$Xi,$T1 # 1st phase 1104 vpsllq \$62,$Xi,$T2 1105 vpxor $T1,$T2,$T2 # 1106 vpsllq \$63,$Xi,$T1 1107 vpxor $T1,$T2,$T2 # 1108 vpslldq \$8,$T2,$T1 # 1109 vpsrldq \$8,$T2,$T2 1110 vpxor $T1,$Xi,$Xi # 1111 vpxor $T2,$Xhi,$Xhi 1112 1113 vpsrlq \$1,$Xi,$T2 # 2nd phase 1114 vpxor $Xi,$Xhi,$Xhi 1115 vpxor $T2,$Xi,$Xi # 1116 vpsrlq \$5,$T2,$T2 1117 vpxor $T2,$Xi,$Xi # 1118 vpsrlq \$1,$Xi,$Xi # 1119 vpxor $Xhi,$Xi,$Xi # 1120___ 1121} 1122 1123$code.=<<___; 1124.align 32 1125.Linit_loop_avx: 1126 vpalignr \$8,$T1,$T2,$T3 # low part is H.lo^H.hi... 1127 vmovdqu $T3,-0x10($Htbl) # save Karatsuba "salt" 1128___ 1129 &clmul64x64_avx ($Xhi,$Xi,$Hkey,$HK); # calculate H^3,5,7 1130 &reduction_avx ($Xhi,$Xi); 1131$code.=<<___; 1132.Linit_start_avx: 1133 vmovdqa $Xi,$T3 1134___ 1135 &clmul64x64_avx ($Xhi,$Xi,$Hkey,$HK); # calculate H^2,4,6,8 1136 &reduction_avx ($Xhi,$Xi); 1137$code.=<<___; 1138 vpshufd \$0b01001110,$T3,$T1 1139 vpshufd \$0b01001110,$Xi,$T2 1140 vpxor $T3,$T1,$T1 # Karatsuba pre-processing 1141 vmovdqu $T3,0x00($Htbl) # save H^1,3,5,7 1142 vpxor $Xi,$T2,$T2 # Karatsuba pre-processing 1143 vmovdqu $Xi,0x10($Htbl) # save H^2,4,6,8 1144 lea 0x30($Htbl),$Htbl 1145 sub \$1,%r10 1146 jnz .Linit_loop_avx 1147 1148 vpalignr \$8,$T2,$T1,$T3 # last "salt" is flipped 1149 vmovdqu $T3,-0x10($Htbl) 1150 1151 vzeroupper 1152___ 1153$code.=<<___ if ($win64); 1154 movaps (%rsp),%xmm6 1155 lea 0x18(%rsp),%rsp 1156.LSEH_end_gcm_init_avx: 1157___ 1158$code.=<<___; 1159 ret 1160.cfi_endproc 1161.size gcm_init_avx,.-gcm_init_avx 1162___ 1163} else { 1164$code.=<<___; 1165 jmp .L_init_clmul 1166.cfi_endproc 1167.size gcm_init_avx,.-gcm_init_avx 1168___ 1169} 1170 1171$code.=<<___; 1172.globl gcm_gmult_avx 1173.type gcm_gmult_avx,\@abi-omnipotent 1174.align 32 1175gcm_gmult_avx: 1176.cfi_startproc 1177 endbranch 1178 jmp .L_gmult_clmul 1179.cfi_endproc 1180.size gcm_gmult_avx,.-gcm_gmult_avx 1181___ 1182 1183$code.=<<___; 1184.globl gcm_ghash_avx 1185.type gcm_ghash_avx,\@abi-omnipotent 1186.align 32 1187gcm_ghash_avx: 1188.cfi_startproc 1189 endbranch 1190___ 1191if ($avx) { 1192my ($Xip,$Htbl,$inp,$len)=@_4args; 1193my ($Xlo,$Xhi,$Xmi, 1194 $Zlo,$Zhi,$Zmi, 1195 $Hkey,$HK,$T1,$T2, 1196 $Xi,$Xo,$Tred,$bswap,$Ii,$Ij) = map("%xmm$_",(0..15)); 1197 1198$code.=<<___ if ($win64); 1199 lea -0x88(%rsp),%rax 1200.LSEH_begin_gcm_ghash_avx: 1201 # I can't trust assembler to use specific encoding:-( 1202 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax),%rsp 1203 .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6,-0x20(%rax) 1204 .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7,-0x10(%rax) 1205 .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8,0(%rax) 1206 .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9,0x10(%rax) 1207 .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10,0x20(%rax) 1208 .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11,0x30(%rax) 1209 .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12,0x40(%rax) 1210 .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13,0x50(%rax) 1211 .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14,0x60(%rax) 1212 .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15,0x70(%rax) 1213___ 1214$code.=<<___; 1215 vzeroupper 1216 1217 vmovdqu ($Xip),$Xi # load $Xi 1218 lea .L0x1c2_polynomial(%rip),%r10 1219 lea 0x40($Htbl),$Htbl # size optimization 1220 vmovdqu .Lbswap_mask(%rip),$bswap 1221 vpshufb $bswap,$Xi,$Xi 1222 cmp \$0x80,$len 1223 jb .Lshort_avx 1224 sub \$0x80,$len 1225 1226 vmovdqu 0x70($inp),$Ii # I[7] 1227 vmovdqu 0x00-0x40($Htbl),$Hkey # $Hkey^1 1228 vpshufb $bswap,$Ii,$Ii 1229 vmovdqu 0x20-0x40($Htbl),$HK 1230 1231 vpunpckhqdq $Ii,$Ii,$T2 1232 vmovdqu 0x60($inp),$Ij # I[6] 1233 vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo 1234 vpxor $Ii,$T2,$T2 1235 vpshufb $bswap,$Ij,$Ij 1236 vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi 1237 vmovdqu 0x10-0x40($Htbl),$Hkey # $Hkey^2 1238 vpunpckhqdq $Ij,$Ij,$T1 1239 vmovdqu 0x50($inp),$Ii # I[5] 1240 vpclmulqdq \$0x00,$HK,$T2,$Xmi 1241 vpxor $Ij,$T1,$T1 1242 1243 vpshufb $bswap,$Ii,$Ii 1244 vpclmulqdq \$0x00,$Hkey,$Ij,$Zlo 1245 vpunpckhqdq $Ii,$Ii,$T2 1246 vpclmulqdq \$0x11,$Hkey,$Ij,$Zhi 1247 vmovdqu 0x30-0x40($Htbl),$Hkey # $Hkey^3 1248 vpxor $Ii,$T2,$T2 1249 vmovdqu 0x40($inp),$Ij # I[4] 1250 vpclmulqdq \$0x10,$HK,$T1,$Zmi 1251 vmovdqu 0x50-0x40($Htbl),$HK 1252 1253 vpshufb $bswap,$Ij,$Ij 1254 vpxor $Xlo,$Zlo,$Zlo 1255 vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo 1256 vpxor $Xhi,$Zhi,$Zhi 1257 vpunpckhqdq $Ij,$Ij,$T1 1258 vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi 1259 vmovdqu 0x40-0x40($Htbl),$Hkey # $Hkey^4 1260 vpxor $Xmi,$Zmi,$Zmi 1261 vpclmulqdq \$0x00,$HK,$T2,$Xmi 1262 vpxor $Ij,$T1,$T1 1263 1264 vmovdqu 0x30($inp),$Ii # I[3] 1265 vpxor $Zlo,$Xlo,$Xlo 1266 vpclmulqdq \$0x00,$Hkey,$Ij,$Zlo 1267 vpxor $Zhi,$Xhi,$Xhi 1268 vpshufb $bswap,$Ii,$Ii 1269 vpclmulqdq \$0x11,$Hkey,$Ij,$Zhi 1270 vmovdqu 0x60-0x40($Htbl),$Hkey # $Hkey^5 1271 vpxor $Zmi,$Xmi,$Xmi 1272 vpunpckhqdq $Ii,$Ii,$T2 1273 vpclmulqdq \$0x10,$HK,$T1,$Zmi 1274 vmovdqu 0x80-0x40($Htbl),$HK 1275 vpxor $Ii,$T2,$T2 1276 1277 vmovdqu 0x20($inp),$Ij # I[2] 1278 vpxor $Xlo,$Zlo,$Zlo 1279 vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo 1280 vpxor $Xhi,$Zhi,$Zhi 1281 vpshufb $bswap,$Ij,$Ij 1282 vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi 1283 vmovdqu 0x70-0x40($Htbl),$Hkey # $Hkey^6 1284 vpxor $Xmi,$Zmi,$Zmi 1285 vpunpckhqdq $Ij,$Ij,$T1 1286 vpclmulqdq \$0x00,$HK,$T2,$Xmi 1287 vpxor $Ij,$T1,$T1 1288 1289 vmovdqu 0x10($inp),$Ii # I[1] 1290 vpxor $Zlo,$Xlo,$Xlo 1291 vpclmulqdq \$0x00,$Hkey,$Ij,$Zlo 1292 vpxor $Zhi,$Xhi,$Xhi 1293 vpshufb $bswap,$Ii,$Ii 1294 vpclmulqdq \$0x11,$Hkey,$Ij,$Zhi 1295 vmovdqu 0x90-0x40($Htbl),$Hkey # $Hkey^7 1296 vpxor $Zmi,$Xmi,$Xmi 1297 vpunpckhqdq $Ii,$Ii,$T2 1298 vpclmulqdq \$0x10,$HK,$T1,$Zmi 1299 vmovdqu 0xb0-0x40($Htbl),$HK 1300 vpxor $Ii,$T2,$T2 1301 1302 vmovdqu ($inp),$Ij # I[0] 1303 vpxor $Xlo,$Zlo,$Zlo 1304 vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo 1305 vpxor $Xhi,$Zhi,$Zhi 1306 vpshufb $bswap,$Ij,$Ij 1307 vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi 1308 vmovdqu 0xa0-0x40($Htbl),$Hkey # $Hkey^8 1309 vpxor $Xmi,$Zmi,$Zmi 1310 vpclmulqdq \$0x10,$HK,$T2,$Xmi 1311 1312 lea 0x80($inp),$inp 1313 cmp \$0x80,$len 1314 jb .Ltail_avx 1315 1316 vpxor $Xi,$Ij,$Ij # accumulate $Xi 1317 sub \$0x80,$len 1318 jmp .Loop8x_avx 1319 1320.align 32 1321.Loop8x_avx: 1322 vpunpckhqdq $Ij,$Ij,$T1 1323 vmovdqu 0x70($inp),$Ii # I[7] 1324 vpxor $Xlo,$Zlo,$Zlo 1325 vpxor $Ij,$T1,$T1 1326 vpclmulqdq \$0x00,$Hkey,$Ij,$Xi 1327 vpshufb $bswap,$Ii,$Ii 1328 vpxor $Xhi,$Zhi,$Zhi 1329 vpclmulqdq \$0x11,$Hkey,$Ij,$Xo 1330 vmovdqu 0x00-0x40($Htbl),$Hkey # $Hkey^1 1331 vpunpckhqdq $Ii,$Ii,$T2 1332 vpxor $Xmi,$Zmi,$Zmi 1333 vpclmulqdq \$0x00,$HK,$T1,$Tred 1334 vmovdqu 0x20-0x40($Htbl),$HK 1335 vpxor $Ii,$T2,$T2 1336 1337 vmovdqu 0x60($inp),$Ij # I[6] 1338 vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo 1339 vpxor $Zlo,$Xi,$Xi # collect result 1340 vpshufb $bswap,$Ij,$Ij 1341 vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi 1342 vxorps $Zhi,$Xo,$Xo 1343 vmovdqu 0x10-0x40($Htbl),$Hkey # $Hkey^2 1344 vpunpckhqdq $Ij,$Ij,$T1 1345 vpclmulqdq \$0x00,$HK, $T2,$Xmi 1346 vpxor $Zmi,$Tred,$Tred 1347 vxorps $Ij,$T1,$T1 1348 1349 vmovdqu 0x50($inp),$Ii # I[5] 1350 vpxor $Xi,$Tred,$Tred # aggregated Karatsuba post-processing 1351 vpclmulqdq \$0x00,$Hkey,$Ij,$Zlo 1352 vpxor $Xo,$Tred,$Tred 1353 vpslldq \$8,$Tred,$T2 1354 vpxor $Xlo,$Zlo,$Zlo 1355 vpclmulqdq \$0x11,$Hkey,$Ij,$Zhi 1356 vpsrldq \$8,$Tred,$Tred 1357 vpxor $T2, $Xi, $Xi 1358 vmovdqu 0x30-0x40($Htbl),$Hkey # $Hkey^3 1359 vpshufb $bswap,$Ii,$Ii 1360 vxorps $Tred,$Xo, $Xo 1361 vpxor $Xhi,$Zhi,$Zhi 1362 vpunpckhqdq $Ii,$Ii,$T2 1363 vpclmulqdq \$0x10,$HK, $T1,$Zmi 1364 vmovdqu 0x50-0x40($Htbl),$HK 1365 vpxor $Ii,$T2,$T2 1366 vpxor $Xmi,$Zmi,$Zmi 1367 1368 vmovdqu 0x40($inp),$Ij # I[4] 1369 vpalignr \$8,$Xi,$Xi,$Tred # 1st phase 1370 vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo 1371 vpshufb $bswap,$Ij,$Ij 1372 vpxor $Zlo,$Xlo,$Xlo 1373 vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi 1374 vmovdqu 0x40-0x40($Htbl),$Hkey # $Hkey^4 1375 vpunpckhqdq $Ij,$Ij,$T1 1376 vpxor $Zhi,$Xhi,$Xhi 1377 vpclmulqdq \$0x00,$HK, $T2,$Xmi 1378 vxorps $Ij,$T1,$T1 1379 vpxor $Zmi,$Xmi,$Xmi 1380 1381 vmovdqu 0x30($inp),$Ii # I[3] 1382 vpclmulqdq \$0x10,(%r10),$Xi,$Xi 1383 vpclmulqdq \$0x00,$Hkey,$Ij,$Zlo 1384 vpshufb $bswap,$Ii,$Ii 1385 vpxor $Xlo,$Zlo,$Zlo 1386 vpclmulqdq \$0x11,$Hkey,$Ij,$Zhi 1387 vmovdqu 0x60-0x40($Htbl),$Hkey # $Hkey^5 1388 vpunpckhqdq $Ii,$Ii,$T2 1389 vpxor $Xhi,$Zhi,$Zhi 1390 vpclmulqdq \$0x10,$HK, $T1,$Zmi 1391 vmovdqu 0x80-0x40($Htbl),$HK 1392 vpxor $Ii,$T2,$T2 1393 vpxor $Xmi,$Zmi,$Zmi 1394 1395 vmovdqu 0x20($inp),$Ij # I[2] 1396 vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo 1397 vpshufb $bswap,$Ij,$Ij 1398 vpxor $Zlo,$Xlo,$Xlo 1399 vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi 1400 vmovdqu 0x70-0x40($Htbl),$Hkey # $Hkey^6 1401 vpunpckhqdq $Ij,$Ij,$T1 1402 vpxor $Zhi,$Xhi,$Xhi 1403 vpclmulqdq \$0x00,$HK, $T2,$Xmi 1404 vpxor $Ij,$T1,$T1 1405 vpxor $Zmi,$Xmi,$Xmi 1406 vxorps $Tred,$Xi,$Xi 1407 1408 vmovdqu 0x10($inp),$Ii # I[1] 1409 vpalignr \$8,$Xi,$Xi,$Tred # 2nd phase 1410 vpclmulqdq \$0x00,$Hkey,$Ij,$Zlo 1411 vpshufb $bswap,$Ii,$Ii 1412 vpxor $Xlo,$Zlo,$Zlo 1413 vpclmulqdq \$0x11,$Hkey,$Ij,$Zhi 1414 vmovdqu 0x90-0x40($Htbl),$Hkey # $Hkey^7 1415 vpclmulqdq \$0x10,(%r10),$Xi,$Xi 1416 vxorps $Xo,$Tred,$Tred 1417 vpunpckhqdq $Ii,$Ii,$T2 1418 vpxor $Xhi,$Zhi,$Zhi 1419 vpclmulqdq \$0x10,$HK, $T1,$Zmi 1420 vmovdqu 0xb0-0x40($Htbl),$HK 1421 vpxor $Ii,$T2,$T2 1422 vpxor $Xmi,$Zmi,$Zmi 1423 1424 vmovdqu ($inp),$Ij # I[0] 1425 vpclmulqdq \$0x00,$Hkey,$Ii,$Xlo 1426 vpshufb $bswap,$Ij,$Ij 1427 vpclmulqdq \$0x11,$Hkey,$Ii,$Xhi 1428 vmovdqu 0xa0-0x40($Htbl),$Hkey # $Hkey^8 1429 vpxor $Tred,$Ij,$Ij 1430 vpclmulqdq \$0x10,$HK, $T2,$Xmi 1431 vpxor $Xi,$Ij,$Ij # accumulate $Xi 1432 1433 lea 0x80($inp),$inp 1434 sub \$0x80,$len 1435 jnc .Loop8x_avx 1436 1437 add \$0x80,$len 1438 jmp .Ltail_no_xor_avx 1439 1440.align 32 1441.Lshort_avx: 1442 vmovdqu -0x10($inp,$len),$Ii # very last word 1443 lea ($inp,$len),$inp 1444 vmovdqu 0x00-0x40($Htbl),$Hkey # $Hkey^1 1445 vmovdqu 0x20-0x40($Htbl),$HK 1446 vpshufb $bswap,$Ii,$Ij 1447 1448 vmovdqa $Xlo,$Zlo # subtle way to zero $Zlo, 1449 vmovdqa $Xhi,$Zhi # $Zhi and 1450 vmovdqa $Xmi,$Zmi # $Zmi 1451 sub \$0x10,$len 1452 jz .Ltail_avx 1453 1454 vpunpckhqdq $Ij,$Ij,$T1 1455 vpxor $Xlo,$Zlo,$Zlo 1456 vpclmulqdq \$0x00,$Hkey,$Ij,$Xlo 1457 vpxor $Ij,$T1,$T1 1458 vmovdqu -0x20($inp),$Ii 1459 vpxor $Xhi,$Zhi,$Zhi 1460 vpclmulqdq \$0x11,$Hkey,$Ij,$Xhi 1461 vmovdqu 0x10-0x40($Htbl),$Hkey # $Hkey^2 1462 vpshufb $bswap,$Ii,$Ij 1463 vpxor $Xmi,$Zmi,$Zmi 1464 vpclmulqdq \$0x00,$HK,$T1,$Xmi 1465 vpsrldq \$8,$HK,$HK 1466 sub \$0x10,$len 1467 jz .Ltail_avx 1468 1469 vpunpckhqdq $Ij,$Ij,$T1 1470 vpxor $Xlo,$Zlo,$Zlo 1471 vpclmulqdq \$0x00,$Hkey,$Ij,$Xlo 1472 vpxor $Ij,$T1,$T1 1473 vmovdqu -0x30($inp),$Ii 1474 vpxor $Xhi,$Zhi,$Zhi 1475 vpclmulqdq \$0x11,$Hkey,$Ij,$Xhi 1476 vmovdqu 0x30-0x40($Htbl),$Hkey # $Hkey^3 1477 vpshufb $bswap,$Ii,$Ij 1478 vpxor $Xmi,$Zmi,$Zmi 1479 vpclmulqdq \$0x00,$HK,$T1,$Xmi 1480 vmovdqu 0x50-0x40($Htbl),$HK 1481 sub \$0x10,$len 1482 jz .Ltail_avx 1483 1484 vpunpckhqdq $Ij,$Ij,$T1 1485 vpxor $Xlo,$Zlo,$Zlo 1486 vpclmulqdq \$0x00,$Hkey,$Ij,$Xlo 1487 vpxor $Ij,$T1,$T1 1488 vmovdqu -0x40($inp),$Ii 1489 vpxor $Xhi,$Zhi,$Zhi 1490 vpclmulqdq \$0x11,$Hkey,$Ij,$Xhi 1491 vmovdqu 0x40-0x40($Htbl),$Hkey # $Hkey^4 1492 vpshufb $bswap,$Ii,$Ij 1493 vpxor $Xmi,$Zmi,$Zmi 1494 vpclmulqdq \$0x00,$HK,$T1,$Xmi 1495 vpsrldq \$8,$HK,$HK 1496 sub \$0x10,$len 1497 jz .Ltail_avx 1498 1499 vpunpckhqdq $Ij,$Ij,$T1 1500 vpxor $Xlo,$Zlo,$Zlo 1501 vpclmulqdq \$0x00,$Hkey,$Ij,$Xlo 1502 vpxor $Ij,$T1,$T1 1503 vmovdqu -0x50($inp),$Ii 1504 vpxor $Xhi,$Zhi,$Zhi 1505 vpclmulqdq \$0x11,$Hkey,$Ij,$Xhi 1506 vmovdqu 0x60-0x40($Htbl),$Hkey # $Hkey^5 1507 vpshufb $bswap,$Ii,$Ij 1508 vpxor $Xmi,$Zmi,$Zmi 1509 vpclmulqdq \$0x00,$HK,$T1,$Xmi 1510 vmovdqu 0x80-0x40($Htbl),$HK 1511 sub \$0x10,$len 1512 jz .Ltail_avx 1513 1514 vpunpckhqdq $Ij,$Ij,$T1 1515 vpxor $Xlo,$Zlo,$Zlo 1516 vpclmulqdq \$0x00,$Hkey,$Ij,$Xlo 1517 vpxor $Ij,$T1,$T1 1518 vmovdqu -0x60($inp),$Ii 1519 vpxor $Xhi,$Zhi,$Zhi 1520 vpclmulqdq \$0x11,$Hkey,$Ij,$Xhi 1521 vmovdqu 0x70-0x40($Htbl),$Hkey # $Hkey^6 1522 vpshufb $bswap,$Ii,$Ij 1523 vpxor $Xmi,$Zmi,$Zmi 1524 vpclmulqdq \$0x00,$HK,$T1,$Xmi 1525 vpsrldq \$8,$HK,$HK 1526 sub \$0x10,$len 1527 jz .Ltail_avx 1528 1529 vpunpckhqdq $Ij,$Ij,$T1 1530 vpxor $Xlo,$Zlo,$Zlo 1531 vpclmulqdq \$0x00,$Hkey,$Ij,$Xlo 1532 vpxor $Ij,$T1,$T1 1533 vmovdqu -0x70($inp),$Ii 1534 vpxor $Xhi,$Zhi,$Zhi 1535 vpclmulqdq \$0x11,$Hkey,$Ij,$Xhi 1536 vmovdqu 0x90-0x40($Htbl),$Hkey # $Hkey^7 1537 vpshufb $bswap,$Ii,$Ij 1538 vpxor $Xmi,$Zmi,$Zmi 1539 vpclmulqdq \$0x00,$HK,$T1,$Xmi 1540 vmovq 0xb8-0x40($Htbl),$HK 1541 sub \$0x10,$len 1542 jmp .Ltail_avx 1543 1544.align 32 1545.Ltail_avx: 1546 vpxor $Xi,$Ij,$Ij # accumulate $Xi 1547.Ltail_no_xor_avx: 1548 vpunpckhqdq $Ij,$Ij,$T1 1549 vpxor $Xlo,$Zlo,$Zlo 1550 vpclmulqdq \$0x00,$Hkey,$Ij,$Xlo 1551 vpxor $Ij,$T1,$T1 1552 vpxor $Xhi,$Zhi,$Zhi 1553 vpclmulqdq \$0x11,$Hkey,$Ij,$Xhi 1554 vpxor $Xmi,$Zmi,$Zmi 1555 vpclmulqdq \$0x00,$HK,$T1,$Xmi 1556 1557 vmovdqu (%r10),$Tred 1558 1559 vpxor $Xlo,$Zlo,$Xi 1560 vpxor $Xhi,$Zhi,$Xo 1561 vpxor $Xmi,$Zmi,$Zmi 1562 1563 vpxor $Xi, $Zmi,$Zmi # aggregated Karatsuba post-processing 1564 vpxor $Xo, $Zmi,$Zmi 1565 vpslldq \$8, $Zmi,$T2 1566 vpsrldq \$8, $Zmi,$Zmi 1567 vpxor $T2, $Xi, $Xi 1568 vpxor $Zmi,$Xo, $Xo 1569 1570 vpclmulqdq \$0x10,$Tred,$Xi,$T2 # 1st phase 1571 vpalignr \$8,$Xi,$Xi,$Xi 1572 vpxor $T2,$Xi,$Xi 1573 1574 vpclmulqdq \$0x10,$Tred,$Xi,$T2 # 2nd phase 1575 vpalignr \$8,$Xi,$Xi,$Xi 1576 vpxor $Xo,$Xi,$Xi 1577 vpxor $T2,$Xi,$Xi 1578 1579 cmp \$0,$len 1580 jne .Lshort_avx 1581 1582 vpshufb $bswap,$Xi,$Xi 1583 vmovdqu $Xi,($Xip) 1584 vzeroupper 1585___ 1586$code.=<<___ if ($win64); 1587 movaps (%rsp),%xmm6 1588 movaps 0x10(%rsp),%xmm7 1589 movaps 0x20(%rsp),%xmm8 1590 movaps 0x30(%rsp),%xmm9 1591 movaps 0x40(%rsp),%xmm10 1592 movaps 0x50(%rsp),%xmm11 1593 movaps 0x60(%rsp),%xmm12 1594 movaps 0x70(%rsp),%xmm13 1595 movaps 0x80(%rsp),%xmm14 1596 movaps 0x90(%rsp),%xmm15 1597 lea 0xa8(%rsp),%rsp 1598.LSEH_end_gcm_ghash_avx: 1599___ 1600$code.=<<___; 1601 ret 1602.cfi_endproc 1603.size gcm_ghash_avx,.-gcm_ghash_avx 1604___ 1605} else { 1606$code.=<<___; 1607 jmp .L_ghash_clmul 1608.cfi_endproc 1609.size gcm_ghash_avx,.-gcm_ghash_avx 1610___ 1611} 1612 1613$code.=<<___; 1614.section .rodata align=64 1615.align 64 1616.Lbswap_mask: 1617 .byte 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 1618.L0x1c2_polynomial: 1619 .byte 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2 1620.L7_mask: 1621 .long 7,0,7,0 1622.L7_mask_poly: 1623 .long 7,0,`0xE1<<1`,0 1624.align 64 1625.type .Lrem_4bit,\@object 1626.Lrem_4bit: 1627 .long 0,`0x0000<<16`,0,`0x1C20<<16`,0,`0x3840<<16`,0,`0x2460<<16` 1628 .long 0,`0x7080<<16`,0,`0x6CA0<<16`,0,`0x48C0<<16`,0,`0x54E0<<16` 1629 .long 0,`0xE100<<16`,0,`0xFD20<<16`,0,`0xD940<<16`,0,`0xC560<<16` 1630 .long 0,`0x9180<<16`,0,`0x8DA0<<16`,0,`0xA9C0<<16`,0,`0xB5E0<<16` 1631.type .Lrem_8bit,\@object 1632.Lrem_8bit: 1633 .value 0x0000,0x01C2,0x0384,0x0246,0x0708,0x06CA,0x048C,0x054E 1634 .value 0x0E10,0x0FD2,0x0D94,0x0C56,0x0918,0x08DA,0x0A9C,0x0B5E 1635 .value 0x1C20,0x1DE2,0x1FA4,0x1E66,0x1B28,0x1AEA,0x18AC,0x196E 1636 .value 0x1230,0x13F2,0x11B4,0x1076,0x1538,0x14FA,0x16BC,0x177E 1637 .value 0x3840,0x3982,0x3BC4,0x3A06,0x3F48,0x3E8A,0x3CCC,0x3D0E 1638 .value 0x3650,0x3792,0x35D4,0x3416,0x3158,0x309A,0x32DC,0x331E 1639 .value 0x2460,0x25A2,0x27E4,0x2626,0x2368,0x22AA,0x20EC,0x212E 1640 .value 0x2A70,0x2BB2,0x29F4,0x2836,0x2D78,0x2CBA,0x2EFC,0x2F3E 1641 .value 0x7080,0x7142,0x7304,0x72C6,0x7788,0x764A,0x740C,0x75CE 1642 .value 0x7E90,0x7F52,0x7D14,0x7CD6,0x7998,0x785A,0x7A1C,0x7BDE 1643 .value 0x6CA0,0x6D62,0x6F24,0x6EE6,0x6BA8,0x6A6A,0x682C,0x69EE 1644 .value 0x62B0,0x6372,0x6134,0x60F6,0x65B8,0x647A,0x663C,0x67FE 1645 .value 0x48C0,0x4902,0x4B44,0x4A86,0x4FC8,0x4E0A,0x4C4C,0x4D8E 1646 .value 0x46D0,0x4712,0x4554,0x4496,0x41D8,0x401A,0x425C,0x439E 1647 .value 0x54E0,0x5522,0x5764,0x56A6,0x53E8,0x522A,0x506C,0x51AE 1648 .value 0x5AF0,0x5B32,0x5974,0x58B6,0x5DF8,0x5C3A,0x5E7C,0x5FBE 1649 .value 0xE100,0xE0C2,0xE284,0xE346,0xE608,0xE7CA,0xE58C,0xE44E 1650 .value 0xEF10,0xEED2,0xEC94,0xED56,0xE818,0xE9DA,0xEB9C,0xEA5E 1651 .value 0xFD20,0xFCE2,0xFEA4,0xFF66,0xFA28,0xFBEA,0xF9AC,0xF86E 1652 .value 0xF330,0xF2F2,0xF0B4,0xF176,0xF438,0xF5FA,0xF7BC,0xF67E 1653 .value 0xD940,0xD882,0xDAC4,0xDB06,0xDE48,0xDF8A,0xDDCC,0xDC0E 1654 .value 0xD750,0xD692,0xD4D4,0xD516,0xD058,0xD19A,0xD3DC,0xD21E 1655 .value 0xC560,0xC4A2,0xC6E4,0xC726,0xC268,0xC3AA,0xC1EC,0xC02E 1656 .value 0xCB70,0xCAB2,0xC8F4,0xC936,0xCC78,0xCDBA,0xCFFC,0xCE3E 1657 .value 0x9180,0x9042,0x9204,0x93C6,0x9688,0x974A,0x950C,0x94CE 1658 .value 0x9F90,0x9E52,0x9C14,0x9DD6,0x9898,0x995A,0x9B1C,0x9ADE 1659 .value 0x8DA0,0x8C62,0x8E24,0x8FE6,0x8AA8,0x8B6A,0x892C,0x88EE 1660 .value 0x83B0,0x8272,0x8034,0x81F6,0x84B8,0x857A,0x873C,0x86FE 1661 .value 0xA9C0,0xA802,0xAA44,0xAB86,0xAEC8,0xAF0A,0xAD4C,0xAC8E 1662 .value 0xA7D0,0xA612,0xA454,0xA596,0xA0D8,0xA11A,0xA35C,0xA29E 1663 .value 0xB5E0,0xB422,0xB664,0xB7A6,0xB2E8,0xB32A,0xB16C,0xB0AE 1664 .value 0xBBF0,0xBA32,0xB874,0xB9B6,0xBCF8,0xBD3A,0xBF7C,0xBEBE 1665 1666.asciz "GHASH for x86_64, CRYPTOGAMS by <appro\@openssl.org>" 1667.align 64 1668.previous 1669___ 1670 1671# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, 1672# CONTEXT *context,DISPATCHER_CONTEXT *disp) 1673if ($win64) { 1674$rec="%rcx"; 1675$frame="%rdx"; 1676$context="%r8"; 1677$disp="%r9"; 1678 1679$code.=<<___; 1680.extern __imp_RtlVirtualUnwind 1681.type se_handler,\@abi-omnipotent 1682.align 16 1683se_handler: 1684 push %rsi 1685 push %rdi 1686 push %rbx 1687 push %rbp 1688 push %r12 1689 push %r13 1690 push %r14 1691 push %r15 1692 pushfq 1693 sub \$64,%rsp 1694 1695 mov 120($context),%rax # pull context->Rax 1696 mov 248($context),%rbx # pull context->Rip 1697 1698 mov 8($disp),%rsi # disp->ImageBase 1699 mov 56($disp),%r11 # disp->HandlerData 1700 1701 mov 0(%r11),%r10d # HandlerData[0] 1702 lea (%rsi,%r10),%r10 # prologue label 1703 cmp %r10,%rbx # context->Rip<prologue label 1704 jb .Lin_prologue 1705 1706 mov 152($context),%rax # pull context->Rsp 1707 1708 mov 4(%r11),%r10d # HandlerData[1] 1709 lea (%rsi,%r10),%r10 # epilogue label 1710 cmp %r10,%rbx # context->Rip>=epilogue label 1711 jae .Lin_prologue 1712 1713 lea 48+280(%rax),%rax # adjust "rsp" 1714 1715 mov -8(%rax),%rbx 1716 mov -16(%rax),%rbp 1717 mov -24(%rax),%r12 1718 mov -32(%rax),%r13 1719 mov -40(%rax),%r14 1720 mov -48(%rax),%r15 1721 mov %rbx,144($context) # restore context->Rbx 1722 mov %rbp,160($context) # restore context->Rbp 1723 mov %r12,216($context) # restore context->R12 1724 mov %r13,224($context) # restore context->R13 1725 mov %r14,232($context) # restore context->R14 1726 mov %r15,240($context) # restore context->R15 1727 1728.Lin_prologue: 1729 mov 8(%rax),%rdi 1730 mov 16(%rax),%rsi 1731 mov %rax,152($context) # restore context->Rsp 1732 mov %rsi,168($context) # restore context->Rsi 1733 mov %rdi,176($context) # restore context->Rdi 1734 1735 mov 40($disp),%rdi # disp->ContextRecord 1736 mov $context,%rsi # context 1737 mov \$`1232/8`,%ecx # sizeof(CONTEXT) 1738 .long 0xa548f3fc # cld; rep movsq 1739 1740 mov $disp,%rsi 1741 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER 1742 mov 8(%rsi),%rdx # arg2, disp->ImageBase 1743 mov 0(%rsi),%r8 # arg3, disp->ControlPc 1744 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry 1745 mov 40(%rsi),%r10 # disp->ContextRecord 1746 lea 56(%rsi),%r11 # &disp->HandlerData 1747 lea 24(%rsi),%r12 # &disp->EstablisherFrame 1748 mov %r10,32(%rsp) # arg5 1749 mov %r11,40(%rsp) # arg6 1750 mov %r12,48(%rsp) # arg7 1751 mov %rcx,56(%rsp) # arg8, (NULL) 1752 call *__imp_RtlVirtualUnwind(%rip) 1753 1754 mov \$1,%eax # ExceptionContinueSearch 1755 add \$64,%rsp 1756 popfq 1757 pop %r15 1758 pop %r14 1759 pop %r13 1760 pop %r12 1761 pop %rbp 1762 pop %rbx 1763 pop %rdi 1764 pop %rsi 1765 ret 1766.size se_handler,.-se_handler 1767 1768.section .pdata 1769.align 4 1770 .rva .LSEH_begin_gcm_gmult_4bit 1771 .rva .LSEH_end_gcm_gmult_4bit 1772 .rva .LSEH_info_gcm_gmult_4bit 1773 1774 .rva .LSEH_begin_gcm_ghash_4bit 1775 .rva .LSEH_end_gcm_ghash_4bit 1776 .rva .LSEH_info_gcm_ghash_4bit 1777 1778 .rva .LSEH_begin_gcm_init_clmul 1779 .rva .LSEH_end_gcm_init_clmul 1780 .rva .LSEH_info_gcm_init_clmul 1781 1782 .rva .LSEH_begin_gcm_ghash_clmul 1783 .rva .LSEH_end_gcm_ghash_clmul 1784 .rva .LSEH_info_gcm_ghash_clmul 1785___ 1786$code.=<<___ if ($avx); 1787 .rva .LSEH_begin_gcm_init_avx 1788 .rva .LSEH_end_gcm_init_avx 1789 .rva .LSEH_info_gcm_init_clmul 1790 1791 .rva .LSEH_begin_gcm_ghash_avx 1792 .rva .LSEH_end_gcm_ghash_avx 1793 .rva .LSEH_info_gcm_ghash_clmul 1794___ 1795$code.=<<___; 1796.section .xdata 1797.align 8 1798.LSEH_info_gcm_gmult_4bit: 1799 .byte 9,0,0,0 1800 .rva se_handler 1801 .rva .Lgmult_prologue,.Lgmult_epilogue # HandlerData 1802.LSEH_info_gcm_ghash_4bit: 1803 .byte 9,0,0,0 1804 .rva se_handler 1805 .rva .Lghash_prologue,.Lghash_epilogue # HandlerData 1806.LSEH_info_gcm_init_clmul: 1807 .byte 0x01,0x08,0x03,0x00 1808 .byte 0x08,0x68,0x00,0x00 #movaps 0x00(rsp),xmm6 1809 .byte 0x04,0x22,0x00,0x00 #sub rsp,0x18 1810.LSEH_info_gcm_ghash_clmul: 1811 .byte 0x01,0x33,0x16,0x00 1812 .byte 0x33,0xf8,0x09,0x00 #movaps 0x90(rsp),xmm15 1813 .byte 0x2e,0xe8,0x08,0x00 #movaps 0x80(rsp),xmm14 1814 .byte 0x29,0xd8,0x07,0x00 #movaps 0x70(rsp),xmm13 1815 .byte 0x24,0xc8,0x06,0x00 #movaps 0x60(rsp),xmm12 1816 .byte 0x1f,0xb8,0x05,0x00 #movaps 0x50(rsp),xmm11 1817 .byte 0x1a,0xa8,0x04,0x00 #movaps 0x40(rsp),xmm10 1818 .byte 0x15,0x98,0x03,0x00 #movaps 0x30(rsp),xmm9 1819 .byte 0x10,0x88,0x02,0x00 #movaps 0x20(rsp),xmm8 1820 .byte 0x0c,0x78,0x01,0x00 #movaps 0x10(rsp),xmm7 1821 .byte 0x08,0x68,0x00,0x00 #movaps 0x00(rsp),xmm6 1822 .byte 0x04,0x01,0x15,0x00 #sub rsp,0xa8 1823___ 1824} 1825 1826$code =~ s/\`([^\`]*)\`/eval($1)/gem; 1827 1828print $code; 1829 1830close STDOUT or die "error closing STDOUT: $!"; 1831
