modes/asm/aes-gcm-avx512.pl

# Copyright 2021-2024 The OpenSSL Project Authors. All Rights Reserved.
# Copyright (c) 2021, Intel Corporation. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
#
# This implementation is based on the AES-GCM code (AVX512VAES + VPCLMULQDQ)
# from Intel(R) Multi-Buffer Crypto for IPsec Library v1.1
# (https://github.com/intel/intel-ipsec-mb).
# Original author is Tomasz Kantecki <tomasz.kantecki@intel.com>.
#
# References:
#  [1] Vinodh Gopal et. al. Optimized Galois-Counter-Mode Implementation on
#      Intel Architecture Processors. August, 2010.
#  [2] Erdinc Ozturk et. al. Enabling High-Performance Galois-Counter-Mode on
#      Intel Architecture Processors. October, 2012.
#  [3] Shay Gueron et. al. Intel Carry-Less Multiplication Instruction and its
#      Usage for Computing the GCM Mode. May, 2010.
#
#
# December 2021
#
# Initial release.
#
# GCM128_CONTEXT structure has storage for 16 hkeys only, but this
# implementation can use up to 48.  To avoid extending the context size,
# precompute and store in the context first 16 hkeys only, and compute the rest
# on demand keeping them in the local frame.
#
#======================================================================
# $output is the last argument if it looks like a file (it has an extension)
# $flavour is the first argument if it doesn't look like a file
$output  = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop   : undef;
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.|          ? shift : undef;

$win64 = 0;
$win64 = 1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

$avx512vaes = 0;

$0 =~ m/(.*[\/\\])[^\/\\]+$/;
$dir = $1;
($xlate = "${dir}x86_64-xlate.pl" and -f $xlate)
  or ($xlate = "${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate)
  or die "can't locate x86_64-xlate.pl";

if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  $avx512vaes = ($1 >= 2.30);
}

if (!$avx512vaes
  && $win64
  && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/)
  && `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/)
{
  $avx512vaes = ($1 == 2.13 && $2 >= 3) + ($1 >= 2.14);
}

if (!$avx512vaes && `$ENV{CC} -v 2>&1`
    =~ /(Apple)?\s*((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)\.([0-9]+)?/) {
    my $ver = $3 + $4/100.0 + $5/10000.0; # 3.1.0->3.01, 3.10.1->3.1001
    if ($1) {
        # Apple conditions, they use a different version series, see
        # https://en.wikipedia.org/wiki/Xcode#Xcode_7.0_-_10.x_(since_Free_On-Device_Development)_2
        # clang 7.0.0 is Apple clang 10.0.1
        $avx512vaes = ($ver>=10.0001)
    } else {
        $avx512vaes = ($ver>=7.0);
    }
}

open OUT, "| \"$^X\" \"$xlate\" $flavour \"$output\""
  or die "can't call $xlate: $!";
*STDOUT = *OUT;

#======================================================================
if ($avx512vaes>0) { #<<<

$code .= <<___;
.extern OPENSSL_ia32cap_P
.globl  ossl_vaes_vpclmulqdq_capable
.type   ossl_vaes_vpclmulqdq_capable,\@abi-omnipotent
.align 32
ossl_vaes_vpclmulqdq_capable:
    mov OPENSSL_ia32cap_P+8(%rip), %rcx
    # avx512vpclmulqdq + avx512vaes + avx512vl + avx512bw + avx512dq + avx512f
    mov \$`1<<42|1<<41|1<<31|1<<30|1<<17|1<<16`,%rdx
    xor %eax,%eax
    and %rdx,%rcx
    cmp %rdx,%rcx
    cmove %rcx,%rax
    ret
.size   ossl_vaes_vpclmulqdq_capable, .-ossl_vaes_vpclmulqdq_capable
___

# ; Mapping key length -> AES rounds count
my %aes_rounds = (
  128 => 9,
  192 => 11,
  256 => 13);

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;;; Code generation control switches
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

# ; ABI-aware zeroing of volatile registers in EPILOG().
# ; Disabled due to performance reasons.
my $CLEAR_SCRATCH_REGISTERS = 0;

# ; Zero HKeys storage from the stack if they are stored there
my $CLEAR_HKEYS_STORAGE_ON_EXIT = 1;

# ; Enable / disable check of function arguments for null pointer
# ; Currently disabled, as this check is handled outside.
my $CHECK_FUNCTION_ARGUMENTS = 0;

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;;; Global constants
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

# AES block size in bytes
my $AES_BLOCK_SIZE = 16;

# Storage capacity in elements
my $HKEYS_STORAGE_CAPACITY = 48;
my $LOCAL_STORAGE_CAPACITY = 48;
my $HKEYS_CONTEXT_CAPACITY = 16;

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;;; Stack frame definition
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

# (1) -> +64(Win)/+48(Lin)-byte space for pushed GPRs
# (2) -> +8-byte space for 16-byte alignment of XMM storage
# (3) -> Frame pointer (%RBP)
# (4) -> +160-byte XMM storage (Windows only, zero on Linux)
# (5) -> +48-byte space for 64-byte alignment of %RSP from p.8
# (6) -> +768-byte LOCAL storage (optional, can be omitted in some functions)
# (7) -> +768-byte HKEYS storage
# (8) -> Stack pointer (%RSP) aligned on 64-byte boundary

my $GP_STORAGE  = $win64 ? 8 * 8     : 8 * 6;    # ; space for saved non-volatile GP registers (pushed on stack)
my $XMM_STORAGE = $win64 ? (10 * 16) : 0;        # ; space for saved XMM registers
my $HKEYS_STORAGE = ($HKEYS_STORAGE_CAPACITY * $AES_BLOCK_SIZE);    # ; space for HKeys^i, i=1..48
my $LOCAL_STORAGE = ($LOCAL_STORAGE_CAPACITY * $AES_BLOCK_SIZE);    # ; space for up to 48 AES blocks

my $STACK_HKEYS_OFFSET = 0;
my $STACK_LOCAL_OFFSET = ($STACK_HKEYS_OFFSET + $HKEYS_STORAGE);

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;;; Function arguments abstraction
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
my ($arg1, $arg2, $arg3, $arg4, $arg5, $arg6, $arg7, $arg8, $arg9, $arg10, $arg11);

# ; Counter used for assembly label generation
my $label_count = 0;

# ; This implementation follows the convention: for non-leaf functions (they
# ; must call PROLOG) %rbp is used as a frame pointer, and has fixed offset from
# ; the function entry: $GP_STORAGE + [8 bytes alignment (Windows only)].  This
# ; helps to facilitate SEH handlers writing.
#
# ; Leaf functions here do not use more than 4 input arguments.
if ($win64) {
  $arg1  = "%rcx";
  $arg2  = "%rdx";
  $arg3  = "%r8";
  $arg4  = "%r9";
  $arg5  = "`$GP_STORAGE + 8 + 8*5`(%rbp)";    # +8 - alignment bytes
  $arg6  = "`$GP_STORAGE + 8 + 8*6`(%rbp)";
  $arg7  = "`$GP_STORAGE + 8 + 8*7`(%rbp)";
  $arg8  = "`$GP_STORAGE + 8 + 8*8`(%rbp)";
  $arg9  = "`$GP_STORAGE + 8 + 8*9`(%rbp)";
  $arg10 = "`$GP_STORAGE + 8 + 8*10`(%rbp)";
  $arg11 = "`$GP_STORAGE + 8 + 8*11`(%rbp)";
} else {
  $arg1  = "%rdi";
  $arg2  = "%rsi";
  $arg3  = "%rdx";
  $arg4  = "%rcx";
  $arg5  = "%r8";
  $arg6  = "%r9";
  $arg7  = "`$GP_STORAGE + 8*1`(%rbp)";
  $arg8  = "`$GP_STORAGE + 8*2`(%rbp)";
  $arg9  = "`$GP_STORAGE + 8*3`(%rbp)";
  $arg10 = "`$GP_STORAGE + 8*4`(%rbp)";
  $arg11 = "`$GP_STORAGE + 8*5`(%rbp)";
}

# ; Offsets in gcm128_context structure (see include/crypto/modes.h)
my $CTX_OFFSET_CurCount  = (16 * 0);          #  ; (Yi) Current counter for generation of encryption key
my $CTX_OFFSET_PEncBlock = (16 * 1);          #  ; (repurposed EKi field) Partial block buffer
my $CTX_OFFSET_EK0       = (16 * 2);          #  ; (EK0) Encrypted Y0 counter (see gcm spec notation)
my $CTX_OFFSET_AadLen    = (16 * 3);          #  ; (len.u[0]) Length of Hash which has been input
my $CTX_OFFSET_InLen     = ((16 * 3) + 8);    #  ; (len.u[1]) Length of input data which will be encrypted or decrypted
my $CTX_OFFSET_AadHash   = (16 * 4);          #  ; (Xi) Current hash
my $CTX_OFFSET_HTable    = (16 * 6);          #  ; (Htable) Precomputed table (allows 16 values)

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;;; Helper functions
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

sub BYTE {
  my ($reg) = @_;
  if ($reg =~ /%r[abcd]x/i) {
    $reg =~ s/%r([abcd])x/%${1}l/i;
  } elsif ($reg =~ /%r[sdb][ip]/i) {
    $reg =~ s/%r([sdb][ip])/%${1}l/i;
  } elsif ($reg =~ /%r[0-9]{1,2}/i) {
    $reg =~ s/%(r[0-9]{1,2})/%${1}b/i;
  } else {
    die "BYTE: unknown register: $reg\n";
  }
  return $reg;
}

sub WORD {
  my ($reg) = @_;
  if ($reg =~ /%r[abcdsdb][xip]/i) {
    $reg =~ s/%r([abcdsdb])([xip])/%${1}${2}/i;
  } elsif ($reg =~ /%r[0-9]{1,2}/) {
    $reg =~ s/%(r[0-9]{1,2})/%${1}w/i;
  } else {
    die "WORD: unknown register: $reg\n";
  }
  return $reg;
}

sub DWORD {
  my ($reg) = @_;
  if ($reg =~ /%r[abcdsdb][xip]/i) {
    $reg =~ s/%r([abcdsdb])([xip])/%e${1}${2}/i;
  } elsif ($reg =~ /%r[0-9]{1,2}/i) {
    $reg =~ s/%(r[0-9]{1,2})/%${1}d/i;
  } else {
    die "DWORD: unknown register: $reg\n";
  }
  return $reg;
}

sub XWORD {
  my ($reg) = @_;
  if ($reg =~ /%[xyz]mm/i) {
    $reg =~ s/%[xyz]mm/%xmm/i;
  } else {
    die "XWORD: unknown register: $reg\n";
  }
  return $reg;
}

sub YWORD {
  my ($reg) = @_;
  if ($reg =~ /%[xyz]mm/i) {
    $reg =~ s/%[xyz]mm/%ymm/i;
  } else {
    die "YWORD: unknown register: $reg\n";
  }
  return $reg;
}

sub ZWORD {
  my ($reg) = @_;
  if ($reg =~ /%[xyz]mm/i) {
    $reg =~ s/%[xyz]mm/%zmm/i;
  } else {
    die "ZWORD: unknown register: $reg\n";
  }
  return $reg;
}

# ; Helper function to construct effective address based on two kinds of
# ; offsets: numerical or located in the register
sub EffectiveAddress {
  my ($base, $offset, $displacement) = @_;
  $displacement = 0 if (!$displacement);

  if ($offset =~ /^\d+\z/) {    # numerical offset
    return "`$offset + $displacement`($base)";
  } else {                      # offset resides in register
    return "$displacement($base,$offset,1)";
  }
}

# ; Provides memory location of corresponding HashKey power
sub HashKeyByIdx {
  my ($idx, $base) = @_;
  my $base_str = ($base eq "%rsp") ? "frame" : "context";

  my $offset = &HashKeyOffsetByIdx($idx, $base_str);
  return "$offset($base)";
}

# ; Provides offset (in bytes) of corresponding HashKey power from the highest key in the storage
sub HashKeyOffsetByIdx {
  my ($idx, $base) = @_;
  die "HashKeyOffsetByIdx: base should be either 'frame' or 'context'; base = $base"
    if (($base ne "frame") && ($base ne "context"));

  my $offset_base;
  my $offset_idx;
  if ($base eq "frame") {    # frame storage
    die "HashKeyOffsetByIdx: idx out of bounds (1..48)! idx = $idx\n" if ($idx > $HKEYS_STORAGE_CAPACITY || $idx < 1);
    $offset_base = $STACK_HKEYS_OFFSET;
    $offset_idx  = ($AES_BLOCK_SIZE * ($HKEYS_STORAGE_CAPACITY - $idx));
  } else {                   # context storage
    die "HashKeyOffsetByIdx: idx out of bounds (1..16)! idx = $idx\n" if ($idx > $HKEYS_CONTEXT_CAPACITY || $idx < 1);
    $offset_base = $CTX_OFFSET_HTable;
    $offset_idx  = ($AES_BLOCK_SIZE * ($HKEYS_CONTEXT_CAPACITY - $idx));
  }
  return $offset_base + $offset_idx;
}

# ; Creates local frame and does back up of non-volatile registers.
# ; Holds stack unwinding directives.
sub PROLOG {
  my ($need_hkeys_stack_storage, $need_aes_stack_storage, $func_name) = @_;

  my $DYNAMIC_STACK_ALLOC_SIZE            = 0;
  my $DYNAMIC_STACK_ALLOC_ALIGNMENT_SPACE = $win64 ? 48 : 52;

  if ($need_hkeys_stack_storage) {
    $DYNAMIC_STACK_ALLOC_SIZE += $HKEYS_STORAGE;
  }

  if ($need_aes_stack_storage) {
    if (!$need_hkeys_stack_storage) {
      die "PROLOG: unsupported case - aes storage without hkeys one";
    }
    $DYNAMIC_STACK_ALLOC_SIZE += $LOCAL_STORAGE;
  }

  $code .= <<___;
    push    %rbx
.cfi_push   %rbx
.L${func_name}_seh_push_rbx:
    push    %rbp
.cfi_push   %rbp
.L${func_name}_seh_push_rbp:
    push    %r12
.cfi_push   %r12
.L${func_name}_seh_push_r12:
    push    %r13
.cfi_push   %r13
.L${func_name}_seh_push_r13:
    push    %r14
.cfi_push   %r14
.L${func_name}_seh_push_r14:
    push    %r15
.cfi_push   %r15
.L${func_name}_seh_push_r15:
___

  if ($win64) {
    $code .= <<___;
    push    %rdi
.L${func_name}_seh_push_rdi:
    push    %rsi
.L${func_name}_seh_push_rsi:

    sub     \$`$XMM_STORAGE+8`,%rsp   # +8 alignment
.L${func_name}_seh_allocstack_xmm:
___
  }
  $code .= <<___;
    # ; %rbp contains stack pointer right after GP regs pushed at stack + [8
    # ; bytes of alignment (Windows only)].  It serves as a frame pointer in SEH
    # ; handlers. The requirement for a frame pointer is that its offset from
    # ; RSP shall be multiple of 16, and not exceed 240 bytes. The frame pointer
    # ; itself seems to be reasonable to use here, because later we do 64-byte stack
    # ; alignment which gives us non-determinate offsets and complicates writing
    # ; SEH handlers.
    #
    # ; It also serves as an anchor for retrieving stack arguments on both Linux
    # ; and Windows.
    lea     `$XMM_STORAGE`(%rsp),%rbp
.cfi_def_cfa_register %rbp
.L${func_name}_seh_setfp:
___
  if ($win64) {

    # ; xmm6:xmm15 need to be preserved on Windows
    foreach my $reg_idx (6 .. 15) {
      my $xmm_reg_offset = ($reg_idx - 6) * 16;
      $code .= <<___;
        vmovdqu           %xmm${reg_idx},$xmm_reg_offset(%rsp)
.L${func_name}_seh_save_xmm${reg_idx}:
___
    }
  }

  $code .= <<___;
# Prolog ends here. Next stack allocation is treated as "dynamic".
.L${func_name}_seh_prolog_end:
___

  if ($DYNAMIC_STACK_ALLOC_SIZE) {
    $code .= <<___;
        sub               \$`$DYNAMIC_STACK_ALLOC_SIZE + $DYNAMIC_STACK_ALLOC_ALIGNMENT_SPACE`,%rsp
        and               \$(-64),%rsp
___
  }
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;;; Restore register content for the caller.
# ;;; And cleanup stack.
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
sub EPILOG {
  my ($hkeys_storage_on_stack, $payload_len) = @_;

  my $label_suffix = $label_count++;

  if ($hkeys_storage_on_stack && $CLEAR_HKEYS_STORAGE_ON_EXIT) {

    # ; There is no need in hkeys cleanup if payload len was small, i.e. no hkeys
    # ; were stored in the local frame storage
    $code .= <<___;
        cmpq              \$`16*16`,$payload_len
        jbe               .Lskip_hkeys_cleanup_${label_suffix}
        vpxor             %xmm0,%xmm0,%xmm0
___
    for (my $i = 0; $i < int($HKEYS_STORAGE / 64); $i++) {
      $code .= "vmovdqa64         %zmm0,`$STACK_HKEYS_OFFSET + 64*$i`(%rsp)\n";
    }
    $code .= ".Lskip_hkeys_cleanup_${label_suffix}:\n";
  }

  if ($CLEAR_SCRATCH_REGISTERS) {
    &clear_scratch_gps_asm();
    &clear_scratch_zmms_asm();
  } else {
    $code .= "vzeroupper\n";
  }

  if ($win64) {

    # ; restore xmm15:xmm6
    for (my $reg_idx = 15; $reg_idx >= 6; $reg_idx--) {
      my $xmm_reg_offset = -$XMM_STORAGE + ($reg_idx - 6) * 16;
      $code .= <<___;
        vmovdqu           $xmm_reg_offset(%rbp),%xmm${reg_idx},
___
    }
  }

  if ($win64) {

    # Forming valid epilog for SEH with use of frame pointer.
    # https://docs.microsoft.com/en-us/cpp/build/prolog-and-epilog?view=msvc-160#epilog-code
    $code .= "lea      8(%rbp),%rsp\n";
  } else {
    $code .= "lea      (%rbp),%rsp\n";
    $code .= ".cfi_def_cfa_register %rsp\n";
  }

  if ($win64) {
    $code .= <<___;
     pop     %rsi
.cfi_pop     %rsi
     pop     %rdi
.cfi_pop     %rdi
___
  }
  $code .= <<___;
     pop     %r15
.cfi_pop     %r15
     pop     %r14
.cfi_pop     %r14
     pop     %r13
.cfi_pop     %r13
     pop     %r12
.cfi_pop     %r12
     pop     %rbp
.cfi_pop     %rbp
     pop     %rbx
.cfi_pop     %rbx
___
}

# ; Clears all scratch ZMM registers
# ;
# ; It should be called before restoring the XMM registers
# ; for Windows (XMM6-XMM15).
# ;
sub clear_scratch_zmms_asm {

  # ; On Linux, all ZMM registers are scratch registers
  if (!$win64) {
    $code .= "vzeroall\n";
  } else {
    foreach my $i (0 .. 5) {
      $code .= "vpxorq  %xmm${i},%xmm${i},%xmm${i}\n";
    }
  }
  foreach my $i (16 .. 31) {
    $code .= "vpxorq  %xmm${i},%xmm${i},%xmm${i}\n";
  }
}

# Clears all scratch GP registers
sub clear_scratch_gps_asm {
  foreach my $reg ("%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11") {
    $code .= "xor $reg,$reg\n";
  }
  if (!$win64) {
    foreach my $reg ("%rsi", "%rdi") {
      $code .= "xor $reg,$reg\n";
    }
  }
}

sub precompute_hkeys_on_stack {
  my $GCM128_CTX  = $_[0];
  my $HKEYS_READY = $_[1];
  my $ZTMP0       = $_[2];
  my $ZTMP1       = $_[3];
  my $ZTMP2       = $_[4];
  my $ZTMP3       = $_[5];
  my $ZTMP4       = $_[6];
  my $ZTMP5       = $_[7];
  my $ZTMP6       = $_[8];
  my $HKEYS_RANGE = $_[9];    # ; "first16", "mid16", "all", "first32", "last32"

  die "precompute_hkeys_on_stack: Unexpected value of HKEYS_RANGE: $HKEYS_RANGE"
    if ($HKEYS_RANGE ne "first16"
    && $HKEYS_RANGE ne "mid16"
    && $HKEYS_RANGE ne "all"
    && $HKEYS_RANGE ne "first32"
    && $HKEYS_RANGE ne "last32");

  my $label_suffix = $label_count++;

  $code .= <<___;
        test              $HKEYS_READY,$HKEYS_READY
        jnz               .L_skip_hkeys_precomputation_${label_suffix}
___

  if ($HKEYS_RANGE eq "first16" || $HKEYS_RANGE eq "first32" || $HKEYS_RANGE eq "all") {

    # ; Fill the stack with the first 16 hkeys from the context
    $code .= <<___;
        # ; Move 16 hkeys from the context to stack
        vmovdqu64         @{[HashKeyByIdx(4,$GCM128_CTX)]},$ZTMP0
        vmovdqu64         $ZTMP0,@{[HashKeyByIdx(4,"%rsp")]}

        vmovdqu64         @{[HashKeyByIdx(8,$GCM128_CTX)]},$ZTMP1
        vmovdqu64         $ZTMP1,@{[HashKeyByIdx(8,"%rsp")]}

        # ; broadcast HashKey^8
        vshufi64x2        \$0x00,$ZTMP1,$ZTMP1,$ZTMP1

        vmovdqu64         @{[HashKeyByIdx(12,$GCM128_CTX)]},$ZTMP2
        vmovdqu64         $ZTMP2,@{[HashKeyByIdx(12,"%rsp")]}

        vmovdqu64         @{[HashKeyByIdx(16,$GCM128_CTX)]},$ZTMP3
        vmovdqu64         $ZTMP3,@{[HashKeyByIdx(16,"%rsp")]}
___
  }

  if ($HKEYS_RANGE eq "mid16" || $HKEYS_RANGE eq "last32") {
    $code .= <<___;
        vmovdqu64         @{[HashKeyByIdx(8,"%rsp")]},$ZTMP1

        # ; broadcast HashKey^8
        vshufi64x2        \$0x00,$ZTMP1,$ZTMP1,$ZTMP1

        vmovdqu64         @{[HashKeyByIdx(12,"%rsp")]},$ZTMP2
        vmovdqu64         @{[HashKeyByIdx(16,"%rsp")]},$ZTMP3
___

  }

  if ($HKEYS_RANGE eq "mid16" || $HKEYS_RANGE eq "first32" || $HKEYS_RANGE eq "last32" || $HKEYS_RANGE eq "all") {

    # ; Precompute hkeys^i, i=17..32
    my $i = 20;
    foreach (1 .. int((32 - 16) / 8)) {

      # ;; compute HashKey^(4 + n), HashKey^(3 + n), ... HashKey^(1 + n)
      &GHASH_MUL($ZTMP2, $ZTMP1, $ZTMP4, $ZTMP5, $ZTMP6);
      $code .= "vmovdqu64         $ZTMP2,@{[HashKeyByIdx($i,\"%rsp\")]}\n";
      $i += 4;

      # ;; compute HashKey^(8 + n), HashKey^(7 + n), ... HashKey^(5 + n)
      &GHASH_MUL($ZTMP3, $ZTMP1, $ZTMP4, $ZTMP5, $ZTMP6);
      $code .= "vmovdqu64         $ZTMP3,@{[HashKeyByIdx($i,\"%rsp\")]}\n";
      $i += 4;
    }
  }

  if ($HKEYS_RANGE eq "last32" || $HKEYS_RANGE eq "all") {

    # ; Precompute hkeys^i, i=33..48 (HKEYS_STORAGE_CAPACITY = 48)
    my $i = 36;
    foreach (1 .. int((48 - 32) / 8)) {

      # ;; compute HashKey^(4 + n), HashKey^(3 + n), ... HashKey^(1 + n)
      &GHASH_MUL($ZTMP2, $ZTMP1, $ZTMP4, $ZTMP5, $ZTMP6);
      $code .= "vmovdqu64         $ZTMP2,@{[HashKeyByIdx($i,\"%rsp\")]}\n";
      $i += 4;

      # ;; compute HashKey^(8 + n), HashKey^(7 + n), ... HashKey^(5 + n)
      &GHASH_MUL($ZTMP3, $ZTMP1, $ZTMP4, $ZTMP5, $ZTMP6);
      $code .= "vmovdqu64         $ZTMP3,@{[HashKeyByIdx($i,\"%rsp\")]}\n";
      $i += 4;
    }
  }

  $code .= ".L_skip_hkeys_precomputation_${label_suffix}:\n";
}

# ;; =============================================================================
# ;; Generic macro to produce code that executes $OPCODE instruction
# ;; on selected number of AES blocks (16 bytes long ) between 0 and 16.
# ;; All three operands of the instruction come from registers.
# ;; Note: if 3 blocks are left at the end instruction is produced to operate all
# ;;       4 blocks (full width of ZMM)
sub ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 {
  my $NUM_BLOCKS = $_[0];    # [in] numerical value, number of AES blocks (0 to 16)
  my $OPCODE     = $_[1];    # [in] instruction name
  my @DST;
  $DST[0] = $_[2];           # [out] destination ZMM register
  $DST[1] = $_[3];           # [out] destination ZMM register
  $DST[2] = $_[4];           # [out] destination ZMM register
  $DST[3] = $_[5];           # [out] destination ZMM register
  my @SRC1;
  $SRC1[0] = $_[6];          # [in] source 1 ZMM register
  $SRC1[1] = $_[7];          # [in] source 1 ZMM register
  $SRC1[2] = $_[8];          # [in] source 1 ZMM register
  $SRC1[3] = $_[9];          # [in] source 1 ZMM register
  my @SRC2;
  $SRC2[0] = $_[10];         # [in] source 2 ZMM register
  $SRC2[1] = $_[11];         # [in] source 2 ZMM register
  $SRC2[2] = $_[12];         # [in] source 2 ZMM register
  $SRC2[3] = $_[13];         # [in] source 2 ZMM register

  die "ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16: num_blocks is out of bounds = $NUM_BLOCKS\n"
    if ($NUM_BLOCKS > 16 || $NUM_BLOCKS < 0);

  my $reg_idx     = 0;
  my $blocks_left = $NUM_BLOCKS;

  foreach (1 .. ($NUM_BLOCKS / 4)) {
    $code .= "$OPCODE        $SRC2[$reg_idx],$SRC1[$reg_idx],$DST[$reg_idx]\n";
    $reg_idx++;
    $blocks_left -= 4;
  }

  my $DSTREG  = $DST[$reg_idx];
  my $SRC1REG = $SRC1[$reg_idx];
  my $SRC2REG = $SRC2[$reg_idx];

  if ($blocks_left == 1) {
    $code .= "$OPCODE         @{[XWORD($SRC2REG)]},@{[XWORD($SRC1REG)]},@{[XWORD($DSTREG)]}\n";
  } elsif ($blocks_left == 2) {
    $code .= "$OPCODE         @{[YWORD($SRC2REG)]},@{[YWORD($SRC1REG)]},@{[YWORD($DSTREG)]}\n";
  } elsif ($blocks_left == 3) {
    $code .= "$OPCODE         $SRC2REG,$SRC1REG,$DSTREG\n";
  }
}

# ;; =============================================================================
# ;; Loads specified number of AES blocks into ZMM registers using mask register
# ;; for the last loaded register (xmm, ymm or zmm).
# ;; Loads take place at 1 byte granularity.
sub ZMM_LOAD_MASKED_BLOCKS_0_16 {
  my $NUM_BLOCKS  = $_[0];    # [in] numerical value, number of AES blocks (0 to 16)
  my $INP         = $_[1];    # [in] input data pointer to read from
  my $DATA_OFFSET = $_[2];    # [in] offset to the output pointer (GP or numerical)
  my @DST;
  $DST[0] = $_[3];            # [out] ZMM register with loaded data
  $DST[1] = $_[4];            # [out] ZMM register with loaded data
  $DST[2] = $_[5];            # [out] ZMM register with loaded data
  $DST[3] = $_[6];            # [out] ZMM register with loaded data
  my $MASK = $_[7];           # [in] mask register

  die "ZMM_LOAD_MASKED_BLOCKS_0_16: num_blocks is out of bounds = $NUM_BLOCKS\n"
    if ($NUM_BLOCKS > 16 || $NUM_BLOCKS < 0);

  my $src_offset  = 0;
  my $dst_idx     = 0;
  my $blocks_left = $NUM_BLOCKS;

  if ($NUM_BLOCKS > 0) {
    foreach (1 .. (int(($NUM_BLOCKS + 3) / 4) - 1)) {
      $code .= "vmovdqu8          @{[EffectiveAddress($INP,$DATA_OFFSET,$src_offset)]},$DST[$dst_idx]\n";
      $src_offset += 64;
      $dst_idx++;
      $blocks_left -= 4;
    }
  }

  my $DSTREG = $DST[$dst_idx];

  if ($blocks_left == 1) {
    $code .= "vmovdqu8          @{[EffectiveAddress($INP,$DATA_OFFSET,$src_offset)]},@{[XWORD($DSTREG)]}\{$MASK\}{z}\n";
  } elsif ($blocks_left == 2) {
    $code .= "vmovdqu8          @{[EffectiveAddress($INP,$DATA_OFFSET,$src_offset)]},@{[YWORD($DSTREG)]}\{$MASK\}{z}\n";
  } elsif (($blocks_left == 3 || $blocks_left == 4)) {
    $code .= "vmovdqu8          @{[EffectiveAddress($INP,$DATA_OFFSET,$src_offset)]},$DSTREG\{$MASK\}{z}\n";
  }
}

# ;; =============================================================================
# ;; Stores specified number of AES blocks from ZMM registers with mask register
# ;; for the last loaded register (xmm, ymm or zmm).
# ;; Stores take place at 1 byte granularity.
sub ZMM_STORE_MASKED_BLOCKS_0_16 {
  my $NUM_BLOCKS  = $_[0];    # [in] numerical value, number of AES blocks (0 to 16)
  my $OUTP        = $_[1];    # [in] output data pointer to write to
  my $DATA_OFFSET = $_[2];    # [in] offset to the output pointer (GP or numerical)
  my @SRC;
  $SRC[0] = $_[3];            # [in] ZMM register with data to store
  $SRC[1] = $_[4];            # [in] ZMM register with data to store
  $SRC[2] = $_[5];            # [in] ZMM register with data to store
  $SRC[3] = $_[6];            # [in] ZMM register with data to store
  my $MASK = $_[7];           # [in] mask register

  die "ZMM_STORE_MASKED_BLOCKS_0_16: num_blocks is out of bounds = $NUM_BLOCKS\n"
    if ($NUM_BLOCKS > 16 || $NUM_BLOCKS < 0);

  my $dst_offset  = 0;
  my $src_idx     = 0;
  my $blocks_left = $NUM_BLOCKS;

  if ($NUM_BLOCKS > 0) {
    foreach (1 .. (int(($NUM_BLOCKS + 3) / 4) - 1)) {
      $code .= "vmovdqu8          $SRC[$src_idx],`$dst_offset`($OUTP,$DATA_OFFSET,1)\n";
      $dst_offset += 64;
      $src_idx++;
      $blocks_left -= 4;
    }
  }

  my $SRCREG = $SRC[$src_idx];

  if ($blocks_left == 1) {
    $code .= "vmovdqu8          @{[XWORD($SRCREG)]},`$dst_offset`($OUTP,$DATA_OFFSET,1){$MASK}\n";
  } elsif ($blocks_left == 2) {
    $code .= "vmovdqu8          @{[YWORD($SRCREG)]},`$dst_offset`($OUTP,$DATA_OFFSET,1){$MASK}\n";
  } elsif ($blocks_left == 3 || $blocks_left == 4) {
    $code .= "vmovdqu8          $SRCREG,`$dst_offset`($OUTP,$DATA_OFFSET,1){$MASK}\n";
  }
}

# ;;; ===========================================================================
# ;;; Handles AES encryption rounds
# ;;; It handles special cases: the last and first rounds
# ;;; Optionally, it performs XOR with data after the last AES round.
# ;;; Uses NROUNDS parameter to check what needs to be done for the current round.
# ;;; If 3 blocks are trailing then operation on whole ZMM is performed (4 blocks).
sub ZMM_AESENC_ROUND_BLOCKS_0_16 {
  my $L0B0_3   = $_[0];     # [in/out] zmm; blocks 0 to 3
  my $L0B4_7   = $_[1];     # [in/out] zmm; blocks 4 to 7
  my $L0B8_11  = $_[2];     # [in/out] zmm; blocks 8 to 11
  my $L0B12_15 = $_[3];     # [in/out] zmm; blocks 12 to 15
  my $KEY      = $_[4];     # [in] zmm containing round key
  my $ROUND    = $_[5];     # [in] round number
  my $D0_3     = $_[6];     # [in] zmm or no_data; plain/cipher text blocks 0-3
  my $D4_7     = $_[7];     # [in] zmm or no_data; plain/cipher text blocks 4-7
  my $D8_11    = $_[8];     # [in] zmm or no_data; plain/cipher text blocks 8-11
  my $D12_15   = $_[9];     # [in] zmm or no_data; plain/cipher text blocks 12-15
  my $NUMBL    = $_[10];    # [in] number of blocks; numerical value
  my $NROUNDS  = $_[11];    # [in] number of rounds; numerical value

  # ;;; === first AES round
  if ($ROUND < 1) {

    # ;;  round 0
    &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
      $NUMBL,  "vpxorq", $L0B0_3,   $L0B4_7, $L0B8_11, $L0B12_15, $L0B0_3,
      $L0B4_7, $L0B8_11, $L0B12_15, $KEY,    $KEY,     $KEY,      $KEY);
  }

  # ;;; === middle AES rounds
  if ($ROUND >= 1 && $ROUND <= $NROUNDS) {

    # ;; rounds 1 to 9/11/13
    &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
      $NUMBL,  "vaesenc", $L0B0_3,   $L0B4_7, $L0B8_11, $L0B12_15, $L0B0_3,
      $L0B4_7, $L0B8_11,  $L0B12_15, $KEY,    $KEY,     $KEY,      $KEY);
  }

  # ;;; === last AES round
  if ($ROUND > $NROUNDS) {

    # ;; the last round - mix enclast with text xor's
    &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
      $NUMBL,  "vaesenclast", $L0B0_3,   $L0B4_7, $L0B8_11, $L0B12_15, $L0B0_3,
      $L0B4_7, $L0B8_11,      $L0B12_15, $KEY,    $KEY,     $KEY,      $KEY);

    # ;;; === XOR with data
    if ( ($D0_3 ne "no_data")
      && ($D4_7 ne "no_data")
      && ($D8_11 ne "no_data")
      && ($D12_15 ne "no_data"))
    {
      &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
        $NUMBL,  "vpxorq", $L0B0_3,   $L0B4_7, $L0B8_11, $L0B12_15, $L0B0_3,
        $L0B4_7, $L0B8_11, $L0B12_15, $D0_3,   $D4_7,    $D8_11,    $D12_15);
    }
  }
}

# ;;; Horizontal XOR - 4 x 128bits xored together
sub VHPXORI4x128 {
  my $REG = $_[0];    # [in/out] ZMM with 4x128bits to xor; 128bit output
  my $TMP = $_[1];    # [clobbered] ZMM temporary register
  $code .= <<___;
        vextracti64x4     \$1,$REG,@{[YWORD($TMP)]}
        vpxorq            @{[YWORD($TMP)]},@{[YWORD($REG)]},@{[YWORD($REG)]}
        vextracti32x4     \$1,@{[YWORD($REG)]},@{[XWORD($TMP)]}
        vpxorq            @{[XWORD($TMP)]},@{[XWORD($REG)]},@{[XWORD($REG)]}
___
}

# ;;; AVX512 reduction macro
sub VCLMUL_REDUCE {
  my $OUT   = $_[0];    # [out] zmm/ymm/xmm: result (must not be $TMP1 or $HI128)
  my $POLY  = $_[1];    # [in] zmm/ymm/xmm: polynomial
  my $HI128 = $_[2];    # [in] zmm/ymm/xmm: high 128b of hash to reduce
  my $LO128 = $_[3];    # [in] zmm/ymm/xmm: low 128b of hash to reduce
  my $TMP0  = $_[4];    # [in] zmm/ymm/xmm: temporary register
  my $TMP1  = $_[5];    # [in] zmm/ymm/xmm: temporary register

  $code .= <<___;
        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; first phase of the reduction
        vpclmulqdq        \$0x01,$LO128,$POLY,$TMP0
        vpslldq           \$8,$TMP0,$TMP0         # ; shift-L 2 DWs
        vpxorq            $TMP0,$LO128,$TMP0      # ; first phase of the reduction complete
        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; second phase of the reduction
        vpclmulqdq        \$0x00,$TMP0,$POLY,$TMP1
        vpsrldq           \$4,$TMP1,$TMP1          # ; shift-R only 1-DW to obtain 2-DWs shift-R
        vpclmulqdq        \$0x10,$TMP0,$POLY,$OUT
        vpslldq           \$4,$OUT,$OUT            # ; shift-L 1-DW to obtain result with no shifts
        vpternlogq        \$0x96,$HI128,$TMP1,$OUT # ; OUT/GHASH = OUT xor TMP1 xor HI128
        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
___
}

# ;; ===========================================================================
# ;; schoolbook multiply of 16 blocks (16 x 16 bytes)
# ;; - it is assumed that data read from $INPTR is already shuffled and
# ;;   $INPTR address is 64 byte aligned
# ;; - there is an option to pass ready blocks through ZMM registers too.
# ;;   4 extra parameters need to be passed in such case and 21st ($ZTMP9) argument can be empty
sub GHASH_16 {
  my $TYPE  = $_[0];     # [in] ghash type: start (xor hash), mid, end (same as mid; no reduction),
                         # end_reduce (end with reduction), start_reduce
  my $GH    = $_[1];     # [in/out] ZMM ghash sum: high 128-bits
  my $GM    = $_[2];     # [in/out] ZMM ghash sum: middle 128-bits
  my $GL    = $_[3];     # [in/out] ZMM ghash sum: low 128-bits
  my $INPTR = $_[4];     # [in] data input pointer
  my $INOFF = $_[5];     # [in] data input offset
  my $INDIS = $_[6];     # [in] data input displacement
  my $HKPTR = $_[7];     # [in] hash key pointer
  my $HKOFF = $_[8];     # [in] hash key offset (can be either numerical offset, or register containing offset)
  my $HKDIS = $_[9];     # [in] hash key displacement
  my $HASH  = $_[10];    # [in/out] ZMM hash value in/out
  my $ZTMP0 = $_[11];    # [clobbered] temporary ZMM
  my $ZTMP1 = $_[12];    # [clobbered] temporary ZMM
  my $ZTMP2 = $_[13];    # [clobbered] temporary ZMM
  my $ZTMP3 = $_[14];    # [clobbered] temporary ZMM
  my $ZTMP4 = $_[15];    # [clobbered] temporary ZMM
  my $ZTMP5 = $_[16];    # [clobbered] temporary ZMM
  my $ZTMP6 = $_[17];    # [clobbered] temporary ZMM
  my $ZTMP7 = $_[18];    # [clobbered] temporary ZMM
  my $ZTMP8 = $_[19];    # [clobbered] temporary ZMM
  my $ZTMP9 = $_[20];    # [clobbered] temporary ZMM, can be empty if 4 extra parameters below are provided
  my $DAT0  = $_[21];    # [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused)
  my $DAT1  = $_[22];    # [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused)
  my $DAT2  = $_[23];    # [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused)
  my $DAT3  = $_[24];    # [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused)

  my $start_ghash  = 0;
  my $do_reduction = 0;
  if ($TYPE eq "start") {
    $start_ghash = 1;
  }

  if ($TYPE eq "start_reduce") {
    $start_ghash  = 1;
    $do_reduction = 1;
  }

  if ($TYPE eq "end_reduce") {
    $do_reduction = 1;
  }

  # ;; ghash blocks 0-3
  if (scalar(@_) == 21) {
    $code .= "vmovdqa64         @{[EffectiveAddress($INPTR,$INOFF,($INDIS+0*64))]},$ZTMP9\n";
  } else {
    $ZTMP9 = $DAT0;
  }

  if ($start_ghash != 0) {
    $code .= "vpxorq            $HASH,$ZTMP9,$ZTMP9\n";
  }
  $code .= <<___;
        vmovdqu64         @{[EffectiveAddress($HKPTR,$HKOFF,($HKDIS+0*64))]},$ZTMP8
        vpclmulqdq        \$0x11,$ZTMP8,$ZTMP9,$ZTMP0      # ; T0H = a1*b1
        vpclmulqdq        \$0x00,$ZTMP8,$ZTMP9,$ZTMP1      # ; T0L = a0*b0
        vpclmulqdq        \$0x01,$ZTMP8,$ZTMP9,$ZTMP2      # ; T0M1 = a1*b0
        vpclmulqdq        \$0x10,$ZTMP8,$ZTMP9,$ZTMP3      # ; T0M2 = a0*b1
___

  # ;; ghash blocks 4-7
  if (scalar(@_) == 21) {
    $code .= "vmovdqa64         @{[EffectiveAddress($INPTR,$INOFF,($INDIS+1*64))]},$ZTMP9\n";
  } else {
    $ZTMP9 = $DAT1;
  }
  $code .= <<___;
        vmovdqu64         @{[EffectiveAddress($HKPTR,$HKOFF,($HKDIS+1*64))]},$ZTMP8
        vpclmulqdq        \$0x11,$ZTMP8,$ZTMP9,$ZTMP4      # ; T1H = a1*b1
        vpclmulqdq        \$0x00,$ZTMP8,$ZTMP9,$ZTMP5      # ; T1L = a0*b0
        vpclmulqdq        \$0x01,$ZTMP8,$ZTMP9,$ZTMP6      # ; T1M1 = a1*b0
        vpclmulqdq        \$0x10,$ZTMP8,$ZTMP9,$ZTMP7      # ; T1M2 = a0*b1
___

  # ;; update sums
  if ($start_ghash != 0) {
    $code .= <<___;
        vpxorq            $ZTMP6,$ZTMP2,$GM             # ; GM = T0M1 + T1M1
        vpxorq            $ZTMP4,$ZTMP0,$GH             # ; GH = T0H + T1H
        vpxorq            $ZTMP5,$ZTMP1,$GL             # ; GL = T0L + T1L
        vpternlogq        \$0x96,$ZTMP7,$ZTMP3,$GM      # ; GM = T0M2 + T1M1
___
  } else {    # ;; mid, end, end_reduce
    $code .= <<___;
        vpternlogq        \$0x96,$ZTMP6,$ZTMP2,$GM      # ; GM += T0M1 + T1M1
        vpternlogq        \$0x96,$ZTMP4,$ZTMP0,$GH      # ; GH += T0H + T1H
        vpternlogq        \$0x96,$ZTMP5,$ZTMP1,$GL      # ; GL += T0L + T1L
        vpternlogq        \$0x96,$ZTMP7,$ZTMP3,$GM      # ; GM += T0M2 + T1M1
___
  }

  # ;; ghash blocks 8-11
  if (scalar(@_) == 21) {
    $code .= "vmovdqa64         @{[EffectiveAddress($INPTR,$INOFF,($INDIS+2*64))]},$ZTMP9\n";
  } else {
    $ZTMP9 = $DAT2;
  }
  $code .= <<___;
        vmovdqu64         @{[EffectiveAddress($HKPTR,$HKOFF,($HKDIS+2*64))]},$ZTMP8
        vpclmulqdq        \$0x11,$ZTMP8,$ZTMP9,$ZTMP0      # ; T0H = a1*b1
        vpclmulqdq        \$0x00,$ZTMP8,$ZTMP9,$ZTMP1      # ; T0L = a0*b0
        vpclmulqdq        \$0x01,$ZTMP8,$ZTMP9,$ZTMP2      # ; T0M1 = a1*b0
        vpclmulqdq        \$0x10,$ZTMP8,$ZTMP9,$ZTMP3      # ; T0M2 = a0*b1
___

  # ;; ghash blocks 12-15
  if (scalar(@_) == 21) {
    $code .= "vmovdqa64         @{[EffectiveAddress($INPTR,$INOFF,($INDIS+3*64))]},$ZTMP9\n";
  } else {
    $ZTMP9 = $DAT3;
  }
  $code .= <<___;
        vmovdqu64         @{[EffectiveAddress($HKPTR,$HKOFF,($HKDIS+3*64))]},$ZTMP8
        vpclmulqdq        \$0x11,$ZTMP8,$ZTMP9,$ZTMP4      # ; T1H = a1*b1
        vpclmulqdq        \$0x00,$ZTMP8,$ZTMP9,$ZTMP5      # ; T1L = a0*b0
        vpclmulqdq        \$0x01,$ZTMP8,$ZTMP9,$ZTMP6      # ; T1M1 = a1*b0
        vpclmulqdq        \$0x10,$ZTMP8,$ZTMP9,$ZTMP7      # ; T1M2 = a0*b1
        # ;; update sums
        vpternlogq        \$0x96,$ZTMP6,$ZTMP2,$GM         # ; GM += T0M1 + T1M1
        vpternlogq        \$0x96,$ZTMP4,$ZTMP0,$GH         # ; GH += T0H + T1H
        vpternlogq        \$0x96,$ZTMP5,$ZTMP1,$GL         # ; GL += T0L + T1L
        vpternlogq        \$0x96,$ZTMP7,$ZTMP3,$GM         # ; GM += T0M2 + T1M1
___
  if ($do_reduction != 0) {
    $code .= <<___;
        # ;; integrate GM into GH and GL
        vpsrldq           \$8,$GM,$ZTMP0
        vpslldq           \$8,$GM,$ZTMP1
        vpxorq            $ZTMP0,$GH,$GH
        vpxorq            $ZTMP1,$GL,$GL
___

    # ;; add GH and GL 128-bit words horizontally
    &VHPXORI4x128($GH, $ZTMP0);
    &VHPXORI4x128($GL, $ZTMP1);

    # ;; reduction
    $code .= "vmovdqa64         POLY2(%rip),@{[XWORD($ZTMP2)]}\n";
    &VCLMUL_REDUCE(&XWORD($HASH), &XWORD($ZTMP2), &XWORD($GH), &XWORD($GL), &XWORD($ZTMP0), &XWORD($ZTMP1));
  }
}

# ;; ===========================================================================
# ;; GHASH 1 to 16 blocks of cipher text
# ;; - performs reduction at the end
# ;; - it doesn't load the data and it assumed it is already loaded and shuffled
sub GHASH_1_TO_16 {
  my $GCM128_CTX  = $_[0];     # [in] pointer to expanded keys
  my $GHASH       = $_[1];     # [out] ghash output
  my $T0H         = $_[2];     # [clobbered] temporary ZMM
  my $T0L         = $_[3];     # [clobbered] temporary ZMM
  my $T0M1        = $_[4];     # [clobbered] temporary ZMM
  my $T0M2        = $_[5];     # [clobbered] temporary ZMM
  my $T1H         = $_[6];     # [clobbered] temporary ZMM
  my $T1L         = $_[7];     # [clobbered] temporary ZMM
  my $T1M1        = $_[8];     # [clobbered] temporary ZMM
  my $T1M2        = $_[9];     # [clobbered] temporary ZMM
  my $HK          = $_[10];    # [clobbered] temporary ZMM
  my $AAD_HASH_IN = $_[11];    # [in] input hash value
  my @CIPHER_IN;
  $CIPHER_IN[0] = $_[12];      # [in] ZMM with cipher text blocks 0-3
  $CIPHER_IN[1] = $_[13];      # [in] ZMM with cipher text blocks 4-7
  $CIPHER_IN[2] = $_[14];      # [in] ZMM with cipher text blocks 8-11
  $CIPHER_IN[3] = $_[15];      # [in] ZMM with cipher text blocks 12-15
  my $NUM_BLOCKS = $_[16];     # [in] numerical value, number of blocks
  my $GH         = $_[17];     # [in] ZMM with hi product part
  my $GM         = $_[18];     # [in] ZMM with mid product part
  my $GL         = $_[19];     # [in] ZMM with lo product part

  die "GHASH_1_TO_16: num_blocks is out of bounds = $NUM_BLOCKS\n" if ($NUM_BLOCKS > 16 || $NUM_BLOCKS < 0);

  if (scalar(@_) == 17) {
    $code .= "vpxorq            $AAD_HASH_IN,$CIPHER_IN[0],$CIPHER_IN[0]\n";
  }

  if ($NUM_BLOCKS == 16) {
    $code .= <<___;
        vmovdqu64         @{[HashKeyByIdx($NUM_BLOCKS, $GCM128_CTX)]},$HK
        vpclmulqdq        \$0x11,$HK,$CIPHER_IN[0],$T0H        # ; H = a1*b1
        vpclmulqdq        \$0x00,$HK,$CIPHER_IN[0],$T0L        # ; L = a0*b0
        vpclmulqdq        \$0x01,$HK,$CIPHER_IN[0],$T0M1       # ; M1 = a1*b0
        vpclmulqdq        \$0x10,$HK,$CIPHER_IN[0],$T0M2       # ; M2 = a0*b1
        vmovdqu64         @{[HashKeyByIdx($NUM_BLOCKS-1*4, $GCM128_CTX)]},$HK
        vpclmulqdq        \$0x11,$HK,$CIPHER_IN[1],$T1H        # ; H = a1*b1
        vpclmulqdq        \$0x00,$HK,$CIPHER_IN[1],$T1L        # ; L = a0*b0
        vpclmulqdq        \$0x01,$HK,$CIPHER_IN[1],$T1M1       # ; M1 = a1*b0
        vpclmulqdq        \$0x10,$HK,$CIPHER_IN[1],$T1M2       # ; M2 = a0*b1
        vmovdqu64         @{[HashKeyByIdx($NUM_BLOCKS-2*4, $GCM128_CTX)]},$HK
        vpclmulqdq        \$0x11,$HK,$CIPHER_IN[2],$CIPHER_IN[0] # ; H = a1*b1
        vpclmulqdq        \$0x00,$HK,$CIPHER_IN[2],$CIPHER_IN[1] # ; L = a0*b0
        vpternlogq        \$0x96,$T1H,$CIPHER_IN[0],$T0H
        vpternlogq        \$0x96,$T1L,$CIPHER_IN[1],$T0L
        vpclmulqdq        \$0x01,$HK,$CIPHER_IN[2],$CIPHER_IN[0] # ; M1 = a1*b0
        vpclmulqdq        \$0x10,$HK,$CIPHER_IN[2],$CIPHER_IN[1] # ; M2 = a0*b1
        vpternlogq        \$0x96,$T1M1,$CIPHER_IN[0],$T0M1
        vpternlogq        \$0x96,$T1M2,$CIPHER_IN[1],$T0M2
        vmovdqu64         @{[HashKeyByIdx($NUM_BLOCKS-3*4, $GCM128_CTX)]},$HK
        vpclmulqdq        \$0x11,$HK,$CIPHER_IN[3],$T1H        # ; H = a1*b1
        vpclmulqdq        \$0x00,$HK,$CIPHER_IN[3],$T1L        # ; L = a0*b0
        vpclmulqdq        \$0x01,$HK,$CIPHER_IN[3],$T1M1       # ; M1 = a1*b0
        vpclmulqdq        \$0x10,$HK,$CIPHER_IN[3],$T1M2       # ; M2 = a0*b1
        vpxorq            $T1H,$T0H,$T1H
        vpxorq            $T1L,$T0L,$T1L
        vpxorq            $T1M1,$T0M1,$T1M1
        vpxorq            $T1M2,$T0M2,$T1M2
___
  } elsif ($NUM_BLOCKS >= 12) {
    $code .= <<___;
        vmovdqu64         @{[HashKeyByIdx($NUM_BLOCKS, $GCM128_CTX)]},$HK
        vpclmulqdq        \$0x11,$HK,$CIPHER_IN[0],$T0H        # ; H = a1*b1
        vpclmulqdq        \$0x00,$HK,$CIPHER_IN[0],$T0L        # ; L = a0*b0
        vpclmulqdq        \$0x01,$HK,$CIPHER_IN[0],$T0M1       # ; M1 = a1*b0
        vpclmulqdq        \$0x10,$HK,$CIPHER_IN[0],$T0M2       # ; M2 = a0*b1
        vmovdqu64         @{[HashKeyByIdx($NUM_BLOCKS-1*4, $GCM128_CTX)]},$HK
        vpclmulqdq        \$0x11,$HK,$CIPHER_IN[1],$T1H        # ; H = a1*b1
        vpclmulqdq        \$0x00,$HK,$CIPHER_IN[1],$T1L        # ; L = a0*b0
        vpclmulqdq        \$0x01,$HK,$CIPHER_IN[1],$T1M1       # ; M1 = a1*b0
        vpclmulqdq        \$0x10,$HK,$CIPHER_IN[1],$T1M2       # ; M2 = a0*b1
        vmovdqu64         @{[HashKeyByIdx($NUM_BLOCKS-2*4, $GCM128_CTX)]},$HK
        vpclmulqdq        \$0x11,$HK,$CIPHER_IN[2],$CIPHER_IN[0] # ; H = a1*b1
        vpclmulqdq        \$0x00,$HK,$CIPHER_IN[2],$CIPHER_IN[1] # ; L = a0*b0
        vpternlogq        \$0x96,$T0H,$CIPHER_IN[0],$T1H
        vpternlogq        \$0x96,$T0L,$CIPHER_IN[1],$T1L
        vpclmulqdq        \$0x01,$HK,$CIPHER_IN[2],$CIPHER_IN[0] # ; M1 = a1*b0
        vpclmulqdq        \$0x10,$HK,$CIPHER_IN[2],$CIPHER_IN[1] # ; M2 = a0*b1
        vpternlogq        \$0x96,$T0M1,$CIPHER_IN[0],$T1M1
        vpternlogq        \$0x96,$T0M2,$CIPHER_IN[1],$T1M2
___
  } elsif ($NUM_BLOCKS >= 8) {
    $code .= <<___;
        vmovdqu64         @{[HashKeyByIdx($NUM_BLOCKS, $GCM128_CTX)]},$HK
        vpclmulqdq        \$0x11,$HK,$CIPHER_IN[0],$T0H        # ; H = a1*b1
        vpclmulqdq        \$0x00,$HK,$CIPHER_IN[0],$T0L        # ; L = a0*b0
        vpclmulqdq        \$0x01,$HK,$CIPHER_IN[0],$T0M1       # ; M1 = a1*b0
        vpclmulqdq        \$0x10,$HK,$CIPHER_IN[0],$T0M2       # ; M2 = a0*b1
        vmovdqu64         @{[HashKeyByIdx($NUM_BLOCKS-1*4, $GCM128_CTX)]},$HK
        vpclmulqdq        \$0x11,$HK,$CIPHER_IN[1],$T1H        # ; H = a1*b1
        vpclmulqdq        \$0x00,$HK,$CIPHER_IN[1],$T1L        # ; L = a0*b0
        vpclmulqdq        \$0x01,$HK,$CIPHER_IN[1],$T1M1       # ; M1 = a1*b0
        vpclmulqdq        \$0x10,$HK,$CIPHER_IN[1],$T1M2       # ; M2 = a0*b1
        vpxorq            $T1H,$T0H,$T1H
        vpxorq            $T1L,$T0L,$T1L
        vpxorq            $T1M1,$T0M1,$T1M1
        vpxorq            $T1M2,$T0M2,$T1M2
___
  } elsif ($NUM_BLOCKS >= 4) {
    $code .= <<___;
        vmovdqu64         @{[HashKeyByIdx($NUM_BLOCKS, $GCM128_CTX)]},$HK
        vpclmulqdq        \$0x11,$HK,$CIPHER_IN[0],$T1H        # ; H = a1*b1
        vpclmulqdq        \$0x00,$HK,$CIPHER_IN[0],$T1L        # ; L = a0*b0
        vpclmulqdq        \$0x01,$HK,$CIPHER_IN[0],$T1M1       # ; M1 = a1*b0
        vpclmulqdq        \$0x10,$HK,$CIPHER_IN[0],$T1M2       # ; M2 = a0*b1
___
  }

  # ;; T1H/L/M1/M2 - hold current product sums (provided $NUM_BLOCKS >= 4)
  my $blocks_left = ($NUM_BLOCKS % 4);
  if ($blocks_left > 0) {

    # ;; =====================================================
    # ;; There are 1, 2 or 3 blocks left to process.
    # ;; It may also be that they are the only blocks to process.

    # ;; Set hash key and register index position for the remaining 1 to 3 blocks
    my $reg_idx = ($NUM_BLOCKS / 4);
    my $REG_IN  = $CIPHER_IN[$reg_idx];

    if ($blocks_left == 1) {
      $code .= <<___;
        vmovdqu64         @{[HashKeyByIdx($blocks_left, $GCM128_CTX)]},@{[XWORD($HK)]}
        vpclmulqdq        \$0x01,@{[XWORD($HK)]},@{[XWORD($REG_IN)]},@{[XWORD($T0M1)]} # ; M1 = a1*b0
        vpclmulqdq        \$0x10,@{[XWORD($HK)]},@{[XWORD($REG_IN)]},@{[XWORD($T0M2)]} # ; M2 = a0*b1
        vpclmulqdq        \$0x11,@{[XWORD($HK)]},@{[XWORD($REG_IN)]},@{[XWORD($T0H)]}  # ; H = a1*b1
        vpclmulqdq        \$0x00,@{[XWORD($HK)]},@{[XWORD($REG_IN)]},@{[XWORD($T0L)]}  # ; L = a0*b0
___
    } elsif ($blocks_left == 2) {
      $code .= <<___;
        vmovdqu64         @{[HashKeyByIdx($blocks_left, $GCM128_CTX)]},@{[YWORD($HK)]}
        vpclmulqdq        \$0x01,@{[YWORD($HK)]},@{[YWORD($REG_IN)]},@{[YWORD($T0M1)]} # ; M1 = a1*b0
        vpclmulqdq        \$0x10,@{[YWORD($HK)]},@{[YWORD($REG_IN)]},@{[YWORD($T0M2)]} # ; M2 = a0*b1
        vpclmulqdq        \$0x11,@{[YWORD($HK)]},@{[YWORD($REG_IN)]},@{[YWORD($T0H)]}  # ; H = a1*b1
        vpclmulqdq        \$0x00,@{[YWORD($HK)]},@{[YWORD($REG_IN)]},@{[YWORD($T0L)]}  # ; L = a0*b0
___
    } else {    # ; blocks_left == 3
      $code .= <<___;
        vmovdqu64         @{[HashKeyByIdx($blocks_left, $GCM128_CTX)]},@{[YWORD($HK)]}
        vinserti64x2      \$2,@{[HashKeyByIdx($blocks_left-2, $GCM128_CTX)]},$HK,$HK
        vpclmulqdq        \$0x01,$HK,$REG_IN,$T0M1                                     # ; M1 = a1*b0
        vpclmulqdq        \$0x10,$HK,$REG_IN,$T0M2                                     # ; M2 = a0*b1
        vpclmulqdq        \$0x11,$HK,$REG_IN,$T0H                                      # ; H = a1*b1
        vpclmulqdq        \$0x00,$HK,$REG_IN,$T0L                                      # ; L = a0*b0
___
    }

    if (scalar(@_) == 20) {

      # ;; *** GH/GM/GL passed as arguments
      if ($NUM_BLOCKS >= 4) {
        $code .= <<___;
        # ;; add ghash product sums from the first 4, 8 or 12 blocks
        vpxorq            $T1M1,$T0M1,$T0M1
        vpternlogq        \$0x96,$T1M2,$GM,$T0M2
        vpternlogq        \$0x96,$T1H,$GH,$T0H
        vpternlogq        \$0x96,$T1L,$GL,$T0L
___
      } else {
        $code .= <<___;
        vpxorq            $GM,$T0M1,$T0M1
        vpxorq            $GH,$T0H,$T0H
        vpxorq            $GL,$T0L,$T0L
___
      }
    } else {

      # ;; *** GH/GM/GL NOT passed as arguments
      if ($NUM_BLOCKS >= 4) {
        $code .= <<___;
        # ;; add ghash product sums from the first 4, 8 or 12 blocks
        vpxorq            $T1M1,$T0M1,$T0M1
        vpxorq            $T1M2,$T0M2,$T0M2
        vpxorq            $T1H,$T0H,$T0H
        vpxorq            $T1L,$T0L,$T0L
___
      }
    }
    $code .= <<___;
        # ;; integrate TM into TH and TL
        vpxorq            $T0M2,$T0M1,$T0M1
        vpsrldq           \$8,$T0M1,$T1M1
        vpslldq           \$8,$T0M1,$T1M2
        vpxorq            $T1M1,$T0H,$T0H
        vpxorq            $T1M2,$T0L,$T0L
___
  } else {

    # ;; =====================================================
    # ;; number of blocks is 4, 8, 12 or 16
    # ;; T1H/L/M1/M2 include product sums not T0H/L/M1/M2
    if (scalar(@_) == 20) {
      $code .= <<___;
        # ;; *** GH/GM/GL passed as arguments
        vpxorq            $GM,$T1M1,$T1M1
        vpxorq            $GH,$T1H,$T1H
        vpxorq            $GL,$T1L,$T1L
___
    }
    $code .= <<___;
        # ;; integrate TM into TH and TL
        vpxorq            $T1M2,$T1M1,$T1M1
        vpsrldq           \$8,$T1M1,$T0M1
        vpslldq           \$8,$T1M1,$T0M2
        vpxorq            $T0M1,$T1H,$T0H
        vpxorq            $T0M2,$T1L,$T0L
___
  }

  # ;; add TH and TL 128-bit words horizontally
  &VHPXORI4x128($T0H, $T1M1);
  &VHPXORI4x128($T0L, $T1M2);

  # ;; reduction
  $code .= "vmovdqa64         POLY2(%rip),@{[XWORD($HK)]}\n";
  &VCLMUL_REDUCE(
    @{[XWORD($GHASH)]},
    @{[XWORD($HK)]},
    @{[XWORD($T0H)]},
    @{[XWORD($T0L)]},
    @{[XWORD($T0M1)]},
    @{[XWORD($T0M2)]});
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;; GHASH_MUL MACRO to implement: Data*HashKey mod (x^128 + x^127 + x^126 +x^121 + 1)
# ;; Input: A and B (128-bits each, bit-reflected)
# ;; Output: C = A*B*x mod poly, (i.e. >>1 )
# ;; To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input
# ;; GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly.
# ;;
# ;; Refer to [3] for more details.
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
sub GHASH_MUL {
  my $GH = $_[0];    #; [in/out] xmm/ymm/zmm with multiply operand(s) (128-bits)
  my $HK = $_[1];    #; [in] xmm/ymm/zmm with hash key value(s) (128-bits)
  my $T1 = $_[2];    #; [clobbered] xmm/ymm/zmm
  my $T2 = $_[3];    #; [clobbered] xmm/ymm/zmm
  my $T3 = $_[4];    #; [clobbered] xmm/ymm/zmm

  $code .= <<___;
        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        vpclmulqdq        \$0x11,$HK,$GH,$T1 # ; $T1 = a1*b1
        vpclmulqdq        \$0x00,$HK,$GH,$T2 # ; $T2 = a0*b0
        vpclmulqdq        \$0x01,$HK,$GH,$T3 # ; $T3 = a1*b0
        vpclmulqdq        \$0x10,$HK,$GH,$GH # ; $GH = a0*b1
        vpxorq            $T3,$GH,$GH

        vpsrldq           \$8,$GH,$T3        # ; shift-R $GH 2 DWs
        vpslldq           \$8,$GH,$GH        # ; shift-L $GH 2 DWs
        vpxorq            $T3,$T1,$T1
        vpxorq            $T2,$GH,$GH

        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;first phase of the reduction
        vmovdqu64         POLY2(%rip),$T3

        vpclmulqdq        \$0x01,$GH,$T3,$T2
        vpslldq           \$8,$T2,$T2        # ; shift-L $T2 2 DWs
        vpxorq            $T2,$GH,$GH        # ; first phase of the reduction complete

        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;second phase of the reduction
        vpclmulqdq        \$0x00,$GH,$T3,$T2
        vpsrldq           \$4,$T2,$T2        # ; shift-R only 1-DW to obtain 2-DWs shift-R
        vpclmulqdq        \$0x10,$GH,$T3,$GH
        vpslldq           \$4,$GH,$GH        # ; Shift-L 1-DW to obtain result with no shifts
                                             # ; second phase of the reduction complete, the result is in $GH
        vpternlogq        \$0x96,$T2,$T1,$GH # ; GH = GH xor T1 xor T2
        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
___
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;;; PRECOMPUTE computes HashKey_i
sub PRECOMPUTE {
  my $GCM128_CTX = $_[0];    #; [in/out] context pointer, hkeys content updated
  my $HK         = $_[1];    #; [in] xmm, hash key
  my $T1         = $_[2];    #; [clobbered] xmm
  my $T2         = $_[3];    #; [clobbered] xmm
  my $T3         = $_[4];    #; [clobbered] xmm
  my $T4         = $_[5];    #; [clobbered] xmm
  my $T5         = $_[6];    #; [clobbered] xmm
  my $T6         = $_[7];    #; [clobbered] xmm

  my $ZT1 = &ZWORD($T1);
  my $ZT2 = &ZWORD($T2);
  my $ZT3 = &ZWORD($T3);
  my $ZT4 = &ZWORD($T4);
  my $ZT5 = &ZWORD($T5);
  my $ZT6 = &ZWORD($T6);

  my $YT1 = &YWORD($T1);
  my $YT2 = &YWORD($T2);
  my $YT3 = &YWORD($T3);
  my $YT4 = &YWORD($T4);
  my $YT5 = &YWORD($T5);
  my $YT6 = &YWORD($T6);

  $code .= <<___;
        vshufi32x4   \$0x00,@{[YWORD($HK)]},@{[YWORD($HK)]},$YT5
        vmovdqa      $YT5,$YT4
___

  # ;; calculate HashKey^2<<1 mod poly
  &GHASH_MUL($YT4, $YT5, $YT1, $YT2, $YT3);

  $code .= <<___;
        vmovdqu64         $T4,@{[HashKeyByIdx(2,$GCM128_CTX)]}
        vinserti64x2      \$1,$HK,$YT4,$YT5
        vmovdqa64         $YT5,$YT6                             # ;; YT6 = HashKey | HashKey^2
___

  # ;; use 2x128-bit computation
  # ;; calculate HashKey^4<<1 mod poly, HashKey^3<<1 mod poly
  &GHASH_MUL($YT5, $YT4, $YT1, $YT2, $YT3);    # ;; YT5 = HashKey^3 | HashKey^4

  $code .= <<___;
        vmovdqu64         $YT5,@{[HashKeyByIdx(4,$GCM128_CTX)]}

        vinserti64x4      \$1,$YT6,$ZT5,$ZT5                    # ;; ZT5 = YT6 | YT5

        # ;; switch to 4x128-bit computations now
        vshufi64x2        \$0x00,$ZT5,$ZT5,$ZT4                 # ;; broadcast HashKey^4 across all ZT4
        vmovdqa64         $ZT5,$ZT6                             # ;; save HashKey^4 to HashKey^1 in ZT6
___

  # ;; calculate HashKey^5<<1 mod poly, HashKey^6<<1 mod poly, ... HashKey^8<<1 mod poly
  &GHASH_MUL($ZT5, $ZT4, $ZT1, $ZT2, $ZT3);
  $code .= <<___;
        vmovdqu64         $ZT5,@{[HashKeyByIdx(8,$GCM128_CTX)]} # ;; HashKey^8 to HashKey^5 in ZT5 now
        vshufi64x2        \$0x00,$ZT5,$ZT5,$ZT4                 # ;; broadcast HashKey^8 across all ZT4
___

  # ;; calculate HashKey^9<<1 mod poly, HashKey^10<<1 mod poly, ... HashKey^16<<1 mod poly
  # ;; use HashKey^8 as multiplier against ZT6 and ZT5 - this allows deeper ooo execution

  # ;; compute HashKey^(12), HashKey^(11), ... HashKey^(9)
  &GHASH_MUL($ZT6, $ZT4, $ZT1, $ZT2, $ZT3);
  $code .= "vmovdqu64         $ZT6,@{[HashKeyByIdx(12,$GCM128_CTX)]}\n";

  # ;; compute HashKey^(16), HashKey^(15), ... HashKey^(13)
  &GHASH_MUL($ZT5, $ZT4, $ZT1, $ZT2, $ZT3);
  $code .= "vmovdqu64         $ZT5,@{[HashKeyByIdx(16,$GCM128_CTX)]}\n";

  # ; Hkeys 17..48 will be precomputed somewhere else as context can hold only 16 hkeys
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;; READ_SMALL_DATA_INPUT
# ;; Packs xmm register with data when data input is less or equal to 16 bytes
# ;; Returns 0 if data has length 0
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
sub READ_SMALL_DATA_INPUT {
  my $OUTPUT = $_[0];    # [out] xmm register
  my $INPUT  = $_[1];    # [in] buffer pointer to read from
  my $LENGTH = $_[2];    # [in] number of bytes to read
  my $TMP1   = $_[3];    # [clobbered]
  my $TMP2   = $_[4];    # [clobbered]
  my $MASK   = $_[5];    # [out] k1 to k7 register to store the partial block mask

  $code .= <<___;
        mov               \$16,@{[DWORD($TMP2)]}
        lea               byte_len_to_mask_table(%rip),$TMP1
        cmp               $TMP2,$LENGTH
        cmovc             $LENGTH,$TMP2
___
  if ($win64) {
    $code .= <<___;
        add               $TMP2,$TMP1
        add               $TMP2,$TMP1
        kmovw             ($TMP1),$MASK
___
  } else {
    $code .= "kmovw           ($TMP1,$TMP2,2),$MASK\n";
  }
  $code .= "vmovdqu8          ($INPUT),${OUTPUT}{$MASK}{z}\n";
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
#  CALC_AAD_HASH: Calculates the hash of the data which will not be encrypted.
#  Input: The input data (A_IN), that data's length (A_LEN), and the hash key (HASH_KEY).
#  Output: The hash of the data (AAD_HASH).
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
sub CALC_AAD_HASH {
  my $A_IN       = $_[0];     # [in] AAD text pointer
  my $A_LEN      = $_[1];     # [in] AAD length
  my $AAD_HASH   = $_[2];     # [in/out] xmm ghash value
  my $GCM128_CTX = $_[3];     # [in] pointer to context
  my $ZT0        = $_[4];     # [clobbered] ZMM register
  my $ZT1        = $_[5];     # [clobbered] ZMM register
  my $ZT2        = $_[6];     # [clobbered] ZMM register
  my $ZT3        = $_[7];     # [clobbered] ZMM register
  my $ZT4        = $_[8];     # [clobbered] ZMM register
  my $ZT5        = $_[9];     # [clobbered] ZMM register
  my $ZT6        = $_[10];    # [clobbered] ZMM register
  my $ZT7        = $_[11];    # [clobbered] ZMM register
  my $ZT8        = $_[12];    # [clobbered] ZMM register
  my $ZT9        = $_[13];    # [clobbered] ZMM register
  my $ZT10       = $_[14];    # [clobbered] ZMM register
  my $ZT11       = $_[15];    # [clobbered] ZMM register
  my $ZT12       = $_[16];    # [clobbered] ZMM register
  my $ZT13       = $_[17];    # [clobbered] ZMM register
  my $ZT14       = $_[18];    # [clobbered] ZMM register
  my $ZT15       = $_[19];    # [clobbered] ZMM register
  my $ZT16       = $_[20];    # [clobbered] ZMM register
  my $T1         = $_[21];    # [clobbered] GP register
  my $T2         = $_[22];    # [clobbered] GP register
  my $T3         = $_[23];    # [clobbered] GP register
  my $MASKREG    = $_[24];    # [clobbered] mask register

  my $HKEYS_READY = "%rbx";

  my $SHFMSK = $ZT13;

  my $label_suffix = $label_count++;

  $code .= <<___;
        mov               $A_IN,$T1      # ; T1 = AAD
        mov               $A_LEN,$T2     # ; T2 = aadLen
        or                $T2,$T2
        jz                .L_CALC_AAD_done_${label_suffix}

        xor               $HKEYS_READY,$HKEYS_READY
        vmovdqa64         SHUF_MASK(%rip),$SHFMSK

.L_get_AAD_loop48x16_${label_suffix}:
        cmp               \$`(48*16)`,$T2
        jl                .L_exit_AAD_loop48x16_${label_suffix}
___

  $code .= <<___;
        vmovdqu64         `64*0`($T1),$ZT1      # ; Blocks 0-3
        vmovdqu64         `64*1`($T1),$ZT2      # ; Blocks 4-7
        vmovdqu64         `64*2`($T1),$ZT3      # ; Blocks 8-11
        vmovdqu64         `64*3`($T1),$ZT4      # ; Blocks 12-15
        vpshufb           $SHFMSK,$ZT1,$ZT1
        vpshufb           $SHFMSK,$ZT2,$ZT2
        vpshufb           $SHFMSK,$ZT3,$ZT3
        vpshufb           $SHFMSK,$ZT4,$ZT4
___

  &precompute_hkeys_on_stack($GCM128_CTX, $HKEYS_READY, $ZT0, $ZT8, $ZT9, $ZT10, $ZT11, $ZT12, $ZT14, "all");
  $code .= "mov     \$1,$HKEYS_READY\n";

  &GHASH_16(
    "start",        $ZT5,           $ZT6,           $ZT7,
    "NO_INPUT_PTR", "NO_INPUT_PTR", "NO_INPUT_PTR", "%rsp",
    &HashKeyOffsetByIdx(48, "frame"), 0, "@{[ZWORD($AAD_HASH)]}", $ZT0,
    $ZT8,     $ZT9,  $ZT10, $ZT11,
    $ZT12,    $ZT14, $ZT15, $ZT16,
    "NO_ZMM", $ZT1,  $ZT2,  $ZT3,
    $ZT4);

  $code .= <<___;
        vmovdqu64         `16*16 + 64*0`($T1),$ZT1      # ; Blocks 16-19
        vmovdqu64         `16*16 + 64*1`($T1),$ZT2      # ; Blocks 20-23
        vmovdqu64         `16*16 + 64*2`($T1),$ZT3      # ; Blocks 24-27
        vmovdqu64         `16*16 + 64*3`($T1),$ZT4      # ; Blocks 28-31
        vpshufb           $SHFMSK,$ZT1,$ZT1
        vpshufb           $SHFMSK,$ZT2,$ZT2
        vpshufb           $SHFMSK,$ZT3,$ZT3
        vpshufb           $SHFMSK,$ZT4,$ZT4
___

  &GHASH_16(
    "mid",          $ZT5,           $ZT6,           $ZT7,
    "NO_INPUT_PTR", "NO_INPUT_PTR", "NO_INPUT_PTR", "%rsp",
    &HashKeyOffsetByIdx(32, "frame"), 0, "NO_HASH_IN_OUT", $ZT0,
    $ZT8,     $ZT9,  $ZT10, $ZT11,
    $ZT12,    $ZT14, $ZT15, $ZT16,
    "NO_ZMM", $ZT1,  $ZT2,  $ZT3,
    $ZT4);

  $code .= <<___;
        vmovdqu64         `32*16 + 64*0`($T1),$ZT1      # ; Blocks 32-35
        vmovdqu64         `32*16 + 64*1`($T1),$ZT2      # ; Blocks 36-39
        vmovdqu64         `32*16 + 64*2`($T1),$ZT3      # ; Blocks 40-43
        vmovdqu64         `32*16 + 64*3`($T1),$ZT4      # ; Blocks 44-47
        vpshufb           $SHFMSK,$ZT1,$ZT1
        vpshufb           $SHFMSK,$ZT2,$ZT2
        vpshufb           $SHFMSK,$ZT3,$ZT3
        vpshufb           $SHFMSK,$ZT4,$ZT4
___

  &GHASH_16(
    "end_reduce",   $ZT5,           $ZT6,           $ZT7,
    "NO_INPUT_PTR", "NO_INPUT_PTR", "NO_INPUT_PTR", "%rsp",
    &HashKeyOffsetByIdx(16, "frame"), 0, &ZWORD($AAD_HASH), $ZT0,
    $ZT8,     $ZT9,  $ZT10, $ZT11,
    $ZT12,    $ZT14, $ZT15, $ZT16,
    "NO_ZMM", $ZT1,  $ZT2,  $ZT3,
    $ZT4);

  $code .= <<___;
        sub               \$`(48*16)`,$T2
        je                .L_CALC_AAD_done_${label_suffix}

        add               \$`(48*16)`,$T1
        jmp               .L_get_AAD_loop48x16_${label_suffix}

.L_exit_AAD_loop48x16_${label_suffix}:
        # ; Less than 48x16 bytes remaining
        cmp               \$`(32*16)`,$T2
        jl                .L_less_than_32x16_${label_suffix}
___

  $code .= <<___;
        # ; Get next 16 blocks
        vmovdqu64         `64*0`($T1),$ZT1
        vmovdqu64         `64*1`($T1),$ZT2
        vmovdqu64         `64*2`($T1),$ZT3
        vmovdqu64         `64*3`($T1),$ZT4
        vpshufb           $SHFMSK,$ZT1,$ZT1
        vpshufb           $SHFMSK,$ZT2,$ZT2
        vpshufb           $SHFMSK,$ZT3,$ZT3
        vpshufb           $SHFMSK,$ZT4,$ZT4
___

  &precompute_hkeys_on_stack($GCM128_CTX, $HKEYS_READY, $ZT0, $ZT8, $ZT9, $ZT10, $ZT11, $ZT12, $ZT14, "first32");
  $code .= "mov     \$1,$HKEYS_READY\n";

  &GHASH_16(
    "start",        $ZT5,           $ZT6,           $ZT7,
    "NO_INPUT_PTR", "NO_INPUT_PTR", "NO_INPUT_PTR", "%rsp",
    &HashKeyOffsetByIdx(32, "frame"), 0, &ZWORD($AAD_HASH), $ZT0,
    $ZT8,     $ZT9,  $ZT10, $ZT11,
    $ZT12,    $ZT14, $ZT15, $ZT16,
    "NO_ZMM", $ZT1,  $ZT2,  $ZT3,
    $ZT4);

  $code .= <<___;
        vmovdqu64         `16*16 + 64*0`($T1),$ZT1
        vmovdqu64         `16*16 + 64*1`($T1),$ZT2
        vmovdqu64         `16*16 + 64*2`($T1),$ZT3
        vmovdqu64         `16*16 + 64*3`($T1),$ZT4
        vpshufb           $SHFMSK,$ZT1,$ZT1
        vpshufb           $SHFMSK,$ZT2,$ZT2
        vpshufb           $SHFMSK,$ZT3,$ZT3
        vpshufb           $SHFMSK,$ZT4,$ZT4
___

  &GHASH_16(
    "end_reduce",   $ZT5,           $ZT6,           $ZT7,
    "NO_INPUT_PTR", "NO_INPUT_PTR", "NO_INPUT_PTR", "%rsp",
    &HashKeyOffsetByIdx(16, "frame"), 0, &ZWORD($AAD_HASH), $ZT0,
    $ZT8,     $ZT9,  $ZT10, $ZT11,
    $ZT12,    $ZT14, $ZT15, $ZT16,
    "NO_ZMM", $ZT1,  $ZT2,  $ZT3,
    $ZT4);

  $code .= <<___;
        sub               \$`(32*16)`,$T2
        je                .L_CALC_AAD_done_${label_suffix}

        add               \$`(32*16)`,$T1
        jmp               .L_less_than_16x16_${label_suffix}

.L_less_than_32x16_${label_suffix}:
        cmp               \$`(16*16)`,$T2
        jl                .L_less_than_16x16_${label_suffix}
        # ; Get next 16 blocks
        vmovdqu64         `64*0`($T1),$ZT1
        vmovdqu64         `64*1`($T1),$ZT2
        vmovdqu64         `64*2`($T1),$ZT3
        vmovdqu64         `64*3`($T1),$ZT4
        vpshufb           $SHFMSK,$ZT1,$ZT1
        vpshufb           $SHFMSK,$ZT2,$ZT2
        vpshufb           $SHFMSK,$ZT3,$ZT3
        vpshufb           $SHFMSK,$ZT4,$ZT4
___

  # ; This code path does not use more than 16 hkeys, so they can be taken from the context
  # ; (not from the stack storage)
  &GHASH_16(
    "start_reduce", $ZT5,           $ZT6,           $ZT7,
    "NO_INPUT_PTR", "NO_INPUT_PTR", "NO_INPUT_PTR", $GCM128_CTX,
    &HashKeyOffsetByIdx(16, "context"), 0, &ZWORD($AAD_HASH), $ZT0,
    $ZT8,     $ZT9,  $ZT10, $ZT11,
    $ZT12,    $ZT14, $ZT15, $ZT16,
    "NO_ZMM", $ZT1,  $ZT2,  $ZT3,
    $ZT4);

  $code .= <<___;
        sub               \$`(16*16)`,$T2
        je                .L_CALC_AAD_done_${label_suffix}

        add               \$`(16*16)`,$T1
        # ; Less than 16x16 bytes remaining
.L_less_than_16x16_${label_suffix}:
        # ;; prep mask source address
        lea               byte64_len_to_mask_table(%rip),$T3
        lea               ($T3,$T2,8),$T3

        # ;; calculate number of blocks to ghash (including partial bytes)
        add               \$15,@{[DWORD($T2)]}
        shr               \$4,@{[DWORD($T2)]}
        cmp               \$2,@{[DWORD($T2)]}
        jb                .L_AAD_blocks_1_${label_suffix}
        je                .L_AAD_blocks_2_${label_suffix}
        cmp               \$4,@{[DWORD($T2)]}
        jb                .L_AAD_blocks_3_${label_suffix}
        je                .L_AAD_blocks_4_${label_suffix}
        cmp               \$6,@{[DWORD($T2)]}
        jb                .L_AAD_blocks_5_${label_suffix}
        je                .L_AAD_blocks_6_${label_suffix}
        cmp               \$8,@{[DWORD($T2)]}
        jb                .L_AAD_blocks_7_${label_suffix}
        je                .L_AAD_blocks_8_${label_suffix}
        cmp               \$10,@{[DWORD($T2)]}
        jb                .L_AAD_blocks_9_${label_suffix}
        je                .L_AAD_blocks_10_${label_suffix}
        cmp               \$12,@{[DWORD($T2)]}
        jb                .L_AAD_blocks_11_${label_suffix}
        je                .L_AAD_blocks_12_${label_suffix}
        cmp               \$14,@{[DWORD($T2)]}
        jb                .L_AAD_blocks_13_${label_suffix}
        je                .L_AAD_blocks_14_${label_suffix}
        cmp               \$15,@{[DWORD($T2)]}
        je                .L_AAD_blocks_15_${label_suffix}
___

  # ;; fall through for 16 blocks

  # ;; The flow of each of these cases is identical:
  # ;; - load blocks plain text
  # ;; - shuffle loaded blocks
  # ;; - xor in current hash value into block 0
  # ;; - perform up multiplications with ghash keys
  # ;; - jump to reduction code

  for (my $aad_blocks = 16; $aad_blocks > 0; $aad_blocks--) {
    $code .= ".L_AAD_blocks_${aad_blocks}_${label_suffix}:\n";
    if ($aad_blocks > 12) {
      $code .= "sub               \$`12*16*8`, $T3\n";
    } elsif ($aad_blocks > 8) {
      $code .= "sub               \$`8*16*8`, $T3\n";
    } elsif ($aad_blocks > 4) {
      $code .= "sub               \$`4*16*8`, $T3\n";
    }
    $code .= "kmovq             ($T3),$MASKREG\n";

    &ZMM_LOAD_MASKED_BLOCKS_0_16($aad_blocks, $T1, 0, $ZT1, $ZT2, $ZT3, $ZT4, $MASKREG);

    &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16($aad_blocks, "vpshufb", $ZT1, $ZT2, $ZT3, $ZT4,
      $ZT1, $ZT2, $ZT3, $ZT4, $SHFMSK, $SHFMSK, $SHFMSK, $SHFMSK);

    &GHASH_1_TO_16($GCM128_CTX, &ZWORD($AAD_HASH),
      $ZT0, $ZT5, $ZT6, $ZT7, $ZT8, $ZT9, $ZT10, $ZT11, $ZT12, &ZWORD($AAD_HASH), $ZT1, $ZT2, $ZT3, $ZT4, $aad_blocks);

    if ($aad_blocks > 1) {

      # ;; fall through to CALC_AAD_done in 1 block case
      $code .= "jmp           .L_CALC_AAD_done_${label_suffix}\n";
    }

  }
  $code .= ".L_CALC_AAD_done_${label_suffix}:\n";

  # ;; result in AAD_HASH
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;; PARTIAL_BLOCK
# ;; Handles encryption/decryption and the tag partial blocks between
# ;; update calls.
# ;; Requires the input data be at least 1 byte long.
# ;; Output:
# ;; A cipher/plain of the first partial block (CIPH_PLAIN_OUT),
# ;; AAD_HASH and updated GCM128_CTX
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
sub PARTIAL_BLOCK {
  my $GCM128_CTX     = $_[0];     # [in] key pointer
  my $PBLOCK_LEN     = $_[1];     # [in] partial block length
  my $CIPH_PLAIN_OUT = $_[2];     # [in] output buffer
  my $PLAIN_CIPH_IN  = $_[3];     # [in] input buffer
  my $PLAIN_CIPH_LEN = $_[4];     # [in] buffer length
  my $DATA_OFFSET    = $_[5];     # [out] data offset (gets set)
  my $AAD_HASH       = $_[6];     # [out] updated GHASH value
  my $ENC_DEC        = $_[7];     # [in] cipher direction
  my $GPTMP0         = $_[8];     # [clobbered] GP temporary register
  my $GPTMP1         = $_[9];     # [clobbered] GP temporary register
  my $GPTMP2         = $_[10];    # [clobbered] GP temporary register
  my $ZTMP0          = $_[11];    # [clobbered] ZMM temporary register
  my $ZTMP1          = $_[12];    # [clobbered] ZMM temporary register
  my $ZTMP2          = $_[13];    # [clobbered] ZMM temporary register
  my $ZTMP3          = $_[14];    # [clobbered] ZMM temporary register
  my $ZTMP4          = $_[15];    # [clobbered] ZMM temporary register
  my $ZTMP5          = $_[16];    # [clobbered] ZMM temporary register
  my $ZTMP6          = $_[17];    # [clobbered] ZMM temporary register
  my $ZTMP7          = $_[18];    # [clobbered] ZMM temporary register
  my $MASKREG        = $_[19];    # [clobbered] mask temporary register

  my $XTMP0 = &XWORD($ZTMP0);
  my $XTMP1 = &XWORD($ZTMP1);
  my $XTMP2 = &XWORD($ZTMP2);
  my $XTMP3 = &XWORD($ZTMP3);
  my $XTMP4 = &XWORD($ZTMP4);
  my $XTMP5 = &XWORD($ZTMP5);
  my $XTMP6 = &XWORD($ZTMP6);
  my $XTMP7 = &XWORD($ZTMP7);

  my $LENGTH = $DATA_OFFSET;
  my $IA0    = $GPTMP1;
  my $IA1    = $GPTMP2;
  my $IA2    = $GPTMP0;

  my $label_suffix = $label_count++;

  $code .= <<___;
        # ;; if no partial block present then LENGTH/DATA_OFFSET will be set to zero
        mov             ($PBLOCK_LEN),$LENGTH
        or              $LENGTH,$LENGTH
        je              .L_partial_block_done_${label_suffix}         #  ;Leave Macro if no partial blocks
___

  &READ_SMALL_DATA_INPUT($XTMP0, $PLAIN_CIPH_IN, $PLAIN_CIPH_LEN, $IA0, $IA2, $MASKREG);

  $code .= <<___;
        # ;; XTMP1 = my_ctx_data.partial_block_enc_key
        vmovdqu64         $CTX_OFFSET_PEncBlock($GCM128_CTX),$XTMP1
        vmovdqu64         @{[HashKeyByIdx(1,$GCM128_CTX)]},$XTMP2

        # ;; adjust the shuffle mask pointer to be able to shift right $LENGTH bytes
        # ;; (16 - $LENGTH) is the number of bytes in plaintext mod 16)
        lea               SHIFT_MASK(%rip),$IA0
        add               $LENGTH,$IA0
        vmovdqu64         ($IA0),$XTMP3         # ; shift right shuffle mask
        vpshufb           $XTMP3,$XTMP1,$XTMP1
___

  if ($ENC_DEC eq "DEC") {
    $code .= <<___;
        # ;;  keep copy of cipher text in $XTMP4
        vmovdqa64         $XTMP0,$XTMP4
___
  }
  $code .= <<___;
        vpxorq            $XTMP0,$XTMP1,$XTMP1  # ; Ciphertext XOR E(K, Yn)
        # ;; Set $IA1 to be the amount of data left in CIPH_PLAIN_IN after filling the block
        # ;; Determine if partial block is not being filled and shift mask accordingly
___
  if ($win64) {
    $code .= <<___;
        mov               $PLAIN_CIPH_LEN,$IA1
        add               $LENGTH,$IA1
___
  } else {
    $code .= "lea               ($PLAIN_CIPH_LEN, $LENGTH, 1),$IA1\n";
  }
  $code .= <<___;
        sub               \$16,$IA1
        jge               .L_no_extra_mask_${label_suffix}
        sub               $IA1,$IA0
.L_no_extra_mask_${label_suffix}:
        # ;; get the appropriate mask to mask out bottom $LENGTH bytes of $XTMP1
        # ;; - mask out bottom $LENGTH bytes of $XTMP1
        # ;; sizeof(SHIFT_MASK) == 16 bytes
        vmovdqu64         16($IA0),$XTMP0
        vpand             $XTMP0,$XTMP1,$XTMP1
___

  if ($ENC_DEC eq "DEC") {
    $code .= <<___;
        vpand             $XTMP0,$XTMP4,$XTMP4
        vpshufb           SHUF_MASK(%rip),$XTMP4,$XTMP4
        vpshufb           $XTMP3,$XTMP4,$XTMP4
        vpxorq            $XTMP4,$AAD_HASH,$AAD_HASH
___
  } else {
    $code .= <<___;
        vpshufb           SHUF_MASK(%rip),$XTMP1,$XTMP1
        vpshufb           $XTMP3,$XTMP1,$XTMP1
        vpxorq            $XTMP1,$AAD_HASH,$AAD_HASH
___
  }
  $code .= <<___;
        cmp               \$0,$IA1
        jl                .L_partial_incomplete_${label_suffix}
___

  # ;; GHASH computation for the last <16 Byte block
  &GHASH_MUL($AAD_HASH, $XTMP2, $XTMP5, $XTMP6, $XTMP7);

  $code .= <<___;
        movq              \$0, ($PBLOCK_LEN)
        # ;;  Set $LENGTH to be the number of bytes to write out
        mov               $LENGTH,$IA0
        mov               \$16,$LENGTH
        sub               $IA0,$LENGTH
        jmp               .L_enc_dec_done_${label_suffix}

.L_partial_incomplete_${label_suffix}:
___
  if ($win64) {
    $code .= <<___;
        mov               $PLAIN_CIPH_LEN,$IA0
        add               $IA0,($PBLOCK_LEN)
___
  } else {
    $code .= "add               $PLAIN_CIPH_LEN,($PBLOCK_LEN)\n";
  }
  $code .= <<___;
        mov               $PLAIN_CIPH_LEN,$LENGTH

.L_enc_dec_done_${label_suffix}:
        # ;; output encrypted Bytes

        lea               byte_len_to_mask_table(%rip),$IA0
        kmovw             ($IA0,$LENGTH,2),$MASKREG
        vmovdqu64         $AAD_HASH,$CTX_OFFSET_AadHash($GCM128_CTX)
___

  if ($ENC_DEC eq "ENC") {
    $code .= <<___;
        # ;; shuffle XTMP1 back to output as ciphertext
        vpshufb           SHUF_MASK(%rip),$XTMP1,$XTMP1
        vpshufb           $XTMP3,$XTMP1,$XTMP1
___
  }
  $code .= <<___;
        mov               $CIPH_PLAIN_OUT,$IA0
        vmovdqu8          $XTMP1,($IA0){$MASKREG}
.L_partial_block_done_${label_suffix}:
___
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;; Ciphers 1 to 16 blocks and prepares them for later GHASH compute operation
sub INITIAL_BLOCKS_PARTIAL_CIPHER {
  my $AES_KEYS        = $_[0];     # [in] key pointer
  my $GCM128_CTX      = $_[1];     # [in] context pointer
  my $CIPH_PLAIN_OUT  = $_[2];     # [in] text output pointer
  my $PLAIN_CIPH_IN   = $_[3];     # [in] text input pointer
  my $LENGTH          = $_[4];     # [in/clobbered] length in bytes
  my $DATA_OFFSET     = $_[5];     # [in/out] current data offset (updated)
  my $NUM_BLOCKS      = $_[6];     # [in] can only be 1, 2, 3, 4, 5, ..., 15 or 16 (not 0)
  my $CTR             = $_[7];     # [in/out] current counter value
  my $ENC_DEC         = $_[8];     # [in] cipher direction (ENC/DEC)
  my $DAT0            = $_[9];     # [out] ZMM with cipher text shuffled for GHASH
  my $DAT1            = $_[10];    # [out] ZMM with cipher text shuffled for GHASH
  my $DAT2            = $_[11];    # [out] ZMM with cipher text shuffled for GHASH
  my $DAT3            = $_[12];    # [out] ZMM with cipher text shuffled for GHASH
  my $LAST_CIPHER_BLK = $_[13];    # [out] XMM to put ciphered counter block partially xor'ed with text
  my $LAST_GHASH_BLK  = $_[14];    # [out] XMM to put last cipher text block shuffled for GHASH
  my $CTR0            = $_[15];    # [clobbered] ZMM temporary
  my $CTR1            = $_[16];    # [clobbered] ZMM temporary
  my $CTR2            = $_[17];    # [clobbered] ZMM temporary
  my $CTR3            = $_[18];    # [clobbered] ZMM temporary
  my $ZT1             = $_[19];    # [clobbered] ZMM temporary
  my $IA0             = $_[20];    # [clobbered] GP temporary
  my $IA1             = $_[21];    # [clobbered] GP temporary
  my $MASKREG         = $_[22];    # [clobbered] mask register
  my $SHUFMASK        = $_[23];    # [out] ZMM loaded with BE/LE shuffle mask

  if ($NUM_BLOCKS == 1) {
    $code .= "vmovdqa64         SHUF_MASK(%rip),@{[XWORD($SHUFMASK)]}\n";
  } elsif ($NUM_BLOCKS == 2) {
    $code .= "vmovdqa64         SHUF_MASK(%rip),@{[YWORD($SHUFMASK)]}\n";
  } else {
    $code .= "vmovdqa64         SHUF_MASK(%rip),$SHUFMASK\n";
  }

  # ;; prepare AES counter blocks
  if ($NUM_BLOCKS == 1) {
    $code .= "vpaddd            ONE(%rip),$CTR,@{[XWORD($CTR0)]}\n";
  } elsif ($NUM_BLOCKS == 2) {
    $code .= <<___;
        vshufi64x2        \$0,@{[YWORD($CTR)]},@{[YWORD($CTR)]},@{[YWORD($CTR0)]}
        vpaddd            ddq_add_1234(%rip),@{[YWORD($CTR0)]},@{[YWORD($CTR0)]}
___
  } else {
    $code .= <<___;
        vshufi64x2        \$0,@{[ZWORD($CTR)]},@{[ZWORD($CTR)]},@{[ZWORD($CTR)]}
        vpaddd            ddq_add_1234(%rip),@{[ZWORD($CTR)]},$CTR0
___
    if ($NUM_BLOCKS > 4) {
      $code .= "vpaddd            ddq_add_5678(%rip),@{[ZWORD($CTR)]},$CTR1\n";
    }
    if ($NUM_BLOCKS > 8) {
      $code .= "vpaddd            ddq_add_8888(%rip),$CTR0,$CTR2\n";
    }
    if ($NUM_BLOCKS > 12) {
      $code .= "vpaddd            ddq_add_8888(%rip),$CTR1,$CTR3\n";
    }
  }

  # ;; get load/store mask
  $code .= <<___;
        lea               byte64_len_to_mask_table(%rip),$IA0
        mov               $LENGTH,$IA1
___
  if ($NUM_BLOCKS > 12) {
    $code .= "sub               \$`3*64`,$IA1\n";
  } elsif ($NUM_BLOCKS > 8) {
    $code .= "sub               \$`2*64`,$IA1\n";
  } elsif ($NUM_BLOCKS > 4) {
    $code .= "sub               \$`1*64`,$IA1\n";
  }
  $code .= "kmovq             ($IA0,$IA1,8),$MASKREG\n";

  # ;; extract new counter value
  # ;; shuffle the counters for AES rounds
  if ($NUM_BLOCKS <= 4) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 1)`,$CTR0,$CTR\n";
  } elsif ($NUM_BLOCKS <= 8) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 5)`,$CTR1,$CTR\n";
  } elsif ($NUM_BLOCKS <= 12) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 9)`,$CTR2,$CTR\n";
  } else {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 13)`,$CTR3,$CTR\n";
  }
  &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
    $NUM_BLOCKS, "vpshufb", $CTR0, $CTR1,     $CTR2,     $CTR3,     $CTR0,
    $CTR1,       $CTR2,     $CTR3, $SHUFMASK, $SHUFMASK, $SHUFMASK, $SHUFMASK);

  # ;; load plain/cipher text
  &ZMM_LOAD_MASKED_BLOCKS_0_16($NUM_BLOCKS, $PLAIN_CIPH_IN, $DATA_OFFSET, $DAT0, $DAT1, $DAT2, $DAT3, $MASKREG);

  # ;; AES rounds and XOR with plain/cipher text
  foreach my $j (0 .. ($NROUNDS + 1)) {
    $code .= "vbroadcastf64x2    `($j * 16)`($AES_KEYS),$ZT1\n";
    &ZMM_AESENC_ROUND_BLOCKS_0_16($CTR0, $CTR1, $CTR2, $CTR3, $ZT1, $j,
      $DAT0, $DAT1, $DAT2, $DAT3, $NUM_BLOCKS, $NROUNDS);
  }

  # ;; retrieve the last cipher counter block (partially XOR'ed with text)
  # ;; - this is needed for partial block cases
  if ($NUM_BLOCKS <= 4) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 1)`,$CTR0,$LAST_CIPHER_BLK\n";
  } elsif ($NUM_BLOCKS <= 8) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 5)`,$CTR1,$LAST_CIPHER_BLK\n";
  } elsif ($NUM_BLOCKS <= 12) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 9)`,$CTR2,$LAST_CIPHER_BLK\n";
  } else {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 13)`,$CTR3,$LAST_CIPHER_BLK\n";
  }

  # ;; write cipher/plain text back to output and
  $code .= "mov       $CIPH_PLAIN_OUT,$IA0\n";
  &ZMM_STORE_MASKED_BLOCKS_0_16($NUM_BLOCKS, $IA0, $DATA_OFFSET, $CTR0, $CTR1, $CTR2, $CTR3, $MASKREG);

  # ;; zero bytes outside the mask before hashing
  if ($NUM_BLOCKS <= 4) {
    $code .= "vmovdqu8          $CTR0,${CTR0}{$MASKREG}{z}\n";
  } elsif ($NUM_BLOCKS <= 8) {
    $code .= "vmovdqu8          $CTR1,${CTR1}{$MASKREG}{z}\n";
  } elsif ($NUM_BLOCKS <= 12) {
    $code .= "vmovdqu8          $CTR2,${CTR2}{$MASKREG}{z}\n";
  } else {
    $code .= "vmovdqu8          $CTR3,${CTR3}{$MASKREG}{z}\n";
  }

  # ;; Shuffle the cipher text blocks for hashing part
  # ;; ZT5 and ZT6 are expected outputs with blocks for hashing
  if ($ENC_DEC eq "DEC") {

    # ;; Decrypt case
    # ;; - cipher blocks are in ZT5 & ZT6
    &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
      $NUM_BLOCKS, "vpshufb", $DAT0, $DAT1,     $DAT2,     $DAT3,     $DAT0,
      $DAT1,       $DAT2,     $DAT3, $SHUFMASK, $SHUFMASK, $SHUFMASK, $SHUFMASK);
  } else {

    # ;; Encrypt case
    # ;; - cipher blocks are in CTR0-CTR3
    &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
      $NUM_BLOCKS, "vpshufb", $DAT0, $DAT1,     $DAT2,     $DAT3,     $CTR0,
      $CTR1,       $CTR2,     $CTR3, $SHUFMASK, $SHUFMASK, $SHUFMASK, $SHUFMASK);
  }

  # ;; Extract the last block for partials and multi_call cases
  if ($NUM_BLOCKS <= 4) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS-1)`,$DAT0,$LAST_GHASH_BLK\n";
  } elsif ($NUM_BLOCKS <= 8) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS-5)`,$DAT1,$LAST_GHASH_BLK\n";
  } elsif ($NUM_BLOCKS <= 12) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS-9)`,$DAT2,$LAST_GHASH_BLK\n";
  } else {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS-13)`,$DAT3,$LAST_GHASH_BLK\n";
  }

}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;; Computes GHASH on 1 to 16 blocks
sub INITIAL_BLOCKS_PARTIAL_GHASH {
  my $AES_KEYS        = $_[0];     # [in] key pointer
  my $GCM128_CTX      = $_[1];     # [in] context pointer
  my $LENGTH          = $_[2];     # [in/clobbered] length in bytes
  my $NUM_BLOCKS      = $_[3];     # [in] can only be 1, 2, 3, 4, 5, ..., 15 or 16 (not 0)
  my $HASH_IN_OUT     = $_[4];     # [in/out] XMM ghash in/out value
  my $ENC_DEC         = $_[5];     # [in] cipher direction (ENC/DEC)
  my $DAT0            = $_[6];     # [in] ZMM with cipher text shuffled for GHASH
  my $DAT1            = $_[7];     # [in] ZMM with cipher text shuffled for GHASH
  my $DAT2            = $_[8];     # [in] ZMM with cipher text shuffled for GHASH
  my $DAT3            = $_[9];     # [in] ZMM with cipher text shuffled for GHASH
  my $LAST_CIPHER_BLK = $_[10];    # [in] XMM with ciphered counter block partially xor'ed with text
  my $LAST_GHASH_BLK  = $_[11];    # [in] XMM with last cipher text block shuffled for GHASH
  my $ZT0             = $_[12];    # [clobbered] ZMM temporary
  my $ZT1             = $_[13];    # [clobbered] ZMM temporary
  my $ZT2             = $_[14];    # [clobbered] ZMM temporary
  my $ZT3             = $_[15];    # [clobbered] ZMM temporary
  my $ZT4             = $_[16];    # [clobbered] ZMM temporary
  my $ZT5             = $_[17];    # [clobbered] ZMM temporary
  my $ZT6             = $_[18];    # [clobbered] ZMM temporary
  my $ZT7             = $_[19];    # [clobbered] ZMM temporary
  my $ZT8             = $_[20];    # [clobbered] ZMM temporary
  my $PBLOCK_LEN      = $_[21];    # [in] partial block length
  my $GH              = $_[22];    # [in] ZMM with hi product part
  my $GM              = $_[23];    # [in] ZMM with mid product part
  my $GL              = $_[24];    # [in] ZMM with lo product part

  my $label_suffix = $label_count++;

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;;; - Hash all but the last partial block of data
  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

  # ;; update data offset
  if ($NUM_BLOCKS > 1) {

    # ;; The final block of data may be <16B
    $code .= "sub               \$16 * ($NUM_BLOCKS - 1),$LENGTH\n";
  }

  if ($NUM_BLOCKS < 16) {
    $code .= <<___;
        # ;; NOTE: the 'jl' is always taken for num_initial_blocks = 16.
        # ;;      This is run in the context of GCM_ENC_DEC_SMALL for length < 256.
        cmp               \$16,$LENGTH
        jl                .L_small_initial_partial_block_${label_suffix}

        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;;; Handle a full length final block - encrypt and hash all blocks
        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

        sub               \$16,$LENGTH
        movq              \$0,($PBLOCK_LEN)
___

    # ;; Hash all of the data
    if (scalar(@_) == 22) {

      # ;; start GHASH compute
      &GHASH_1_TO_16($GCM128_CTX, $HASH_IN_OUT, $ZT0, $ZT1, $ZT2, $ZT3, $ZT4,
        $ZT5, $ZT6, $ZT7, $ZT8, &ZWORD($HASH_IN_OUT), $DAT0, $DAT1, $DAT2, $DAT3, $NUM_BLOCKS);
    } elsif (scalar(@_) == 25) {

      # ;; continue GHASH compute
      &GHASH_1_TO_16($GCM128_CTX, $HASH_IN_OUT, $ZT0, $ZT1, $ZT2, $ZT3, $ZT4,
        $ZT5, $ZT6, $ZT7, $ZT8, &ZWORD($HASH_IN_OUT), $DAT0, $DAT1, $DAT2, $DAT3, $NUM_BLOCKS, $GH, $GM, $GL);
    }
    $code .= "jmp           .L_small_initial_compute_done_${label_suffix}\n";
  }

  $code .= <<___;
.L_small_initial_partial_block_${label_suffix}:

        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;;; Handle ghash for a <16B final block
        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

        # ;; As it's an init / update / finalize series we need to leave the
        # ;; last block if it's less than a full block of data.

        mov               $LENGTH,($PBLOCK_LEN)
        vmovdqu64         $LAST_CIPHER_BLK,$CTX_OFFSET_PEncBlock($GCM128_CTX)
___

  my $k                  = ($NUM_BLOCKS - 1);
  my $last_block_to_hash = 1;
  if (($NUM_BLOCKS > $last_block_to_hash)) {

    # ;; ZT12-ZT20 - temporary registers
    if (scalar(@_) == 22) {

      # ;; start GHASH compute
      &GHASH_1_TO_16($GCM128_CTX, $HASH_IN_OUT, $ZT0, $ZT1, $ZT2, $ZT3, $ZT4,
        $ZT5, $ZT6, $ZT7, $ZT8, &ZWORD($HASH_IN_OUT), $DAT0, $DAT1, $DAT2, $DAT3, $k);
    } elsif (scalar(@_) == 25) {

      # ;; continue GHASH compute
      &GHASH_1_TO_16($GCM128_CTX, $HASH_IN_OUT, $ZT0, $ZT1, $ZT2, $ZT3, $ZT4,
        $ZT5, $ZT6, $ZT7, $ZT8, &ZWORD($HASH_IN_OUT), $DAT0, $DAT1, $DAT2, $DAT3, $k, $GH, $GM, $GL);
    }

    # ;; just fall through no jmp needed
  } else {

    if (scalar(@_) == 25) {
      $code .= <<___;
        # ;; Reduction is required in this case.
        # ;; Integrate GM into GH and GL.
        vpsrldq           \$8,$GM,$ZT0
        vpslldq           \$8,$GM,$ZT1
        vpxorq            $ZT0,$GH,$GH
        vpxorq            $ZT1,$GL,$GL
___

      # ;; Add GH and GL 128-bit words horizontally
      &VHPXORI4x128($GH, $ZT0);
      &VHPXORI4x128($GL, $ZT1);

      # ;; 256-bit to 128-bit reduction
      $code .= "vmovdqa64         POLY2(%rip),@{[XWORD($ZT0)]}\n";
      &VCLMUL_REDUCE(&XWORD($HASH_IN_OUT), &XWORD($ZT0), &XWORD($GH), &XWORD($GL), &XWORD($ZT1), &XWORD($ZT2));
    }
    $code .= <<___;
        # ;; Record that a reduction is not needed -
        # ;; In this case no hashes are computed because there
        # ;; is only one initial block and it is < 16B in length.
        # ;; We only need to check if a reduction is needed if
        # ;; initial_blocks == 1 and init/update/final is being used.
        # ;; In this case we may just have a partial block, and that
        # ;; gets hashed in finalize.

        # ;; The hash should end up in HASH_IN_OUT.
        # ;; The only way we should get here is if there is
        # ;; a partial block of data, so xor that into the hash.
        vpxorq            $LAST_GHASH_BLK,$HASH_IN_OUT,$HASH_IN_OUT
        # ;; The result is in $HASH_IN_OUT
        jmp               .L_after_reduction_${label_suffix}
___
  }

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;;; After GHASH reduction
  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

  $code .= ".L_small_initial_compute_done_${label_suffix}:\n";

  # ;; If using init/update/finalize, we need to xor any partial block data
  # ;; into the hash.
  if ($NUM_BLOCKS > 1) {

    # ;; NOTE: for $NUM_BLOCKS = 0 the xor never takes place
    if ($NUM_BLOCKS != 16) {
      $code .= <<___;
        # ;; NOTE: for $NUM_BLOCKS = 16, $LENGTH, stored in [PBlockLen] is never zero
        or                $LENGTH,$LENGTH
        je                .L_after_reduction_${label_suffix}
___
    }
    $code .= "vpxorq            $LAST_GHASH_BLK,$HASH_IN_OUT,$HASH_IN_OUT\n";
  }

  $code .= ".L_after_reduction_${label_suffix}:\n";

  # ;; Final hash is now in HASH_IN_OUT
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;; INITIAL_BLOCKS_PARTIAL macro with support for a partial final block.
# ;; It may look similar to INITIAL_BLOCKS but its usage is different:
# ;; - first encrypts/decrypts required number of blocks and then
# ;;   ghashes these blocks
# ;; - Small packets or left over data chunks (<256 bytes)
# ;; - Remaining data chunks below 256 bytes (multi buffer code)
# ;;
# ;; num_initial_blocks is expected to include the partial final block
# ;; in the count.
sub INITIAL_BLOCKS_PARTIAL {
  my $AES_KEYS        = $_[0];     # [in] key pointer
  my $GCM128_CTX      = $_[1];     # [in] context pointer
  my $CIPH_PLAIN_OUT  = $_[2];     # [in] text output pointer
  my $PLAIN_CIPH_IN   = $_[3];     # [in] text input pointer
  my $LENGTH          = $_[4];     # [in/clobbered] length in bytes
  my $DATA_OFFSET     = $_[5];     # [in/out] current data offset (updated)
  my $NUM_BLOCKS      = $_[6];     # [in] can only be 1, 2, 3, 4, 5, ..., 15 or 16 (not 0)
  my $CTR             = $_[7];     # [in/out] current counter value
  my $HASH_IN_OUT     = $_[8];     # [in/out] XMM ghash in/out value
  my $ENC_DEC         = $_[9];     # [in] cipher direction (ENC/DEC)
  my $CTR0            = $_[10];    # [clobbered] ZMM temporary
  my $CTR1            = $_[11];    # [clobbered] ZMM temporary
  my $CTR2            = $_[12];    # [clobbered] ZMM temporary
  my $CTR3            = $_[13];    # [clobbered] ZMM temporary
  my $DAT0            = $_[14];    # [clobbered] ZMM temporary
  my $DAT1            = $_[15];    # [clobbered] ZMM temporary
  my $DAT2            = $_[16];    # [clobbered] ZMM temporary
  my $DAT3            = $_[17];    # [clobbered] ZMM temporary
  my $LAST_CIPHER_BLK = $_[18];    # [clobbered] ZMM temporary
  my $LAST_GHASH_BLK  = $_[19];    # [clobbered] ZMM temporary
  my $ZT0             = $_[20];    # [clobbered] ZMM temporary
  my $ZT1             = $_[21];    # [clobbered] ZMM temporary
  my $ZT2             = $_[22];    # [clobbered] ZMM temporary
  my $ZT3             = $_[23];    # [clobbered] ZMM temporary
  my $ZT4             = $_[24];    # [clobbered] ZMM temporary
  my $IA0             = $_[25];    # [clobbered] GP temporary
  my $IA1             = $_[26];    # [clobbered] GP temporary
  my $MASKREG         = $_[27];    # [clobbered] mask register
  my $SHUFMASK        = $_[28];    # [clobbered] ZMM for BE/LE shuffle mask
  my $PBLOCK_LEN      = $_[29];    # [in] partial block length

  &INITIAL_BLOCKS_PARTIAL_CIPHER(
    $AES_KEYS, $GCM128_CTX,              $CIPH_PLAIN_OUT,         $PLAIN_CIPH_IN,
    $LENGTH,   $DATA_OFFSET,             $NUM_BLOCKS,             $CTR,
    $ENC_DEC,  $DAT0,                    $DAT1,                   $DAT2,
    $DAT3,     &XWORD($LAST_CIPHER_BLK), &XWORD($LAST_GHASH_BLK), $CTR0,
    $CTR1,     $CTR2,                    $CTR3,                   $ZT0,
    $IA0,      $IA1,                     $MASKREG,                $SHUFMASK);

  &INITIAL_BLOCKS_PARTIAL_GHASH($AES_KEYS, $GCM128_CTX, $LENGTH, $NUM_BLOCKS, $HASH_IN_OUT, $ENC_DEC, $DAT0,
    $DAT1, $DAT2, $DAT3, &XWORD($LAST_CIPHER_BLK),
    &XWORD($LAST_GHASH_BLK), $CTR0, $CTR1, $CTR2, $CTR3, $ZT0, $ZT1, $ZT2, $ZT3, $ZT4, $PBLOCK_LEN);
}

# ;; ===========================================================================
# ;; Stitched GHASH of 16 blocks (with reduction) with encryption of N blocks
# ;; followed with GHASH of the N blocks.
sub GHASH_16_ENCRYPT_N_GHASH_N {
  my $AES_KEYS           = $_[0];     # [in] key pointer
  my $GCM128_CTX         = $_[1];     # [in] context pointer
  my $CIPH_PLAIN_OUT     = $_[2];     # [in] pointer to output buffer
  my $PLAIN_CIPH_IN      = $_[3];     # [in] pointer to input buffer
  my $DATA_OFFSET        = $_[4];     # [in] data offset
  my $LENGTH             = $_[5];     # [in] data length
  my $CTR_BE             = $_[6];     # [in/out] ZMM counter blocks (last 4) in big-endian
  my $CTR_CHECK          = $_[7];     # [in/out] GP with 8-bit counter for overflow check
  my $HASHKEY_OFFSET     = $_[8];     # [in] numerical offset for the highest hash key
                                      # (can be in form of register or numerical value)
  my $GHASHIN_BLK_OFFSET = $_[9];     # [in] numerical offset for GHASH blocks in
  my $SHFMSK             = $_[10];    # [in] ZMM with byte swap mask for pshufb
  my $B00_03             = $_[11];    # [clobbered] temporary ZMM
  my $B04_07             = $_[12];    # [clobbered] temporary ZMM
  my $B08_11             = $_[13];    # [clobbered] temporary ZMM
  my $B12_15             = $_[14];    # [clobbered] temporary ZMM
  my $GH1H_UNUSED        = $_[15];    # [clobbered] temporary ZMM
  my $GH1L               = $_[16];    # [clobbered] temporary ZMM
  my $GH1M               = $_[17];    # [clobbered] temporary ZMM
  my $GH1T               = $_[18];    # [clobbered] temporary ZMM
  my $GH2H               = $_[19];    # [clobbered] temporary ZMM
  my $GH2L               = $_[20];    # [clobbered] temporary ZMM
  my $GH2M               = $_[21];    # [clobbered] temporary ZMM
  my $GH2T               = $_[22];    # [clobbered] temporary ZMM
  my $GH3H               = $_[23];    # [clobbered] temporary ZMM
  my $GH3L               = $_[24];    # [clobbered] temporary ZMM
  my $GH3M               = $_[25];    # [clobbered] temporary ZMM
  my $GH3T               = $_[26];    # [clobbered] temporary ZMM
  my $AESKEY1            = $_[27];    # [clobbered] temporary ZMM
  my $AESKEY2            = $_[28];    # [clobbered] temporary ZMM
  my $GHKEY1             = $_[29];    # [clobbered] temporary ZMM
  my $GHKEY2             = $_[30];    # [clobbered] temporary ZMM
  my $GHDAT1             = $_[31];    # [clobbered] temporary ZMM
  my $GHDAT2             = $_[32];    # [clobbered] temporary ZMM
  my $ZT01               = $_[33];    # [clobbered] temporary ZMM
  my $ADDBE_4x4          = $_[34];    # [in] ZMM with 4x128bits 4 in big-endian
  my $ADDBE_1234         = $_[35];    # [in] ZMM with 4x128bits 1, 2, 3 and 4 in big-endian
  my $GHASH_TYPE         = $_[36];    # [in] "start", "start_reduce", "mid", "end_reduce"
  my $TO_REDUCE_L        = $_[37];    # [in] ZMM for low 4x128-bit GHASH sum
  my $TO_REDUCE_H        = $_[38];    # [in] ZMM for hi 4x128-bit GHASH sum
  my $TO_REDUCE_M        = $_[39];    # [in] ZMM for medium 4x128-bit GHASH sum
  my $ENC_DEC            = $_[40];    # [in] cipher direction
  my $HASH_IN_OUT        = $_[41];    # [in/out] XMM ghash in/out value
  my $IA0                = $_[42];    # [clobbered] GP temporary
  my $IA1                = $_[43];    # [clobbered] GP temporary
  my $MASKREG            = $_[44];    # [clobbered] mask register
  my $NUM_BLOCKS         = $_[45];    # [in] numerical value with number of blocks to be encrypted/ghashed (1 to 16)
  my $PBLOCK_LEN         = $_[46];    # [in] partial block length

  die "GHASH_16_ENCRYPT_N_GHASH_N: num_blocks is out of bounds = $NUM_BLOCKS\n"
    if ($NUM_BLOCKS > 16 || $NUM_BLOCKS < 0);

  my $label_suffix = $label_count++;

  my $GH1H = $HASH_IN_OUT;

  # ; this is to avoid additional move in do_reduction case

  my $LAST_GHASH_BLK  = $GH1L;
  my $LAST_CIPHER_BLK = $GH1T;

  my $RED_POLY = $GH2T;
  my $RED_P1   = $GH2L;
  my $RED_T1   = $GH2H;
  my $RED_T2   = $GH2M;

  my $DATA1 = $GH3H;
  my $DATA2 = $GH3L;
  my $DATA3 = $GH3M;
  my $DATA4 = $GH3T;

  # ;; do reduction after the 16 blocks ?
  my $do_reduction = 0;

  # ;; is 16 block chunk a start?
  my $is_start = 0;

  if ($GHASH_TYPE eq "start_reduce") {
    $is_start     = 1;
    $do_reduction = 1;
  }

  if ($GHASH_TYPE eq "start") {
    $is_start = 1;
  }

  if ($GHASH_TYPE eq "end_reduce") {
    $do_reduction = 1;
  }

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; - get load/store mask
  # ;; - load plain/cipher text
  # ;; get load/store mask
  $code .= <<___;
        lea               byte64_len_to_mask_table(%rip),$IA0
        mov               $LENGTH,$IA1
___
  if ($NUM_BLOCKS > 12) {
    $code .= "sub               \$`3*64`,$IA1\n";
  } elsif ($NUM_BLOCKS > 8) {
    $code .= "sub               \$`2*64`,$IA1\n";
  } elsif ($NUM_BLOCKS > 4) {
    $code .= "sub               \$`1*64`,$IA1\n";
  }
  $code .= "kmovq             ($IA0,$IA1,8),$MASKREG\n";

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; prepare counter blocks

  $code .= <<___;
        cmp               \$`(256 - $NUM_BLOCKS)`,@{[DWORD($CTR_CHECK)]}
        jae               .L_16_blocks_overflow_${label_suffix}
___

  &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
    $NUM_BLOCKS, "vpaddd", $B00_03, $B04_07,     $B08_11,    $B12_15,    $CTR_BE,
    $B00_03,     $B04_07,  $B08_11, $ADDBE_1234, $ADDBE_4x4, $ADDBE_4x4, $ADDBE_4x4);
  $code .= <<___;
        jmp               .L_16_blocks_ok_${label_suffix}

.L_16_blocks_overflow_${label_suffix}:
        vpshufb           $SHFMSK,$CTR_BE,$CTR_BE
        vpaddd            ddq_add_1234(%rip),$CTR_BE,$B00_03
___
  if ($NUM_BLOCKS > 4) {
    $code .= <<___;
        vmovdqa64         ddq_add_4444(%rip),$B12_15
        vpaddd            $B12_15,$B00_03,$B04_07
___
  }
  if ($NUM_BLOCKS > 8) {
    $code .= "vpaddd            $B12_15,$B04_07,$B08_11\n";
  }
  if ($NUM_BLOCKS > 12) {
    $code .= "vpaddd            $B12_15,$B08_11,$B12_15\n";
  }
  &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
    $NUM_BLOCKS, "vpshufb", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03,
    $B04_07,     $B08_11,   $B12_15, $SHFMSK, $SHFMSK, $SHFMSK, $SHFMSK);
  $code .= <<___;
.L_16_blocks_ok_${label_suffix}:

        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; - pre-load constants
        # ;; - add current hash into the 1st block
        vbroadcastf64x2    `(16 * 0)`($AES_KEYS),$AESKEY1
___
  if ($is_start != 0) {
    $code .= "vpxorq            `$GHASHIN_BLK_OFFSET + (0*64)`(%rsp),$HASH_IN_OUT,$GHDAT1\n";
  } else {
    $code .= "vmovdqa64         `$GHASHIN_BLK_OFFSET + (0*64)`(%rsp),$GHDAT1\n";
  }

  $code .= "vmovdqu64         @{[EffectiveAddress(\"%rsp\",$HASHKEY_OFFSET,0*64)]},$GHKEY1\n";

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; save counter for the next round
  # ;; increment counter overflow check register
  if ($NUM_BLOCKS <= 4) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 1)`,$B00_03,@{[XWORD($CTR_BE)]}\n";
  } elsif ($NUM_BLOCKS <= 8) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 5)`,$B04_07,@{[XWORD($CTR_BE)]}\n";
  } elsif ($NUM_BLOCKS <= 12) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 9)`,$B08_11,@{[XWORD($CTR_BE)]}\n";
  } else {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 13)`,$B12_15,@{[XWORD($CTR_BE)]}\n";
  }
  $code .= "vshufi64x2        \$0b00000000,$CTR_BE,$CTR_BE,$CTR_BE\n";

  $code .= <<___;
        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; pre-load constants
        vbroadcastf64x2    `(16 * 1)`($AES_KEYS),$AESKEY2
        vmovdqu64         @{[EffectiveAddress("%rsp",$HASHKEY_OFFSET,1*64)]},$GHKEY2
        vmovdqa64         `$GHASHIN_BLK_OFFSET + (1*64)`(%rsp),$GHDAT2
___

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; stitch AES rounds with GHASH

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; AES round 0 - ARK

  &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
    $NUM_BLOCKS, "vpxorq", $B00_03, $B04_07,  $B08_11,  $B12_15,  $B00_03,
    $B04_07,     $B08_11,  $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1);
  $code .= "vbroadcastf64x2    `(16 * 2)`($AES_KEYS),$AESKEY1\n";

  $code .= <<___;
        # ;;==================================================
        # ;; GHASH 4 blocks (15 to 12)
        vpclmulqdq        \$0x11,$GHKEY1,$GHDAT1,$GH1H      # ; a1*b1
        vpclmulqdq        \$0x00,$GHKEY1,$GHDAT1,$GH1L      # ; a0*b0
        vpclmulqdq        \$0x01,$GHKEY1,$GHDAT1,$GH1M      # ; a1*b0
        vpclmulqdq        \$0x10,$GHKEY1,$GHDAT1,$GH1T      # ; a0*b1
        vmovdqu64         @{[EffectiveAddress("%rsp",$HASHKEY_OFFSET,2*64)]},$GHKEY1
        vmovdqa64         `$GHASHIN_BLK_OFFSET + (2*64)`(%rsp),$GHDAT1
___

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; AES round 1
  &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
    $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07,  $B08_11,  $B12_15,  $B00_03,
    $B04_07,     $B08_11,   $B12_15, $AESKEY2, $AESKEY2, $AESKEY2, $AESKEY2);
  $code .= "vbroadcastf64x2    `(16 * 3)`($AES_KEYS),$AESKEY2\n";

  $code .= <<___;
        # ;; =================================================
        # ;; GHASH 4 blocks (11 to 8)
        vpclmulqdq        \$0x10,$GHKEY2,$GHDAT2,$GH2M      # ; a0*b1
        vpclmulqdq        \$0x01,$GHKEY2,$GHDAT2,$GH2T      # ; a1*b0
        vpclmulqdq        \$0x11,$GHKEY2,$GHDAT2,$GH2H      # ; a1*b1
        vpclmulqdq        \$0x00,$GHKEY2,$GHDAT2,$GH2L      # ; a0*b0
        vmovdqu64         @{[EffectiveAddress("%rsp",$HASHKEY_OFFSET,3*64)]},$GHKEY2
        vmovdqa64         `$GHASHIN_BLK_OFFSET + (3*64)`(%rsp),$GHDAT2
___

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; AES round 2
  &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
    $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07,  $B08_11,  $B12_15,  $B00_03,
    $B04_07,     $B08_11,   $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1);
  $code .= "vbroadcastf64x2    `(16 * 4)`($AES_KEYS),$AESKEY1\n";

  $code .= <<___;
        # ;; =================================================
        # ;; GHASH 4 blocks (7 to 4)
        vpclmulqdq        \$0x10,$GHKEY1,$GHDAT1,$GH3M      # ; a0*b1
        vpclmulqdq        \$0x01,$GHKEY1,$GHDAT1,$GH3T      # ; a1*b0
        vpclmulqdq        \$0x11,$GHKEY1,$GHDAT1,$GH3H      # ; a1*b1
        vpclmulqdq        \$0x00,$GHKEY1,$GHDAT1,$GH3L      # ; a0*b0
___

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; AES rounds 3
  &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
    $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07,  $B08_11,  $B12_15,  $B00_03,
    $B04_07,     $B08_11,   $B12_15, $AESKEY2, $AESKEY2, $AESKEY2, $AESKEY2);
  $code .= "vbroadcastf64x2    `(16 * 5)`($AES_KEYS),$AESKEY2\n";

  $code .= <<___;
        # ;; =================================================
        # ;; Gather (XOR) GHASH for 12 blocks
        vpternlogq        \$0x96,$GH3H,$GH2H,$GH1H
        vpternlogq        \$0x96,$GH3L,$GH2L,$GH1L
        vpternlogq        \$0x96,$GH3T,$GH2T,$GH1T
        vpternlogq        \$0x96,$GH3M,$GH2M,$GH1M
___

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; AES rounds 4
  &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
    $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07,  $B08_11,  $B12_15,  $B00_03,
    $B04_07,     $B08_11,   $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1);
  $code .= "vbroadcastf64x2    `(16 * 6)`($AES_KEYS),$AESKEY1\n";

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; load plain/cipher text
  &ZMM_LOAD_MASKED_BLOCKS_0_16($NUM_BLOCKS, $PLAIN_CIPH_IN, $DATA_OFFSET, $DATA1, $DATA2, $DATA3, $DATA4, $MASKREG);

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; AES rounds 5
  &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
    $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07,  $B08_11,  $B12_15,  $B00_03,
    $B04_07,     $B08_11,   $B12_15, $AESKEY2, $AESKEY2, $AESKEY2, $AESKEY2);
  $code .= "vbroadcastf64x2    `(16 * 7)`($AES_KEYS),$AESKEY2\n";

  $code .= <<___;
        # ;; =================================================
        # ;; GHASH 4 blocks (3 to 0)
        vpclmulqdq        \$0x10,$GHKEY2,$GHDAT2,$GH2M      # ; a0*b1
        vpclmulqdq        \$0x01,$GHKEY2,$GHDAT2,$GH2T      # ; a1*b0
        vpclmulqdq        \$0x11,$GHKEY2,$GHDAT2,$GH2H      # ; a1*b1
        vpclmulqdq        \$0x00,$GHKEY2,$GHDAT2,$GH2L      # ; a0*b0
___

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; AES round 6
  &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
    $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07,  $B08_11,  $B12_15,  $B00_03,
    $B04_07,     $B08_11,   $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1);
  $code .= "vbroadcastf64x2    `(16 * 8)`($AES_KEYS),$AESKEY1\n";

  # ;; =================================================
  # ;; gather GHASH in GH1L (low), GH1H (high), GH1M (mid)
  # ;; - add GH2[MTLH] to GH1[MTLH]
  $code .= "vpternlogq        \$0x96,$GH2T,$GH1T,$GH1M\n";
  if ($do_reduction != 0) {

    if ($is_start != 0) {
      $code .= "vpxorq            $GH2M,$GH1M,$GH1M\n";
    } else {
      $code .= <<___;
        vpternlogq        \$0x96,$GH2H,$TO_REDUCE_H,$GH1H
        vpternlogq        \$0x96,$GH2L,$TO_REDUCE_L,$GH1L
        vpternlogq        \$0x96,$GH2M,$TO_REDUCE_M,$GH1M
___
    }

  } else {

    # ;; Update H/M/L hash sums if not carrying reduction
    if ($is_start != 0) {
      $code .= <<___;
        vpxorq            $GH2H,$GH1H,$TO_REDUCE_H
        vpxorq            $GH2L,$GH1L,$TO_REDUCE_L
        vpxorq            $GH2M,$GH1M,$TO_REDUCE_M
___
    } else {
      $code .= <<___;
        vpternlogq        \$0x96,$GH2H,$GH1H,$TO_REDUCE_H
        vpternlogq        \$0x96,$GH2L,$GH1L,$TO_REDUCE_L
        vpternlogq        \$0x96,$GH2M,$GH1M,$TO_REDUCE_M
___
    }

  }

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; AES round 7
  &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
    $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07,  $B08_11,  $B12_15,  $B00_03,
    $B04_07,     $B08_11,   $B12_15, $AESKEY2, $AESKEY2, $AESKEY2, $AESKEY2);
  $code .= "vbroadcastf64x2    `(16 * 9)`($AES_KEYS),$AESKEY2\n";

  # ;; =================================================
  # ;; prepare mid sum for adding to high & low
  # ;; load polynomial constant for reduction
  if ($do_reduction != 0) {
    $code .= <<___;
        vpsrldq           \$8,$GH1M,$GH2M
        vpslldq           \$8,$GH1M,$GH1M

        vmovdqa64         POLY2(%rip),@{[XWORD($RED_POLY)]}
___
  }

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; AES round 8
  &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
    $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07,  $B08_11,  $B12_15,  $B00_03,
    $B04_07,     $B08_11,   $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1);
  $code .= "vbroadcastf64x2    `(16 * 10)`($AES_KEYS),$AESKEY1\n";

  # ;; =================================================
  # ;; Add mid product to high and low
  if ($do_reduction != 0) {
    if ($is_start != 0) {
      $code .= <<___;
        vpternlogq        \$0x96,$GH2M,$GH2H,$GH1H      # ; TH = TH1 + TH2 + TM>>64
        vpternlogq        \$0x96,$GH1M,$GH2L,$GH1L      # ; TL = TL1 + TL2 + TM<<64
___
    } else {
      $code .= <<___;
        vpxorq            $GH2M,$GH1H,$GH1H      # ; TH = TH1 + TM>>64
        vpxorq            $GH1M,$GH1L,$GH1L      # ; TL = TL1 + TM<<64
___
    }
  }

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; AES round 9
  &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
    $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07,  $B08_11,  $B12_15,  $B00_03,
    $B04_07,     $B08_11,   $B12_15, $AESKEY2, $AESKEY2, $AESKEY2, $AESKEY2);

  # ;; =================================================
  # ;; horizontal xor of low and high 4x128
  if ($do_reduction != 0) {
    &VHPXORI4x128($GH1H, $GH2H);
    &VHPXORI4x128($GH1L, $GH2L);
  }

  if (($NROUNDS >= 11)) {
    $code .= "vbroadcastf64x2    `(16 * 11)`($AES_KEYS),$AESKEY2\n";
  }

  # ;; =================================================
  # ;; first phase of reduction
  if ($do_reduction != 0) {
    $code .= <<___;
        vpclmulqdq        \$0x01,@{[XWORD($GH1L)]},@{[XWORD($RED_POLY)]},@{[XWORD($RED_P1)]}
        vpslldq           \$8,@{[XWORD($RED_P1)]},@{[XWORD($RED_P1)]}                    # ; shift-L 2 DWs
        vpxorq            @{[XWORD($RED_P1)]},@{[XWORD($GH1L)]},@{[XWORD($RED_P1)]}      # ; first phase of the reduct
___
  }

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; AES rounds up to 11 (AES192) or 13 (AES256)
  # ;; AES128 is done
  if (($NROUNDS >= 11)) {
    &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
      $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07,  $B08_11,  $B12_15,  $B00_03,
      $B04_07,     $B08_11,   $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1);
    $code .= "vbroadcastf64x2    `(16 * 12)`($AES_KEYS),$AESKEY1\n";

    &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
      $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07,  $B08_11,  $B12_15,  $B00_03,
      $B04_07,     $B08_11,   $B12_15, $AESKEY2, $AESKEY2, $AESKEY2, $AESKEY2);
    if (($NROUNDS == 13)) {
      $code .= "vbroadcastf64x2    `(16 * 13)`($AES_KEYS),$AESKEY2\n";

      &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
        $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07,  $B08_11,  $B12_15,  $B00_03,
        $B04_07,     $B08_11,   $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1);
      $code .= "vbroadcastf64x2    `(16 * 14)`($AES_KEYS),$AESKEY1\n";

      &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
        $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07,  $B08_11,  $B12_15,  $B00_03,
        $B04_07,     $B08_11,   $B12_15, $AESKEY2, $AESKEY2, $AESKEY2, $AESKEY2);
    }
  }

  # ;; =================================================
  # ;; second phase of the reduction
  if ($do_reduction != 0) {
    $code .= <<___;
        vpclmulqdq        \$0x00,@{[XWORD($RED_P1)]},@{[XWORD($RED_POLY)]},@{[XWORD($RED_T1)]}
        vpsrldq           \$4,@{[XWORD($RED_T1)]},@{[XWORD($RED_T1)]}      # ; shift-R 1-DW to obtain 2-DWs shift-R
        vpclmulqdq        \$0x10,@{[XWORD($RED_P1)]},@{[XWORD($RED_POLY)]},@{[XWORD($RED_T2)]}
        vpslldq           \$4,@{[XWORD($RED_T2)]},@{[XWORD($RED_T2)]}      # ; shift-L 1-DW for result without shifts
        # ;; GH1H = GH1H + RED_T1 + RED_T2
        vpternlogq        \$0x96,@{[XWORD($RED_T1)]},@{[XWORD($RED_T2)]},@{[XWORD($GH1H)]}
___
  }

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; the last AES round
  &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
    $NUM_BLOCKS, "vaesenclast", $B00_03, $B04_07,  $B08_11,  $B12_15,  $B00_03,
    $B04_07,     $B08_11,       $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1);

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; XOR against plain/cipher text
  &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
    $NUM_BLOCKS, "vpxorq", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03,
    $B04_07,     $B08_11,  $B12_15, $DATA1,  $DATA2,  $DATA3,  $DATA4);

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; retrieve the last cipher counter block (partially XOR'ed with text)
  # ;; - this is needed for partial block cases
  if ($NUM_BLOCKS <= 4) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 1)`,$B00_03,@{[XWORD($LAST_CIPHER_BLK)]}\n";
  } elsif ($NUM_BLOCKS <= 8) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 5)`,$B04_07,@{[XWORD($LAST_CIPHER_BLK)]}\n";
  } elsif ($NUM_BLOCKS <= 12) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 9)`,$B08_11,@{[XWORD($LAST_CIPHER_BLK)]}\n";
  } else {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS - 13)`,$B12_15,@{[XWORD($LAST_CIPHER_BLK)]}\n";
  }

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; store cipher/plain text
  $code .= "mov       $CIPH_PLAIN_OUT,$IA0\n";
  &ZMM_STORE_MASKED_BLOCKS_0_16($NUM_BLOCKS, $IA0, $DATA_OFFSET, $B00_03, $B04_07, $B08_11, $B12_15, $MASKREG);

  # ;; =================================================
  # ;; shuffle cipher text blocks for GHASH computation
  if ($ENC_DEC eq "ENC") {

    # ;; zero bytes outside the mask before hashing
    if ($NUM_BLOCKS <= 4) {
      $code .= "vmovdqu8           $B00_03,${B00_03}{$MASKREG}{z}\n";
    } elsif ($NUM_BLOCKS <= 8) {
      $code .= "vmovdqu8          $B04_07,${B04_07}{$MASKREG}{z}\n";
    } elsif ($NUM_BLOCKS <= 12) {
      $code .= "vmovdqu8          $B08_11,${B08_11}{$MASKREG}{z}\n";
    } else {
      $code .= "vmovdqu8          $B12_15,${B12_15}{$MASKREG}{z}\n";
    }

    &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
      $NUM_BLOCKS, "vpshufb", $DATA1,  $DATA2,  $DATA3,  $DATA4,  $B00_03,
      $B04_07,     $B08_11,   $B12_15, $SHFMSK, $SHFMSK, $SHFMSK, $SHFMSK);
  } else {

    # ;; zero bytes outside the mask before hashing
    if ($NUM_BLOCKS <= 4) {
      $code .= "vmovdqu8          $DATA1,${DATA1}{$MASKREG}{z}\n";
    } elsif ($NUM_BLOCKS <= 8) {
      $code .= "vmovdqu8          $DATA2,${DATA2}{$MASKREG}{z}\n";
    } elsif ($NUM_BLOCKS <= 12) {
      $code .= "vmovdqu8          $DATA3,${DATA3}{$MASKREG}{z}\n";
    } else {
      $code .= "vmovdqu8          $DATA4,${DATA4}{$MASKREG}{z}\n";
    }

    &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(
      $NUM_BLOCKS, "vpshufb", $DATA1, $DATA2,  $DATA3,  $DATA4,  $DATA1,
      $DATA2,      $DATA3,    $DATA4, $SHFMSK, $SHFMSK, $SHFMSK, $SHFMSK);
  }

  # ;; =================================================
  # ;; Extract the last block for partial / multi_call cases
  if ($NUM_BLOCKS <= 4) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS-1)`,$DATA1,@{[XWORD($LAST_GHASH_BLK)]}\n";
  } elsif ($NUM_BLOCKS <= 8) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS-5)`,$DATA2,@{[XWORD($LAST_GHASH_BLK)]}\n";
  } elsif ($NUM_BLOCKS <= 12) {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS-9)`,$DATA3,@{[XWORD($LAST_GHASH_BLK)]}\n";
  } else {
    $code .= "vextracti32x4     \$`($NUM_BLOCKS-13)`,$DATA4,@{[XWORD($LAST_GHASH_BLK)]}\n";
  }

  if ($do_reduction != 0) {

    # ;; GH1H holds reduced hash value
    # ;; - normally do "vmovdqa64 &XWORD($GH1H), &XWORD($HASH_IN_OUT)"
    # ;; - register rename trick obsoletes the above move
  }

  # ;; =================================================
  # ;; GHASH last N blocks
  # ;; - current hash value in HASH_IN_OUT or
  # ;;   product parts in TO_REDUCE_H/M/L
  # ;; - DATA1-DATA4 include blocks for GHASH

  if ($do_reduction == 0) {
    &INITIAL_BLOCKS_PARTIAL_GHASH(
      $AES_KEYS,            $GCM128_CTX, $LENGTH,                  $NUM_BLOCKS,
      &XWORD($HASH_IN_OUT), $ENC_DEC,    $DATA1,                   $DATA2,
      $DATA3,               $DATA4,      &XWORD($LAST_CIPHER_BLK), &XWORD($LAST_GHASH_BLK),
      $B00_03,              $B04_07,     $B08_11,                  $B12_15,
      $GHDAT1,              $GHDAT2,     $AESKEY1,                 $AESKEY2,
      $GHKEY1,              $PBLOCK_LEN, $TO_REDUCE_H,             $TO_REDUCE_M,
      $TO_REDUCE_L);
  } else {
    &INITIAL_BLOCKS_PARTIAL_GHASH(
      $AES_KEYS,            $GCM128_CTX, $LENGTH,                  $NUM_BLOCKS,
      &XWORD($HASH_IN_OUT), $ENC_DEC,    $DATA1,                   $DATA2,
      $DATA3,               $DATA4,      &XWORD($LAST_CIPHER_BLK), &XWORD($LAST_GHASH_BLK),
      $B00_03,              $B04_07,     $B08_11,                  $B12_15,
      $GHDAT1,              $GHDAT2,     $AESKEY1,                 $AESKEY2,
      $GHKEY1,              $PBLOCK_LEN);
  }
}

# ;; ===========================================================================
# ;; ===========================================================================
# ;; Stitched GHASH of 16 blocks (with reduction) with encryption of N blocks
# ;; followed with GHASH of the N blocks.
sub GCM_ENC_DEC_LAST {
  my $AES_KEYS           = $_[0];     # [in] key pointer
  my $GCM128_CTX         = $_[1];     # [in] context pointer
  my $CIPH_PLAIN_OUT     = $_[2];     # [in] pointer to output buffer
  my $PLAIN_CIPH_IN      = $_[3];     # [in] pointer to input buffer
  my $DATA_OFFSET        = $_[4];     # [in] data offset
  my $LENGTH             = $_[5];     # [in/clobbered] data length
  my $CTR_BE             = $_[6];     # [in/out] ZMM counter blocks (last 4) in big-endian
  my $CTR_CHECK          = $_[7];     # [in/out] GP with 8-bit counter for overflow check
  my $HASHKEY_OFFSET     = $_[8];     # [in] numerical offset for the highest hash key
                                      # (can be register or numerical offset)
  my $GHASHIN_BLK_OFFSET = $_[9];     # [in] numerical offset for GHASH blocks in
  my $SHFMSK             = $_[10];    # [in] ZMM with byte swap mask for pshufb
  my $ZT00               = $_[11];    # [clobbered] temporary ZMM
  my $ZT01               = $_[12];    # [clobbered] temporary ZMM
  my $ZT02               = $_[13];    # [clobbered] temporary ZMM
  my $ZT03               = $_[14];    # [clobbered] temporary ZMM
  my $ZT04               = $_[15];    # [clobbered] temporary ZMM
  my $ZT05               = $_[16];    # [clobbered] temporary ZMM
  my $ZT06               = $_[17];    # [clobbered] temporary ZMM
  my $ZT07               = $_[18];    # [clobbered] temporary ZMM
  my $ZT08               = $_[19];    # [clobbered] temporary ZMM
  my $ZT09               = $_[20];    # [clobbered] temporary ZMM
  my $ZT10               = $_[21];    # [clobbered] temporary ZMM
  my $ZT11               = $_[22];    # [clobbered] temporary ZMM
  my $ZT12               = $_[23];    # [clobbered] temporary ZMM
  my $ZT13               = $_[24];    # [clobbered] temporary ZMM
  my $ZT14               = $_[25];    # [clobbered] temporary ZMM
  my $ZT15               = $_[26];    # [clobbered] temporary ZMM
  my $ZT16               = $_[27];    # [clobbered] temporary ZMM
  my $ZT17               = $_[28];    # [clobbered] temporary ZMM
  my $ZT18               = $_[29];    # [clobbered] temporary ZMM
  my $ZT19               = $_[30];    # [clobbered] temporary ZMM
  my $ZT20               = $_[31];    # [clobbered] temporary ZMM
  my $ZT21               = $_[32];    # [clobbered] temporary ZMM
  my $ZT22               = $_[33];    # [clobbered] temporary ZMM
  my $ADDBE_4x4          = $_[34];    # [in] ZMM with 4x128bits 4 in big-endian
  my $ADDBE_1234         = $_[35];    # [in] ZMM with 4x128bits 1, 2, 3 and 4 in big-endian
  my $GHASH_TYPE         = $_[36];    # [in] "start", "start_reduce", "mid", "end_reduce"
  my $TO_REDUCE_L        = $_[37];    # [in] ZMM for low 4x128-bit GHASH sum
  my $TO_REDUCE_H        = $_[38];    # [in] ZMM for hi 4x128-bit GHASH sum
  my $TO_REDUCE_M        = $_[39];    # [in] ZMM for medium 4x128-bit GHASH sum
  my $ENC_DEC            = $_[40];    # [in] cipher direction
  my $HASH_IN_OUT        = $_[41];    # [in/out] XMM ghash in/out value
  my $IA0                = $_[42];    # [clobbered] GP temporary
  my $IA1                = $_[43];    # [clobbered] GP temporary
  my $MASKREG            = $_[44];    # [clobbered] mask register
  my $PBLOCK_LEN         = $_[45];    # [in] partial block length

  my $label_suffix = $label_count++;

  $code .= <<___;
        mov               @{[DWORD($LENGTH)]},@{[DWORD($IA0)]}
        add               \$15,@{[DWORD($IA0)]}
        shr               \$4,@{[DWORD($IA0)]}
        je                .L_last_num_blocks_is_0_${label_suffix}

        cmp               \$8,@{[DWORD($IA0)]}
        je                .L_last_num_blocks_is_8_${label_suffix}
        jb                .L_last_num_blocks_is_7_1_${label_suffix}


        cmp               \$12,@{[DWORD($IA0)]}
        je                .L_last_num_blocks_is_12_${label_suffix}
        jb                .L_last_num_blocks_is_11_9_${label_suffix}

        # ;; 16, 15, 14 or 13
        cmp               \$15,@{[DWORD($IA0)]}
        je                .L_last_num_blocks_is_15_${label_suffix}
        ja                .L_last_num_blocks_is_16_${label_suffix}
        cmp               \$14,@{[DWORD($IA0)]}
        je                .L_last_num_blocks_is_14_${label_suffix}
        jmp               .L_last_num_blocks_is_13_${label_suffix}

.L_last_num_blocks_is_11_9_${label_suffix}:
        # ;; 11, 10 or 9
        cmp               \$10,@{[DWORD($IA0)]}
        je                .L_last_num_blocks_is_10_${label_suffix}
        ja                .L_last_num_blocks_is_11_${label_suffix}
        jmp               .L_last_num_blocks_is_9_${label_suffix}

.L_last_num_blocks_is_7_1_${label_suffix}:
        cmp               \$4,@{[DWORD($IA0)]}
        je                .L_last_num_blocks_is_4_${label_suffix}
        jb                .L_last_num_blocks_is_3_1_${label_suffix}
        # ;; 7, 6 or 5
        cmp               \$6,@{[DWORD($IA0)]}
        ja                .L_last_num_blocks_is_7_${label_suffix}
        je                .L_last_num_blocks_is_6_${label_suffix}
        jmp               .L_last_num_blocks_is_5_${label_suffix}

.L_last_num_blocks_is_3_1_${label_suffix}:
        # ;; 3, 2 or 1
        cmp               \$2,@{[DWORD($IA0)]}
        ja                .L_last_num_blocks_is_3_${label_suffix}
        je                .L_last_num_blocks_is_2_${label_suffix}
___

  # ;; fall through for `jmp .L_last_num_blocks_is_1`

  # ;; Use rep to generate different block size variants
  # ;; - one block size has to be the first one
  for my $num_blocks (1 .. 16) {
    $code .= ".L_last_num_blocks_is_${num_blocks}_${label_suffix}:\n";
    &GHASH_16_ENCRYPT_N_GHASH_N(
      $AES_KEYS,   $GCM128_CTX,  $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN,  $DATA_OFFSET,
      $LENGTH,     $CTR_BE,      $CTR_CHECK,      $HASHKEY_OFFSET, $GHASHIN_BLK_OFFSET,
      $SHFMSK,     $ZT00,        $ZT01,           $ZT02,           $ZT03,
      $ZT04,       $ZT05,        $ZT06,           $ZT07,           $ZT08,
      $ZT09,       $ZT10,        $ZT11,           $ZT12,           $ZT13,
      $ZT14,       $ZT15,        $ZT16,           $ZT17,           $ZT18,
      $ZT19,       $ZT20,        $ZT21,           $ZT22,           $ADDBE_4x4,
      $ADDBE_1234, $GHASH_TYPE,  $TO_REDUCE_L,    $TO_REDUCE_H,    $TO_REDUCE_M,
      $ENC_DEC,    $HASH_IN_OUT, $IA0,            $IA1,            $MASKREG,
      $num_blocks, $PBLOCK_LEN);

    $code .= "jmp           .L_last_blocks_done_${label_suffix}\n";
  }

  $code .= ".L_last_num_blocks_is_0_${label_suffix}:\n";

  # ;; if there is 0 blocks to cipher then there are only 16 blocks for ghash and reduction
  # ;; - convert mid into end_reduce
  # ;; - convert start into start_reduce
  if ($GHASH_TYPE eq "mid") {
    $GHASH_TYPE = "end_reduce";
  }
  if ($GHASH_TYPE eq "start") {
    $GHASH_TYPE = "start_reduce";
  }

  &GHASH_16($GHASH_TYPE, $TO_REDUCE_H, $TO_REDUCE_M, $TO_REDUCE_L, "%rsp",
    $GHASHIN_BLK_OFFSET, 0, "%rsp", $HASHKEY_OFFSET, 0, $HASH_IN_OUT, $ZT00, $ZT01,
    $ZT02, $ZT03, $ZT04, $ZT05, $ZT06, $ZT07, $ZT08, $ZT09);

  $code .= ".L_last_blocks_done_${label_suffix}:\n";
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;; Main GCM macro stitching cipher with GHASH
# ;; - operates on single stream
# ;; - encrypts 16 blocks at a time
# ;; - ghash the 16 previously encrypted ciphertext blocks
# ;; - no partial block or multi_call handling here
sub GHASH_16_ENCRYPT_16_PARALLEL {
  my $AES_KEYS           = $_[0];     # [in] key pointer
  my $CIPH_PLAIN_OUT     = $_[1];     # [in] pointer to output buffer
  my $PLAIN_CIPH_IN      = $_[2];     # [in] pointer to input buffer
  my $DATA_OFFSET        = $_[3];     # [in] data offset
  my $CTR_BE             = $_[4];     # [in/out] ZMM counter blocks (last 4) in big-endian
  my $CTR_CHECK          = $_[5];     # [in/out] GP with 8-bit counter for overflow check
  my $HASHKEY_OFFSET     = $_[6];     # [in] numerical offset for the highest hash key (hash key index value)
  my $AESOUT_BLK_OFFSET  = $_[7];     # [in] numerical offset for AES-CTR out
  my $GHASHIN_BLK_OFFSET = $_[8];     # [in] numerical offset for GHASH blocks in
  my $SHFMSK             = $_[9];     # [in] ZMM with byte swap mask for pshufb
  my $ZT1                = $_[10];    # [clobbered] temporary ZMM (cipher)
  my $ZT2                = $_[11];    # [clobbered] temporary ZMM (cipher)
  my $ZT3                = $_[12];    # [clobbered] temporary ZMM (cipher)
  my $ZT4                = $_[13];    # [clobbered] temporary ZMM (cipher)
  my $ZT5                = $_[14];    # [clobbered/out] temporary ZMM or GHASH OUT (final_reduction)
  my $ZT6                = $_[15];    # [clobbered] temporary ZMM (cipher)
  my $ZT7                = $_[16];    # [clobbered] temporary ZMM (cipher)
  my $ZT8                = $_[17];    # [clobbered] temporary ZMM (cipher)
  my $ZT9                = $_[18];    # [clobbered] temporary ZMM (cipher)
  my $ZT10               = $_[19];    # [clobbered] temporary ZMM (ghash)
  my $ZT11               = $_[20];    # [clobbered] temporary ZMM (ghash)
  my $ZT12               = $_[21];    # [clobbered] temporary ZMM (ghash)
  my $ZT13               = $_[22];    # [clobbered] temporary ZMM (ghash)
  my $ZT14               = $_[23];    # [clobbered] temporary ZMM (ghash)
  my $ZT15               = $_[24];    # [clobbered] temporary ZMM (ghash)
  my $ZT16               = $_[25];    # [clobbered] temporary ZMM (ghash)
  my $ZT17               = $_[26];    # [clobbered] temporary ZMM (ghash)
  my $ZT18               = $_[27];    # [clobbered] temporary ZMM (ghash)
  my $ZT19               = $_[28];    # [clobbered] temporary ZMM
  my $ZT20               = $_[29];    # [clobbered] temporary ZMM
  my $ZT21               = $_[30];    # [clobbered] temporary ZMM
  my $ZT22               = $_[31];    # [clobbered] temporary ZMM
  my $ZT23               = $_[32];    # [clobbered] temporary ZMM
  my $ADDBE_4x4          = $_[33];    # [in] ZMM with 4x128bits 4 in big-endian
  my $ADDBE_1234         = $_[34];    # [in] ZMM with 4x128bits 1, 2, 3 and 4 in big-endian
  my $TO_REDUCE_L        = $_[35];    # [in/out] ZMM for low 4x128-bit GHASH sum
  my $TO_REDUCE_H        = $_[36];    # [in/out] ZMM for hi 4x128-bit GHASH sum
  my $TO_REDUCE_M        = $_[37];    # [in/out] ZMM for medium 4x128-bit GHASH sum
  my $DO_REDUCTION       = $_[38];    # [in] "no_reduction", "final_reduction", "first_time"
  my $ENC_DEC            = $_[39];    # [in] cipher direction
  my $DATA_DISPL         = $_[40];    # [in] fixed numerical data displacement/offset
  my $GHASH_IN           = $_[41];    # [in] current GHASH value or "no_ghash_in"
  my $IA0                = $_[42];    # [clobbered] temporary GPR

  my $B00_03 = $ZT1;
  my $B04_07 = $ZT2;
  my $B08_11 = $ZT3;
  my $B12_15 = $ZT4;

  my $GH1H = $ZT5;

  # ; @note: do not change this mapping
  my $GH1L = $ZT6;
  my $GH1M = $ZT7;
  my $GH1T = $ZT8;

  my $GH2H = $ZT9;
  my $GH2L = $ZT10;
  my $GH2M = $ZT11;
  my $GH2T = $ZT12;

  my $RED_POLY = $GH2T;
  my $RED_P1   = $GH2L;
  my $RED_T1   = $GH2H;
  my $RED_T2   = $GH2M;

  my $GH3H = $ZT13;
  my $GH3L = $ZT14;
  my $GH3M = $ZT15;
  my $GH3T = $ZT16;

  my $DATA1 = $ZT13;
  my $DATA2 = $ZT14;
  my $DATA3 = $ZT15;
  my $DATA4 = $ZT16;

  my $AESKEY1 = $ZT17;
  my $AESKEY2 = $ZT18;

  my $GHKEY1 = $ZT19;
  my $GHKEY2 = $ZT20;
  my $GHDAT1 = $ZT21;
  my $GHDAT2 = $ZT22;

  my $label_suffix = $label_count++;

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; prepare counter blocks

  $code .= <<___;
        cmpb              \$`(256 - 16)`,@{[BYTE($CTR_CHECK)]}
        jae               .L_16_blocks_overflow_${label_suffix}
        vpaddd            $ADDBE_1234,$CTR_BE,$B00_03
        vpaddd            $ADDBE_4x4,$B00_03,$B04_07
        vpaddd            $ADDBE_4x4,$B04_07,$B08_11
        vpaddd            $ADDBE_4x4,$B08_11,$B12_15
        jmp               .L_16_blocks_ok_${label_suffix}
.L_16_blocks_overflow_${label_suffix}:
        vpshufb           $SHFMSK,$CTR_BE,$CTR_BE
        vmovdqa64         ddq_add_4444(%rip),$B12_15
        vpaddd            ddq_add_1234(%rip),$CTR_BE,$B00_03
        vpaddd            $B12_15,$B00_03,$B04_07
        vpaddd            $B12_15,$B04_07,$B08_11
        vpaddd            $B12_15,$B08_11,$B12_15
        vpshufb           $SHFMSK,$B00_03,$B00_03
        vpshufb           $SHFMSK,$B04_07,$B04_07
        vpshufb           $SHFMSK,$B08_11,$B08_11
        vpshufb           $SHFMSK,$B12_15,$B12_15
.L_16_blocks_ok_${label_suffix}:
___

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; pre-load constants
  $code .= "vbroadcastf64x2    `(16 * 0)`($AES_KEYS),$AESKEY1\n";
  if ($GHASH_IN ne "no_ghash_in") {
    $code .= "vpxorq            `$GHASHIN_BLK_OFFSET + (0*64)`(%rsp),$GHASH_IN,$GHDAT1\n";
  } else {
    $code .= "vmovdqa64         `$GHASHIN_BLK_OFFSET + (0*64)`(%rsp),$GHDAT1\n";
  }

  $code .= <<___;
        vmovdqu64         @{[HashKeyByIdx(($HASHKEY_OFFSET - (0*4)),"%rsp")]},$GHKEY1

        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; save counter for the next round
        # ;; increment counter overflow check register
        vshufi64x2        \$0b11111111,$B12_15,$B12_15,$CTR_BE
        addb              \$16,@{[BYTE($CTR_CHECK)]}
        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; pre-load constants
        vbroadcastf64x2    `(16 * 1)`($AES_KEYS),$AESKEY2
        vmovdqu64         @{[HashKeyByIdx(($HASHKEY_OFFSET - (1*4)),"%rsp")]},$GHKEY2
        vmovdqa64         `$GHASHIN_BLK_OFFSET + (1*64)`(%rsp),$GHDAT2

        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; stitch AES rounds with GHASH

        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; AES round 0 - ARK

        vpxorq            $AESKEY1,$B00_03,$B00_03
        vpxorq            $AESKEY1,$B04_07,$B04_07
        vpxorq            $AESKEY1,$B08_11,$B08_11
        vpxorq            $AESKEY1,$B12_15,$B12_15
        vbroadcastf64x2    `(16 * 2)`($AES_KEYS),$AESKEY1

        # ;;==================================================
        # ;; GHASH 4 blocks (15 to 12)
        vpclmulqdq        \$0x11,$GHKEY1,$GHDAT1,$GH1H      # ; a1*b1
        vpclmulqdq        \$0x00,$GHKEY1,$GHDAT1,$GH1L      # ; a0*b0
        vpclmulqdq        \$0x01,$GHKEY1,$GHDAT1,$GH1M      # ; a1*b0
        vpclmulqdq        \$0x10,$GHKEY1,$GHDAT1,$GH1T      # ; a0*b1
        vmovdqu64         @{[HashKeyByIdx(($HASHKEY_OFFSET - (2*4)),"%rsp")]},$GHKEY1
        vmovdqa64         `$GHASHIN_BLK_OFFSET + (2*64)`(%rsp),$GHDAT1

        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; AES round 1
        vaesenc           $AESKEY2,$B00_03,$B00_03
        vaesenc           $AESKEY2,$B04_07,$B04_07
        vaesenc           $AESKEY2,$B08_11,$B08_11
        vaesenc           $AESKEY2,$B12_15,$B12_15
        vbroadcastf64x2    `(16 * 3)`($AES_KEYS),$AESKEY2

        # ;; =================================================
        # ;; GHASH 4 blocks (11 to 8)
        vpclmulqdq        \$0x10,$GHKEY2,$GHDAT2,$GH2M      # ; a0*b1
        vpclmulqdq        \$0x01,$GHKEY2,$GHDAT2,$GH2T      # ; a1*b0
        vpclmulqdq        \$0x11,$GHKEY2,$GHDAT2,$GH2H      # ; a1*b1
        vpclmulqdq        \$0x00,$GHKEY2,$GHDAT2,$GH2L      # ; a0*b0
        vmovdqu64         @{[HashKeyByIdx(($HASHKEY_OFFSET - (3*4)),"%rsp")]},$GHKEY2
        vmovdqa64         `$GHASHIN_BLK_OFFSET + (3*64)`(%rsp),$GHDAT2

        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; AES round 2
        vaesenc           $AESKEY1,$B00_03,$B00_03
        vaesenc           $AESKEY1,$B04_07,$B04_07
        vaesenc           $AESKEY1,$B08_11,$B08_11
        vaesenc           $AESKEY1,$B12_15,$B12_15
        vbroadcastf64x2    `(16 * 4)`($AES_KEYS),$AESKEY1

        # ;; =================================================
        # ;; GHASH 4 blocks (7 to 4)
        vpclmulqdq        \$0x10,$GHKEY1,$GHDAT1,$GH3M      # ; a0*b1
        vpclmulqdq        \$0x01,$GHKEY1,$GHDAT1,$GH3T      # ; a1*b0
        vpclmulqdq        \$0x11,$GHKEY1,$GHDAT1,$GH3H      # ; a1*b1
        vpclmulqdq        \$0x00,$GHKEY1,$GHDAT1,$GH3L      # ; a0*b0
        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; AES rounds 3
        vaesenc           $AESKEY2,$B00_03,$B00_03
        vaesenc           $AESKEY2,$B04_07,$B04_07
        vaesenc           $AESKEY2,$B08_11,$B08_11
        vaesenc           $AESKEY2,$B12_15,$B12_15
        vbroadcastf64x2    `(16 * 5)`($AES_KEYS),$AESKEY2

        # ;; =================================================
        # ;; Gather (XOR) GHASH for 12 blocks
        vpternlogq        \$0x96,$GH3H,$GH2H,$GH1H
        vpternlogq        \$0x96,$GH3L,$GH2L,$GH1L
        vpternlogq        \$0x96,$GH3T,$GH2T,$GH1T
        vpternlogq        \$0x96,$GH3M,$GH2M,$GH1M

        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; AES rounds 4
        vaesenc           $AESKEY1,$B00_03,$B00_03
        vaesenc           $AESKEY1,$B04_07,$B04_07
        vaesenc           $AESKEY1,$B08_11,$B08_11
        vaesenc           $AESKEY1,$B12_15,$B12_15
        vbroadcastf64x2    `(16 * 6)`($AES_KEYS),$AESKEY1

        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; load plain/cipher text (recycle GH3xx registers)
        vmovdqu8          `$DATA_DISPL + (0 * 64)`($PLAIN_CIPH_IN,$DATA_OFFSET),$DATA1
        vmovdqu8          `$DATA_DISPL + (1 * 64)`($PLAIN_CIPH_IN,$DATA_OFFSET),$DATA2
        vmovdqu8          `$DATA_DISPL + (2 * 64)`($PLAIN_CIPH_IN,$DATA_OFFSET),$DATA3
        vmovdqu8          `$DATA_DISPL + (3 * 64)`($PLAIN_CIPH_IN,$DATA_OFFSET),$DATA4

        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; AES rounds 5
        vaesenc           $AESKEY2,$B00_03,$B00_03
        vaesenc           $AESKEY2,$B04_07,$B04_07
        vaesenc           $AESKEY2,$B08_11,$B08_11
        vaesenc           $AESKEY2,$B12_15,$B12_15
        vbroadcastf64x2    `(16 * 7)`($AES_KEYS),$AESKEY2

        # ;; =================================================
        # ;; GHASH 4 blocks (3 to 0)
        vpclmulqdq        \$0x10,$GHKEY2,$GHDAT2,$GH2M      # ; a0*b1
        vpclmulqdq        \$0x01,$GHKEY2,$GHDAT2,$GH2T      # ; a1*b0
        vpclmulqdq        \$0x11,$GHKEY2,$GHDAT2,$GH2H      # ; a1*b1
        vpclmulqdq        \$0x00,$GHKEY2,$GHDAT2,$GH2L      # ; a0*b0
        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; AES round 6
        vaesenc           $AESKEY1,$B00_03,$B00_03
        vaesenc           $AESKEY1,$B04_07,$B04_07
        vaesenc           $AESKEY1,$B08_11,$B08_11
        vaesenc           $AESKEY1,$B12_15,$B12_15
        vbroadcastf64x2    `(16 * 8)`($AES_KEYS),$AESKEY1
___

  # ;; =================================================
  # ;; gather GHASH in GH1L (low) and GH1H (high)
  if ($DO_REDUCTION eq "first_time") {
    $code .= <<___;
        vpternlogq        \$0x96,$GH2T,$GH1T,$GH1M      # ; TM
        vpxorq            $GH2M,$GH1M,$TO_REDUCE_M      # ; TM
        vpxorq            $GH2H,$GH1H,$TO_REDUCE_H      # ; TH
        vpxorq            $GH2L,$GH1L,$TO_REDUCE_L      # ; TL
___
  }
  if ($DO_REDUCTION eq "no_reduction") {
    $code .= <<___;
        vpternlogq        \$0x96,$GH2T,$GH1T,$GH1M             # ; TM
        vpternlogq        \$0x96,$GH2M,$GH1M,$TO_REDUCE_M      # ; TM
        vpternlogq        \$0x96,$GH2H,$GH1H,$TO_REDUCE_H      # ; TH
        vpternlogq        \$0x96,$GH2L,$GH1L,$TO_REDUCE_L      # ; TL
___
  }
  if ($DO_REDUCTION eq "final_reduction") {
    $code .= <<___;
        # ;; phase 1: add mid products together
        # ;; also load polynomial constant for reduction
        vpternlogq        \$0x96,$GH2T,$GH1T,$GH1M      # ; TM
        vpternlogq        \$0x96,$GH2M,$TO_REDUCE_M,$GH1M

        vpsrldq           \$8,$GH1M,$GH2M
        vpslldq           \$8,$GH1M,$GH1M

        vmovdqa64         POLY2(%rip),@{[XWORD($RED_POLY)]}
___
  }

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; AES round 7
  $code .= <<___;
        vaesenc           $AESKEY2,$B00_03,$B00_03
        vaesenc           $AESKEY2,$B04_07,$B04_07
        vaesenc           $AESKEY2,$B08_11,$B08_11
        vaesenc           $AESKEY2,$B12_15,$B12_15
        vbroadcastf64x2    `(16 * 9)`($AES_KEYS),$AESKEY2
___

  # ;; =================================================
  # ;; Add mid product to high and low
  if ($DO_REDUCTION eq "final_reduction") {
    $code .= <<___;
        vpternlogq        \$0x96,$GH2M,$GH2H,$GH1H      # ; TH = TH1 + TH2 + TM>>64
        vpxorq            $TO_REDUCE_H,$GH1H,$GH1H
        vpternlogq        \$0x96,$GH1M,$GH2L,$GH1L      # ; TL = TL1 + TL2 + TM<<64
        vpxorq            $TO_REDUCE_L,$GH1L,$GH1L
___
  }

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; AES round 8
  $code .= <<___;
        vaesenc           $AESKEY1,$B00_03,$B00_03
        vaesenc           $AESKEY1,$B04_07,$B04_07
        vaesenc           $AESKEY1,$B08_11,$B08_11
        vaesenc           $AESKEY1,$B12_15,$B12_15
        vbroadcastf64x2    `(16 * 10)`($AES_KEYS),$AESKEY1
___

  # ;; =================================================
  # ;; horizontal xor of low and high 4x128
  if ($DO_REDUCTION eq "final_reduction") {
    &VHPXORI4x128($GH1H, $GH2H);
    &VHPXORI4x128($GH1L, $GH2L);
  }

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; AES round 9
  $code .= <<___;
        vaesenc           $AESKEY2,$B00_03,$B00_03
        vaesenc           $AESKEY2,$B04_07,$B04_07
        vaesenc           $AESKEY2,$B08_11,$B08_11
        vaesenc           $AESKEY2,$B12_15,$B12_15
___
  if (($NROUNDS >= 11)) {
    $code .= "vbroadcastf64x2    `(16 * 11)`($AES_KEYS),$AESKEY2\n";
  }

  # ;; =================================================
  # ;; first phase of reduction
  if ($DO_REDUCTION eq "final_reduction") {
    $code .= <<___;
        vpclmulqdq        \$0x01,@{[XWORD($GH1L)]},@{[XWORD($RED_POLY)]},@{[XWORD($RED_P1)]}
        vpslldq           \$8,@{[XWORD($RED_P1)]},@{[XWORD($RED_P1)]}                    # ; shift-L 2 DWs
        vpxorq            @{[XWORD($RED_P1)]},@{[XWORD($GH1L)]},@{[XWORD($RED_P1)]}      # ; first phase of the reduct
___
  }

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; AES rounds up to 11 (AES192) or 13 (AES256)
  # ;; AES128 is done
  if (($NROUNDS >= 11)) {
    $code .= <<___;
        vaesenc           $AESKEY1,$B00_03,$B00_03
        vaesenc           $AESKEY1,$B04_07,$B04_07
        vaesenc           $AESKEY1,$B08_11,$B08_11
        vaesenc           $AESKEY1,$B12_15,$B12_15
        vbroadcastf64x2    `(16 * 12)`($AES_KEYS),$AESKEY1

        vaesenc           $AESKEY2,$B00_03,$B00_03
        vaesenc           $AESKEY2,$B04_07,$B04_07
        vaesenc           $AESKEY2,$B08_11,$B08_11
        vaesenc           $AESKEY2,$B12_15,$B12_15
___
    if (($NROUNDS == 13)) {
      $code .= <<___;
        vbroadcastf64x2    `(16 * 13)`($AES_KEYS),$AESKEY2

        vaesenc           $AESKEY1,$B00_03,$B00_03
        vaesenc           $AESKEY1,$B04_07,$B04_07
        vaesenc           $AESKEY1,$B08_11,$B08_11
        vaesenc           $AESKEY1,$B12_15,$B12_15
        vbroadcastf64x2    `(16 * 14)`($AES_KEYS),$AESKEY1

        vaesenc           $AESKEY2,$B00_03,$B00_03
        vaesenc           $AESKEY2,$B04_07,$B04_07
        vaesenc           $AESKEY2,$B08_11,$B08_11
        vaesenc           $AESKEY2,$B12_15,$B12_15
___
    }
  }

  # ;; =================================================
  # ;; second phase of the reduction
  if ($DO_REDUCTION eq "final_reduction") {
    $code .= <<___;
        vpclmulqdq        \$0x00,@{[XWORD($RED_P1)]},@{[XWORD($RED_POLY)]},@{[XWORD($RED_T1)]}
        vpsrldq           \$4,@{[XWORD($RED_T1)]},@{[XWORD($RED_T1)]}      # ; shift-R 1-DW to obtain 2-DWs shift-R
        vpclmulqdq        \$0x10,@{[XWORD($RED_P1)]},@{[XWORD($RED_POLY)]},@{[XWORD($RED_T2)]}
        vpslldq           \$4,@{[XWORD($RED_T2)]},@{[XWORD($RED_T2)]}      # ; shift-L 1-DW for result without shifts
        # ;; GH1H = GH1H x RED_T1 x RED_T2
        vpternlogq        \$0x96,@{[XWORD($RED_T1)]},@{[XWORD($RED_T2)]},@{[XWORD($GH1H)]}
___
  }

  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;; the last AES round
  $code .= <<___;
        vaesenclast       $AESKEY1,$B00_03,$B00_03
        vaesenclast       $AESKEY1,$B04_07,$B04_07
        vaesenclast       $AESKEY1,$B08_11,$B08_11
        vaesenclast       $AESKEY1,$B12_15,$B12_15

        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; XOR against plain/cipher text
        vpxorq            $DATA1,$B00_03,$B00_03
        vpxorq            $DATA2,$B04_07,$B04_07
        vpxorq            $DATA3,$B08_11,$B08_11
        vpxorq            $DATA4,$B12_15,$B12_15

        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; store cipher/plain text
        mov               $CIPH_PLAIN_OUT,$IA0
        vmovdqu8          $B00_03,`$DATA_DISPL + (0 * 64)`($IA0,$DATA_OFFSET,1)
        vmovdqu8          $B04_07,`$DATA_DISPL + (1 * 64)`($IA0,$DATA_OFFSET,1)
        vmovdqu8          $B08_11,`$DATA_DISPL + (2 * 64)`($IA0,$DATA_OFFSET,1)
        vmovdqu8          $B12_15,`$DATA_DISPL + (3 * 64)`($IA0,$DATA_OFFSET,1)
___

  # ;; =================================================
  # ;; shuffle cipher text blocks for GHASH computation
  if ($ENC_DEC eq "ENC") {
    $code .= <<___;
        vpshufb           $SHFMSK,$B00_03,$B00_03
        vpshufb           $SHFMSK,$B04_07,$B04_07
        vpshufb           $SHFMSK,$B08_11,$B08_11
        vpshufb           $SHFMSK,$B12_15,$B12_15
___
  } else {
    $code .= <<___;
        vpshufb           $SHFMSK,$DATA1,$B00_03
        vpshufb           $SHFMSK,$DATA2,$B04_07
        vpshufb           $SHFMSK,$DATA3,$B08_11
        vpshufb           $SHFMSK,$DATA4,$B12_15
___
  }

  # ;; =================================================
  # ;; store shuffled cipher text for ghashing
  $code .= <<___;
        vmovdqa64         $B00_03,`$AESOUT_BLK_OFFSET + (0*64)`(%rsp)
        vmovdqa64         $B04_07,`$AESOUT_BLK_OFFSET + (1*64)`(%rsp)
        vmovdqa64         $B08_11,`$AESOUT_BLK_OFFSET + (2*64)`(%rsp)
        vmovdqa64         $B12_15,`$AESOUT_BLK_OFFSET + (3*64)`(%rsp)
___
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;;; Encryption of a single block
sub ENCRYPT_SINGLE_BLOCK {
  my $AES_KEY = $_[0];    # ; [in]
  my $XMM0    = $_[1];    # ; [in/out]
  my $GPR1    = $_[2];    # ; [clobbered]

  my $label_suffix = $label_count++;

  $code .= <<___;
        # ; load number of rounds from AES_KEY structure (offset in bytes is
        # ; size of the |rd_key| buffer)
        mov             `4*15*4`($AES_KEY),@{[DWORD($GPR1)]}
        cmp             \$9,@{[DWORD($GPR1)]}
        je              .Laes_128_${label_suffix}
        cmp             \$11,@{[DWORD($GPR1)]}
        je              .Laes_192_${label_suffix}
        cmp             \$13,@{[DWORD($GPR1)]}
        je              .Laes_256_${label_suffix}
        jmp             .Lexit_aes_${label_suffix}
___
  for my $keylen (sort keys %aes_rounds) {
    my $nr = $aes_rounds{$keylen};
    $code .= <<___;
.align 32
.Laes_${keylen}_${label_suffix}:
___
    $code .= "vpxorq          `16*0`($AES_KEY),$XMM0, $XMM0\n\n";
    for (my $i = 1; $i <= $nr; $i++) {
      $code .= "vaesenc         `16*$i`($AES_KEY),$XMM0,$XMM0\n\n";
    }
    $code .= <<___;
        vaesenclast     `16*($nr+1)`($AES_KEY),$XMM0,$XMM0
        jmp .Lexit_aes_${label_suffix}
___
  }
  $code .= ".Lexit_aes_${label_suffix}:\n\n";
}

sub CALC_J0 {
  my $GCM128_CTX = $_[0];     #; [in] Pointer to GCM context
  my $IV         = $_[1];     #; [in] Pointer to IV
  my $IV_LEN     = $_[2];     #; [in] IV length
  my $J0         = $_[3];     #; [out] XMM reg to contain J0
  my $ZT0        = $_[4];     #; [clobbered] ZMM register
  my $ZT1        = $_[5];     #; [clobbered] ZMM register
  my $ZT2        = $_[6];     #; [clobbered] ZMM register
  my $ZT3        = $_[7];     #; [clobbered] ZMM register
  my $ZT4        = $_[8];     #; [clobbered] ZMM register
  my $ZT5        = $_[9];     #; [clobbered] ZMM register
  my $ZT6        = $_[10];    #; [clobbered] ZMM register
  my $ZT7        = $_[11];    #; [clobbered] ZMM register
  my $ZT8        = $_[12];    #; [clobbered] ZMM register
  my $ZT9        = $_[13];    #; [clobbered] ZMM register
  my $ZT10       = $_[14];    #; [clobbered] ZMM register
  my $ZT11       = $_[15];    #; [clobbered] ZMM register
  my $ZT12       = $_[16];    #; [clobbered] ZMM register
  my $ZT13       = $_[17];    #; [clobbered] ZMM register
  my $ZT14       = $_[18];    #; [clobbered] ZMM register
  my $ZT15       = $_[19];    #; [clobbered] ZMM register
  my $ZT16       = $_[20];    #; [clobbered] ZMM register
  my $T1         = $_[21];    #; [clobbered] GP register
  my $T2         = $_[22];    #; [clobbered] GP register
  my $T3         = $_[23];    #; [clobbered] GP register
  my $MASKREG    = $_[24];    #; [clobbered] mask register

  # ;; J0 = GHASH(IV || 0s+64 || len(IV)64)
  # ;; s = 16 * RoundUp(len(IV)/16) -  len(IV) */

  # ;; Calculate GHASH of (IV || 0s)
  $code .= "vpxor             $J0,$J0,$J0\n";
  &CALC_AAD_HASH($IV, $IV_LEN, $J0, $GCM128_CTX, $ZT0, $ZT1, $ZT2, $ZT3, $ZT4,
    $ZT5, $ZT6, $ZT7, $ZT8, $ZT9, $ZT10, $ZT11, $ZT12, $ZT13, $ZT14, $ZT15, $ZT16, $T1, $T2, $T3, $MASKREG);

  # ;; Calculate GHASH of last 16-byte block (0 || len(IV)64)
  $code .= <<___;
        mov               $IV_LEN,$T1
        shl               \$3,$T1      # ; IV length in bits
        vmovq             $T1,@{[XWORD($ZT2)]}

        # ;; Might need shuffle of ZT2
        vpxorq            $J0,@{[XWORD($ZT2)]},$J0

        vmovdqu64         @{[HashKeyByIdx(1,$GCM128_CTX)]},@{[XWORD($ZT0)]}
___
  &GHASH_MUL($J0, @{[XWORD($ZT0)]}, @{[XWORD($ZT1)]}, @{[XWORD($ZT2)]}, @{[XWORD($ZT3)]});

  $code .= "vpshufb           SHUF_MASK(%rip),$J0,$J0      # ; perform a 16Byte swap\n";
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;;; GCM_INIT_IV performs an initialization of gcm128_ctx struct to prepare for
# ;;; encoding/decoding.
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
sub GCM_INIT_IV {
  my $AES_KEYS   = $_[0];     # [in] AES key schedule
  my $GCM128_CTX = $_[1];     # [in/out] GCM context
  my $IV         = $_[2];     # [in] IV pointer
  my $IV_LEN     = $_[3];     # [in] IV length
  my $GPR1       = $_[4];     # [clobbered] GP register
  my $GPR2       = $_[5];     # [clobbered] GP register
  my $GPR3       = $_[6];     # [clobbered] GP register
  my $MASKREG    = $_[7];     # [clobbered] mask register
  my $CUR_COUNT  = $_[8];     # [out] XMM with current counter
  my $ZT0        = $_[9];     # [clobbered] ZMM register
  my $ZT1        = $_[10];    # [clobbered] ZMM register
  my $ZT2        = $_[11];    # [clobbered] ZMM register
  my $ZT3        = $_[12];    # [clobbered] ZMM register
  my $ZT4        = $_[13];    # [clobbered] ZMM register
  my $ZT5        = $_[14];    # [clobbered] ZMM register
  my $ZT6        = $_[15];    # [clobbered] ZMM register
  my $ZT7        = $_[16];    # [clobbered] ZMM register
  my $ZT8        = $_[17];    # [clobbered] ZMM register
  my $ZT9        = $_[18];    # [clobbered] ZMM register
  my $ZT10       = $_[19];    # [clobbered] ZMM register
  my $ZT11       = $_[20];    # [clobbered] ZMM register
  my $ZT12       = $_[21];    # [clobbered] ZMM register
  my $ZT13       = $_[22];    # [clobbered] ZMM register
  my $ZT14       = $_[23];    # [clobbered] ZMM register
  my $ZT15       = $_[24];    # [clobbered] ZMM register
  my $ZT16       = $_[25];    # [clobbered] ZMM register

  my $ZT0x = $ZT0;
  $ZT0x =~ s/zmm/xmm/;

  $code .= <<___;
        cmp     \$12,$IV_LEN
        je      iv_len_12_init_IV
___

  # ;; IV is different than 12 bytes
  &CALC_J0($GCM128_CTX, $IV, $IV_LEN, $CUR_COUNT, $ZT0, $ZT1, $ZT2, $ZT3, $ZT4, $ZT5, $ZT6, $ZT7,
    $ZT8, $ZT9, $ZT10, $ZT11, $ZT12, $ZT13, $ZT14, $ZT15, $ZT16, $GPR1, $GPR2, $GPR3, $MASKREG);
  $code .= <<___;
       jmp      skip_iv_len_12_init_IV
iv_len_12_init_IV:   # ;; IV is 12 bytes
        # ;; read 12 IV bytes and pad with 0x00000001
        vmovdqu8          ONEf(%rip),$CUR_COUNT
        mov               $IV,$GPR2
        mov               \$0x0000000000000fff,@{[DWORD($GPR1)]}
        kmovq             $GPR1,$MASKREG
        vmovdqu8          ($GPR2),${CUR_COUNT}{$MASKREG}         # ; ctr = IV | 0x1
skip_iv_len_12_init_IV:
        vmovdqu           $CUR_COUNT,$ZT0x
___
  &ENCRYPT_SINGLE_BLOCK($AES_KEYS, "$ZT0x", "$GPR1");    # ; E(K, Y0)
  $code .= <<___;
        vmovdqu           $ZT0x,`$CTX_OFFSET_EK0`($GCM128_CTX)   # ; save EK0 for finalization stage

        # ;; store IV as counter in LE format
        vpshufb           SHUF_MASK(%rip),$CUR_COUNT,$CUR_COUNT
        vmovdqu           $CUR_COUNT,`$CTX_OFFSET_CurCount`($GCM128_CTX)   # ; save current counter Yi
___
}

sub GCM_UPDATE_AAD {
  my $GCM128_CTX = $_[0];  # [in] GCM context pointer
  my $A_IN       = $_[1];  # [in] AAD pointer
  my $A_LEN      = $_[2];  # [in] AAD length in bytes
  my $GPR1       = $_[3];  # [clobbered] GP register
  my $GPR2       = $_[4];  # [clobbered] GP register
  my $GPR3       = $_[5];  # [clobbered] GP register
  my $MASKREG    = $_[6];  # [clobbered] mask register
  my $AAD_HASH   = $_[7];  # [out] XMM for AAD_HASH value
  my $ZT0        = $_[8];  # [clobbered] ZMM register
  my $ZT1        = $_[9];  # [clobbered] ZMM register
  my $ZT2        = $_[10]; # [clobbered] ZMM register
  my $ZT3        = $_[11]; # [clobbered] ZMM register
  my $ZT4        = $_[12]; # [clobbered] ZMM register
  my $ZT5        = $_[13]; # [clobbered] ZMM register
  my $ZT6        = $_[14]; # [clobbered] ZMM register
  my $ZT7        = $_[15]; # [clobbered] ZMM register
  my $ZT8        = $_[16]; # [clobbered] ZMM register
  my $ZT9        = $_[17]; # [clobbered] ZMM register
  my $ZT10       = $_[18]; # [clobbered] ZMM register
  my $ZT11       = $_[19]; # [clobbered] ZMM register
  my $ZT12       = $_[20]; # [clobbered] ZMM register
  my $ZT13       = $_[21]; # [clobbered] ZMM register
  my $ZT14       = $_[22]; # [clobbered] ZMM register
  my $ZT15       = $_[23]; # [clobbered] ZMM register
  my $ZT16       = $_[24]; # [clobbered] ZMM register

  # ; load current hash
  $code .= "vmovdqu64         $CTX_OFFSET_AadHash($GCM128_CTX),$AAD_HASH\n";

  &CALC_AAD_HASH($A_IN, $A_LEN, $AAD_HASH, $GCM128_CTX, $ZT0, $ZT1, $ZT2,
    $ZT3, $ZT4, $ZT5, $ZT6, $ZT7, $ZT8, $ZT9, $ZT10, $ZT11, $ZT12, $ZT13,
    $ZT14, $ZT15, $ZT16, $GPR1, $GPR2, $GPR3, $MASKREG);

  # ; load current hash
  $code .= "vmovdqu64         $AAD_HASH,$CTX_OFFSET_AadHash($GCM128_CTX)\n";
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;;; Cipher and ghash of payloads shorter than 256 bytes
# ;;; - number of blocks in the message comes as argument
# ;;; - depending on the number of blocks an optimized variant of
# ;;;   INITIAL_BLOCKS_PARTIAL is invoked
sub GCM_ENC_DEC_SMALL {
  my $AES_KEYS       = $_[0];     # [in] key pointer
  my $GCM128_CTX     = $_[1];     # [in] context pointer
  my $CIPH_PLAIN_OUT = $_[2];     # [in] output buffer
  my $PLAIN_CIPH_IN  = $_[3];     # [in] input buffer
  my $PLAIN_CIPH_LEN = $_[4];     # [in] buffer length
  my $ENC_DEC        = $_[5];     # [in] cipher direction
  my $DATA_OFFSET    = $_[6];     # [in] data offset
  my $LENGTH         = $_[7];     # [in] data length
  my $NUM_BLOCKS     = $_[8];     # [in] number of blocks to process 1 to 16
  my $CTR            = $_[9];     # [in/out] XMM counter block
  my $HASH_IN_OUT    = $_[10];    # [in/out] XMM GHASH value
  my $ZTMP0          = $_[11];    # [clobbered] ZMM register
  my $ZTMP1          = $_[12];    # [clobbered] ZMM register
  my $ZTMP2          = $_[13];    # [clobbered] ZMM register
  my $ZTMP3          = $_[14];    # [clobbered] ZMM register
  my $ZTMP4          = $_[15];    # [clobbered] ZMM register
  my $ZTMP5          = $_[16];    # [clobbered] ZMM register
  my $ZTMP6          = $_[17];    # [clobbered] ZMM register
  my $ZTMP7          = $_[18];    # [clobbered] ZMM register
  my $ZTMP8          = $_[19];    # [clobbered] ZMM register
  my $ZTMP9          = $_[20];    # [clobbered] ZMM register
  my $ZTMP10         = $_[21];    # [clobbered] ZMM register
  my $ZTMP11         = $_[22];    # [clobbered] ZMM register
  my $ZTMP12         = $_[23];    # [clobbered] ZMM register
  my $ZTMP13         = $_[24];    # [clobbered] ZMM register
  my $ZTMP14         = $_[25];    # [clobbered] ZMM register
  my $IA0            = $_[26];    # [clobbered] GP register
  my $IA1            = $_[27];    # [clobbered] GP register
  my $MASKREG        = $_[28];    # [clobbered] mask register
  my $SHUFMASK       = $_[29];    # [in] ZMM with BE/LE shuffle mask
  my $PBLOCK_LEN     = $_[30];    # [in] partial block length

  my $label_suffix = $label_count++;

  $code .= <<___;
        cmp               \$8,$NUM_BLOCKS
        je                .L_small_initial_num_blocks_is_8_${label_suffix}
        jl                .L_small_initial_num_blocks_is_7_1_${label_suffix}


        cmp               \$12,$NUM_BLOCKS
        je                .L_small_initial_num_blocks_is_12_${label_suffix}
        jl                .L_small_initial_num_blocks_is_11_9_${label_suffix}

        # ;; 16, 15, 14 or 13
        cmp               \$16,$NUM_BLOCKS
        je                .L_small_initial_num_blocks_is_16_${label_suffix}
        cmp               \$15,$NUM_BLOCKS
        je                .L_small_initial_num_blocks_is_15_${label_suffix}
        cmp               \$14,$NUM_BLOCKS
        je                .L_small_initial_num_blocks_is_14_${label_suffix}
        jmp               .L_small_initial_num_blocks_is_13_${label_suffix}

.L_small_initial_num_blocks_is_11_9_${label_suffix}:
        # ;; 11, 10 or 9
        cmp               \$11,$NUM_BLOCKS
        je                .L_small_initial_num_blocks_is_11_${label_suffix}
        cmp               \$10,$NUM_BLOCKS
        je                .L_small_initial_num_blocks_is_10_${label_suffix}
        jmp               .L_small_initial_num_blocks_is_9_${label_suffix}

.L_small_initial_num_blocks_is_7_1_${label_suffix}:
        cmp               \$4,$NUM_BLOCKS
        je                .L_small_initial_num_blocks_is_4_${label_suffix}
        jl                .L_small_initial_num_blocks_is_3_1_${label_suffix}
        # ;; 7, 6 or 5
        cmp               \$7,$NUM_BLOCKS
        je                .L_small_initial_num_blocks_is_7_${label_suffix}
        cmp               \$6,$NUM_BLOCKS
        je                .L_small_initial_num_blocks_is_6_${label_suffix}
        jmp               .L_small_initial_num_blocks_is_5_${label_suffix}

.L_small_initial_num_blocks_is_3_1_${label_suffix}:
        # ;; 3, 2 or 1
        cmp               \$3,$NUM_BLOCKS
        je                .L_small_initial_num_blocks_is_3_${label_suffix}
        cmp               \$2,$NUM_BLOCKS
        je                .L_small_initial_num_blocks_is_2_${label_suffix}

        # ;; for $NUM_BLOCKS == 1, just fall through and no 'jmp' needed

        # ;; Generation of different block size variants
        # ;; - one block size has to be the first one
___

  for (my $num_blocks = 1; $num_blocks <= 16; $num_blocks++) {
    $code .= ".L_small_initial_num_blocks_is_${num_blocks}_${label_suffix}:\n";
    &INITIAL_BLOCKS_PARTIAL(
      $AES_KEYS,   $GCM128_CTX, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $LENGTH,   $DATA_OFFSET,
      $num_blocks, $CTR,        $HASH_IN_OUT,    $ENC_DEC,       $ZTMP0,    $ZTMP1,
      $ZTMP2,      $ZTMP3,      $ZTMP4,          $ZTMP5,         $ZTMP6,    $ZTMP7,
      $ZTMP8,      $ZTMP9,      $ZTMP10,         $ZTMP11,        $ZTMP12,   $ZTMP13,
      $ZTMP14,     $IA0,        $IA1,            $MASKREG,       $SHUFMASK, $PBLOCK_LEN);

    if ($num_blocks != 16) {
      $code .= "jmp           .L_small_initial_blocks_encrypted_${label_suffix}\n";
    }
  }

  $code .= ".L_small_initial_blocks_encrypted_${label_suffix}:\n";
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ; GCM_ENC_DEC Encrypts/Decrypts given data. Assumes that the passed gcm128_context
# ; struct has been initialized by GCM_INIT_IV
# ; Requires the input data be at least 1 byte long because of READ_SMALL_INPUT_DATA.
# ; Clobbers rax, r10-r15, and zmm0-zmm31, k1
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
sub GCM_ENC_DEC {
  my $AES_KEYS       = $_[0];    # [in] AES Key schedule
  my $GCM128_CTX     = $_[1];    # [in] context pointer
  my $PBLOCK_LEN     = $_[2];    # [in] length of partial block at the moment of previous update
  my $PLAIN_CIPH_IN  = $_[3];    # [in] input buffer pointer
  my $PLAIN_CIPH_LEN = $_[4];    # [in] buffer length
  my $CIPH_PLAIN_OUT = $_[5];    # [in] output buffer pointer
  my $ENC_DEC        = $_[6];    # [in] cipher direction

  my $IA0 = "%r10";
  my $IA1 = "%r12";
  my $IA2 = "%r13";
  my $IA3 = "%r15";
  my $IA4 = "%r11";
  my $IA5 = "%rax";
  my $IA6 = "%rbx";
  my $IA7 = "%r14";

  my $LENGTH = $win64 ? $IA2 : $PLAIN_CIPH_LEN;

  my $CTR_CHECK   = $IA3;
  my $DATA_OFFSET = $IA4;
  my $HASHK_PTR   = $IA6;

  my $HKEYS_READY = $IA7;

  my $CTR_BLOCKz = "%zmm2";
  my $CTR_BLOCKx = "%xmm2";

  # ; hardcoded in GCM_INIT

  my $AAD_HASHz = "%zmm14";
  my $AAD_HASHx = "%xmm14";

  # ; hardcoded in GCM_COMPLETE

  my $ZTMP0  = "%zmm0";
  my $ZTMP1  = "%zmm3";
  my $ZTMP2  = "%zmm4";
  my $ZTMP3  = "%zmm5";
  my $ZTMP4  = "%zmm6";
  my $ZTMP5  = "%zmm7";
  my $ZTMP6  = "%zmm10";
  my $ZTMP7  = "%zmm11";
  my $ZTMP8  = "%zmm12";
  my $ZTMP9  = "%zmm13";
  my $ZTMP10 = "%zmm15";
  my $ZTMP11 = "%zmm16";
  my $ZTMP12 = "%zmm17";

  my $ZTMP13 = "%zmm19";
  my $ZTMP14 = "%zmm20";
  my $ZTMP15 = "%zmm21";
  my $ZTMP16 = "%zmm30";
  my $ZTMP17 = "%zmm31";
  my $ZTMP18 = "%zmm1";
  my $ZTMP19 = "%zmm18";
  my $ZTMP20 = "%zmm8";
  my $ZTMP21 = "%zmm22";
  my $ZTMP22 = "%zmm23";

  my $GH        = "%zmm24";
  my $GL        = "%zmm25";
  my $GM        = "%zmm26";
  my $SHUF_MASK = "%zmm29";

  # ; Unused in the small packet path
  my $ADDBE_4x4  = "%zmm27";
  my $ADDBE_1234 = "%zmm28";

  my $MASKREG = "%k1";

  my $label_suffix = $label_count++;

  # ;; reduction every 48 blocks, depth 32 blocks
  # ;; @note 48 blocks is the maximum capacity of the stack frame
  my $big_loop_nblocks = 48;
  my $big_loop_depth   = 32;

  # ;;; Macro flow depending on packet size
  # ;;; - LENGTH <= 16 blocks
  # ;;;   - cipher followed by hashing (reduction)
  # ;;; - 16 blocks < LENGTH < 32 blocks
  # ;;;   - cipher 16 blocks
  # ;;;   - cipher N blocks & hash 16 blocks, hash N blocks (reduction)
  # ;;; - 32 blocks < LENGTH < 48 blocks
  # ;;;   - cipher 2 x 16 blocks
  # ;;;   - hash 16 blocks
  # ;;;   - cipher N blocks & hash 16 blocks, hash N blocks (reduction)
  # ;;; - LENGTH >= 48 blocks
  # ;;;   - cipher 2 x 16 blocks
  # ;;;   - while (data_to_cipher >= 48 blocks):
  # ;;;     - cipher 16 blocks & hash 16 blocks
  # ;;;     - cipher 16 blocks & hash 16 blocks
  # ;;;     - cipher 16 blocks & hash 16 blocks (reduction)
  # ;;;   - if (data_to_cipher >= 32 blocks):
  # ;;;     - cipher 16 blocks & hash 16 blocks
  # ;;;     - cipher 16 blocks & hash 16 blocks
  # ;;;     - hash 16 blocks (reduction)
  # ;;;     - cipher N blocks & hash 16 blocks, hash N blocks (reduction)
  # ;;;   - elif (data_to_cipher >= 16 blocks):
  # ;;;     - cipher 16 blocks & hash 16 blocks
  # ;;;     - hash 16 blocks
  # ;;;     - cipher N blocks & hash 16 blocks, hash N blocks (reduction)
  # ;;;   - else:
  # ;;;     - hash 16 blocks
  # ;;;     - cipher N blocks & hash 16 blocks, hash N blocks (reduction)

  if ($win64) {
    $code .= "cmpq              \$0,$PLAIN_CIPH_LEN\n";
  } else {
    $code .= "or                $PLAIN_CIPH_LEN,$PLAIN_CIPH_LEN\n";
  }
  $code .= "je            .L_enc_dec_done_${label_suffix}\n";

  # Length value from context $CTX_OFFSET_InLen`($GCM128_CTX) is updated in
  # 'providers/implementations/ciphers/cipher_aes_gcm_hw_vaes_avx512.inc'

  $code .= "xor                $HKEYS_READY, $HKEYS_READY\n";
  $code .= "vmovdqu64         `$CTX_OFFSET_AadHash`($GCM128_CTX),$AAD_HASHx\n";

  # ;; Used for the update flow - if there was a previous partial
  # ;; block fill the remaining bytes here.
  &PARTIAL_BLOCK(
    $GCM128_CTX,  $PBLOCK_LEN, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $PLAIN_CIPH_LEN,
    $DATA_OFFSET, $AAD_HASHx,  $ENC_DEC,        $IA0,           $IA1,
    $IA2,         $ZTMP0,      $ZTMP1,          $ZTMP2,         $ZTMP3,
    $ZTMP4,       $ZTMP5,      $ZTMP6,          $ZTMP7,         $MASKREG);

  $code .= "vmovdqu64         `$CTX_OFFSET_CurCount`($GCM128_CTX),$CTR_BLOCKx\n";

  # ;; Save the amount of data left to process in $LENGTH
  # ;; NOTE: PLAIN_CIPH_LEN is a register on linux;
  if ($win64) {
    $code .= "mov               $PLAIN_CIPH_LEN,$LENGTH\n";
  }

  # ;; There may be no more data if it was consumed in the partial block.
  $code .= <<___;
        sub               $DATA_OFFSET,$LENGTH
        je                .L_enc_dec_done_${label_suffix}
___

  $code .= <<___;
        cmp               \$`(16 * 16)`,$LENGTH
        jbe              .L_message_below_equal_16_blocks_${label_suffix}

        vmovdqa64         SHUF_MASK(%rip),$SHUF_MASK
        vmovdqa64         ddq_addbe_4444(%rip),$ADDBE_4x4
        vmovdqa64         ddq_addbe_1234(%rip),$ADDBE_1234

        # ;; start the pipeline
        # ;; - 32 blocks aes-ctr
        # ;; - 16 blocks ghash + aes-ctr

        # ;; set up CTR_CHECK
        vmovd             $CTR_BLOCKx,@{[DWORD($CTR_CHECK)]}
        and               \$255,@{[DWORD($CTR_CHECK)]}
        # ;; in LE format after init, convert to BE
        vshufi64x2        \$0,$CTR_BLOCKz,$CTR_BLOCKz,$CTR_BLOCKz
        vpshufb           $SHUF_MASK,$CTR_BLOCKz,$CTR_BLOCKz
___

  # ;; ==== AES-CTR - first 16 blocks
  my $aesout_offset      = ($STACK_LOCAL_OFFSET + (0 * 16));
  my $data_in_out_offset = 0;
  &INITIAL_BLOCKS_16(
    $PLAIN_CIPH_IN, $CIPH_PLAIN_OUT, $AES_KEYS,      $DATA_OFFSET,        "no_ghash", $CTR_BLOCKz,
    $CTR_CHECK,     $ADDBE_4x4,      $ADDBE_1234,    $ZTMP0,              $ZTMP1,     $ZTMP2,
    $ZTMP3,         $ZTMP4,          $ZTMP5,         $ZTMP6,              $ZTMP7,     $ZTMP8,
    $SHUF_MASK,     $ENC_DEC,        $aesout_offset, $data_in_out_offset, $IA0);

  &precompute_hkeys_on_stack($GCM128_CTX, $HKEYS_READY, $ZTMP0, $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6,
    "first16");

  $code .= <<___;
        cmp               \$`(32 * 16)`,$LENGTH
        jb                .L_message_below_32_blocks_${label_suffix}
___

  # ;; ==== AES-CTR - next 16 blocks
  $aesout_offset      = ($STACK_LOCAL_OFFSET + (16 * 16));
  $data_in_out_offset = (16 * 16);
  &INITIAL_BLOCKS_16(
    $PLAIN_CIPH_IN, $CIPH_PLAIN_OUT, $AES_KEYS,      $DATA_OFFSET,        "no_ghash", $CTR_BLOCKz,
    $CTR_CHECK,     $ADDBE_4x4,      $ADDBE_1234,    $ZTMP0,              $ZTMP1,     $ZTMP2,
    $ZTMP3,         $ZTMP4,          $ZTMP5,         $ZTMP6,              $ZTMP7,     $ZTMP8,
    $SHUF_MASK,     $ENC_DEC,        $aesout_offset, $data_in_out_offset, $IA0);

  &precompute_hkeys_on_stack($GCM128_CTX, $HKEYS_READY, $ZTMP0, $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6,
    "last32");
  $code .= "mov     \$1,$HKEYS_READY\n";

  $code .= <<___;
        add               \$`(32 * 16)`,$DATA_OFFSET
        sub               \$`(32 * 16)`,$LENGTH

        cmp               \$`($big_loop_nblocks * 16)`,$LENGTH
        jb                .L_no_more_big_nblocks_${label_suffix}
___

  # ;; ====
  # ;; ==== AES-CTR + GHASH - 48 blocks loop
  # ;; ====
  $code .= ".L_encrypt_big_nblocks_${label_suffix}:\n";

  # ;; ==== AES-CTR + GHASH - 16 blocks, start
  $aesout_offset      = ($STACK_LOCAL_OFFSET + (32 * 16));
  $data_in_out_offset = (0 * 16);
  my $ghashin_offset = ($STACK_LOCAL_OFFSET + (0 * 16));
  &GHASH_16_ENCRYPT_16_PARALLEL(
    $AES_KEYS, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN,  $DATA_OFFSET, $CTR_BLOCKz,         $CTR_CHECK,
    48,        $aesout_offset,  $ghashin_offset, $SHUF_MASK,   $ZTMP0,              $ZTMP1,
    $ZTMP2,    $ZTMP3,          $ZTMP4,          $ZTMP5,       $ZTMP6,              $ZTMP7,
    $ZTMP8,    $ZTMP9,          $ZTMP10,         $ZTMP11,      $ZTMP12,             $ZTMP13,
    $ZTMP14,   $ZTMP15,         $ZTMP16,         $ZTMP17,      $ZTMP18,             $ZTMP19,
    $ZTMP20,   $ZTMP21,         $ZTMP22,         $ADDBE_4x4,   $ADDBE_1234,         $GL,
    $GH,       $GM,             "first_time",    $ENC_DEC,     $data_in_out_offset, $AAD_HASHz,
    $IA0);

  # ;; ==== AES-CTR + GHASH - 16 blocks, no reduction
  $aesout_offset      = ($STACK_LOCAL_OFFSET + (0 * 16));
  $data_in_out_offset = (16 * 16);
  $ghashin_offset     = ($STACK_LOCAL_OFFSET + (16 * 16));
  &GHASH_16_ENCRYPT_16_PARALLEL(
    $AES_KEYS, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN,  $DATA_OFFSET, $CTR_BLOCKz,         $CTR_CHECK,
    32,        $aesout_offset,  $ghashin_offset, $SHUF_MASK,   $ZTMP0,              $ZTMP1,
    $ZTMP2,    $ZTMP3,          $ZTMP4,          $ZTMP5,       $ZTMP6,              $ZTMP7,
    $ZTMP8,    $ZTMP9,          $ZTMP10,         $ZTMP11,      $ZTMP12,             $ZTMP13,
    $ZTMP14,   $ZTMP15,         $ZTMP16,         $ZTMP17,      $ZTMP18,             $ZTMP19,
    $ZTMP20,   $ZTMP21,         $ZTMP22,         $ADDBE_4x4,   $ADDBE_1234,         $GL,
    $GH,       $GM,             "no_reduction",  $ENC_DEC,     $data_in_out_offset, "no_ghash_in",
    $IA0);

  # ;; ==== AES-CTR + GHASH - 16 blocks, reduction
  $aesout_offset      = ($STACK_LOCAL_OFFSET + (16 * 16));
  $data_in_out_offset = (32 * 16);
  $ghashin_offset     = ($STACK_LOCAL_OFFSET + (32 * 16));
  &GHASH_16_ENCRYPT_16_PARALLEL(
    $AES_KEYS, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN,    $DATA_OFFSET, $CTR_BLOCKz,         $CTR_CHECK,
    16,        $aesout_offset,  $ghashin_offset,   $SHUF_MASK,   $ZTMP0,              $ZTMP1,
    $ZTMP2,    $ZTMP3,          $ZTMP4,            $ZTMP5,       $ZTMP6,              $ZTMP7,
    $ZTMP8,    $ZTMP9,          $ZTMP10,           $ZTMP11,      $ZTMP12,             $ZTMP13,
    $ZTMP14,   $ZTMP15,         $ZTMP16,           $ZTMP17,      $ZTMP18,             $ZTMP19,
    $ZTMP20,   $ZTMP21,         $ZTMP22,           $ADDBE_4x4,   $ADDBE_1234,         $GL,
    $GH,       $GM,             "final_reduction", $ENC_DEC,     $data_in_out_offset, "no_ghash_in",
    $IA0);

  # ;; === xor cipher block 0 with GHASH (ZT4)
  $code .= <<___;
        vmovdqa64         $ZTMP4,$AAD_HASHz

        add               \$`($big_loop_nblocks * 16)`,$DATA_OFFSET
        sub               \$`($big_loop_nblocks * 16)`,$LENGTH
        cmp               \$`($big_loop_nblocks * 16)`,$LENGTH
        jae               .L_encrypt_big_nblocks_${label_suffix}

.L_no_more_big_nblocks_${label_suffix}:

        cmp               \$`(32 * 16)`,$LENGTH
        jae               .L_encrypt_32_blocks_${label_suffix}

        cmp               \$`(16 * 16)`,$LENGTH
        jae               .L_encrypt_16_blocks_${label_suffix}
___

  # ;; =====================================================
  # ;; =====================================================
  # ;; ==== GHASH 1 x 16 blocks
  # ;; ==== GHASH 1 x 16 blocks (reduction) & encrypt N blocks
  # ;; ====      then GHASH N blocks
  $code .= ".L_encrypt_0_blocks_ghash_32_${label_suffix}:\n";

  # ;; calculate offset to the right hash key
  $code .= <<___;
mov               @{[DWORD($LENGTH)]},@{[DWORD($IA0)]}
and               \$~15,@{[DWORD($IA0)]}
mov               \$`@{[HashKeyOffsetByIdx(32,"frame")]}`,@{[DWORD($HASHK_PTR)]}
sub               @{[DWORD($IA0)]},@{[DWORD($HASHK_PTR)]}
___

  # ;; ==== GHASH 32 blocks and follow with reduction
  &GHASH_16("start", $GH, $GM, $GL, "%rsp", $STACK_LOCAL_OFFSET, (0 * 16),
    "%rsp", $HASHK_PTR, 0, $AAD_HASHz, $ZTMP0, $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, $ZTMP8, $ZTMP9);

  # ;; ==== GHASH 1 x 16 blocks with reduction + cipher and ghash on the reminder
  $ghashin_offset = ($STACK_LOCAL_OFFSET + (16 * 16));
  $code .= "add               \$`(16 * 16)`,@{[DWORD($HASHK_PTR)]}\n";
  &GCM_ENC_DEC_LAST(
    $AES_KEYS,   $GCM128_CTX, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN,  $DATA_OFFSET, $LENGTH,
    $CTR_BLOCKz, $CTR_CHECK,  $HASHK_PTR,      $ghashin_offset, $SHUF_MASK,   $ZTMP0,
    $ZTMP1,      $ZTMP2,      $ZTMP3,          $ZTMP4,          $ZTMP5,       $ZTMP6,
    $ZTMP7,      $ZTMP8,      $ZTMP9,          $ZTMP10,         $ZTMP11,      $ZTMP12,
    $ZTMP13,     $ZTMP14,     $ZTMP15,         $ZTMP16,         $ZTMP17,      $ZTMP18,
    $ZTMP19,     $ZTMP20,     $ZTMP21,         $ZTMP22,         $ADDBE_4x4,   $ADDBE_1234,
    "mid",       $GL,         $GH,             $GM,             $ENC_DEC,     $AAD_HASHz,
    $IA0,        $IA5,        $MASKREG,        $PBLOCK_LEN);

  $code .= "vpshufb           @{[XWORD($SHUF_MASK)]},$CTR_BLOCKx,$CTR_BLOCKx\n";
  $code .= "jmp           .L_ghash_done_${label_suffix}\n";

  # ;; =====================================================
  # ;; =====================================================
  # ;; ==== GHASH & encrypt 1 x 16 blocks
  # ;; ==== GHASH & encrypt 1 x 16 blocks
  # ;; ==== GHASH 1 x 16 blocks (reduction)
  # ;; ==== GHASH 1 x 16 blocks (reduction) & encrypt N blocks
  # ;; ====      then GHASH N blocks
  $code .= ".L_encrypt_32_blocks_${label_suffix}:\n";

  # ;; ==== AES-CTR + GHASH - 16 blocks, start
  $aesout_offset  = ($STACK_LOCAL_OFFSET + (32 * 16));
  $ghashin_offset = ($STACK_LOCAL_OFFSET + (0 * 16));
  $data_in_out_offset = (0 * 16);
  &GHASH_16_ENCRYPT_16_PARALLEL(
    $AES_KEYS, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN,  $DATA_OFFSET, $CTR_BLOCKz,         $CTR_CHECK,
    48,        $aesout_offset,  $ghashin_offset, $SHUF_MASK,   $ZTMP0,              $ZTMP1,
    $ZTMP2,    $ZTMP3,          $ZTMP4,          $ZTMP5,       $ZTMP6,              $ZTMP7,
    $ZTMP8,    $ZTMP9,          $ZTMP10,         $ZTMP11,      $ZTMP12,             $ZTMP13,
    $ZTMP14,   $ZTMP15,         $ZTMP16,         $ZTMP17,      $ZTMP18,             $ZTMP19,
    $ZTMP20,   $ZTMP21,         $ZTMP22,         $ADDBE_4x4,   $ADDBE_1234,         $GL,
    $GH,       $GM,             "first_time",    $ENC_DEC,     $data_in_out_offset, $AAD_HASHz,
    $IA0);

  # ;; ==== AES-CTR + GHASH - 16 blocks, no reduction
  $aesout_offset  = ($STACK_LOCAL_OFFSET + (0 * 16));
  $ghashin_offset = ($STACK_LOCAL_OFFSET + (16 * 16));
  $data_in_out_offset = (16 * 16);
  &GHASH_16_ENCRYPT_16_PARALLEL(
    $AES_KEYS, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN,  $DATA_OFFSET, $CTR_BLOCKz,         $CTR_CHECK,
    32,        $aesout_offset,  $ghashin_offset, $SHUF_MASK,   $ZTMP0,              $ZTMP1,
    $ZTMP2,    $ZTMP3,          $ZTMP4,          $ZTMP5,       $ZTMP6,              $ZTMP7,
    $ZTMP8,    $ZTMP9,          $ZTMP10,         $ZTMP11,      $ZTMP12,             $ZTMP13,
    $ZTMP14,   $ZTMP15,         $ZTMP16,         $ZTMP17,      $ZTMP18,             $ZTMP19,
    $ZTMP20,   $ZTMP21,         $ZTMP22,         $ADDBE_4x4,   $ADDBE_1234,         $GL,
    $GH,       $GM,             "no_reduction",  $ENC_DEC,     $data_in_out_offset, "no_ghash_in",
    $IA0);

  # ;; ==== GHASH 16 blocks with reduction
  &GHASH_16(
    "end_reduce", $GH, $GM, $GL, "%rsp", $STACK_LOCAL_OFFSET, (32 * 16),
    "%rsp", &HashKeyOffsetByIdx(16, "frame"),
    0, $AAD_HASHz, $ZTMP0, $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, $ZTMP8, $ZTMP9);

  # ;; ==== GHASH 1 x 16 blocks with reduction + cipher and ghash on the reminder
  $ghashin_offset = ($STACK_LOCAL_OFFSET + (0 * 16));
  $code .= <<___;
        sub               \$`(32 * 16)`,$LENGTH
        add               \$`(32 * 16)`,$DATA_OFFSET
___

  # ;; calculate offset to the right hash key
  $code .= "mov               @{[DWORD($LENGTH)]},@{[DWORD($IA0)]}\n";
  $code .= <<___;
        and               \$~15,@{[DWORD($IA0)]}
        mov               \$`@{[HashKeyOffsetByIdx(16,"frame")]}`,@{[DWORD($HASHK_PTR)]}
        sub               @{[DWORD($IA0)]},@{[DWORD($HASHK_PTR)]}
___
  &GCM_ENC_DEC_LAST(
    $AES_KEYS,   $GCM128_CTX, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN,  $DATA_OFFSET, $LENGTH,
    $CTR_BLOCKz, $CTR_CHECK,  $HASHK_PTR,      $ghashin_offset, $SHUF_MASK,   $ZTMP0,
    $ZTMP1,      $ZTMP2,      $ZTMP3,          $ZTMP4,          $ZTMP5,       $ZTMP6,
    $ZTMP7,      $ZTMP8,      $ZTMP9,          $ZTMP10,         $ZTMP11,      $ZTMP12,
    $ZTMP13,     $ZTMP14,     $ZTMP15,         $ZTMP16,         $ZTMP17,      $ZTMP18,
    $ZTMP19,     $ZTMP20,     $ZTMP21,         $ZTMP22,         $ADDBE_4x4,   $ADDBE_1234,
    "start",     $GL,         $GH,             $GM,             $ENC_DEC,     $AAD_HASHz,
    $IA0,        $IA5,        $MASKREG,        $PBLOCK_LEN);

  $code .= "vpshufb           @{[XWORD($SHUF_MASK)]},$CTR_BLOCKx,$CTR_BLOCKx\n";
  $code .= "jmp           .L_ghash_done_${label_suffix}\n";

  # ;; =====================================================
  # ;; =====================================================
  # ;; ==== GHASH & encrypt 16 blocks (done before)
  # ;; ==== GHASH 1 x 16 blocks
  # ;; ==== GHASH 1 x 16 blocks (reduction) & encrypt N blocks
  # ;; ====      then GHASH N blocks
  $code .= ".L_encrypt_16_blocks_${label_suffix}:\n";

  # ;; ==== AES-CTR + GHASH - 16 blocks, start
  $aesout_offset  = ($STACK_LOCAL_OFFSET + (32 * 16));
  $ghashin_offset = ($STACK_LOCAL_OFFSET + (0 * 16));
  $data_in_out_offset = (0 * 16);
  &GHASH_16_ENCRYPT_16_PARALLEL(
    $AES_KEYS, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN,  $DATA_OFFSET, $CTR_BLOCKz,         $CTR_CHECK,
    48,        $aesout_offset,  $ghashin_offset, $SHUF_MASK,   $ZTMP0,              $ZTMP1,
    $ZTMP2,    $ZTMP3,          $ZTMP4,          $ZTMP5,       $ZTMP6,              $ZTMP7,
    $ZTMP8,    $ZTMP9,          $ZTMP10,         $ZTMP11,      $ZTMP12,             $ZTMP13,
    $ZTMP14,   $ZTMP15,         $ZTMP16,         $ZTMP17,      $ZTMP18,             $ZTMP19,
    $ZTMP20,   $ZTMP21,         $ZTMP22,         $ADDBE_4x4,   $ADDBE_1234,         $GL,
    $GH,       $GM,             "first_time",    $ENC_DEC,     $data_in_out_offset, $AAD_HASHz,
    $IA0);

  # ;; ==== GHASH 1 x 16 blocks
  &GHASH_16(
    "mid", $GH, $GM, $GL, "%rsp", $STACK_LOCAL_OFFSET, (16 * 16),
    "%rsp", &HashKeyOffsetByIdx(32, "frame"),
    0, "no_hash_input", $ZTMP0, $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, $ZTMP8, $ZTMP9);

  # ;; ==== GHASH 1 x 16 blocks with reduction + cipher and ghash on the reminder
  $ghashin_offset = ($STACK_LOCAL_OFFSET + (32 * 16));
  $code .= <<___;
        sub               \$`(16 * 16)`,$LENGTH
        add               \$`(16 * 16)`,$DATA_OFFSET
___
  &GCM_ENC_DEC_LAST(
    $AES_KEYS,    $GCM128_CTX, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN,
    $DATA_OFFSET, $LENGTH,     $CTR_BLOCKz,     $CTR_CHECK,
    &HashKeyOffsetByIdx(16, "frame"), $ghashin_offset, $SHUF_MASK, $ZTMP0,
    $ZTMP1,       $ZTMP2,     $ZTMP3,     $ZTMP4,
    $ZTMP5,       $ZTMP6,     $ZTMP7,     $ZTMP8,
    $ZTMP9,       $ZTMP10,    $ZTMP11,    $ZTMP12,
    $ZTMP13,      $ZTMP14,    $ZTMP15,    $ZTMP16,
    $ZTMP17,      $ZTMP18,    $ZTMP19,    $ZTMP20,
    $ZTMP21,      $ZTMP22,    $ADDBE_4x4, $ADDBE_1234,
    "end_reduce", $GL,        $GH,        $GM,
    $ENC_DEC,     $AAD_HASHz, $IA0,       $IA5,
    $MASKREG,     $PBLOCK_LEN);

  $code .= "vpshufb           @{[XWORD($SHUF_MASK)]},$CTR_BLOCKx,$CTR_BLOCKx\n";
  $code .= <<___;
        jmp               .L_ghash_done_${label_suffix}

.L_message_below_32_blocks_${label_suffix}:
        # ;; 32 > number of blocks > 16

        sub               \$`(16 * 16)`,$LENGTH
        add               \$`(16 * 16)`,$DATA_OFFSET
___
  $ghashin_offset = ($STACK_LOCAL_OFFSET + (0 * 16));

  # ;; calculate offset to the right hash key
  $code .= "mov               @{[DWORD($LENGTH)]},@{[DWORD($IA0)]}\n";

  &precompute_hkeys_on_stack($GCM128_CTX, $HKEYS_READY, $ZTMP0, $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6,
    "mid16");
  $code .= "mov     \$1,$HKEYS_READY\n";

  $code .= <<___;
and               \$~15,@{[DWORD($IA0)]}
mov               \$`@{[HashKeyOffsetByIdx(16,"frame")]}`,@{[DWORD($HASHK_PTR)]}
sub               @{[DWORD($IA0)]},@{[DWORD($HASHK_PTR)]}
___

  &GCM_ENC_DEC_LAST(
    $AES_KEYS,   $GCM128_CTX, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN,  $DATA_OFFSET, $LENGTH,
    $CTR_BLOCKz, $CTR_CHECK,  $HASHK_PTR,      $ghashin_offset, $SHUF_MASK,   $ZTMP0,
    $ZTMP1,      $ZTMP2,      $ZTMP3,          $ZTMP4,          $ZTMP5,       $ZTMP6,
    $ZTMP7,      $ZTMP8,      $ZTMP9,          $ZTMP10,         $ZTMP11,      $ZTMP12,
    $ZTMP13,     $ZTMP14,     $ZTMP15,         $ZTMP16,         $ZTMP17,      $ZTMP18,
    $ZTMP19,     $ZTMP20,     $ZTMP21,         $ZTMP22,         $ADDBE_4x4,   $ADDBE_1234,
    "start",     $GL,         $GH,             $GM,             $ENC_DEC,     $AAD_HASHz,
    $IA0,        $IA5,        $MASKREG,        $PBLOCK_LEN);

  $code .= "vpshufb           @{[XWORD($SHUF_MASK)]},$CTR_BLOCKx,$CTR_BLOCKx\n";
  $code .= <<___;
        jmp           .L_ghash_done_${label_suffix}

.L_message_below_equal_16_blocks_${label_suffix}:
        # ;; Determine how many blocks to process
        # ;; - process one additional block if there is a partial block
        mov               @{[DWORD($LENGTH)]},@{[DWORD($IA1)]}
        add               \$15,@{[DWORD($IA1)]}
        shr               \$4, @{[DWORD($IA1)]}     # ; $IA1 can be in the range from 0 to 16
___
  &GCM_ENC_DEC_SMALL(
    $AES_KEYS,    $GCM128_CTX, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $PLAIN_CIPH_LEN, $ENC_DEC,
    $DATA_OFFSET, $LENGTH,     $IA1,            $CTR_BLOCKx,    $AAD_HASHx,      $ZTMP0,
    $ZTMP1,       $ZTMP2,      $ZTMP3,          $ZTMP4,         $ZTMP5,          $ZTMP6,
    $ZTMP7,       $ZTMP8,      $ZTMP9,          $ZTMP10,        $ZTMP11,         $ZTMP12,
    $ZTMP13,      $ZTMP14,     $IA0,            $IA3,           $MASKREG,        $SHUF_MASK,
    $PBLOCK_LEN);

  # ;; fall through to exit

  $code .= ".L_ghash_done_${label_suffix}:\n";

  # ;; save the last counter block
  $code .= "vmovdqu64         $CTR_BLOCKx,`$CTX_OFFSET_CurCount`($GCM128_CTX)\n";
  $code .= <<___;
        vmovdqu64         $AAD_HASHx,`$CTX_OFFSET_AadHash`($GCM128_CTX)
.L_enc_dec_done_${label_suffix}:
___
}

# ;;; ===========================================================================
# ;;; Encrypt/decrypt the initial 16 blocks
sub INITIAL_BLOCKS_16 {
  my $IN          = $_[0];     # [in] input buffer
  my $OUT         = $_[1];     # [in] output buffer
  my $AES_KEYS    = $_[2];     # [in] pointer to expanded keys
  my $DATA_OFFSET = $_[3];     # [in] data offset
  my $GHASH       = $_[4];     # [in] ZMM with AAD (low 128 bits)
  my $CTR         = $_[5];     # [in] ZMM with CTR BE blocks 4x128 bits
  my $CTR_CHECK   = $_[6];     # [in/out] GPR with counter overflow check
  my $ADDBE_4x4   = $_[7];     # [in] ZMM 4x128bits with value 4 (big endian)
  my $ADDBE_1234  = $_[8];     # [in] ZMM 4x128bits with values 1, 2, 3 & 4 (big endian)
  my $T0          = $_[9];     # [clobered] temporary ZMM register
  my $T1          = $_[10];    # [clobered] temporary ZMM register
  my $T2          = $_[11];    # [clobered] temporary ZMM register
  my $T3          = $_[12];    # [clobered] temporary ZMM register
  my $T4          = $_[13];    # [clobered] temporary ZMM register
  my $T5          = $_[14];    # [clobered] temporary ZMM register
  my $T6          = $_[15];    # [clobered] temporary ZMM register
  my $T7          = $_[16];    # [clobered] temporary ZMM register
  my $T8          = $_[17];    # [clobered] temporary ZMM register
  my $SHUF_MASK   = $_[18];    # [in] ZMM with BE/LE shuffle mask
  my $ENC_DEC     = $_[19];    # [in] ENC (encrypt) or DEC (decrypt) selector
  my $BLK_OFFSET  = $_[20];    # [in] stack frame offset to ciphered blocks
  my $DATA_DISPL  = $_[21];    # [in] fixed numerical data displacement/offset
  my $IA0         = $_[22];    # [clobered] temporary GP register

  my $B00_03 = $T5;
  my $B04_07 = $T6;
  my $B08_11 = $T7;
  my $B12_15 = $T8;

  my $label_suffix = $label_count++;

  my $stack_offset = $BLK_OFFSET;
  $code .= <<___;
        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        # ;; prepare counter blocks

        cmpb              \$`(256 - 16)`,@{[BYTE($CTR_CHECK)]}
        jae               .L_next_16_overflow_${label_suffix}
        vpaddd            $ADDBE_1234,$CTR,$B00_03
        vpaddd            $ADDBE_4x4,$B00_03,$B04_07
        vpaddd            $ADDBE_4x4,$B04_07,$B08_11
        vpaddd            $ADDBE_4x4,$B08_11,$B12_15
        jmp               .L_next_16_ok_${label_suffix}
.L_next_16_overflow_${label_suffix}:
        vpshufb           $SHUF_MASK,$CTR,$CTR
        vmovdqa64         ddq_add_4444(%rip),$B12_15
        vpaddd            ddq_add_1234(%rip),$CTR,$B00_03
        vpaddd            $B12_15,$B00_03,$B04_07
        vpaddd            $B12_15,$B04_07,$B08_11
        vpaddd            $B12_15,$B08_11,$B12_15
        vpshufb           $SHUF_MASK,$B00_03,$B00_03
        vpshufb           $SHUF_MASK,$B04_07,$B04_07
        vpshufb           $SHUF_MASK,$B08_11,$B08_11
        vpshufb           $SHUF_MASK,$B12_15,$B12_15
.L_next_16_ok_${label_suffix}:
        vshufi64x2        \$0b11111111,$B12_15,$B12_15,$CTR
        addb               \$16,@{[BYTE($CTR_CHECK)]}
        # ;; === load 16 blocks of data
        vmovdqu8          `$DATA_DISPL + (64*0)`($IN,$DATA_OFFSET,1),$T0
        vmovdqu8          `$DATA_DISPL + (64*1)`($IN,$DATA_OFFSET,1),$T1
        vmovdqu8          `$DATA_DISPL + (64*2)`($IN,$DATA_OFFSET,1),$T2
        vmovdqu8          `$DATA_DISPL + (64*3)`($IN,$DATA_OFFSET,1),$T3

        # ;; move to AES encryption rounds
        vbroadcastf64x2    `(16*0)`($AES_KEYS),$T4
        vpxorq            $T4,$B00_03,$B00_03
        vpxorq            $T4,$B04_07,$B04_07
        vpxorq            $T4,$B08_11,$B08_11
        vpxorq            $T4,$B12_15,$B12_15
___
  foreach (1 .. ($NROUNDS)) {
    $code .= <<___;
        vbroadcastf64x2    `(16*$_)`($AES_KEYS),$T4
        vaesenc            $T4,$B00_03,$B00_03
        vaesenc            $T4,$B04_07,$B04_07
        vaesenc            $T4,$B08_11,$B08_11
        vaesenc            $T4,$B12_15,$B12_15
___
  }
  $code .= <<___;
        vbroadcastf64x2    `(16*($NROUNDS+1))`($AES_KEYS),$T4
        vaesenclast         $T4,$B00_03,$B00_03
        vaesenclast         $T4,$B04_07,$B04_07
        vaesenclast         $T4,$B08_11,$B08_11
        vaesenclast         $T4,$B12_15,$B12_15

        # ;;  xor against text
        vpxorq            $T0,$B00_03,$B00_03
        vpxorq            $T1,$B04_07,$B04_07
        vpxorq            $T2,$B08_11,$B08_11
        vpxorq            $T3,$B12_15,$B12_15

        # ;; store
        mov               $OUT, $IA0
        vmovdqu8          $B00_03,`$DATA_DISPL + (64*0)`($IA0,$DATA_OFFSET,1)
        vmovdqu8          $B04_07,`$DATA_DISPL + (64*1)`($IA0,$DATA_OFFSET,1)
        vmovdqu8          $B08_11,`$DATA_DISPL + (64*2)`($IA0,$DATA_OFFSET,1)
        vmovdqu8          $B12_15,`$DATA_DISPL + (64*3)`($IA0,$DATA_OFFSET,1)
___
  if ($ENC_DEC eq "DEC") {
    $code .= <<___;
        # ;; decryption - cipher text needs to go to GHASH phase
        vpshufb           $SHUF_MASK,$T0,$B00_03
        vpshufb           $SHUF_MASK,$T1,$B04_07
        vpshufb           $SHUF_MASK,$T2,$B08_11
        vpshufb           $SHUF_MASK,$T3,$B12_15
___
  } else {
    $code .= <<___;
        # ;; encryption
        vpshufb           $SHUF_MASK,$B00_03,$B00_03
        vpshufb           $SHUF_MASK,$B04_07,$B04_07
        vpshufb           $SHUF_MASK,$B08_11,$B08_11
        vpshufb           $SHUF_MASK,$B12_15,$B12_15
___
  }

  if ($GHASH ne "no_ghash") {
    $code .= <<___;
        # ;; === xor cipher block 0 with GHASH for the next GHASH round
        vpxorq            $GHASH,$B00_03,$B00_03
___
  }
  $code .= <<___;
        vmovdqa64         $B00_03,`$stack_offset + (0 * 64)`(%rsp)
        vmovdqa64         $B04_07,`$stack_offset + (1 * 64)`(%rsp)
        vmovdqa64         $B08_11,`$stack_offset + (2 * 64)`(%rsp)
        vmovdqa64         $B12_15,`$stack_offset + (3 * 64)`(%rsp)
___
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ; GCM_COMPLETE Finishes ghash calculation
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
sub GCM_COMPLETE {
  my $GCM128_CTX = $_[0];
  my $PBLOCK_LEN = $_[1];

  my $label_suffix = $label_count++;

  $code .= <<___;
        vmovdqu           @{[HashKeyByIdx(1,$GCM128_CTX)]},%xmm2
        vmovdqu           $CTX_OFFSET_EK0($GCM128_CTX),%xmm3      # ; xmm3 = E(K,Y0)
___

  $code .= <<___;
        vmovdqu           `$CTX_OFFSET_AadHash`($GCM128_CTX),%xmm4

        # ;; Process the final partial block.
        cmp               \$0,$PBLOCK_LEN
        je                .L_partial_done_${label_suffix}
___

  #  ;GHASH computation for the last <16 Byte block
  &GHASH_MUL("%xmm4", "%xmm2", "%xmm0", "%xmm16", "%xmm17");

  $code .= <<___;
.L_partial_done_${label_suffix}:
        vmovq           `$CTX_OFFSET_InLen`($GCM128_CTX), %xmm5
        vpinsrq         \$1, `$CTX_OFFSET_AadLen`($GCM128_CTX), %xmm5, %xmm5    #  ; xmm5 = len(A)||len(C)
        vpsllq          \$3, %xmm5, %xmm5                                       #  ; convert bytes into bits

        vpxor           %xmm5,%xmm4,%xmm4
___

  &GHASH_MUL("%xmm4", "%xmm2", "%xmm0", "%xmm16", "%xmm17");

  $code .= <<___;
        vpshufb         SHUF_MASK(%rip),%xmm4,%xmm4      # ; perform a 16Byte swap
        vpxor           %xmm4,%xmm3,%xmm3

.L_return_T_${label_suffix}:
        vmovdqu           %xmm3,`$CTX_OFFSET_AadHash`($GCM128_CTX)
___
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;;; Functions definitions
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

$code .= ".text\n";
{
  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  # ;void   ossl_aes_gcm_init_avx512 /
  # ;       (const void *aes_keys,
  # ;        void *gcm128ctx)
  # ;
  # ; Precomputes hashkey table for GHASH optimization.
  # ; Leaf function (does not allocate stack space, does not use non-volatile registers).
  # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  $code .= <<___;
.globl ossl_aes_gcm_init_avx512
.type ossl_aes_gcm_init_avx512,\@abi-omnipotent
.align 32
ossl_aes_gcm_init_avx512:
.cfi_startproc
        endbranch
___
  if ($CHECK_FUNCTION_ARGUMENTS) {
    $code .= <<___;
        # ;; Check aes_keys != NULL
        test               $arg1,$arg1
        jz                .Labort_init

        # ;; Check gcm128ctx != NULL
        test               $arg2,$arg2
        jz                .Labort_init
___
  }
  $code .= "vpxorq            %xmm16,%xmm16,%xmm16\n";
  &ENCRYPT_SINGLE_BLOCK("$arg1", "%xmm16", "%rax");    # ; xmm16 = HashKey
  $code .= <<___;
        vpshufb           SHUF_MASK(%rip),%xmm16,%xmm16
        # ;;;  PRECOMPUTATION of HashKey<<1 mod poly from the HashKey ;;;
        vmovdqa64         %xmm16,%xmm2
        vpsllq            \$1,%xmm16,%xmm16
        vpsrlq            \$63,%xmm2,%xmm2
        vmovdqa           %xmm2,%xmm1
        vpslldq           \$8,%xmm2,%xmm2
        vpsrldq           \$8,%xmm1,%xmm1
        vporq             %xmm2,%xmm16,%xmm16
        # ;reduction
        vpshufd           \$0b00100100,%xmm1,%xmm2
        vpcmpeqd          TWOONE(%rip),%xmm2,%xmm2
        vpand             POLY(%rip),%xmm2,%xmm2
        vpxorq            %xmm2,%xmm16,%xmm16                  # ; xmm16 holds the HashKey<<1 mod poly
        # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        vmovdqu64         %xmm16,@{[HashKeyByIdx(1,$arg2)]} # ; store HashKey<<1 mod poly
___
  &PRECOMPUTE("$arg2", "%xmm16", "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5");
  if ($CLEAR_SCRATCH_REGISTERS) {
    &clear_scratch_gps_asm();
    &clear_scratch_zmms_asm();
  } else {
    $code .= "vzeroupper\n";
  }
  $code .= <<___;
.Labort_init:
ret
.cfi_endproc
.size ossl_aes_gcm_init_avx512, .-ossl_aes_gcm_init_avx512
___
}

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;void   ossl_aes_gcm_setiv_avx512
# ;       (const void *aes_keys,
# ;        void *gcm128ctx,
# ;        const unsigned char *iv,
# ;        size_t ivlen)
# ;
# ; Computes E(K,Y0) for finalization, updates current counter Yi in gcm128_context structure.
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
$code .= <<___;
.globl ossl_aes_gcm_setiv_avx512
.type ossl_aes_gcm_setiv_avx512,\@abi-omnipotent
.align 32
ossl_aes_gcm_setiv_avx512:
.cfi_startproc
.Lsetiv_seh_begin:
        endbranch
___
if ($CHECK_FUNCTION_ARGUMENTS) {
  $code .= <<___;
        # ;; Check aes_keys != NULL
        test               $arg1,$arg1
        jz                 .Labort_setiv

        # ;; Check gcm128ctx != NULL
        test               $arg2,$arg2
        jz                 .Labort_setiv

        # ;; Check iv != NULL
        test               $arg3,$arg3
        jz                 .Labort_setiv

        # ;; Check ivlen != 0
        test               $arg4,$arg4
        jz                 .Labort_setiv
___
}

# ; NOTE: code before PROLOG() must not modify any registers
&PROLOG(
  1,    # allocate stack space for hkeys
  0,    # do not allocate stack space for AES blocks
  "setiv");
&GCM_INIT_IV(
  "$arg1",  "$arg2",  "$arg3",  "$arg4",  "%r10",   "%r11",  "%r12",  "%k1",   "%xmm2",  "%zmm1",
  "%zmm11", "%zmm3",  "%zmm4",  "%zmm5",  "%zmm6",  "%zmm7", "%zmm8", "%zmm9", "%zmm10", "%zmm12",
  "%zmm13", "%zmm15", "%zmm16", "%zmm17", "%zmm18", "%zmm19");
&EPILOG(
  1,    # hkeys were allocated
  $arg4);
$code .= <<___;
.Labort_setiv:
ret
.Lsetiv_seh_end:
.cfi_endproc
.size ossl_aes_gcm_setiv_avx512, .-ossl_aes_gcm_setiv_avx512
___

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;void ossl_aes_gcm_update_aad_avx512
# ;     (unsigned char *gcm128ctx,
# ;      const unsigned char *aad,
# ;      size_t aadlen)
# ;
# ; Updates AAD hash in gcm128_context structure.
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
$code .= <<___;
.globl ossl_aes_gcm_update_aad_avx512
.type ossl_aes_gcm_update_aad_avx512,\@abi-omnipotent
.align 32
ossl_aes_gcm_update_aad_avx512:
.cfi_startproc
.Lghash_seh_begin:
        endbranch
___
if ($CHECK_FUNCTION_ARGUMENTS) {
  $code .= <<___;
        # ;; Check gcm128ctx != NULL
        test               $arg1,$arg1
        jz                 .Lexit_update_aad

        # ;; Check aad != NULL
        test               $arg2,$arg2
        jz                 .Lexit_update_aad

        # ;; Check aadlen != 0
        test               $arg3,$arg3
        jz                 .Lexit_update_aad
___
}

# ; NOTE: code before PROLOG() must not modify any registers
&PROLOG(
  1,    # allocate stack space for hkeys,
  0,    # do not allocate stack space for AES blocks
  "ghash");
&GCM_UPDATE_AAD(
  "$arg1",  "$arg2",  "$arg3",  "%r10",   "%r11",  "%r12",  "%k1",   "%xmm14", "%zmm1",  "%zmm11",
  "%zmm3",  "%zmm4",  "%zmm5",  "%zmm6",  "%zmm7", "%zmm8", "%zmm9", "%zmm10", "%zmm12", "%zmm13",
  "%zmm15", "%zmm16", "%zmm17", "%zmm18", "%zmm19");
&EPILOG(
  1,    # hkeys were allocated
  $arg3);
$code .= <<___;
.Lexit_update_aad:
ret
.Lghash_seh_end:
.cfi_endproc
.size ossl_aes_gcm_update_aad_avx512, .-ossl_aes_gcm_update_aad_avx512
___

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;void   ossl_aes_gcm_encrypt_avx512
# ;       (const void* aes_keys,
# ;        void *gcm128ctx,
# ;        unsigned int *pblocklen,
# ;        const unsigned char *in,
# ;        size_t len,
# ;        unsigned char *out);
# ;
# ; Performs encryption of data |in| of len |len|, and stores the output in |out|.
# ; Stores encrypted partial block (if any) in gcm128ctx and its length in |pblocklen|.
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
$code .= <<___;
.globl ossl_aes_gcm_encrypt_avx512
.type ossl_aes_gcm_encrypt_avx512,\@abi-omnipotent
.align 32
ossl_aes_gcm_encrypt_avx512:
.cfi_startproc
.Lencrypt_seh_begin:
        endbranch
___

# ; NOTE: code before PROLOG() must not modify any registers
&PROLOG(
  1,    # allocate stack space for hkeys
  1,    # allocate stack space for AES blocks
  "encrypt");
if ($CHECK_FUNCTION_ARGUMENTS) {
  $code .= <<___;
        # ;; Check aes_keys != NULL
        test               $arg1,$arg1
        jz                 .Lexit_gcm_encrypt

        # ;; Check gcm128ctx != NULL
        test               $arg2,$arg2
        jz                 .Lexit_gcm_encrypt

        # ;; Check pblocklen != NULL
        test               $arg3,$arg3
        jz                 .Lexit_gcm_encrypt

        # ;; Check in != NULL
        test               $arg4,$arg4
        jz                 .Lexit_gcm_encrypt

        # ;; Check if len != 0
        cmp                \$0,$arg5
        jz                 .Lexit_gcm_encrypt

        # ;; Check out != NULL
        cmp                \$0,$arg6
        jz                 .Lexit_gcm_encrypt
___
}
$code .= <<___;
        # ; load number of rounds from AES_KEY structure (offset in bytes is
        # ; size of the |rd_key| buffer)
        mov             `4*15*4`($arg1),%eax
        cmp             \$9,%eax
        je              .Laes_gcm_encrypt_128_avx512
        cmp             \$11,%eax
        je              .Laes_gcm_encrypt_192_avx512
        cmp             \$13,%eax
        je              .Laes_gcm_encrypt_256_avx512
        xor             %eax,%eax
        jmp             .Lexit_gcm_encrypt
___
for my $keylen (sort keys %aes_rounds) {
  $NROUNDS = $aes_rounds{$keylen};
  $code .= <<___;
.align 32
.Laes_gcm_encrypt_${keylen}_avx512:
___
  &GCM_ENC_DEC("$arg1", "$arg2", "$arg3", "$arg4", "$arg5", "$arg6", "ENC");
  $code .= "jmp .Lexit_gcm_encrypt\n";
}
$code .= ".Lexit_gcm_encrypt:\n";
&EPILOG(1, $arg5);
$code .= <<___;
ret
.Lencrypt_seh_end:
.cfi_endproc
.size ossl_aes_gcm_encrypt_avx512, .-ossl_aes_gcm_encrypt_avx512
___

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;void   ossl_aes_gcm_decrypt_avx512
# ;       (const void* keys,
# ;        void *gcm128ctx,
# ;        unsigned int *pblocklen,
# ;        const unsigned char *in,
# ;        size_t len,
# ;        unsigned char *out);
# ;
# ; Performs decryption of data |in| of len |len|, and stores the output in |out|.
# ; Stores decrypted partial block (if any) in gcm128ctx and its length in |pblocklen|.
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
$code .= <<___;
.globl ossl_aes_gcm_decrypt_avx512
.type ossl_aes_gcm_decrypt_avx512,\@abi-omnipotent
.align 32
ossl_aes_gcm_decrypt_avx512:
.cfi_startproc
.Ldecrypt_seh_begin:
        endbranch
___

# ; NOTE: code before PROLOG() must not modify any registers
&PROLOG(
  1,    # allocate stack space for hkeys
  1,    # allocate stack space for AES blocks
  "decrypt");
if ($CHECK_FUNCTION_ARGUMENTS) {
  $code .= <<___;
        # ;; Check keys != NULL
        test               $arg1,$arg1
        jz                 .Lexit_gcm_decrypt

        # ;; Check gcm128ctx != NULL
        test               $arg2,$arg2
        jz                 .Lexit_gcm_decrypt

        # ;; Check pblocklen != NULL
        test               $arg3,$arg3
        jz                 .Lexit_gcm_decrypt

        # ;; Check in != NULL
        test               $arg4,$arg4
        jz                 .Lexit_gcm_decrypt

        # ;; Check if len != 0
        cmp                \$0,$arg5
        jz                 .Lexit_gcm_decrypt

        # ;; Check out != NULL
        cmp                \$0,$arg6
        jz                 .Lexit_gcm_decrypt
___
}
$code .= <<___;
        # ; load number of rounds from AES_KEY structure (offset in bytes is
        # ; size of the |rd_key| buffer)
        mov             `4*15*4`($arg1),%eax
        cmp             \$9,%eax
        je              .Laes_gcm_decrypt_128_avx512
        cmp             \$11,%eax
        je              .Laes_gcm_decrypt_192_avx512
        cmp             \$13,%eax
        je              .Laes_gcm_decrypt_256_avx512
        xor             %eax,%eax
        jmp             .Lexit_gcm_decrypt
___
for my $keylen (sort keys %aes_rounds) {
  $NROUNDS = $aes_rounds{$keylen};
  $code .= <<___;
.align 32
.Laes_gcm_decrypt_${keylen}_avx512:
___
  &GCM_ENC_DEC("$arg1", "$arg2", "$arg3", "$arg4", "$arg5", "$arg6", "DEC");
  $code .= "jmp .Lexit_gcm_decrypt\n";
}
$code .= ".Lexit_gcm_decrypt:\n";
&EPILOG(1, $arg5);
$code .= <<___;
ret
.Ldecrypt_seh_end:
.cfi_endproc
.size ossl_aes_gcm_decrypt_avx512, .-ossl_aes_gcm_decrypt_avx512
___

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;void   ossl_aes_gcm_finalize_vaes_avx512
# ;       (void *gcm128ctx,
# ;       unsigned int pblocklen);
# ;
# ; Finalizes encryption / decryption
# ; Leaf function (does not allocate stack space, does not use non-volatile registers).
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
$code .= <<___;
.globl ossl_aes_gcm_finalize_avx512
.type ossl_aes_gcm_finalize_avx512,\@abi-omnipotent
.align 32
ossl_aes_gcm_finalize_avx512:
.cfi_startproc
        endbranch
___
if ($CHECK_FUNCTION_ARGUMENTS) {
  $code .= <<___;
        # ;; Check gcm128ctx != NULL
        test               $arg1,$arg1
        jz                 .Labort_finalize
___
}

&GCM_COMPLETE("$arg1", "$arg2");

$code .= <<___;
.Labort_finalize:
ret
.cfi_endproc
.size ossl_aes_gcm_finalize_avx512, .-ossl_aes_gcm_finalize_avx512
___

# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ;void ossl_gcm_gmult_avx512(u64 Xi[2],
# ;                           const void* gcm128ctx)
# ;
# ; Leaf function (does not allocate stack space, does not use non-volatile registers).
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
$code .= <<___;
.globl ossl_gcm_gmult_avx512
.hidden ossl_gcm_gmult_avx512
.type ossl_gcm_gmult_avx512,\@abi-omnipotent
.align 32
ossl_gcm_gmult_avx512:
.cfi_startproc
        endbranch
___
if ($CHECK_FUNCTION_ARGUMENTS) {
  $code .= <<___;
        # ;; Check Xi != NULL
        test               $arg1,$arg1
        jz                 .Labort_gmult

        # ;; Check gcm128ctx != NULL
        test               $arg2,$arg2
        jz                 .Labort_gmult
___
}
$code .= "vmovdqu64         ($arg1),%xmm1\n";
$code .= "vmovdqu64         @{[HashKeyByIdx(1,$arg2)]},%xmm2\n";

&GHASH_MUL("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5");

$code .= "vmovdqu64         %xmm1,($arg1)\n";
if ($CLEAR_SCRATCH_REGISTERS) {
  &clear_scratch_gps_asm();
  &clear_scratch_zmms_asm();
} else {
  $code .= "vzeroupper\n";
}
$code .= <<___;
.Labort_gmult:
ret
.cfi_endproc
.size ossl_gcm_gmult_avx512, .-ossl_gcm_gmult_avx512
___

if ($win64) {

  # Add unwind metadata for SEH.

  # See https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64?view=msvc-160
  my $UWOP_PUSH_NONVOL = 0;
  my $UWOP_ALLOC_LARGE = 1;
  my $UWOP_SET_FPREG   = 3;
  my $UWOP_SAVE_XMM128 = 8;
  my %UWOP_REG_NUMBER  = (
    rax => 0,
    rcx => 1,
    rdx => 2,
    rbx => 3,
    rsp => 4,
    rbp => 5,
    rsi => 6,
    rdi => 7,
    map(("r$_" => $_), (8 .. 15)));

  $code .= <<___;
.section    .pdata
.align  4
    .rva    .Lsetiv_seh_begin
    .rva    .Lsetiv_seh_end
    .rva    .Lsetiv_seh_info

    .rva    .Lghash_seh_begin
    .rva    .Lghash_seh_end
    .rva    .Lghash_seh_info

    .rva    .Lencrypt_seh_begin
    .rva    .Lencrypt_seh_end
    .rva    .Lencrypt_seh_info

    .rva    .Ldecrypt_seh_begin
    .rva    .Ldecrypt_seh_end
    .rva    .Ldecrypt_seh_info

.section    .xdata
___

  foreach my $func_name ("setiv", "ghash", "encrypt", "decrypt") {
    $code .= <<___;
.align  8
.L${func_name}_seh_info:
    .byte   1   # version 1, no flags
    .byte   .L${func_name}_seh_prolog_end-.L${func_name}_seh_begin
    .byte   31 # num_slots = 1*8 + 2 + 1 + 2*10
    # FR = rbp; Offset from RSP = $XMM_STORAGE scaled on 16
    .byte   @{[$UWOP_REG_NUMBER{rbp} | (($XMM_STORAGE / 16 ) << 4)]}
___

    # Metadata for %xmm15-%xmm6
    # Occupy 2 slots each
    for (my $reg_idx = 15; $reg_idx >= 6; $reg_idx--) {

      # Scaled-by-16 stack offset
      my $xmm_reg_offset = ($reg_idx - 6);
      $code .= <<___;
    .byte   .L${func_name}_seh_save_xmm${reg_idx}-.L${func_name}_seh_begin
    .byte   @{[$UWOP_SAVE_XMM128 | (${reg_idx} << 4)]}
    .value  $xmm_reg_offset
___
    }

    $code .= <<___;
    # Frame pointer (occupy 1 slot)
    .byte   .L${func_name}_seh_setfp-.L${func_name}_seh_begin
    .byte   $UWOP_SET_FPREG

    # Occupy 2 slots, as stack allocation < 512K, but > 128 bytes
    .byte   .L${func_name}_seh_allocstack_xmm-.L${func_name}_seh_begin
    .byte   $UWOP_ALLOC_LARGE
    .value  `($XMM_STORAGE + 8) / 8`
___

    # Metadata for GPR regs
    # Occupy 1 slot each
    foreach my $reg ("rsi", "rdi", "r15", "r14", "r13", "r12", "rbp", "rbx") {
      $code .= <<___;
    .byte   .L${func_name}_seh_push_${reg}-.L${func_name}_seh_begin
    .byte   @{[$UWOP_PUSH_NONVOL | ($UWOP_REG_NUMBER{$reg} << 4)]}
___
    }
  }
}

$code .= <<___;
.section .rodata align=16
.align 16
POLY:   .quad     0x0000000000000001, 0xC200000000000000

.align 64
POLY2:
        .quad     0x00000001C2000000, 0xC200000000000000
        .quad     0x00000001C2000000, 0xC200000000000000
        .quad     0x00000001C2000000, 0xC200000000000000
        .quad     0x00000001C2000000, 0xC200000000000000

.align 16
TWOONE: .quad     0x0000000000000001, 0x0000000100000000

# ;;; Order of these constants should not change.
# ;;; More specifically, ALL_F should follow SHIFT_MASK, and ZERO should follow ALL_F
.align 64
SHUF_MASK:
        .quad     0x08090A0B0C0D0E0F, 0x0001020304050607
        .quad     0x08090A0B0C0D0E0F, 0x0001020304050607
        .quad     0x08090A0B0C0D0E0F, 0x0001020304050607
        .quad     0x08090A0B0C0D0E0F, 0x0001020304050607

.align 16
SHIFT_MASK:
        .quad     0x0706050403020100, 0x0f0e0d0c0b0a0908

ALL_F:
        .quad     0xffffffffffffffff, 0xffffffffffffffff

ZERO:
        .quad     0x0000000000000000, 0x0000000000000000

.align 16
ONE:
        .quad     0x0000000000000001, 0x0000000000000000

.align 16
ONEf:
        .quad     0x0000000000000000, 0x0100000000000000

.align 64
ddq_add_1234:
        .quad  0x0000000000000001, 0x0000000000000000
        .quad  0x0000000000000002, 0x0000000000000000
        .quad  0x0000000000000003, 0x0000000000000000
        .quad  0x0000000000000004, 0x0000000000000000

.align 64
ddq_add_5678:
        .quad  0x0000000000000005, 0x0000000000000000
        .quad  0x0000000000000006, 0x0000000000000000
        .quad  0x0000000000000007, 0x0000000000000000
        .quad  0x0000000000000008, 0x0000000000000000

.align 64
ddq_add_4444:
        .quad  0x0000000000000004, 0x0000000000000000
        .quad  0x0000000000000004, 0x0000000000000000
        .quad  0x0000000000000004, 0x0000000000000000
        .quad  0x0000000000000004, 0x0000000000000000

.align 64
ddq_add_8888:
        .quad  0x0000000000000008, 0x0000000000000000
        .quad  0x0000000000000008, 0x0000000000000000
        .quad  0x0000000000000008, 0x0000000000000000
        .quad  0x0000000000000008, 0x0000000000000000

.align 64
ddq_addbe_1234:
        .quad  0x0000000000000000, 0x0100000000000000
        .quad  0x0000000000000000, 0x0200000000000000
        .quad  0x0000000000000000, 0x0300000000000000
        .quad  0x0000000000000000, 0x0400000000000000

.align 64
ddq_addbe_4444:
        .quad  0x0000000000000000, 0x0400000000000000
        .quad  0x0000000000000000, 0x0400000000000000
        .quad  0x0000000000000000, 0x0400000000000000
        .quad  0x0000000000000000, 0x0400000000000000

.align 64
byte_len_to_mask_table:
        .value      0x0000, 0x0001, 0x0003, 0x0007
        .value      0x000f, 0x001f, 0x003f, 0x007f
        .value      0x00ff, 0x01ff, 0x03ff, 0x07ff
        .value      0x0fff, 0x1fff, 0x3fff, 0x7fff
        .value      0xffff

.align 64
byte64_len_to_mask_table:
        .quad      0x0000000000000000, 0x0000000000000001
        .quad      0x0000000000000003, 0x0000000000000007
        .quad      0x000000000000000f, 0x000000000000001f
        .quad      0x000000000000003f, 0x000000000000007f
        .quad      0x00000000000000ff, 0x00000000000001ff
        .quad      0x00000000000003ff, 0x00000000000007ff
        .quad      0x0000000000000fff, 0x0000000000001fff
        .quad      0x0000000000003fff, 0x0000000000007fff
        .quad      0x000000000000ffff, 0x000000000001ffff
        .quad      0x000000000003ffff, 0x000000000007ffff
        .quad      0x00000000000fffff, 0x00000000001fffff
        .quad      0x00000000003fffff, 0x00000000007fffff
        .quad      0x0000000000ffffff, 0x0000000001ffffff
        .quad      0x0000000003ffffff, 0x0000000007ffffff
        .quad      0x000000000fffffff, 0x000000001fffffff
        .quad      0x000000003fffffff, 0x000000007fffffff
        .quad      0x00000000ffffffff, 0x00000001ffffffff
        .quad      0x00000003ffffffff, 0x00000007ffffffff
        .quad      0x0000000fffffffff, 0x0000001fffffffff
        .quad      0x0000003fffffffff, 0x0000007fffffffff
        .quad      0x000000ffffffffff, 0x000001ffffffffff
        .quad      0x000003ffffffffff, 0x000007ffffffffff
        .quad      0x00000fffffffffff, 0x00001fffffffffff
        .quad      0x00003fffffffffff, 0x00007fffffffffff
        .quad      0x0000ffffffffffff, 0x0001ffffffffffff
        .quad      0x0003ffffffffffff, 0x0007ffffffffffff
        .quad      0x000fffffffffffff, 0x001fffffffffffff
        .quad      0x003fffffffffffff, 0x007fffffffffffff
        .quad      0x00ffffffffffffff, 0x01ffffffffffffff
        .quad      0x03ffffffffffffff, 0x07ffffffffffffff
        .quad      0x0fffffffffffffff, 0x1fffffffffffffff
        .quad      0x3fffffffffffffff, 0x7fffffffffffffff
        .quad      0xffffffffffffffff
___

} else {
# Fallback for old assembler
$code .= <<___;
.text
.globl  ossl_vaes_vpclmulqdq_capable
.type   ossl_vaes_vpclmulqdq_capable,\@abi-omnipotent
ossl_vaes_vpclmulqdq_capable:
    xor     %eax,%eax
    ret
.size   ossl_vaes_vpclmulqdq_capable, .-ossl_vaes_vpclmulqdq_capable

.globl ossl_aes_gcm_init_avx512
.globl ossl_aes_gcm_setiv_avx512
.globl ossl_aes_gcm_update_aad_avx512
.globl ossl_aes_gcm_encrypt_avx512
.globl ossl_aes_gcm_decrypt_avx512
.globl ossl_aes_gcm_finalize_avx512
.globl ossl_gcm_gmult_avx512

.type ossl_aes_gcm_init_avx512,\@abi-omnipotent
ossl_aes_gcm_init_avx512:
ossl_aes_gcm_setiv_avx512:
ossl_aes_gcm_update_aad_avx512:
ossl_aes_gcm_encrypt_avx512:
ossl_aes_gcm_decrypt_avx512:
ossl_aes_gcm_finalize_avx512:
ossl_gcm_gmult_avx512:
    .byte   0x0f,0x0b    # ud2
    ret
.size   ossl_aes_gcm_init_avx512, .-ossl_aes_gcm_init_avx512
___
}

$code =~ s/\`([^\`]*)\`/eval $1/gem;
print $code;
close STDOUT or die "error closing STDOUT: $!";