1/* 2 * Copyright 2021-2022 The OpenSSL Project Authors. All Rights Reserved. 3 * Copyright (c) 2021, Intel Corporation. All Rights Reserved. 4 * 5 * Licensed under the Apache License 2.0 (the "License"). You may not use 6 * this file except in compliance with the License. You can obtain a copy 7 * in the file LICENSE in the source distribution or at 8 * https://www.openssl.org/source/license.html 9 */ 10 11/*- 12 * AVX512 VAES + VPCLMULDQD support for AES GCM. 13 * This file is included by cipher_aes_gcm_hw_aesni.inc 14 */ 15 16#undef VAES_GCM_ENABLED 17#if (defined(__x86_64) || defined(__x86_64__) || \ 18 defined(_M_AMD64) || defined(_M_X64)) 19# define VAES_GCM_ENABLED 20 21/* Returns non-zero when AVX512F + VAES + VPCLMULDQD combination is available */ 22int ossl_vaes_vpclmulqdq_capable(void); 23 24# define OSSL_AES_GCM_UPDATE(direction) \ 25 void ossl_aes_gcm_ ## direction ## _avx512(const void *ks, \ 26 void *gcm128ctx, \ 27 unsigned int *pblocklen, \ 28 const unsigned char *in, \ 29 size_t len, \ 30 unsigned char *out); 31 32OSSL_AES_GCM_UPDATE(encrypt) 33OSSL_AES_GCM_UPDATE(decrypt) 34 35void ossl_aes_gcm_init_avx512(const void *ks, void *gcm128ctx); 36void ossl_aes_gcm_setiv_avx512(const void *ks, void *gcm128ctx, 37 const unsigned char *iv, size_t ivlen); 38void ossl_aes_gcm_update_aad_avx512(void *gcm128ctx, const unsigned char *aad, 39 size_t aadlen); 40void ossl_aes_gcm_finalize_avx512(void *gcm128ctx, unsigned int pblocklen); 41 42void ossl_gcm_gmult_avx512(u64 Xi[2], const void *gcm128ctx); 43 44static int vaes_gcm_setkey(PROV_GCM_CTX *ctx, const unsigned char *key, 45 size_t keylen) 46{ 47 GCM128_CONTEXT *gcmctx = &ctx->gcm; 48 PROV_AES_GCM_CTX *actx = (PROV_AES_GCM_CTX *)ctx; 49 AES_KEY *ks = &actx->ks.ks; 50 51 ctx->ks = ks; 52 aesni_set_encrypt_key(key, keylen * 8, ks); 53 memset(gcmctx, 0, sizeof(*gcmctx)); 54 gcmctx->key = ks; 55 ctx->key_set = 1; 56 57 ossl_aes_gcm_init_avx512(ks, gcmctx); 58 59 return 1; 60} 61 62static int vaes_gcm_setiv(PROV_GCM_CTX *ctx, const unsigned char *iv, 63 size_t ivlen) 64{ 65 GCM128_CONTEXT *gcmctx = &ctx->gcm; 66 67 gcmctx->Yi.u[0] = 0; /* Current counter */ 68 gcmctx->Yi.u[1] = 0; 69 gcmctx->Xi.u[0] = 0; /* AAD hash */ 70 gcmctx->Xi.u[1] = 0; 71 gcmctx->len.u[0] = 0; /* AAD length */ 72 gcmctx->len.u[1] = 0; /* Message length */ 73 gcmctx->ares = 0; 74 gcmctx->mres = 0; 75 76 /* IV is limited by 2^64 bits, thus 2^61 bytes */ 77 if (ivlen > (U64(1) << 61)) 78 return 0; 79 80 ossl_aes_gcm_setiv_avx512(ctx->ks, gcmctx, iv, ivlen); 81 82 return 1; 83} 84 85static int vaes_gcm_aadupdate(PROV_GCM_CTX *ctx, 86 const unsigned char *aad, 87 size_t aad_len) 88{ 89 GCM128_CONTEXT *gcmctx = &ctx->gcm; 90 u64 alen = gcmctx->len.u[0]; 91 unsigned int ares; 92 size_t i, lenBlks; 93 94 /* Bad sequence: call of AAD update after message processing */ 95 if (gcmctx->len.u[1] > 0) 96 return 0; 97 98 alen += aad_len; 99 /* AAD is limited by 2^64 bits, thus 2^61 bytes */ 100 if ((alen > (U64(1) << 61)) || (alen < aad_len)) 101 return 0; 102 103 gcmctx->len.u[0] = alen; 104 105 ares = gcmctx->ares; 106 /* Partial AAD block left from previous AAD update calls */ 107 if (ares > 0) { 108 /* 109 * Fill partial block buffer till full block 110 * (note, the hash is stored reflected) 111 */ 112 while (ares > 0 && aad_len > 0) { 113 gcmctx->Xi.c[15 - ares] ^= *(aad++); 114 --aad_len; 115 ares = (ares + 1) % AES_BLOCK_SIZE; 116 } 117 /* Full block gathered */ 118 if (ares == 0) { 119 ossl_gcm_gmult_avx512(gcmctx->Xi.u, gcmctx); 120 } else { /* no more AAD */ 121 gcmctx->ares = ares; 122 return 1; 123 } 124 } 125 126 /* Bulk AAD processing */ 127 lenBlks = aad_len & ((size_t)(-AES_BLOCK_SIZE)); 128 if (lenBlks > 0) { 129 ossl_aes_gcm_update_aad_avx512(gcmctx, aad, lenBlks); 130 aad += lenBlks; 131 aad_len -= lenBlks; 132 } 133 134 /* Add remaining AAD to the hash (note, the hash is stored reflected) */ 135 if (aad_len > 0) { 136 ares = aad_len; 137 for (i = 0; i < aad_len; i++) 138 gcmctx->Xi.c[15 - i] ^= aad[i]; 139 } 140 141 gcmctx->ares = ares; 142 143 return 1; 144} 145 146static int vaes_gcm_cipherupdate(PROV_GCM_CTX *ctx, const unsigned char *in, 147 size_t len, unsigned char *out) 148{ 149 GCM128_CONTEXT *gcmctx = &ctx->gcm; 150 u64 mlen = gcmctx->len.u[1]; 151 152 mlen += len; 153 if (mlen > ((U64(1) << 36) - 32) || (mlen < len)) 154 return 0; 155 156 gcmctx->len.u[1] = mlen; 157 158 /* Finalize GHASH(AAD) if AAD partial blocks left unprocessed */ 159 if (gcmctx->ares > 0) { 160 ossl_gcm_gmult_avx512(gcmctx->Xi.u, gcmctx); 161 gcmctx->ares = 0; 162 } 163 164 if (ctx->enc) 165 ossl_aes_gcm_encrypt_avx512(ctx->ks, gcmctx, &gcmctx->mres, in, len, out); 166 else 167 ossl_aes_gcm_decrypt_avx512(ctx->ks, gcmctx, &gcmctx->mres, in, len, out); 168 169 return 1; 170} 171 172static int vaes_gcm_cipherfinal(PROV_GCM_CTX *ctx, unsigned char *tag) 173{ 174 GCM128_CONTEXT *gcmctx = &ctx->gcm; 175 unsigned int *res = &gcmctx->mres; 176 177 /* Finalize AAD processing */ 178 if (gcmctx->ares > 0) 179 res = &gcmctx->ares; 180 181 ossl_aes_gcm_finalize_avx512(gcmctx, *res); 182 183 if (ctx->enc) { 184 ctx->taglen = GCM_TAG_MAX_SIZE; 185 memcpy(tag, gcmctx->Xi.c, 186 ctx->taglen <= sizeof(gcmctx->Xi.c) ? ctx->taglen : 187 sizeof(gcmctx->Xi.c)); 188 *res = 0; 189 } else { 190 return !CRYPTO_memcmp(gcmctx->Xi.c, tag, ctx->taglen); 191 } 192 193 return 1; 194} 195 196static const PROV_GCM_HW vaes_gcm = { 197 vaes_gcm_setkey, 198 vaes_gcm_setiv, 199 vaes_gcm_aadupdate, 200 vaes_gcm_cipherupdate, 201 vaes_gcm_cipherfinal, 202 ossl_gcm_one_shot 203}; 204 205#endif 206