otx_cptvf_algs.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* Marvell OcteonTX CPT driver
   3 *
   4 * Copyright (C) 2019 Marvell International Ltd.
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License version 2 as
   8 * published by the Free Software Foundation.
   9 */
  10
  11#include <crypto/aes.h>
  12#include <crypto/authenc.h>
  13#include <crypto/cryptd.h>
  14#include <crypto/des.h>
  15#include <crypto/internal/aead.h>
  16#include <crypto/sha1.h>
  17#include <crypto/sha2.h>
  18#include <crypto/xts.h>
  19#include <crypto/scatterwalk.h>
  20#include <linux/sort.h>
  21#include <linux/module.h>
  22#include "otx_cptvf.h"
  23#include "otx_cptvf_algs.h"
  24#include "otx_cptvf_reqmgr.h"
  25
  26#define CPT_MAX_VF_NUM	64
  27/* Size of salt in AES GCM mode */
  28#define AES_GCM_SALT_SIZE	4
  29/* Size of IV in AES GCM mode */
  30#define AES_GCM_IV_SIZE		8
  31/* Size of ICV (Integrity Check Value) in AES GCM mode */
  32#define AES_GCM_ICV_SIZE	16
  33/* Offset of IV in AES GCM mode */
  34#define AES_GCM_IV_OFFSET	8
  35#define CONTROL_WORD_LEN	8
  36#define KEY2_OFFSET		48
  37#define DMA_MODE_FLAG(dma_mode) \
  38	(((dma_mode) == OTX_CPT_DMA_GATHER_SCATTER) ? (1 << 7) : 0)
  39
  40/* Truncated SHA digest size */
  41#define SHA1_TRUNC_DIGEST_SIZE		12
  42#define SHA256_TRUNC_DIGEST_SIZE	16
  43#define SHA384_TRUNC_DIGEST_SIZE	24
  44#define SHA512_TRUNC_DIGEST_SIZE	32
  45
  46static DEFINE_MUTEX(mutex);
  47static int is_crypto_registered;
  48
  49struct cpt_device_desc {
  50	enum otx_cptpf_type pf_type;
  51	struct pci_dev *dev;
  52	int num_queues;
  53};
  54
  55struct cpt_device_table {
  56	atomic_t count;
  57	struct cpt_device_desc desc[CPT_MAX_VF_NUM];
  58};
  59
  60static struct cpt_device_table se_devices = {
  61	.count = ATOMIC_INIT(0)
  62};
  63
  64static struct cpt_device_table ae_devices = {
  65	.count = ATOMIC_INIT(0)
  66};
  67
  68static struct otx_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg);
  69
  70static inline int get_se_device(struct pci_dev **pdev, int *cpu_num)
  71{
  72	int count, ret = 0;
  73
  74	count = atomic_read(&se_devices.count);
  75	if (count < 1)
  76		return -ENODEV;
  77
  78	*cpu_num = get_cpu();
  79
  80	if (se_devices.desc[0].pf_type == OTX_CPT_SE) {
  81		/*
  82		 * On OcteonTX platform there is one CPT instruction queue bound
  83		 * to each VF. We get maximum performance if one CPT queue
  84		 * is available for each cpu otherwise CPT queues need to be
  85		 * shared between cpus.
  86		 */
  87		if (*cpu_num >= count)
  88			*cpu_num %= count;
  89		*pdev = se_devices.desc[*cpu_num].dev;
  90	} else {
  91		pr_err("Unknown PF type %d\n", se_devices.desc[0].pf_type);
  92		ret = -EINVAL;
  93	}
  94	put_cpu();
  95
  96	return ret;
  97}
  98
  99static inline int validate_hmac_cipher_null(struct otx_cpt_req_info *cpt_req)
 100{
 101	struct otx_cpt_req_ctx *rctx;
 102	struct aead_request *req;
 103	struct crypto_aead *tfm;
 104
 105	req = container_of(cpt_req->areq, struct aead_request, base);
 106	tfm = crypto_aead_reqtfm(req);
 107	rctx = aead_request_ctx_dma(req);
 108	if (memcmp(rctx->fctx.hmac.s.hmac_calc,
 109		   rctx->fctx.hmac.s.hmac_recv,
 110		   crypto_aead_authsize(tfm)) != 0)
 111		return -EBADMSG;
 112
 113	return 0;
 114}
 115
 116static void otx_cpt_aead_callback(int status, void *arg1, void *arg2)
 117{
 118	struct otx_cpt_info_buffer *cpt_info = arg2;
 119	struct crypto_async_request *areq = arg1;
 120	struct otx_cpt_req_info *cpt_req;
 121	struct pci_dev *pdev;
 122
 123	if (!cpt_info)
 124		goto complete;
 125
 126	cpt_req = cpt_info->req;
 127	if (!status) {
 128		/*
 129		 * When selected cipher is NULL we need to manually
 130		 * verify whether calculated hmac value matches
 131		 * received hmac value
 132		 */
 133		if (cpt_req->req_type == OTX_CPT_AEAD_ENC_DEC_NULL_REQ &&
 134		    !cpt_req->is_enc)
 135			status = validate_hmac_cipher_null(cpt_req);
 136	}
 137	pdev = cpt_info->pdev;
 138	do_request_cleanup(pdev, cpt_info);
 139
 140complete:
 141	if (areq)
 142		crypto_request_complete(areq, status);
 143}
 144
 145static void output_iv_copyback(struct crypto_async_request *areq)
 146{
 147	struct otx_cpt_req_info *req_info;
 148	struct skcipher_request *sreq;
 149	struct crypto_skcipher *stfm;
 150	struct otx_cpt_req_ctx *rctx;
 151	struct otx_cpt_enc_ctx *ctx;
 152	u32 start, ivsize;
 153
 154	sreq = container_of(areq, struct skcipher_request, base);
 155	stfm = crypto_skcipher_reqtfm(sreq);
 156	ctx = crypto_skcipher_ctx(stfm);
 157	if (ctx->cipher_type == OTX_CPT_AES_CBC ||
 158	    ctx->cipher_type == OTX_CPT_DES3_CBC) {
 159		rctx = skcipher_request_ctx_dma(sreq);
 160		req_info = &rctx->cpt_req;
 161		ivsize = crypto_skcipher_ivsize(stfm);
 162		start = sreq->cryptlen - ivsize;
 163
 164		if (req_info->is_enc) {
 165			scatterwalk_map_and_copy(sreq->iv, sreq->dst, start,
 166						 ivsize, 0);
 167		} else {
 168			if (sreq->src != sreq->dst) {
 169				scatterwalk_map_and_copy(sreq->iv, sreq->src,
 170							 start, ivsize, 0);
 171			} else {
 172				memcpy(sreq->iv, req_info->iv_out, ivsize);
 173				kfree(req_info->iv_out);
 174			}
 175		}
 176	}
 177}
 178
 179static void otx_cpt_skcipher_callback(int status, void *arg1, void *arg2)
 180{
 181	struct otx_cpt_info_buffer *cpt_info = arg2;
 182	struct crypto_async_request *areq = arg1;
 183	struct pci_dev *pdev;
 184
 185	if (areq) {
 186		if (!status)
 187			output_iv_copyback(areq);
 188		if (cpt_info) {
 189			pdev = cpt_info->pdev;
 190			do_request_cleanup(pdev, cpt_info);
 191		}
 192		crypto_request_complete(areq, status);
 193	}
 194}
 195
 196static inline void update_input_data(struct otx_cpt_req_info *req_info,
 197				     struct scatterlist *inp_sg,
 198				     u32 nbytes, u32 *argcnt)
 199{
 200	req_info->req.dlen += nbytes;
 201
 202	while (nbytes) {
 203		u32 len = min(nbytes, inp_sg->length);
 204		u8 *ptr = sg_virt(inp_sg);
 205
 206		req_info->in[*argcnt].vptr = (void *)ptr;
 207		req_info->in[*argcnt].size = len;
 208		nbytes -= len;
 209		++(*argcnt);
 210		inp_sg = sg_next(inp_sg);
 211	}
 212}
 213
 214static inline void update_output_data(struct otx_cpt_req_info *req_info,
 215				      struct scatterlist *outp_sg,
 216				      u32 offset, u32 nbytes, u32 *argcnt)
 217{
 218	req_info->rlen += nbytes;
 219
 220	while (nbytes) {
 221		u32 len = min(nbytes, outp_sg->length - offset);
 222		u8 *ptr = sg_virt(outp_sg);
 223
 224		req_info->out[*argcnt].vptr = (void *) (ptr + offset);
 225		req_info->out[*argcnt].size = len;
 226		nbytes -= len;
 227		++(*argcnt);
 228		offset = 0;
 229		outp_sg = sg_next(outp_sg);
 230	}
 231}
 232
 233static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc,
 234				 u32 *argcnt)
 235{
 236	struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
 237	struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
 238	struct otx_cpt_req_info *req_info = &rctx->cpt_req;
 239	struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
 240	struct otx_cpt_enc_ctx *ctx = crypto_tfm_ctx(tfm);
 241	struct otx_cpt_fc_ctx *fctx = &rctx->fctx;
 242	int ivsize = crypto_skcipher_ivsize(stfm);
 243	u32 start = req->cryptlen - ivsize;
 244	gfp_t flags;
 245
 246	flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 247			GFP_KERNEL : GFP_ATOMIC;
 248	req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER;
 249	req_info->ctrl.s.se_req = OTX_CPT_SE_CORE_REQ;
 250
 251	req_info->req.opcode.s.major = OTX_CPT_MAJOR_OP_FC |
 252				DMA_MODE_FLAG(OTX_CPT_DMA_GATHER_SCATTER);
 253	if (enc) {
 254		req_info->req.opcode.s.minor = 2;
 255	} else {
 256		req_info->req.opcode.s.minor = 3;
 257		if ((ctx->cipher_type == OTX_CPT_AES_CBC ||
 258		    ctx->cipher_type == OTX_CPT_DES3_CBC) &&
 259		    req->src == req->dst) {
 260			req_info->iv_out = kmalloc(ivsize, flags);
 261			if (!req_info->iv_out)
 262				return -ENOMEM;
 263
 264			scatterwalk_map_and_copy(req_info->iv_out, req->src,
 265						 start, ivsize, 0);
 266		}
 267	}
 268	/* Encryption data length */
 269	req_info->req.param1 = req->cryptlen;
 270	/* Authentication data length */
 271	req_info->req.param2 = 0;
 272
 273	fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
 274	fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
 275	fctx->enc.enc_ctrl.e.iv_source = OTX_CPT_FROM_CPTR;
 276
 277	if (ctx->cipher_type == OTX_CPT_AES_XTS)
 278		memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
 279	else
 280		memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
 281
 282	memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm));
 283
 284	fctx->enc.enc_ctrl.flags = cpu_to_be64(fctx->enc.enc_ctrl.cflags);
 285
 286	/*
 287	 * Storing  Packet Data Information in offset
 288	 * Control Word First 8 bytes
 289	 */
 290	req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
 291	req_info->in[*argcnt].size = CONTROL_WORD_LEN;
 292	req_info->req.dlen += CONTROL_WORD_LEN;
 293	++(*argcnt);
 294
 295	req_info->in[*argcnt].vptr = (u8 *)fctx;
 296	req_info->in[*argcnt].size = sizeof(struct otx_cpt_fc_ctx);
 297	req_info->req.dlen += sizeof(struct otx_cpt_fc_ctx);
 298
 299	++(*argcnt);
 300
 301	return 0;
 302}
 303
 304static inline u32 create_input_list(struct skcipher_request *req, u32 enc,
 305				    u32 enc_iv_len)
 306{
 307	struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
 308	struct otx_cpt_req_info *req_info = &rctx->cpt_req;
 309	u32 argcnt =  0;
 310	int ret;
 311
 312	ret = create_ctx_hdr(req, enc, &argcnt);
 313	if (ret)
 314		return ret;
 315
 316	update_input_data(req_info, req->src, req->cryptlen, &argcnt);
 317	req_info->incnt = argcnt;
 318
 319	return 0;
 320}
 321
 322static inline void create_output_list(struct skcipher_request *req,
 323				      u32 enc_iv_len)
 324{
 325	struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
 326	struct otx_cpt_req_info *req_info = &rctx->cpt_req;
 327	u32 argcnt = 0;
 328
 329	/*
 330	 * OUTPUT Buffer Processing
 331	 * AES encryption/decryption output would be
 332	 * received in the following format
 333	 *
 334	 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
 335	 * [ 16 Bytes/     [   Request Enc/Dec/ DATA Len AES CBC ]
 336	 */
 337	update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt);
 338	req_info->outcnt = argcnt;
 339}
 340
 341static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc)
 342{
 343	struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
 344	struct otx_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
 345	struct otx_cpt_req_info *req_info = &rctx->cpt_req;
 346	u32 enc_iv_len = crypto_skcipher_ivsize(stfm);
 347	struct pci_dev *pdev;
 348	int status, cpu_num;
 349
 350	/* Validate that request doesn't exceed maximum CPT supported size */
 351	if (req->cryptlen > OTX_CPT_MAX_REQ_SIZE)
 352		return -E2BIG;
 353
 354	/* Clear control words */
 355	rctx->ctrl_word.flags = 0;
 356	rctx->fctx.enc.enc_ctrl.flags = 0;
 357
 358	status = create_input_list(req, enc, enc_iv_len);
 359	if (status)
 360		return status;
 361	create_output_list(req, enc_iv_len);
 362
 363	status = get_se_device(&pdev, &cpu_num);
 364	if (status)
 365		return status;
 366
 367	req_info->callback = (void *)otx_cpt_skcipher_callback;
 368	req_info->areq = &req->base;
 369	req_info->req_type = OTX_CPT_ENC_DEC_REQ;
 370	req_info->is_enc = enc;
 371	req_info->is_trunc_hmac = false;
 372	req_info->ctrl.s.grp = 0;
 373
 374	/*
 375	 * We perform an asynchronous send and once
 376	 * the request is completed the driver would
 377	 * intimate through registered call back functions
 378	 */
 379	status = otx_cpt_do_request(pdev, req_info, cpu_num);
 380
 381	return status;
 382}
 383
 384static int otx_cpt_skcipher_encrypt(struct skcipher_request *req)
 385{
 386	return cpt_enc_dec(req, true);
 387}
 388
 389static int otx_cpt_skcipher_decrypt(struct skcipher_request *req)
 390{
 391	return cpt_enc_dec(req, false);
 392}
 393
 394static int otx_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm,
 395				       const u8 *key, u32 keylen)
 396{
 397	struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
 398	const u8 *key2 = key + (keylen / 2);
 399	const u8 *key1 = key;
 400	int ret;
 401
 402	ret = xts_verify_key(tfm, key, keylen);
 403	if (ret)
 404		return ret;
 405	ctx->key_len = keylen;
 406	memcpy(ctx->enc_key, key1, keylen / 2);
 407	memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
 408	ctx->cipher_type = OTX_CPT_AES_XTS;
 409	switch (ctx->key_len) {
 410	case 2 * AES_KEYSIZE_128:
 411		ctx->key_type = OTX_CPT_AES_128_BIT;
 412		break;
 413	case 2 * AES_KEYSIZE_256:
 414		ctx->key_type = OTX_CPT_AES_256_BIT;
 415		break;
 416	default:
 417		return -EINVAL;
 418	}
 419
 420	return 0;
 421}
 422
 423static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
 424			  u32 keylen, u8 cipher_type)
 425{
 426	struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
 427
 428	if (keylen != DES3_EDE_KEY_SIZE)
 429		return -EINVAL;
 430
 431	ctx->key_len = keylen;
 432	ctx->cipher_type = cipher_type;
 433
 434	memcpy(ctx->enc_key, key, keylen);
 435
 436	return 0;
 437}
 438
 439static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
 440			  u32 keylen, u8 cipher_type)
 441{
 442	struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
 443
 444	switch (keylen) {
 445	case AES_KEYSIZE_128:
 446		ctx->key_type = OTX_CPT_AES_128_BIT;
 447		break;
 448	case AES_KEYSIZE_192:
 449		ctx->key_type = OTX_CPT_AES_192_BIT;
 450		break;
 451	case AES_KEYSIZE_256:
 452		ctx->key_type = OTX_CPT_AES_256_BIT;
 453		break;
 454	default:
 455		return -EINVAL;
 456	}
 457	ctx->key_len = keylen;
 458	ctx->cipher_type = cipher_type;
 459
 460	memcpy(ctx->enc_key, key, keylen);
 461
 462	return 0;
 463}
 464
 465static int otx_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm,
 466					   const u8 *key, u32 keylen)
 467{
 468	return cpt_aes_setkey(tfm, key, keylen, OTX_CPT_AES_CBC);
 469}
 470
 471static int otx_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm,
 472					   const u8 *key, u32 keylen)
 473{
 474	return cpt_aes_setkey(tfm, key, keylen, OTX_CPT_AES_ECB);
 475}
 476
 477static int otx_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm,
 478					    const u8 *key, u32 keylen)
 479{
 480	return cpt_des_setkey(tfm, key, keylen, OTX_CPT_DES3_CBC);
 481}
 482
 483static int otx_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm,
 484					    const u8 *key, u32 keylen)
 485{
 486	return cpt_des_setkey(tfm, key, keylen, OTX_CPT_DES3_ECB);
 487}
 488
 489static int otx_cpt_enc_dec_init(struct crypto_skcipher *tfm)
 490{
 491	struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
 492
 493	memset(ctx, 0, sizeof(*ctx));
 494	/*
 495	 * Additional memory for skcipher_request is
 496	 * allocated since the cryptd daemon uses
 497	 * this memory for request_ctx information
 498	 */
 499	crypto_skcipher_set_reqsize_dma(
 500		tfm, sizeof(struct otx_cpt_req_ctx) +
 501		     sizeof(struct skcipher_request));
 502
 503	return 0;
 504}
 505
 506static int cpt_aead_init(struct crypto_aead *tfm, u8 cipher_type, u8 mac_type)
 507{
 508	struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
 509
 510	ctx->cipher_type = cipher_type;
 511	ctx->mac_type = mac_type;
 512
 513	switch (ctx->mac_type) {
 514	case OTX_CPT_SHA1:
 515		ctx->hashalg = crypto_alloc_shash("sha1", 0, 0);
 516		break;
 517
 518	case OTX_CPT_SHA256:
 519		ctx->hashalg = crypto_alloc_shash("sha256", 0, 0);
 520		break;
 521
 522	case OTX_CPT_SHA384:
 523		ctx->hashalg = crypto_alloc_shash("sha384", 0, 0);
 524		break;
 525
 526	case OTX_CPT_SHA512:
 527		ctx->hashalg = crypto_alloc_shash("sha512", 0, 0);
 528		break;
 529	}
 530
 531	if (IS_ERR(ctx->hashalg))
 532		return PTR_ERR(ctx->hashalg);
 533
 534	crypto_aead_set_reqsize_dma(tfm, sizeof(struct otx_cpt_req_ctx));
 535
 536	if (!ctx->hashalg)
 537		return 0;
 538
 539	/*
 540	 * When selected cipher is NULL we use HMAC opcode instead of
 541	 * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms
 542	 * for calculating ipad and opad
 543	 */
 544	if (ctx->cipher_type != OTX_CPT_CIPHER_NULL) {
 545		int ss = crypto_shash_statesize(ctx->hashalg);
 546
 547		ctx->ipad = kzalloc(ss, GFP_KERNEL);
 548		if (!ctx->ipad) {
 549			crypto_free_shash(ctx->hashalg);
 550			return -ENOMEM;
 551		}
 552
 553		ctx->opad = kzalloc(ss, GFP_KERNEL);
 554		if (!ctx->opad) {
 555			kfree(ctx->ipad);
 556			crypto_free_shash(ctx->hashalg);
 557			return -ENOMEM;
 558		}
 559	}
 560
 561	ctx->sdesc = alloc_sdesc(ctx->hashalg);
 562	if (!ctx->sdesc) {
 563		kfree(ctx->opad);
 564		kfree(ctx->ipad);
 565		crypto_free_shash(ctx->hashalg);
 566		return -ENOMEM;
 567	}
 568
 569	return 0;
 570}
 571
 572static int otx_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm)
 573{
 574	return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA1);
 575}
 576
 577static int otx_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm)
 578{
 579	return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA256);
 580}
 581
 582static int otx_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm)
 583{
 584	return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA384);
 585}
 586
 587static int otx_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm)
 588{
 589	return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA512);
 590}
 591
 592static int otx_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm)
 593{
 594	return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA1);
 595}
 596
 597static int otx_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm)
 598{
 599	return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA256);
 600}
 601
 602static int otx_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm)
 603{
 604	return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA384);
 605}
 606
 607static int otx_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm)
 608{
 609	return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA512);
 610}
 611
 612static int otx_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)
 613{
 614	return cpt_aead_init(tfm, OTX_CPT_AES_GCM, OTX_CPT_MAC_NULL);
 615}
 616
 617static void otx_cpt_aead_exit(struct crypto_aead *tfm)
 618{
 619	struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
 620
 621	kfree(ctx->ipad);
 622	kfree(ctx->opad);
 623	crypto_free_shash(ctx->hashalg);
 624	kfree(ctx->sdesc);
 625}
 626
 627/*
 628 * This is the Integrity Check Value validation (aka the authentication tag
 629 * length)
 630 */
 631static int otx_cpt_aead_set_authsize(struct crypto_aead *tfm,
 632				     unsigned int authsize)
 633{
 634	struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
 635
 636	switch (ctx->mac_type) {
 637	case OTX_CPT_SHA1:
 638		if (authsize != SHA1_DIGEST_SIZE &&
 639		    authsize != SHA1_TRUNC_DIGEST_SIZE)
 640			return -EINVAL;
 641
 642		if (authsize == SHA1_TRUNC_DIGEST_SIZE)
 643			ctx->is_trunc_hmac = true;
 644		break;
 645
 646	case OTX_CPT_SHA256:
 647		if (authsize != SHA256_DIGEST_SIZE &&
 648		    authsize != SHA256_TRUNC_DIGEST_SIZE)
 649			return -EINVAL;
 650
 651		if (authsize == SHA256_TRUNC_DIGEST_SIZE)
 652			ctx->is_trunc_hmac = true;
 653		break;
 654
 655	case OTX_CPT_SHA384:
 656		if (authsize != SHA384_DIGEST_SIZE &&
 657		    authsize != SHA384_TRUNC_DIGEST_SIZE)
 658			return -EINVAL;
 659
 660		if (authsize == SHA384_TRUNC_DIGEST_SIZE)
 661			ctx->is_trunc_hmac = true;
 662		break;
 663
 664	case OTX_CPT_SHA512:
 665		if (authsize != SHA512_DIGEST_SIZE &&
 666		    authsize != SHA512_TRUNC_DIGEST_SIZE)
 667			return -EINVAL;
 668
 669		if (authsize == SHA512_TRUNC_DIGEST_SIZE)
 670			ctx->is_trunc_hmac = true;
 671		break;
 672
 673	case OTX_CPT_MAC_NULL:
 674		if (ctx->cipher_type == OTX_CPT_AES_GCM) {
 675			if (authsize != AES_GCM_ICV_SIZE)
 676				return -EINVAL;
 677		} else
 678			return -EINVAL;
 679		break;
 680
 681	default:
 682		return -EINVAL;
 683	}
 684
 685	tfm->authsize = authsize;
 686	return 0;
 687}
 688
 689static struct otx_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg)
 690{
 691	struct otx_cpt_sdesc *sdesc;
 692	int size;
 693
 694	size = sizeof(struct shash_desc) + crypto_shash_descsize(alg);
 695	sdesc = kmalloc(size, GFP_KERNEL);
 696	if (!sdesc)
 697		return NULL;
 698
 699	sdesc->shash.tfm = alg;
 700
 701	return sdesc;
 702}
 703
 704static inline void swap_data32(void *buf, u32 len)
 705{
 706	cpu_to_be32_array(buf, buf, len / 4);
 707}
 708
 709static inline void swap_data64(void *buf, u32 len)
 710{
 711	__be64 *dst = buf;
 712	u64 *src = buf;
 713	int i = 0;
 714
 715	for (i = 0 ; i < len / 8; i++, src++, dst++)
 716		*dst = cpu_to_be64p(src);
 717}
 718
 719static int swap_pad(u8 mac_type, u8 *pad)
 720{
 721	struct sha512_state *sha512;
 722	struct sha256_state *sha256;
 723	struct sha1_state *sha1;
 724
 725	switch (mac_type) {
 726	case OTX_CPT_SHA1:
 727		sha1 = (struct sha1_state *)pad;
 728		swap_data32(sha1->state, SHA1_DIGEST_SIZE);
 729		break;
 730
 731	case OTX_CPT_SHA256:
 732		sha256 = (struct sha256_state *)pad;
 733		swap_data32(sha256->state, SHA256_DIGEST_SIZE);
 734		break;
 735
 736	case OTX_CPT_SHA384:
 737	case OTX_CPT_SHA512:
 738		sha512 = (struct sha512_state *)pad;
 739		swap_data64(sha512->state, SHA512_DIGEST_SIZE);
 740		break;
 741
 742	default:
 743		return -EINVAL;
 744	}
 745
 746	return 0;
 747}
 748
 749static int aead_hmac_init(struct crypto_aead *cipher,
 750			  struct crypto_authenc_keys *keys)
 751{
 752	struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
 753	int ds = crypto_shash_digestsize(ctx->hashalg);
 754	int bs = crypto_shash_blocksize(ctx->hashalg);
 755	int authkeylen = keys->authkeylen;
 756	u8 *ipad = NULL, *opad = NULL;
 757	int icount = 0;
 758	int ret;
 759
 760	if (authkeylen > bs) {
 761		ret = crypto_shash_digest(&ctx->sdesc->shash, keys->authkey,
 762					  authkeylen, ctx->key);
 763		if (ret)
 764			return ret;
 765		authkeylen = ds;
 766	} else
 767		memcpy(ctx->key, keys->authkey, authkeylen);
 768
 769	ctx->enc_key_len = keys->enckeylen;
 770	ctx->auth_key_len = authkeylen;
 771
 772	if (ctx->cipher_type == OTX_CPT_CIPHER_NULL)
 773		return keys->enckeylen ? -EINVAL : 0;
 774
 775	switch (keys->enckeylen) {
 776	case AES_KEYSIZE_128:
 777		ctx->key_type = OTX_CPT_AES_128_BIT;
 778		break;
 779	case AES_KEYSIZE_192:
 780		ctx->key_type = OTX_CPT_AES_192_BIT;
 781		break;
 782	case AES_KEYSIZE_256:
 783		ctx->key_type = OTX_CPT_AES_256_BIT;
 784		break;
 785	default:
 786		/* Invalid key length */
 787		return -EINVAL;
 788	}
 789
 790	memcpy(ctx->key + authkeylen, keys->enckey, keys->enckeylen);
 791
 792	ipad = ctx->ipad;
 793	opad = ctx->opad;
 794
 795	memcpy(ipad, ctx->key, authkeylen);
 796	memset(ipad + authkeylen, 0, bs - authkeylen);
 797	memcpy(opad, ipad, bs);
 798
 799	for (icount = 0; icount < bs; icount++) {
 800		ipad[icount] ^= 0x36;
 801		opad[icount] ^= 0x5c;
 802	}
 803
 804	/*
 805	 * Partial Hash calculated from the software
 806	 * algorithm is retrieved for IPAD & OPAD
 807	 */
 808
 809	/* IPAD Calculation */
 810	crypto_shash_init(&ctx->sdesc->shash);
 811	crypto_shash_update(&ctx->sdesc->shash, ipad, bs);
 812	crypto_shash_export(&ctx->sdesc->shash, ipad);
 813	ret = swap_pad(ctx->mac_type, ipad);
 814	if (ret)
 815		goto calc_fail;
 816
 817	/* OPAD Calculation */
 818	crypto_shash_init(&ctx->sdesc->shash);
 819	crypto_shash_update(&ctx->sdesc->shash, opad, bs);
 820	crypto_shash_export(&ctx->sdesc->shash, opad);
 821	ret = swap_pad(ctx->mac_type, opad);
 822
 823calc_fail:
 824	return ret;
 825}
 826
 827static int otx_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher,
 828					   const unsigned char *key,
 829					   unsigned int keylen)
 830{
 831	struct crypto_authenc_keys authenc_keys;
 832	int status;
 833
 834	status = crypto_authenc_extractkeys(&authenc_keys, key, keylen);
 835	if (status)
 836		goto badkey;
 837
 838	status = aead_hmac_init(cipher, &authenc_keys);
 839
 840badkey:
 841	return status;
 842}
 843
 844static int otx_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher,
 845					    const unsigned char *key,
 846					    unsigned int keylen)
 847{
 848	return otx_cpt_aead_cbc_aes_sha_setkey(cipher, key, keylen);
 849}
 850
 851static int otx_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,
 852				       const unsigned char *key,
 853				       unsigned int keylen)
 854{
 855	struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
 856
 857	/*
 858	 * For aes gcm we expect to get encryption key (16, 24, 32 bytes)
 859	 * and salt (4 bytes)
 860	 */
 861	switch (keylen) {
 862	case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE:
 863		ctx->key_type = OTX_CPT_AES_128_BIT;
 864		ctx->enc_key_len = AES_KEYSIZE_128;
 865		break;
 866	case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE:
 867		ctx->key_type = OTX_CPT_AES_192_BIT;
 868		ctx->enc_key_len = AES_KEYSIZE_192;
 869		break;
 870	case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE:
 871		ctx->key_type = OTX_CPT_AES_256_BIT;
 872		ctx->enc_key_len = AES_KEYSIZE_256;
 873		break;
 874	default:
 875		/* Invalid key and salt length */
 876		return -EINVAL;
 877	}
 878
 879	/* Store encryption key and salt */
 880	memcpy(ctx->key, key, keylen);
 881
 882	return 0;
 883}
 884
 885static inline u32 create_aead_ctx_hdr(struct aead_request *req, u32 enc,
 886				      u32 *argcnt)
 887{
 888	struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
 889	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 890	struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
 891	struct otx_cpt_req_info *req_info = &rctx->cpt_req;
 892	struct otx_cpt_fc_ctx *fctx = &rctx->fctx;
 893	int mac_len = crypto_aead_authsize(tfm);
 894	int ds;
 895
 896	rctx->ctrl_word.e.enc_data_offset = req->assoclen;
 897
 898	switch (ctx->cipher_type) {
 899	case OTX_CPT_AES_CBC:
 900		fctx->enc.enc_ctrl.e.iv_source = OTX_CPT_FROM_CPTR;
 901		/* Copy encryption key to context */
 902		memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len,
 903		       ctx->enc_key_len);
 904		/* Copy IV to context */
 905		memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm));
 906
 907		ds = crypto_shash_digestsize(ctx->hashalg);
 908		if (ctx->mac_type == OTX_CPT_SHA384)
 909			ds = SHA512_DIGEST_SIZE;
 910		if (ctx->ipad)
 911			memcpy(fctx->hmac.e.ipad, ctx->ipad, ds);
 912		if (ctx->opad)
 913			memcpy(fctx->hmac.e.opad, ctx->opad, ds);
 914		break;
 915
 916	case OTX_CPT_AES_GCM:
 917		fctx->enc.enc_ctrl.e.iv_source = OTX_CPT_FROM_DPTR;
 918		/* Copy encryption key to context */
 919		memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len);
 920		/* Copy salt to context */
 921		memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len,
 922		       AES_GCM_SALT_SIZE);
 923
 924		rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET;
 925		break;
 926
 927	default:
 928		/* Unknown cipher type */
 929		return -EINVAL;
 930	}
 931	rctx->ctrl_word.flags = cpu_to_be64(rctx->ctrl_word.cflags);
 932
 933	req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER;
 934	req_info->ctrl.s.se_req = OTX_CPT_SE_CORE_REQ;
 935	req_info->req.opcode.s.major = OTX_CPT_MAJOR_OP_FC |
 936				 DMA_MODE_FLAG(OTX_CPT_DMA_GATHER_SCATTER);
 937	if (enc) {
 938		req_info->req.opcode.s.minor = 2;
 939		req_info->req.param1 = req->cryptlen;
 940		req_info->req.param2 = req->cryptlen + req->assoclen;
 941	} else {
 942		req_info->req.opcode.s.minor = 3;
 943		req_info->req.param1 = req->cryptlen - mac_len;
 944		req_info->req.param2 = req->cryptlen + req->assoclen - mac_len;
 945	}
 946
 947	fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
 948	fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
 949	fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type;
 950	fctx->enc.enc_ctrl.e.mac_len = mac_len;
 951	fctx->enc.enc_ctrl.flags = cpu_to_be64(fctx->enc.enc_ctrl.cflags);
 952
 953	/*
 954	 * Storing Packet Data Information in offset
 955	 * Control Word First 8 bytes
 956	 */
 957	req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
 958	req_info->in[*argcnt].size = CONTROL_WORD_LEN;
 959	req_info->req.dlen += CONTROL_WORD_LEN;
 960	++(*argcnt);
 961
 962	req_info->in[*argcnt].vptr = (u8 *)fctx;
 963	req_info->in[*argcnt].size = sizeof(struct otx_cpt_fc_ctx);
 964	req_info->req.dlen += sizeof(struct otx_cpt_fc_ctx);
 965	++(*argcnt);
 966
 967	return 0;
 968}
 969
 970static inline u32 create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,
 971				      u32 enc)
 972{
 973	struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
 974	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 975	struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
 976	struct otx_cpt_req_info *req_info = &rctx->cpt_req;
 977
 978	req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER;
 979	req_info->ctrl.s.se_req = OTX_CPT_SE_CORE_REQ;
 980	req_info->req.opcode.s.major = OTX_CPT_MAJOR_OP_HMAC |
 981				 DMA_MODE_FLAG(OTX_CPT_DMA_GATHER_SCATTER);
 982	req_info->is_trunc_hmac = ctx->is_trunc_hmac;
 983
 984	req_info->req.opcode.s.minor = 0;
 985	req_info->req.param1 = ctx->auth_key_len;
 986	req_info->req.param2 = ctx->mac_type << 8;
 987
 988	/* Add authentication key */
 989	req_info->in[*argcnt].vptr = ctx->key;
 990	req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8);
 991	req_info->req.dlen += round_up(ctx->auth_key_len, 8);
 992	++(*argcnt);
 993
 994	return 0;
 995}
 996
 997static inline u32 create_aead_input_list(struct aead_request *req, u32 enc)
 998{
 999	struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1000	struct otx_cpt_req_info *req_info = &rctx->cpt_req;
1001	u32 inputlen =  req->cryptlen + req->assoclen;
1002	u32 status, argcnt = 0;
1003
1004	status = create_aead_ctx_hdr(req, enc, &argcnt);
1005	if (status)
1006		return status;
1007	update_input_data(req_info, req->src, inputlen, &argcnt);
1008	req_info->incnt = argcnt;
1009
1010	return 0;
1011}
1012
1013static inline u32 create_aead_output_list(struct aead_request *req, u32 enc,
1014					  u32 mac_len)
1015{
1016	struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1017	struct otx_cpt_req_info *req_info =  &rctx->cpt_req;
1018	u32 argcnt = 0, outputlen = 0;
1019
1020	if (enc)
1021		outputlen = req->cryptlen +  req->assoclen + mac_len;
1022	else
1023		outputlen = req->cryptlen + req->assoclen - mac_len;
1024
1025	update_output_data(req_info, req->dst, 0, outputlen, &argcnt);
1026	req_info->outcnt = argcnt;
1027
1028	return 0;
1029}
1030
1031static inline u32 create_aead_null_input_list(struct aead_request *req,
1032					      u32 enc, u32 mac_len)
1033{
1034	struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1035	struct otx_cpt_req_info *req_info = &rctx->cpt_req;
1036	u32 inputlen, argcnt = 0;
1037
1038	if (enc)
1039		inputlen =  req->cryptlen + req->assoclen;
1040	else
1041		inputlen =  req->cryptlen + req->assoclen - mac_len;
1042
1043	create_hmac_ctx_hdr(req, &argcnt, enc);
1044	update_input_data(req_info, req->src, inputlen, &argcnt);
1045	req_info->incnt = argcnt;
1046
1047	return 0;
1048}
1049
1050static inline u32 create_aead_null_output_list(struct aead_request *req,
1051					       u32 enc, u32 mac_len)
1052{
1053	struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1054	struct otx_cpt_req_info *req_info =  &rctx->cpt_req;
1055	struct scatterlist *dst;
1056	u8 *ptr = NULL;
1057	int argcnt = 0, status, offset;
1058	u32 inputlen;
1059
1060	if (enc)
1061		inputlen =  req->cryptlen + req->assoclen;
1062	else
1063		inputlen =  req->cryptlen + req->assoclen - mac_len;
1064
1065	/*
1066	 * If source and destination are different
1067	 * then copy payload to destination
1068	 */
1069	if (req->src != req->dst) {
1070
1071		ptr = kmalloc(inputlen, (req_info->areq->flags &
1072					 CRYPTO_TFM_REQ_MAY_SLEEP) ?
1073					 GFP_KERNEL : GFP_ATOMIC);
1074		if (!ptr) {
1075			status = -ENOMEM;
1076			goto error;
1077		}
1078
1079		status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr,
1080					   inputlen);
1081		if (status != inputlen) {
1082			status = -EINVAL;
1083			goto error_free;
1084		}
1085		status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr,
1086					     inputlen);
1087		if (status != inputlen) {
1088			status = -EINVAL;
1089			goto error_free;
1090		}
1091		kfree(ptr);
1092	}
1093
1094	if (enc) {
1095		/*
1096		 * In an encryption scenario hmac needs
1097		 * to be appended after payload
1098		 */
1099		dst = req->dst;
1100		offset = inputlen;
1101		while (offset >= dst->length) {
1102			offset -= dst->length;
1103			dst = sg_next(dst);
1104			if (!dst) {
1105				status = -ENOENT;
1106				goto error;
1107			}
1108		}
1109
1110		update_output_data(req_info, dst, offset, mac_len, &argcnt);
1111	} else {
1112		/*
1113		 * In a decryption scenario calculated hmac for received
1114		 * payload needs to be compare with hmac received
1115		 */
1116		status = sg_copy_buffer(req->src, sg_nents(req->src),
1117					rctx->fctx.hmac.s.hmac_recv, mac_len,
1118					inputlen, true);
1119		if (status != mac_len) {
1120			status = -EINVAL;
1121			goto error;
1122		}
1123
1124		req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc;
1125		req_info->out[argcnt].size = mac_len;
1126		argcnt++;
1127	}
1128
1129	req_info->outcnt = argcnt;
1130	return 0;
1131
1132error_free:
1133	kfree(ptr);
1134error:
1135	return status;
1136}
1137
1138static u32 cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc)
1139{
1140	struct otx_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1141	struct otx_cpt_req_info *req_info = &rctx->cpt_req;
1142	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1143	struct pci_dev *pdev;
1144	u32 status, cpu_num;
1145
1146	/* Clear control words */
1147	rctx->ctrl_word.flags = 0;
1148	rctx->fctx.enc.enc_ctrl.flags = 0;
1149
1150	req_info->callback = otx_cpt_aead_callback;
1151	req_info->areq = &req->base;
1152	req_info->req_type = reg_type;
1153	req_info->is_enc = enc;
1154	req_info->is_trunc_hmac = false;
1155
1156	switch (reg_type) {
1157	case OTX_CPT_AEAD_ENC_DEC_REQ:
1158		status = create_aead_input_list(req, enc);
1159		if (status)
1160			return status;
1161		status = create_aead_output_list(req, enc,
1162						 crypto_aead_authsize(tfm));
1163		if (status)
1164			return status;
1165		break;
1166
1167	case OTX_CPT_AEAD_ENC_DEC_NULL_REQ:
1168		status = create_aead_null_input_list(req, enc,
1169						     crypto_aead_authsize(tfm));
1170		if (status)
1171			return status;
1172		status = create_aead_null_output_list(req, enc,
1173						crypto_aead_authsize(tfm));
1174		if (status)
1175			return status;
1176		break;
1177
1178	default:
1179		return -EINVAL;
1180	}
1181
1182	/* Validate that request doesn't exceed maximum CPT supported size */
1183	if (req_info->req.param1 > OTX_CPT_MAX_REQ_SIZE ||
1184	    req_info->req.param2 > OTX_CPT_MAX_REQ_SIZE)
1185		return -E2BIG;
1186
1187	status = get_se_device(&pdev, &cpu_num);
1188	if (status)
1189		return status;
1190
1191	req_info->ctrl.s.grp = 0;
1192
1193	status = otx_cpt_do_request(pdev, req_info, cpu_num);
1194	/*
1195	 * We perform an asynchronous send and once
1196	 * the request is completed the driver would
1197	 * intimate through registered call back functions
1198	 */
1199	return status;
1200}
1201
1202static int otx_cpt_aead_encrypt(struct aead_request *req)
1203{
1204	return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_REQ, true);
1205}
1206
1207static int otx_cpt_aead_decrypt(struct aead_request *req)
1208{
1209	return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_REQ, false);
1210}
1211
1212static int otx_cpt_aead_null_encrypt(struct aead_request *req)
1213{
1214	return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_NULL_REQ, true);
1215}
1216
1217static int otx_cpt_aead_null_decrypt(struct aead_request *req)
1218{
1219	return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_NULL_REQ, false);
1220}
1221
1222static struct skcipher_alg otx_cpt_skciphers[] = { {
1223	.base.cra_name = "xts(aes)",
1224	.base.cra_driver_name = "cpt_xts_aes",
1225	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1226	.base.cra_blocksize = AES_BLOCK_SIZE,
1227	.base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx),
1228	.base.cra_alignmask = 7,
1229	.base.cra_priority = 4001,
1230	.base.cra_module = THIS_MODULE,
1231
1232	.init = otx_cpt_enc_dec_init,
1233	.ivsize = AES_BLOCK_SIZE,
1234	.min_keysize = 2 * AES_MIN_KEY_SIZE,
1235	.max_keysize = 2 * AES_MAX_KEY_SIZE,
1236	.setkey = otx_cpt_skcipher_xts_setkey,
1237	.encrypt = otx_cpt_skcipher_encrypt,
1238	.decrypt = otx_cpt_skcipher_decrypt,
1239}, {
1240	.base.cra_name = "cbc(aes)",
1241	.base.cra_driver_name = "cpt_cbc_aes",
1242	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1243	.base.cra_blocksize = AES_BLOCK_SIZE,
1244	.base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx),
1245	.base.cra_alignmask = 7,
1246	.base.cra_priority = 4001,
1247	.base.cra_module = THIS_MODULE,
1248
1249	.init = otx_cpt_enc_dec_init,
1250	.ivsize = AES_BLOCK_SIZE,
1251	.min_keysize = AES_MIN_KEY_SIZE,
1252	.max_keysize = AES_MAX_KEY_SIZE,
1253	.setkey = otx_cpt_skcipher_cbc_aes_setkey,
1254	.encrypt = otx_cpt_skcipher_encrypt,
1255	.decrypt = otx_cpt_skcipher_decrypt,
1256}, {
1257	.base.cra_name = "ecb(aes)",
1258	.base.cra_driver_name = "cpt_ecb_aes",
1259	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1260	.base.cra_blocksize = AES_BLOCK_SIZE,
1261	.base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx),
1262	.base.cra_alignmask = 7,
1263	.base.cra_priority = 4001,
1264	.base.cra_module = THIS_MODULE,
1265
1266	.init = otx_cpt_enc_dec_init,
1267	.ivsize = 0,
1268	.min_keysize = AES_MIN_KEY_SIZE,
1269	.max_keysize = AES_MAX_KEY_SIZE,
1270	.setkey = otx_cpt_skcipher_ecb_aes_setkey,
1271	.encrypt = otx_cpt_skcipher_encrypt,
1272	.decrypt = otx_cpt_skcipher_decrypt,
1273}, {
1274	.base.cra_name = "cbc(des3_ede)",
1275	.base.cra_driver_name = "cpt_cbc_des3_ede",
1276	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1277	.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1278	.base.cra_ctxsize = sizeof(struct otx_cpt_des3_ctx),
1279	.base.cra_alignmask = 7,
1280	.base.cra_priority = 4001,
1281	.base.cra_module = THIS_MODULE,
1282
1283	.init = otx_cpt_enc_dec_init,
1284	.min_keysize = DES3_EDE_KEY_SIZE,
1285	.max_keysize = DES3_EDE_KEY_SIZE,
1286	.ivsize = DES_BLOCK_SIZE,
1287	.setkey = otx_cpt_skcipher_cbc_des3_setkey,
1288	.encrypt = otx_cpt_skcipher_encrypt,
1289	.decrypt = otx_cpt_skcipher_decrypt,
1290}, {
1291	.base.cra_name = "ecb(des3_ede)",
1292	.base.cra_driver_name = "cpt_ecb_des3_ede",
1293	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1294	.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1295	.base.cra_ctxsize = sizeof(struct otx_cpt_des3_ctx),
1296	.base.cra_alignmask = 7,
1297	.base.cra_priority = 4001,
1298	.base.cra_module = THIS_MODULE,
1299
1300	.init = otx_cpt_enc_dec_init,
1301	.min_keysize = DES3_EDE_KEY_SIZE,
1302	.max_keysize = DES3_EDE_KEY_SIZE,
1303	.ivsize = 0,
1304	.setkey = otx_cpt_skcipher_ecb_des3_setkey,
1305	.encrypt = otx_cpt_skcipher_encrypt,
1306	.decrypt = otx_cpt_skcipher_decrypt,
1307} };
1308
1309static struct aead_alg otx_cpt_aeads[] = { {
1310	.base = {
1311		.cra_name = "authenc(hmac(sha1),cbc(aes))",
1312		.cra_driver_name = "cpt_hmac_sha1_cbc_aes",
1313		.cra_blocksize = AES_BLOCK_SIZE,
1314		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1315		.cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1316		.cra_priority = 4001,
1317		.cra_alignmask = 0,
1318		.cra_module = THIS_MODULE,
1319	},
1320	.init = otx_cpt_aead_cbc_aes_sha1_init,
1321	.exit = otx_cpt_aead_exit,
1322	.setkey = otx_cpt_aead_cbc_aes_sha_setkey,
1323	.setauthsize = otx_cpt_aead_set_authsize,
1324	.encrypt = otx_cpt_aead_encrypt,
1325	.decrypt = otx_cpt_aead_decrypt,
1326	.ivsize = AES_BLOCK_SIZE,
1327	.maxauthsize = SHA1_DIGEST_SIZE,
1328}, {
1329	.base = {
1330		.cra_name = "authenc(hmac(sha256),cbc(aes))",
1331		.cra_driver_name = "cpt_hmac_sha256_cbc_aes",
1332		.cra_blocksize = AES_BLOCK_SIZE,
1333		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1334		.cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1335		.cra_priority = 4001,
1336		.cra_alignmask = 0,
1337		.cra_module = THIS_MODULE,
1338	},
1339	.init = otx_cpt_aead_cbc_aes_sha256_init,
1340	.exit = otx_cpt_aead_exit,
1341	.setkey = otx_cpt_aead_cbc_aes_sha_setkey,
1342	.setauthsize = otx_cpt_aead_set_authsize,
1343	.encrypt = otx_cpt_aead_encrypt,
1344	.decrypt = otx_cpt_aead_decrypt,
1345	.ivsize = AES_BLOCK_SIZE,
1346	.maxauthsize = SHA256_DIGEST_SIZE,
1347}, {
1348	.base = {
1349		.cra_name = "authenc(hmac(sha384),cbc(aes))",
1350		.cra_driver_name = "cpt_hmac_sha384_cbc_aes",
1351		.cra_blocksize = AES_BLOCK_SIZE,
1352		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1353		.cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1354		.cra_priority = 4001,
1355		.cra_alignmask = 0,
1356		.cra_module = THIS_MODULE,
1357	},
1358	.init = otx_cpt_aead_cbc_aes_sha384_init,
1359	.exit = otx_cpt_aead_exit,
1360	.setkey = otx_cpt_aead_cbc_aes_sha_setkey,
1361	.setauthsize = otx_cpt_aead_set_authsize,
1362	.encrypt = otx_cpt_aead_encrypt,
1363	.decrypt = otx_cpt_aead_decrypt,
1364	.ivsize = AES_BLOCK_SIZE,
1365	.maxauthsize = SHA384_DIGEST_SIZE,
1366}, {
1367	.base = {
1368		.cra_name = "authenc(hmac(sha512),cbc(aes))",
1369		.cra_driver_name = "cpt_hmac_sha512_cbc_aes",
1370		.cra_blocksize = AES_BLOCK_SIZE,
1371		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1372		.cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1373		.cra_priority = 4001,
1374		.cra_alignmask = 0,
1375		.cra_module = THIS_MODULE,
1376	},
1377	.init = otx_cpt_aead_cbc_aes_sha512_init,
1378	.exit = otx_cpt_aead_exit,
1379	.setkey = otx_cpt_aead_cbc_aes_sha_setkey,
1380	.setauthsize = otx_cpt_aead_set_authsize,
1381	.encrypt = otx_cpt_aead_encrypt,
1382	.decrypt = otx_cpt_aead_decrypt,
1383	.ivsize = AES_BLOCK_SIZE,
1384	.maxauthsize = SHA512_DIGEST_SIZE,
1385}, {
1386	.base = {
1387		.cra_name = "authenc(hmac(sha1),ecb(cipher_null))",
1388		.cra_driver_name = "cpt_hmac_sha1_ecb_null",
1389		.cra_blocksize = 1,
1390		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1391		.cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1392		.cra_priority = 4001,
1393		.cra_alignmask = 0,
1394		.cra_module = THIS_MODULE,
1395	},
1396	.init = otx_cpt_aead_ecb_null_sha1_init,
1397	.exit = otx_cpt_aead_exit,
1398	.setkey = otx_cpt_aead_ecb_null_sha_setkey,
1399	.setauthsize = otx_cpt_aead_set_authsize,
1400	.encrypt = otx_cpt_aead_null_encrypt,
1401	.decrypt = otx_cpt_aead_null_decrypt,
1402	.ivsize = 0,
1403	.maxauthsize = SHA1_DIGEST_SIZE,
1404}, {
1405	.base = {
1406		.cra_name = "authenc(hmac(sha256),ecb(cipher_null))",
1407		.cra_driver_name = "cpt_hmac_sha256_ecb_null",
1408		.cra_blocksize = 1,
1409		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1410		.cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1411		.cra_priority = 4001,
1412		.cra_alignmask = 0,
1413		.cra_module = THIS_MODULE,
1414	},
1415	.init = otx_cpt_aead_ecb_null_sha256_init,
1416	.exit = otx_cpt_aead_exit,
1417	.setkey = otx_cpt_aead_ecb_null_sha_setkey,
1418	.setauthsize = otx_cpt_aead_set_authsize,
1419	.encrypt = otx_cpt_aead_null_encrypt,
1420	.decrypt = otx_cpt_aead_null_decrypt,
1421	.ivsize = 0,
1422	.maxauthsize = SHA256_DIGEST_SIZE,
1423}, {
1424	.base = {
1425		.cra_name = "authenc(hmac(sha384),ecb(cipher_null))",
1426		.cra_driver_name = "cpt_hmac_sha384_ecb_null",
1427		.cra_blocksize = 1,
1428		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1429		.cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1430		.cra_priority = 4001,
1431		.cra_alignmask = 0,
1432		.cra_module = THIS_MODULE,
1433	},
1434	.init = otx_cpt_aead_ecb_null_sha384_init,
1435	.exit = otx_cpt_aead_exit,
1436	.setkey = otx_cpt_aead_ecb_null_sha_setkey,
1437	.setauthsize = otx_cpt_aead_set_authsize,
1438	.encrypt = otx_cpt_aead_null_encrypt,
1439	.decrypt = otx_cpt_aead_null_decrypt,
1440	.ivsize = 0,
1441	.maxauthsize = SHA384_DIGEST_SIZE,
1442}, {
1443	.base = {
1444		.cra_name = "authenc(hmac(sha512),ecb(cipher_null))",
1445		.cra_driver_name = "cpt_hmac_sha512_ecb_null",
1446		.cra_blocksize = 1,
1447		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1448		.cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1449		.cra_priority = 4001,
1450		.cra_alignmask = 0,
1451		.cra_module = THIS_MODULE,
1452	},
1453	.init = otx_cpt_aead_ecb_null_sha512_init,
1454	.exit = otx_cpt_aead_exit,
1455	.setkey = otx_cpt_aead_ecb_null_sha_setkey,
1456	.setauthsize = otx_cpt_aead_set_authsize,
1457	.encrypt = otx_cpt_aead_null_encrypt,
1458	.decrypt = otx_cpt_aead_null_decrypt,
1459	.ivsize = 0,
1460	.maxauthsize = SHA512_DIGEST_SIZE,
1461}, {
1462	.base = {
1463		.cra_name = "rfc4106(gcm(aes))",
1464		.cra_driver_name = "cpt_rfc4106_gcm_aes",
1465		.cra_blocksize = 1,
1466		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1467		.cra_ctxsize = sizeof(struct otx_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1468		.cra_priority = 4001,
1469		.cra_alignmask = 0,
1470		.cra_module = THIS_MODULE,
1471	},
1472	.init = otx_cpt_aead_gcm_aes_init,
1473	.exit = otx_cpt_aead_exit,
1474	.setkey = otx_cpt_aead_gcm_aes_setkey,
1475	.setauthsize = otx_cpt_aead_set_authsize,
1476	.encrypt = otx_cpt_aead_encrypt,
1477	.decrypt = otx_cpt_aead_decrypt,
1478	.ivsize = AES_GCM_IV_SIZE,
1479	.maxauthsize = AES_GCM_ICV_SIZE,
1480} };
1481
1482static inline int is_any_alg_used(void)
1483{
1484	int i;
1485
1486	for (i = 0; i < ARRAY_SIZE(otx_cpt_skciphers); i++)
1487		if (refcount_read(&otx_cpt_skciphers[i].base.cra_refcnt) != 1)
1488			return true;
1489	for (i = 0; i < ARRAY_SIZE(otx_cpt_aeads); i++)
1490		if (refcount_read(&otx_cpt_aeads[i].base.cra_refcnt) != 1)
1491			return true;
1492	return false;
1493}
1494
1495static inline int cpt_register_algs(void)
1496{
1497	int i, err = 0;
1498
1499	if (!IS_ENABLED(CONFIG_DM_CRYPT)) {
1500		for (i = 0; i < ARRAY_SIZE(otx_cpt_skciphers); i++)
1501			otx_cpt_skciphers[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
1502
1503		err = crypto_register_skciphers(otx_cpt_skciphers,
1504						ARRAY_SIZE(otx_cpt_skciphers));
1505		if (err)
1506			return err;
1507	}
1508
1509	for (i = 0; i < ARRAY_SIZE(otx_cpt_aeads); i++)
1510		otx_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
1511
1512	err = crypto_register_aeads(otx_cpt_aeads, ARRAY_SIZE(otx_cpt_aeads));
1513	if (err) {
1514		crypto_unregister_skciphers(otx_cpt_skciphers,
1515					    ARRAY_SIZE(otx_cpt_skciphers));
1516		return err;
1517	}
1518
1519	return 0;
1520}
1521
1522static inline void cpt_unregister_algs(void)
1523{
1524	crypto_unregister_skciphers(otx_cpt_skciphers,
1525				    ARRAY_SIZE(otx_cpt_skciphers));
1526	crypto_unregister_aeads(otx_cpt_aeads, ARRAY_SIZE(otx_cpt_aeads));
1527}
1528
1529static int compare_func(const void *lptr, const void *rptr)
1530{
1531	struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr;
1532	struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr;
1533
1534	if (ldesc->dev->devfn < rdesc->dev->devfn)
1535		return -1;
1536	if (ldesc->dev->devfn > rdesc->dev->devfn)
1537		return 1;
1538	return 0;
1539}
1540
1541int otx_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,
1542			enum otx_cptpf_type pf_type,
1543			enum otx_cptvf_type engine_type,
1544			int num_queues, int num_devices)
1545{
1546	int ret = 0;
1547	int count;
1548
1549	mutex_lock(&mutex);
1550	switch (engine_type) {
1551	case OTX_CPT_SE_TYPES:
1552		count = atomic_read(&se_devices.count);
1553		if (count >= CPT_MAX_VF_NUM) {
1554			dev_err(&pdev->dev, "No space to add a new device\n");
1555			ret = -ENOSPC;
1556			goto err;
1557		}
1558		se_devices.desc[count].pf_type = pf_type;
1559		se_devices.desc[count].num_queues = num_queues;
1560		se_devices.desc[count++].dev = pdev;
1561		atomic_inc(&se_devices.count);
1562
1563		if (atomic_read(&se_devices.count) == num_devices &&
1564		    is_crypto_registered == false) {
1565			if (cpt_register_algs()) {
1566				dev_err(&pdev->dev,
1567				   "Error in registering crypto algorithms\n");
1568				ret =  -EINVAL;
1569				goto err;
1570			}
1571			try_module_get(mod);
1572			is_crypto_registered = true;
1573		}
1574		sort(se_devices.desc, count, sizeof(struct cpt_device_desc),
1575		     compare_func, NULL);
1576		break;
1577
1578	case OTX_CPT_AE_TYPES:
1579		count = atomic_read(&ae_devices.count);
1580		if (count >= CPT_MAX_VF_NUM) {
1581			dev_err(&pdev->dev, "No space to a add new device\n");
1582			ret = -ENOSPC;
1583			goto err;
1584		}
1585		ae_devices.desc[count].pf_type = pf_type;
1586		ae_devices.desc[count].num_queues = num_queues;
1587		ae_devices.desc[count++].dev = pdev;
1588		atomic_inc(&ae_devices.count);
1589		sort(ae_devices.desc, count, sizeof(struct cpt_device_desc),
1590		     compare_func, NULL);
1591		break;
1592
1593	default:
1594		dev_err(&pdev->dev, "Unknown VF type %d\n", engine_type);
1595		ret = BAD_OTX_CPTVF_TYPE;
1596	}
1597err:
1598	mutex_unlock(&mutex);
1599	return ret;
1600}
1601
1602void otx_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod,
1603			 enum otx_cptvf_type engine_type)
1604{
1605	struct cpt_device_table *dev_tbl;
1606	bool dev_found = false;
1607	int i, j, count;
1608
1609	mutex_lock(&mutex);
1610
1611	dev_tbl = (engine_type == OTX_CPT_AE_TYPES) ? &ae_devices : &se_devices;
1612	count = atomic_read(&dev_tbl->count);
1613	for (i = 0; i < count; i++)
1614		if (pdev == dev_tbl->desc[i].dev) {
1615			for (j = i; j < count-1; j++)
1616				dev_tbl->desc[j] = dev_tbl->desc[j+1];
1617			dev_found = true;
1618			break;
1619		}
1620
1621	if (!dev_found) {
1622		dev_err(&pdev->dev, "%s device not found\n", __func__);
1623		goto exit;
1624	}
1625
1626	if (engine_type != OTX_CPT_AE_TYPES) {
1627		if (atomic_dec_and_test(&se_devices.count) &&
1628		    !is_any_alg_used()) {
1629			cpt_unregister_algs();
1630			module_put(mod);
1631			is_crypto_registered = false;
1632		}
1633	} else
1634		atomic_dec(&ae_devices.count);
1635exit:
1636	mutex_unlock(&mutex);
1637}