safexcel_hash.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 2017 Marvell
   4 *
   5 * Antoine Tenart <antoine.tenart@free-electrons.com>
   6 */
   7
   8#include <crypto/aes.h>
   9#include <crypto/hmac.h>
  10#include <crypto/md5.h>
  11#include <crypto/sha1.h>
  12#include <crypto/sha2.h>
  13#include <crypto/sha3.h>
  14#include <crypto/skcipher.h>
  15#include <crypto/sm3.h>
  16#include <crypto/internal/cipher.h>
  17#include <linux/device.h>
  18#include <linux/dma-mapping.h>
  19#include <linux/dmapool.h>
  20
  21#include "safexcel.h"
  22
  23struct safexcel_ahash_ctx {
  24	struct safexcel_context base;
  25
  26	u32 alg;
  27	u8  key_sz;
  28	bool cbcmac;
  29	bool do_fallback;
  30	bool fb_init_done;
  31	bool fb_do_setkey;
  32
  33	struct crypto_aes_ctx *aes;
  34	struct crypto_ahash *fback;
  35	struct crypto_shash *shpre;
  36	struct shash_desc *shdesc;
  37};
  38
  39struct safexcel_ahash_req {
  40	bool last_req;
  41	bool finish;
  42	bool hmac;
  43	bool needs_inv;
  44	bool hmac_zlen;
  45	bool len_is_le;
  46	bool not_first;
  47	bool xcbcmac;
  48
  49	int nents;
  50	dma_addr_t result_dma;
  51
  52	u32 digest;
  53
  54	u8 state_sz;    /* expected state size, only set once */
  55	u8 block_sz;    /* block size, only set once */
  56	u8 digest_sz;   /* output digest size, only set once */
  57	__le32 state[SHA3_512_BLOCK_SIZE /
  58		     sizeof(__le32)] __aligned(sizeof(__le32));
  59
  60	u64 len;
  61	u64 processed;
  62
  63	u8 cache[HASH_CACHE_SIZE] __aligned(sizeof(u32));
  64	dma_addr_t cache_dma;
  65	unsigned int cache_sz;
  66
  67	u8 cache_next[HASH_CACHE_SIZE] __aligned(sizeof(u32));
  68};
  69
  70static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req)
  71{
  72	return req->len - req->processed;
  73}
  74
  75static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
  76				u32 input_length, u32 result_length,
  77				bool cbcmac)
  78{
  79	struct safexcel_token *token =
  80		(struct safexcel_token *)cdesc->control_data.token;
  81
  82	token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
  83	token[0].packet_length = input_length;
  84	token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH;
  85
  86	input_length &= 15;
  87	if (unlikely(cbcmac && input_length)) {
  88		token[0].stat =  0;
  89		token[1].opcode = EIP197_TOKEN_OPCODE_INSERT;
  90		token[1].packet_length = 16 - input_length;
  91		token[1].stat = EIP197_TOKEN_STAT_LAST_HASH;
  92		token[1].instructions = EIP197_TOKEN_INS_TYPE_HASH;
  93	} else {
  94		token[0].stat = EIP197_TOKEN_STAT_LAST_HASH;
  95		eip197_noop_token(&token[1]);
  96	}
  97
  98	token[2].opcode = EIP197_TOKEN_OPCODE_INSERT;
  99	token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
 100			EIP197_TOKEN_STAT_LAST_PACKET;
 101	token[2].packet_length = result_length;
 102	token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
 103				EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
 104
 105	eip197_noop_token(&token[3]);
 106}
 107
 108static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
 109				     struct safexcel_ahash_req *req,
 110				     struct safexcel_command_desc *cdesc)
 111{
 112	struct safexcel_crypto_priv *priv = ctx->base.priv;
 113	u64 count = 0;
 114
 115	cdesc->control_data.control0 = ctx->alg;
 116	cdesc->control_data.control1 = 0;
 117
 118	/*
 119	 * Copy the input digest if needed, and setup the context
 120	 * fields. Do this now as we need it to setup the first command
 121	 * descriptor.
 122	 */
 123	if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) {
 124		if (req->xcbcmac)
 125			memcpy(ctx->base.ctxr->data, &ctx->base.ipad, ctx->key_sz);
 126		else
 127			memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
 128
 129		if (!req->finish && req->xcbcmac)
 130			cdesc->control_data.control0 |=
 131				CONTEXT_CONTROL_DIGEST_XCM |
 132				CONTEXT_CONTROL_TYPE_HASH_OUT  |
 133				CONTEXT_CONTROL_NO_FINISH_HASH |
 134				CONTEXT_CONTROL_SIZE(req->state_sz /
 135						     sizeof(u32));
 136		else
 137			cdesc->control_data.control0 |=
 138				CONTEXT_CONTROL_DIGEST_XCM |
 139				CONTEXT_CONTROL_TYPE_HASH_OUT  |
 140				CONTEXT_CONTROL_SIZE(req->state_sz /
 141						     sizeof(u32));
 142		return;
 143	} else if (!req->processed) {
 144		/* First - and possibly only - block of basic hash only */
 145		if (req->finish)
 146			cdesc->control_data.control0 |= req->digest |
 147				CONTEXT_CONTROL_TYPE_HASH_OUT |
 148				CONTEXT_CONTROL_RESTART_HASH  |
 149				/* ensure its not 0! */
 150				CONTEXT_CONTROL_SIZE(1);
 151		else
 152			cdesc->control_data.control0 |= req->digest |
 153				CONTEXT_CONTROL_TYPE_HASH_OUT  |
 154				CONTEXT_CONTROL_RESTART_HASH   |
 155				CONTEXT_CONTROL_NO_FINISH_HASH |
 156				/* ensure its not 0! */
 157				CONTEXT_CONTROL_SIZE(1);
 158		return;
 159	}
 160
 161	/* Hash continuation or HMAC, setup (inner) digest from state */
 162	memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
 163
 164	if (req->finish) {
 165		/* Compute digest count for hash/HMAC finish operations */
 166		if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
 167		    req->hmac_zlen || (req->processed != req->block_sz)) {
 168			count = req->processed / EIP197_COUNTER_BLOCK_SIZE;
 169
 170			/* This is a hardware limitation, as the
 171			 * counter must fit into an u32. This represents
 172			 * a fairly big amount of input data, so we
 173			 * shouldn't see this.
 174			 */
 175			if (unlikely(count & 0xffffffff00000000ULL)) {
 176				dev_warn(priv->dev,
 177					 "Input data is too big\n");
 178				return;
 179			}
 180		}
 181
 182		if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
 183		    /* Special case: zero length HMAC */
 184		    req->hmac_zlen ||
 185		    /* PE HW < 4.4 cannot do HMAC continue, fake using hash */
 186		    (req->processed != req->block_sz)) {
 187			/* Basic hash continue operation, need digest + cnt */
 188			cdesc->control_data.control0 |=
 189				CONTEXT_CONTROL_SIZE((req->state_sz >> 2) + 1) |
 190				CONTEXT_CONTROL_TYPE_HASH_OUT |
 191				CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
 192			/* For zero-len HMAC, don't finalize, already padded! */
 193			if (req->hmac_zlen)
 194				cdesc->control_data.control0 |=
 195					CONTEXT_CONTROL_NO_FINISH_HASH;
 196			cdesc->control_data.control1 |=
 197				CONTEXT_CONTROL_DIGEST_CNT;
 198			ctx->base.ctxr->data[req->state_sz >> 2] =
 199				cpu_to_le32(count);
 200			req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
 201
 202			/* Clear zero-length HMAC flag for next operation! */
 203			req->hmac_zlen = false;
 204		} else { /* HMAC */
 205			/* Need outer digest for HMAC finalization */
 206			memcpy(ctx->base.ctxr->data + (req->state_sz >> 2),
 207			       &ctx->base.opad, req->state_sz);
 208
 209			/* Single pass HMAC - no digest count */
 210			cdesc->control_data.control0 |=
 211				CONTEXT_CONTROL_SIZE(req->state_sz >> 1) |
 212				CONTEXT_CONTROL_TYPE_HASH_OUT |
 213				CONTEXT_CONTROL_DIGEST_HMAC;
 214		}
 215	} else { /* Hash continuation, do not finish yet */
 216		cdesc->control_data.control0 |=
 217			CONTEXT_CONTROL_SIZE(req->state_sz >> 2) |
 218			CONTEXT_CONTROL_DIGEST_PRECOMPUTED |
 219			CONTEXT_CONTROL_TYPE_HASH_OUT |
 220			CONTEXT_CONTROL_NO_FINISH_HASH;
 221	}
 222}
 223
 224static int safexcel_ahash_enqueue(struct ahash_request *areq);
 225
 226static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv,
 227				      int ring,
 228				      struct crypto_async_request *async,
 229				      bool *should_complete, int *ret)
 230{
 231	struct safexcel_result_desc *rdesc;
 232	struct ahash_request *areq = ahash_request_cast(async);
 233	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 234	struct safexcel_ahash_req *sreq = ahash_request_ctx_dma(areq);
 235	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
 236	u64 cache_len;
 237
 238	*ret = 0;
 239
 240	rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 241	if (IS_ERR(rdesc)) {
 242		dev_err(priv->dev,
 243			"hash: result: could not retrieve the result descriptor\n");
 244		*ret = PTR_ERR(rdesc);
 245	} else {
 246		*ret = safexcel_rdesc_check_errors(priv, rdesc);
 247	}
 248
 249	safexcel_complete(priv, ring);
 250
 251	if (sreq->nents) {
 252		dma_unmap_sg(priv->dev, areq->src,
 253			     sg_nents_for_len(areq->src, areq->nbytes),
 254			     DMA_TO_DEVICE);
 255		sreq->nents = 0;
 256	}
 257
 258	if (sreq->result_dma) {
 259		dma_unmap_single(priv->dev, sreq->result_dma, sreq->digest_sz,
 260				 DMA_FROM_DEVICE);
 261		sreq->result_dma = 0;
 262	}
 263
 264	if (sreq->cache_dma) {
 265		dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz,
 266				 DMA_TO_DEVICE);
 267		sreq->cache_dma = 0;
 268		sreq->cache_sz = 0;
 269	}
 270
 271	if (sreq->finish) {
 272		if (sreq->hmac &&
 273		    (sreq->digest != CONTEXT_CONTROL_DIGEST_HMAC)) {
 274			/* Faking HMAC using hash - need to do outer hash */
 275			memcpy(sreq->cache, sreq->state,
 276			       crypto_ahash_digestsize(ahash));
 277
 278			memcpy(sreq->state, &ctx->base.opad, sreq->digest_sz);
 279
 280			sreq->len = sreq->block_sz +
 281				    crypto_ahash_digestsize(ahash);
 282			sreq->processed = sreq->block_sz;
 283			sreq->hmac = 0;
 284
 285			if (priv->flags & EIP197_TRC_CACHE)
 286				ctx->base.needs_inv = true;
 287			areq->nbytes = 0;
 288			safexcel_ahash_enqueue(areq);
 289
 290			*should_complete = false; /* Not done yet */
 291			return 1;
 292		}
 293
 294		memcpy(areq->result, sreq->state,
 295		       crypto_ahash_digestsize(ahash));
 296	}
 297
 298	cache_len = safexcel_queued_len(sreq);
 299	if (cache_len)
 300		memcpy(sreq->cache, sreq->cache_next, cache_len);
 301
 302	*should_complete = true;
 303
 304	return 1;
 305}
 306
 307static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
 308				   int *commands, int *results)
 309{
 310	struct ahash_request *areq = ahash_request_cast(async);
 311	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 312	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 313	struct safexcel_crypto_priv *priv = ctx->base.priv;
 314	struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
 315	struct safexcel_result_desc *rdesc;
 316	struct scatterlist *sg;
 317	struct safexcel_token *dmmy;
 318	int i, extra = 0, n_cdesc = 0, ret = 0, cache_len, skip = 0;
 319	u64 queued, len;
 320
 321	queued = safexcel_queued_len(req);
 322	if (queued <= HASH_CACHE_SIZE)
 323		cache_len = queued;
 324	else
 325		cache_len = queued - areq->nbytes;
 326
 327	if (!req->finish && !req->last_req) {
 328		/* If this is not the last request and the queued data does not
 329		 * fit into full cache blocks, cache it for the next send call.
 330		 */
 331		extra = queued & (HASH_CACHE_SIZE - 1);
 332
 333		/* If this is not the last request and the queued data
 334		 * is a multiple of a block, cache the last one for now.
 335		 */
 336		if (!extra)
 337			extra = HASH_CACHE_SIZE;
 338
 339		sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
 340				   req->cache_next, extra,
 341				   areq->nbytes - extra);
 342
 343		queued -= extra;
 344
 345		if (!queued) {
 346			*commands = 0;
 347			*results = 0;
 348			return 0;
 349		}
 350
 351		extra = 0;
 352	}
 353
 354	if (unlikely(req->xcbcmac && req->processed > AES_BLOCK_SIZE)) {
 355		if (unlikely(cache_len < AES_BLOCK_SIZE)) {
 356			/*
 357			 * Cache contains less than 1 full block, complete.
 358			 */
 359			extra = AES_BLOCK_SIZE - cache_len;
 360			if (queued > cache_len) {
 361				/* More data follows: borrow bytes */
 362				u64 tmp = queued - cache_len;
 363
 364				skip = min_t(u64, tmp, extra);
 365				sg_pcopy_to_buffer(areq->src,
 366					sg_nents(areq->src),
 367					req->cache + cache_len,
 368					skip, 0);
 369			}
 370			extra -= skip;
 371			memset(req->cache + cache_len + skip, 0, extra);
 372			if (!ctx->cbcmac && extra) {
 373				// 10- padding for XCBCMAC & CMAC
 374				req->cache[cache_len + skip] = 0x80;
 375				// HW will use K2 iso K3 - compensate!
 376				for (i = 0; i < AES_BLOCK_SIZE / 4; i++) {
 377					u32 *cache = (void *)req->cache;
 378					u32 *ipad = ctx->base.ipad.word;
 379					u32 x;
 380
 381					x = ipad[i] ^ ipad[i + 4];
 382					cache[i] ^= swab32(x);
 383				}
 384			}
 385			cache_len = AES_BLOCK_SIZE;
 386			queued = queued + extra;
 387		}
 388
 389		/* XCBC continue: XOR previous result into 1st word */
 390		crypto_xor(req->cache, (const u8 *)req->state, AES_BLOCK_SIZE);
 391	}
 392
 393	len = queued;
 394	/* Add a command descriptor for the cached data, if any */
 395	if (cache_len) {
 396		req->cache_dma = dma_map_single(priv->dev, req->cache,
 397						cache_len, DMA_TO_DEVICE);
 398		if (dma_mapping_error(priv->dev, req->cache_dma))
 399			return -EINVAL;
 400
 401		req->cache_sz = cache_len;
 402		first_cdesc = safexcel_add_cdesc(priv, ring, 1,
 403						 (cache_len == len),
 404						 req->cache_dma, cache_len,
 405						 len, ctx->base.ctxr_dma,
 406						 &dmmy);
 407		if (IS_ERR(first_cdesc)) {
 408			ret = PTR_ERR(first_cdesc);
 409			goto unmap_cache;
 410		}
 411		n_cdesc++;
 412
 413		queued -= cache_len;
 414		if (!queued)
 415			goto send_command;
 416	}
 417
 418	/* Now handle the current ahash request buffer(s) */
 419	req->nents = dma_map_sg(priv->dev, areq->src,
 420				sg_nents_for_len(areq->src,
 421						 areq->nbytes),
 422				DMA_TO_DEVICE);
 423	if (!req->nents) {
 424		ret = -ENOMEM;
 425		goto cdesc_rollback;
 426	}
 427
 428	for_each_sg(areq->src, sg, req->nents, i) {
 429		int sglen = sg_dma_len(sg);
 430
 431		if (unlikely(sglen <= skip)) {
 432			skip -= sglen;
 433			continue;
 434		}
 435
 436		/* Do not overflow the request */
 437		if ((queued + skip) <= sglen)
 438			sglen = queued;
 439		else
 440			sglen -= skip;
 441
 442		cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
 443					   !(queued - sglen),
 444					   sg_dma_address(sg) + skip, sglen,
 445					   len, ctx->base.ctxr_dma, &dmmy);
 446		if (IS_ERR(cdesc)) {
 447			ret = PTR_ERR(cdesc);
 448			goto unmap_sg;
 449		}
 450
 451		if (!n_cdesc)
 452			first_cdesc = cdesc;
 453		n_cdesc++;
 454
 455		queued -= sglen;
 456		if (!queued)
 457			break;
 458		skip = 0;
 459	}
 460
 461send_command:
 462	/* Setup the context options */
 463	safexcel_context_control(ctx, req, first_cdesc);
 464
 465	/* Add the token */
 466	safexcel_hash_token(first_cdesc, len, req->digest_sz, ctx->cbcmac);
 467
 468	req->result_dma = dma_map_single(priv->dev, req->state, req->digest_sz,
 469					 DMA_FROM_DEVICE);
 470	if (dma_mapping_error(priv->dev, req->result_dma)) {
 471		ret = -EINVAL;
 472		goto unmap_sg;
 473	}
 474
 475	/* Add a result descriptor */
 476	rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma,
 477				   req->digest_sz);
 478	if (IS_ERR(rdesc)) {
 479		ret = PTR_ERR(rdesc);
 480		goto unmap_result;
 481	}
 482
 483	safexcel_rdr_req_set(priv, ring, rdesc, &areq->base);
 484
 485	req->processed += len - extra;
 486
 487	*commands = n_cdesc;
 488	*results = 1;
 489	return 0;
 490
 491unmap_result:
 492	dma_unmap_single(priv->dev, req->result_dma, req->digest_sz,
 493			 DMA_FROM_DEVICE);
 494unmap_sg:
 495	if (req->nents) {
 496		dma_unmap_sg(priv->dev, areq->src,
 497			     sg_nents_for_len(areq->src, areq->nbytes),
 498			     DMA_TO_DEVICE);
 499		req->nents = 0;
 500	}
 501cdesc_rollback:
 502	for (i = 0; i < n_cdesc; i++)
 503		safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
 504unmap_cache:
 505	if (req->cache_dma) {
 506		dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz,
 507				 DMA_TO_DEVICE);
 508		req->cache_dma = 0;
 509		req->cache_sz = 0;
 510	}
 511
 512	return ret;
 513}
 514
 515static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 516				      int ring,
 517				      struct crypto_async_request *async,
 518				      bool *should_complete, int *ret)
 519{
 520	struct safexcel_result_desc *rdesc;
 521	struct ahash_request *areq = ahash_request_cast(async);
 522	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 523	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
 524	int enq_ret;
 525
 526	*ret = 0;
 527
 528	rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 529	if (IS_ERR(rdesc)) {
 530		dev_err(priv->dev,
 531			"hash: invalidate: could not retrieve the result descriptor\n");
 532		*ret = PTR_ERR(rdesc);
 533	} else {
 534		*ret = safexcel_rdesc_check_errors(priv, rdesc);
 535	}
 536
 537	safexcel_complete(priv, ring);
 538
 539	if (ctx->base.exit_inv) {
 540		dma_pool_free(priv->context_pool, ctx->base.ctxr,
 541			      ctx->base.ctxr_dma);
 542
 543		*should_complete = true;
 544		return 1;
 545	}
 546
 547	ring = safexcel_select_ring(priv);
 548	ctx->base.ring = ring;
 549
 550	spin_lock_bh(&priv->ring[ring].queue_lock);
 551	enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async);
 552	spin_unlock_bh(&priv->ring[ring].queue_lock);
 553
 554	if (enq_ret != -EINPROGRESS)
 555		*ret = enq_ret;
 556
 557	queue_work(priv->ring[ring].workqueue,
 558		   &priv->ring[ring].work_data.work);
 559
 560	*should_complete = false;
 561
 562	return 1;
 563}
 564
 565static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
 566				  struct crypto_async_request *async,
 567				  bool *should_complete, int *ret)
 568{
 569	struct ahash_request *areq = ahash_request_cast(async);
 570	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 571	int err;
 572
 573	BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv);
 574
 575	if (req->needs_inv) {
 576		req->needs_inv = false;
 577		err = safexcel_handle_inv_result(priv, ring, async,
 578						 should_complete, ret);
 579	} else {
 580		err = safexcel_handle_req_result(priv, ring, async,
 581						 should_complete, ret);
 582	}
 583
 584	return err;
 585}
 586
 587static int safexcel_ahash_send_inv(struct crypto_async_request *async,
 588				   int ring, int *commands, int *results)
 589{
 590	struct ahash_request *areq = ahash_request_cast(async);
 591	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 592	int ret;
 593
 594	ret = safexcel_invalidate_cache(async, ctx->base.priv,
 595					ctx->base.ctxr_dma, ring);
 596	if (unlikely(ret))
 597		return ret;
 598
 599	*commands = 1;
 600	*results = 1;
 601
 602	return 0;
 603}
 604
 605static int safexcel_ahash_send(struct crypto_async_request *async,
 606			       int ring, int *commands, int *results)
 607{
 608	struct ahash_request *areq = ahash_request_cast(async);
 609	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 610	int ret;
 611
 612	if (req->needs_inv)
 613		ret = safexcel_ahash_send_inv(async, ring, commands, results);
 614	else
 615		ret = safexcel_ahash_send_req(async, ring, commands, results);
 616
 617	return ret;
 618}
 619
 620static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
 621{
 622	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 623	struct safexcel_crypto_priv *priv = ctx->base.priv;
 624	EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE);
 625	struct safexcel_ahash_req *rctx = ahash_request_ctx_dma(req);
 626	DECLARE_CRYPTO_WAIT(result);
 627	int ring = ctx->base.ring;
 628	int err;
 629
 630	memset(req, 0, EIP197_AHASH_REQ_SIZE);
 631
 632	/* create invalidation request */
 633	init_completion(&result.completion);
 634	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 635				   crypto_req_done, &result);
 636
 637	ahash_request_set_tfm(req, __crypto_ahash_cast(tfm));
 638	ctx = crypto_tfm_ctx(req->base.tfm);
 639	ctx->base.exit_inv = true;
 640	rctx->needs_inv = true;
 641
 642	spin_lock_bh(&priv->ring[ring].queue_lock);
 643	crypto_enqueue_request(&priv->ring[ring].queue, &req->base);
 644	spin_unlock_bh(&priv->ring[ring].queue_lock);
 645
 646	queue_work(priv->ring[ring].workqueue,
 647		   &priv->ring[ring].work_data.work);
 648
 649	err = crypto_wait_req(-EINPROGRESS, &result);
 650
 651	if (err) {
 652		dev_warn(priv->dev, "hash: completion error (%d)\n", err);
 653		return err;
 654	}
 655
 656	return 0;
 657}
 658
 659/* safexcel_ahash_cache: cache data until at least one request can be sent to
 660 * the engine, aka. when there is at least 1 block size in the pipe.
 661 */
 662static int safexcel_ahash_cache(struct ahash_request *areq)
 663{
 664	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 665	u64 cache_len;
 666
 667	/* cache_len: everything accepted by the driver but not sent yet,
 668	 * tot sz handled by update() - last req sz - tot sz handled by send()
 669	 */
 670	cache_len = safexcel_queued_len(req);
 671
 672	/*
 673	 * In case there isn't enough bytes to proceed (less than a
 674	 * block size), cache the data until we have enough.
 675	 */
 676	if (cache_len + areq->nbytes <= HASH_CACHE_SIZE) {
 677		sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
 678				   req->cache + cache_len,
 679				   areq->nbytes, 0);
 680		return 0;
 681	}
 682
 683	/* We couldn't cache all the data */
 684	return -E2BIG;
 685}
 686
 687static int safexcel_ahash_enqueue(struct ahash_request *areq)
 688{
 689	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 690	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 691	struct safexcel_crypto_priv *priv = ctx->base.priv;
 692	int ret, ring;
 693
 694	req->needs_inv = false;
 695
 696	if (ctx->base.ctxr) {
 697		if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv &&
 698		     /* invalidate for *any* non-XCBC continuation */
 699		   ((req->not_first && !req->xcbcmac) ||
 700		     /* invalidate if (i)digest changed */
 701		     memcmp(ctx->base.ctxr->data, req->state, req->state_sz) ||
 702		     /* invalidate for HMAC finish with odigest changed */
 703		     (req->finish && req->hmac &&
 704		      memcmp(ctx->base.ctxr->data + (req->state_sz>>2),
 705			     &ctx->base.opad, req->state_sz))))
 706			/*
 707			 * We're still setting needs_inv here, even though it is
 708			 * cleared right away, because the needs_inv flag can be
 709			 * set in other functions and we want to keep the same
 710			 * logic.
 711			 */
 712			ctx->base.needs_inv = true;
 713
 714		if (ctx->base.needs_inv) {
 715			ctx->base.needs_inv = false;
 716			req->needs_inv = true;
 717		}
 718	} else {
 719		ctx->base.ring = safexcel_select_ring(priv);
 720		ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
 721						 EIP197_GFP_FLAGS(areq->base),
 722						 &ctx->base.ctxr_dma);
 723		if (!ctx->base.ctxr)
 724			return -ENOMEM;
 725	}
 726	req->not_first = true;
 727
 728	ring = ctx->base.ring;
 729
 730	spin_lock_bh(&priv->ring[ring].queue_lock);
 731	ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base);
 732	spin_unlock_bh(&priv->ring[ring].queue_lock);
 733
 734	queue_work(priv->ring[ring].workqueue,
 735		   &priv->ring[ring].work_data.work);
 736
 737	return ret;
 738}
 739
 740static int safexcel_ahash_update(struct ahash_request *areq)
 741{
 742	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 743	int ret;
 744
 745	/* If the request is 0 length, do nothing */
 746	if (!areq->nbytes)
 747		return 0;
 748
 749	/* Add request to the cache if it fits */
 750	ret = safexcel_ahash_cache(areq);
 751
 752	/* Update total request length */
 753	req->len += areq->nbytes;
 754
 755	/* If not all data could fit into the cache, go process the excess.
 756	 * Also go process immediately for an HMAC IV precompute, which
 757	 * will never be finished at all, but needs to be processed anyway.
 758	 */
 759	if ((ret && !req->finish) || req->last_req)
 760		return safexcel_ahash_enqueue(areq);
 761
 762	return 0;
 763}
 764
 765static int safexcel_ahash_final(struct ahash_request *areq)
 766{
 767	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 768	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 769
 770	req->finish = true;
 771
 772	if (unlikely(!req->len && !areq->nbytes)) {
 773		/*
 774		 * If we have an overall 0 length *hash* request:
 775		 * The HW cannot do 0 length hash, so we provide the correct
 776		 * result directly here.
 777		 */
 778		if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5)
 779			memcpy(areq->result, md5_zero_message_hash,
 780			       MD5_DIGEST_SIZE);
 781		else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
 782			memcpy(areq->result, sha1_zero_message_hash,
 783			       SHA1_DIGEST_SIZE);
 784		else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224)
 785			memcpy(areq->result, sha224_zero_message_hash,
 786			       SHA224_DIGEST_SIZE);
 787		else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
 788			memcpy(areq->result, sha256_zero_message_hash,
 789			       SHA256_DIGEST_SIZE);
 790		else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384)
 791			memcpy(areq->result, sha384_zero_message_hash,
 792			       SHA384_DIGEST_SIZE);
 793		else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512)
 794			memcpy(areq->result, sha512_zero_message_hash,
 795			       SHA512_DIGEST_SIZE);
 796		else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SM3) {
 797			memcpy(areq->result,
 798			       EIP197_SM3_ZEROM_HASH, SM3_DIGEST_SIZE);
 799		}
 800
 801		return 0;
 802	} else if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM &&
 803			    ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5 &&
 804			    req->len == sizeof(u32) && !areq->nbytes)) {
 805		/* Zero length CRC32 */
 806		memcpy(areq->result, &ctx->base.ipad, sizeof(u32));
 807		return 0;
 808	} else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE &&
 809			    !areq->nbytes)) {
 810		/* Zero length CBC MAC */
 811		memset(areq->result, 0, AES_BLOCK_SIZE);
 812		return 0;
 813	} else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE &&
 814			    !areq->nbytes)) {
 815		/* Zero length (X)CBC/CMAC */
 816		int i;
 817
 818		for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) {
 819			u32 *result = (void *)areq->result;
 820
 821			/* K3 */
 822			result[i] = swab32(ctx->base.ipad.word[i + 4]);
 823		}
 824		areq->result[0] ^= 0x80;			// 10- padding
 825		aes_encrypt(ctx->aes, areq->result, areq->result);
 826		return 0;
 827	} else if (unlikely(req->hmac &&
 828			    (req->len == req->block_sz) &&
 829			    !areq->nbytes)) {
 830		/*
 831		 * If we have an overall 0 length *HMAC* request:
 832		 * For HMAC, we need to finalize the inner digest
 833		 * and then perform the outer hash.
 834		 */
 835
 836		/* generate pad block in the cache */
 837		/* start with a hash block of all zeroes */
 838		memset(req->cache, 0, req->block_sz);
 839		/* set the first byte to 0x80 to 'append a 1 bit' */
 840		req->cache[0] = 0x80;
 841		/* add the length in bits in the last 2 bytes */
 842		if (req->len_is_le) {
 843			/* Little endian length word (e.g. MD5) */
 844			req->cache[req->block_sz-8] = (req->block_sz << 3) &
 845						      255;
 846			req->cache[req->block_sz-7] = (req->block_sz >> 5);
 847		} else {
 848			/* Big endian length word (e.g. any SHA) */
 849			req->cache[req->block_sz-2] = (req->block_sz >> 5);
 850			req->cache[req->block_sz-1] = (req->block_sz << 3) &
 851						      255;
 852		}
 853
 854		req->len += req->block_sz; /* plus 1 hash block */
 855
 856		/* Set special zero-length HMAC flag */
 857		req->hmac_zlen = true;
 858
 859		/* Finalize HMAC */
 860		req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
 861	} else if (req->hmac) {
 862		/* Finalize HMAC */
 863		req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
 864	}
 865
 866	return safexcel_ahash_enqueue(areq);
 867}
 868
 869static int safexcel_ahash_finup(struct ahash_request *areq)
 870{
 871	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 872
 873	req->finish = true;
 874
 875	safexcel_ahash_update(areq);
 876	return safexcel_ahash_final(areq);
 877}
 878
 879static int safexcel_ahash_export(struct ahash_request *areq, void *out)
 880{
 881	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 882	struct safexcel_ahash_export_state *export = out;
 883
 884	export->len = req->len;
 885	export->processed = req->processed;
 886
 887	export->digest = req->digest;
 888
 889	memcpy(export->state, req->state, req->state_sz);
 890	memcpy(export->cache, req->cache, HASH_CACHE_SIZE);
 891
 892	return 0;
 893}
 894
 895static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
 896{
 897	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 898	const struct safexcel_ahash_export_state *export = in;
 899	int ret;
 900
 901	ret = crypto_ahash_init(areq);
 902	if (ret)
 903		return ret;
 904
 905	req->len = export->len;
 906	req->processed = export->processed;
 907
 908	req->digest = export->digest;
 909
 910	memcpy(req->cache, export->cache, HASH_CACHE_SIZE);
 911	memcpy(req->state, export->state, req->state_sz);
 912
 913	return 0;
 914}
 915
 916static int safexcel_ahash_cra_init(struct crypto_tfm *tfm)
 917{
 918	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 919	struct safexcel_alg_template *tmpl =
 920		container_of(__crypto_ahash_alg(tfm->__crt_alg),
 921			     struct safexcel_alg_template, alg.ahash);
 922
 923	ctx->base.priv = tmpl->priv;
 924	ctx->base.send = safexcel_ahash_send;
 925	ctx->base.handle_result = safexcel_handle_result;
 926	ctx->fb_do_setkey = false;
 927
 928	crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm),
 929				     sizeof(struct safexcel_ahash_req));
 930	return 0;
 931}
 932
 933static int safexcel_sha1_init(struct ahash_request *areq)
 934{
 935	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 936	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
 937
 938	memset(req, 0, sizeof(*req));
 939
 940	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
 941	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
 942	req->state_sz = SHA1_DIGEST_SIZE;
 943	req->digest_sz = SHA1_DIGEST_SIZE;
 944	req->block_sz = SHA1_BLOCK_SIZE;
 945
 946	return 0;
 947}
 948
 949static int safexcel_sha1_digest(struct ahash_request *areq)
 950{
 951	int ret = safexcel_sha1_init(areq);
 952
 953	if (ret)
 954		return ret;
 955
 956	return safexcel_ahash_finup(areq);
 957}
 958
 959static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
 960{
 961	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 962	struct safexcel_crypto_priv *priv = ctx->base.priv;
 963	int ret;
 964
 965	/* context not allocated, skip invalidation */
 966	if (!ctx->base.ctxr)
 967		return;
 968
 969	if (priv->flags & EIP197_TRC_CACHE) {
 970		ret = safexcel_ahash_exit_inv(tfm);
 971		if (ret)
 972			dev_warn(priv->dev, "hash: invalidation error %d\n", ret);
 973	} else {
 974		dma_pool_free(priv->context_pool, ctx->base.ctxr,
 975			      ctx->base.ctxr_dma);
 976	}
 977}
 978
 979struct safexcel_alg_template safexcel_alg_sha1 = {
 980	.type = SAFEXCEL_ALG_TYPE_AHASH,
 981	.algo_mask = SAFEXCEL_ALG_SHA1,
 982	.alg.ahash = {
 983		.init = safexcel_sha1_init,
 984		.update = safexcel_ahash_update,
 985		.final = safexcel_ahash_final,
 986		.finup = safexcel_ahash_finup,
 987		.digest = safexcel_sha1_digest,
 988		.export = safexcel_ahash_export,
 989		.import = safexcel_ahash_import,
 990		.halg = {
 991			.digestsize = SHA1_DIGEST_SIZE,
 992			.statesize = sizeof(struct safexcel_ahash_export_state),
 993			.base = {
 994				.cra_name = "sha1",
 995				.cra_driver_name = "safexcel-sha1",
 996				.cra_priority = SAFEXCEL_CRA_PRIORITY,
 997				.cra_flags = CRYPTO_ALG_ASYNC |
 998					     CRYPTO_ALG_ALLOCATES_MEMORY |
 999					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1000				.cra_blocksize = SHA1_BLOCK_SIZE,
1001				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1002				.cra_init = safexcel_ahash_cra_init,
1003				.cra_exit = safexcel_ahash_cra_exit,
1004				.cra_module = THIS_MODULE,
1005			},
1006		},
1007	},
1008};
1009
1010static int safexcel_hmac_sha1_init(struct ahash_request *areq)
1011{
1012	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1013	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1014
1015	memset(req, 0, sizeof(*req));
1016
1017	/* Start from ipad precompute */
1018	memcpy(req->state, &ctx->base.ipad, SHA1_DIGEST_SIZE);
1019	/* Already processed the key^ipad part now! */
1020	req->len	= SHA1_BLOCK_SIZE;
1021	req->processed	= SHA1_BLOCK_SIZE;
1022
1023	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
1024	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1025	req->state_sz = SHA1_DIGEST_SIZE;
1026	req->digest_sz = SHA1_DIGEST_SIZE;
1027	req->block_sz = SHA1_BLOCK_SIZE;
1028	req->hmac = true;
1029
1030	return 0;
1031}
1032
1033static int safexcel_hmac_sha1_digest(struct ahash_request *areq)
1034{
1035	int ret = safexcel_hmac_sha1_init(areq);
1036
1037	if (ret)
1038		return ret;
1039
1040	return safexcel_ahash_finup(areq);
1041}
1042
1043static int safexcel_hmac_init_pad(struct ahash_request *areq,
1044				  unsigned int blocksize, const u8 *key,
1045				  unsigned int keylen, u8 *ipad, u8 *opad)
1046{
1047	DECLARE_CRYPTO_WAIT(result);
1048	struct scatterlist sg;
1049	int ret, i;
1050	u8 *keydup;
1051
1052	if (keylen <= blocksize) {
1053		memcpy(ipad, key, keylen);
1054	} else {
1055		keydup = kmemdup(key, keylen, GFP_KERNEL);
1056		if (!keydup)
1057			return -ENOMEM;
1058
1059		ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1060					   crypto_req_done, &result);
1061		sg_init_one(&sg, keydup, keylen);
1062		ahash_request_set_crypt(areq, &sg, ipad, keylen);
1063
1064		ret = crypto_ahash_digest(areq);
1065		ret = crypto_wait_req(ret, &result);
1066
1067		/* Avoid leaking */
1068		kfree_sensitive(keydup);
1069
1070		if (ret)
1071			return ret;
1072
1073		keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
1074	}
1075
1076	memset(ipad + keylen, 0, blocksize - keylen);
1077	memcpy(opad, ipad, blocksize);
1078
1079	for (i = 0; i < blocksize; i++) {
1080		ipad[i] ^= HMAC_IPAD_VALUE;
1081		opad[i] ^= HMAC_OPAD_VALUE;
1082	}
1083
1084	return 0;
1085}
1086
1087static int safexcel_hmac_init_iv(struct ahash_request *areq,
1088				 unsigned int blocksize, u8 *pad, void *state)
1089{
1090	struct safexcel_ahash_req *req;
1091	DECLARE_CRYPTO_WAIT(result);
1092	struct scatterlist sg;
1093	int ret;
1094
1095	ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1096				   crypto_req_done, &result);
1097	sg_init_one(&sg, pad, blocksize);
1098	ahash_request_set_crypt(areq, &sg, pad, blocksize);
1099
1100	ret = crypto_ahash_init(areq);
1101	if (ret)
1102		return ret;
1103
1104	req = ahash_request_ctx_dma(areq);
1105	req->hmac = true;
1106	req->last_req = true;
1107
1108	ret = crypto_ahash_update(areq);
1109	ret = crypto_wait_req(ret, &result);
1110
1111	return ret ?: crypto_ahash_export(areq, state);
1112}
1113
1114static int __safexcel_hmac_setkey(const char *alg, const u8 *key,
1115				  unsigned int keylen,
1116				  void *istate, void *ostate)
1117{
1118	struct ahash_request *areq;
1119	struct crypto_ahash *tfm;
1120	unsigned int blocksize;
1121	u8 *ipad, *opad;
1122	int ret;
1123
1124	tfm = crypto_alloc_ahash(alg, 0, 0);
1125	if (IS_ERR(tfm))
1126		return PTR_ERR(tfm);
1127
1128	areq = ahash_request_alloc(tfm, GFP_KERNEL);
1129	if (!areq) {
1130		ret = -ENOMEM;
1131		goto free_ahash;
1132	}
1133
1134	crypto_ahash_clear_flags(tfm, ~0);
1135	blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1136
1137	ipad = kcalloc(2, blocksize, GFP_KERNEL);
1138	if (!ipad) {
1139		ret = -ENOMEM;
1140		goto free_request;
1141	}
1142
1143	opad = ipad + blocksize;
1144
1145	ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad);
1146	if (ret)
1147		goto free_ipad;
1148
1149	ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate);
1150	if (ret)
1151		goto free_ipad;
1152
1153	ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate);
1154
1155free_ipad:
1156	kfree(ipad);
1157free_request:
1158	ahash_request_free(areq);
1159free_ahash:
1160	crypto_free_ahash(tfm);
1161
1162	return ret;
1163}
1164
1165int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key,
1166			 unsigned int keylen, const char *alg,
1167			 unsigned int state_sz)
1168{
1169	struct safexcel_crypto_priv *priv = base->priv;
1170	struct safexcel_ahash_export_state istate, ostate;
1171	int ret;
1172
1173	ret = __safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate);
1174	if (ret)
1175		return ret;
1176
1177	if (priv->flags & EIP197_TRC_CACHE && base->ctxr &&
1178	    (memcmp(&base->ipad, istate.state, state_sz) ||
1179	     memcmp(&base->opad, ostate.state, state_sz)))
1180		base->needs_inv = true;
1181
1182	memcpy(&base->ipad, &istate.state, state_sz);
1183	memcpy(&base->opad, &ostate.state, state_sz);
1184
1185	return 0;
1186}
1187
1188static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key,
1189				    unsigned int keylen, const char *alg,
1190				    unsigned int state_sz)
1191{
1192	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
1193
1194	return safexcel_hmac_setkey(&ctx->base, key, keylen, alg, state_sz);
1195}
1196
1197static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
1198				     unsigned int keylen)
1199{
1200	return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha1",
1201					SHA1_DIGEST_SIZE);
1202}
1203
1204struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
1205	.type = SAFEXCEL_ALG_TYPE_AHASH,
1206	.algo_mask = SAFEXCEL_ALG_SHA1,
1207	.alg.ahash = {
1208		.init = safexcel_hmac_sha1_init,
1209		.update = safexcel_ahash_update,
1210		.final = safexcel_ahash_final,
1211		.finup = safexcel_ahash_finup,
1212		.digest = safexcel_hmac_sha1_digest,
1213		.setkey = safexcel_hmac_sha1_setkey,
1214		.export = safexcel_ahash_export,
1215		.import = safexcel_ahash_import,
1216		.halg = {
1217			.digestsize = SHA1_DIGEST_SIZE,
1218			.statesize = sizeof(struct safexcel_ahash_export_state),
1219			.base = {
1220				.cra_name = "hmac(sha1)",
1221				.cra_driver_name = "safexcel-hmac-sha1",
1222				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1223				.cra_flags = CRYPTO_ALG_ASYNC |
1224					     CRYPTO_ALG_ALLOCATES_MEMORY |
1225					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1226				.cra_blocksize = SHA1_BLOCK_SIZE,
1227				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1228				.cra_init = safexcel_ahash_cra_init,
1229				.cra_exit = safexcel_ahash_cra_exit,
1230				.cra_module = THIS_MODULE,
1231			},
1232		},
1233	},
1234};
1235
1236static int safexcel_sha256_init(struct ahash_request *areq)
1237{
1238	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1239	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1240
1241	memset(req, 0, sizeof(*req));
1242
1243	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1244	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1245	req->state_sz = SHA256_DIGEST_SIZE;
1246	req->digest_sz = SHA256_DIGEST_SIZE;
1247	req->block_sz = SHA256_BLOCK_SIZE;
1248
1249	return 0;
1250}
1251
1252static int safexcel_sha256_digest(struct ahash_request *areq)
1253{
1254	int ret = safexcel_sha256_init(areq);
1255
1256	if (ret)
1257		return ret;
1258
1259	return safexcel_ahash_finup(areq);
1260}
1261
1262struct safexcel_alg_template safexcel_alg_sha256 = {
1263	.type = SAFEXCEL_ALG_TYPE_AHASH,
1264	.algo_mask = SAFEXCEL_ALG_SHA2_256,
1265	.alg.ahash = {
1266		.init = safexcel_sha256_init,
1267		.update = safexcel_ahash_update,
1268		.final = safexcel_ahash_final,
1269		.finup = safexcel_ahash_finup,
1270		.digest = safexcel_sha256_digest,
1271		.export = safexcel_ahash_export,
1272		.import = safexcel_ahash_import,
1273		.halg = {
1274			.digestsize = SHA256_DIGEST_SIZE,
1275			.statesize = sizeof(struct safexcel_ahash_export_state),
1276			.base = {
1277				.cra_name = "sha256",
1278				.cra_driver_name = "safexcel-sha256",
1279				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1280				.cra_flags = CRYPTO_ALG_ASYNC |
1281					     CRYPTO_ALG_ALLOCATES_MEMORY |
1282					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1283				.cra_blocksize = SHA256_BLOCK_SIZE,
1284				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1285				.cra_init = safexcel_ahash_cra_init,
1286				.cra_exit = safexcel_ahash_cra_exit,
1287				.cra_module = THIS_MODULE,
1288			},
1289		},
1290	},
1291};
1292
1293static int safexcel_sha224_init(struct ahash_request *areq)
1294{
1295	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1296	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1297
1298	memset(req, 0, sizeof(*req));
1299
1300	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1301	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1302	req->state_sz = SHA256_DIGEST_SIZE;
1303	req->digest_sz = SHA256_DIGEST_SIZE;
1304	req->block_sz = SHA256_BLOCK_SIZE;
1305
1306	return 0;
1307}
1308
1309static int safexcel_sha224_digest(struct ahash_request *areq)
1310{
1311	int ret = safexcel_sha224_init(areq);
1312
1313	if (ret)
1314		return ret;
1315
1316	return safexcel_ahash_finup(areq);
1317}
1318
1319struct safexcel_alg_template safexcel_alg_sha224 = {
1320	.type = SAFEXCEL_ALG_TYPE_AHASH,
1321	.algo_mask = SAFEXCEL_ALG_SHA2_256,
1322	.alg.ahash = {
1323		.init = safexcel_sha224_init,
1324		.update = safexcel_ahash_update,
1325		.final = safexcel_ahash_final,
1326		.finup = safexcel_ahash_finup,
1327		.digest = safexcel_sha224_digest,
1328		.export = safexcel_ahash_export,
1329		.import = safexcel_ahash_import,
1330		.halg = {
1331			.digestsize = SHA224_DIGEST_SIZE,
1332			.statesize = sizeof(struct safexcel_ahash_export_state),
1333			.base = {
1334				.cra_name = "sha224",
1335				.cra_driver_name = "safexcel-sha224",
1336				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1337				.cra_flags = CRYPTO_ALG_ASYNC |
1338					     CRYPTO_ALG_ALLOCATES_MEMORY |
1339					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1340				.cra_blocksize = SHA224_BLOCK_SIZE,
1341				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1342				.cra_init = safexcel_ahash_cra_init,
1343				.cra_exit = safexcel_ahash_cra_exit,
1344				.cra_module = THIS_MODULE,
1345			},
1346		},
1347	},
1348};
1349
1350static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key,
1351				       unsigned int keylen)
1352{
1353	return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha224",
1354					SHA256_DIGEST_SIZE);
1355}
1356
1357static int safexcel_hmac_sha224_init(struct ahash_request *areq)
1358{
1359	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1360	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1361
1362	memset(req, 0, sizeof(*req));
1363
1364	/* Start from ipad precompute */
1365	memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
1366	/* Already processed the key^ipad part now! */
1367	req->len	= SHA256_BLOCK_SIZE;
1368	req->processed	= SHA256_BLOCK_SIZE;
1369
1370	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1371	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1372	req->state_sz = SHA256_DIGEST_SIZE;
1373	req->digest_sz = SHA256_DIGEST_SIZE;
1374	req->block_sz = SHA256_BLOCK_SIZE;
1375	req->hmac = true;
1376
1377	return 0;
1378}
1379
1380static int safexcel_hmac_sha224_digest(struct ahash_request *areq)
1381{
1382	int ret = safexcel_hmac_sha224_init(areq);
1383
1384	if (ret)
1385		return ret;
1386
1387	return safexcel_ahash_finup(areq);
1388}
1389
1390struct safexcel_alg_template safexcel_alg_hmac_sha224 = {
1391	.type = SAFEXCEL_ALG_TYPE_AHASH,
1392	.algo_mask = SAFEXCEL_ALG_SHA2_256,
1393	.alg.ahash = {
1394		.init = safexcel_hmac_sha224_init,
1395		.update = safexcel_ahash_update,
1396		.final = safexcel_ahash_final,
1397		.finup = safexcel_ahash_finup,
1398		.digest = safexcel_hmac_sha224_digest,
1399		.setkey = safexcel_hmac_sha224_setkey,
1400		.export = safexcel_ahash_export,
1401		.import = safexcel_ahash_import,
1402		.halg = {
1403			.digestsize = SHA224_DIGEST_SIZE,
1404			.statesize = sizeof(struct safexcel_ahash_export_state),
1405			.base = {
1406				.cra_name = "hmac(sha224)",
1407				.cra_driver_name = "safexcel-hmac-sha224",
1408				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1409				.cra_flags = CRYPTO_ALG_ASYNC |
1410					     CRYPTO_ALG_ALLOCATES_MEMORY |
1411					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1412				.cra_blocksize = SHA224_BLOCK_SIZE,
1413				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1414				.cra_init = safexcel_ahash_cra_init,
1415				.cra_exit = safexcel_ahash_cra_exit,
1416				.cra_module = THIS_MODULE,
1417			},
1418		},
1419	},
1420};
1421
1422static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key,
1423				     unsigned int keylen)
1424{
1425	return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha256",
1426					SHA256_DIGEST_SIZE);
1427}
1428
1429static int safexcel_hmac_sha256_init(struct ahash_request *areq)
1430{
1431	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1432	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1433
1434	memset(req, 0, sizeof(*req));
1435
1436	/* Start from ipad precompute */
1437	memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
1438	/* Already processed the key^ipad part now! */
1439	req->len	= SHA256_BLOCK_SIZE;
1440	req->processed	= SHA256_BLOCK_SIZE;
1441
1442	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1443	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1444	req->state_sz = SHA256_DIGEST_SIZE;
1445	req->digest_sz = SHA256_DIGEST_SIZE;
1446	req->block_sz = SHA256_BLOCK_SIZE;
1447	req->hmac = true;
1448
1449	return 0;
1450}
1451
1452static int safexcel_hmac_sha256_digest(struct ahash_request *areq)
1453{
1454	int ret = safexcel_hmac_sha256_init(areq);
1455
1456	if (ret)
1457		return ret;
1458
1459	return safexcel_ahash_finup(areq);
1460}
1461
1462struct safexcel_alg_template safexcel_alg_hmac_sha256 = {
1463	.type = SAFEXCEL_ALG_TYPE_AHASH,
1464	.algo_mask = SAFEXCEL_ALG_SHA2_256,
1465	.alg.ahash = {
1466		.init = safexcel_hmac_sha256_init,
1467		.update = safexcel_ahash_update,
1468		.final = safexcel_ahash_final,
1469		.finup = safexcel_ahash_finup,
1470		.digest = safexcel_hmac_sha256_digest,
1471		.setkey = safexcel_hmac_sha256_setkey,
1472		.export = safexcel_ahash_export,
1473		.import = safexcel_ahash_import,
1474		.halg = {
1475			.digestsize = SHA256_DIGEST_SIZE,
1476			.statesize = sizeof(struct safexcel_ahash_export_state),
1477			.base = {
1478				.cra_name = "hmac(sha256)",
1479				.cra_driver_name = "safexcel-hmac-sha256",
1480				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1481				.cra_flags = CRYPTO_ALG_ASYNC |
1482					     CRYPTO_ALG_ALLOCATES_MEMORY |
1483					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1484				.cra_blocksize = SHA256_BLOCK_SIZE,
1485				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1486				.cra_init = safexcel_ahash_cra_init,
1487				.cra_exit = safexcel_ahash_cra_exit,
1488				.cra_module = THIS_MODULE,
1489			},
1490		},
1491	},
1492};
1493
1494static int safexcel_sha512_init(struct ahash_request *areq)
1495{
1496	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1497	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1498
1499	memset(req, 0, sizeof(*req));
1500
1501	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1502	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1503	req->state_sz = SHA512_DIGEST_SIZE;
1504	req->digest_sz = SHA512_DIGEST_SIZE;
1505	req->block_sz = SHA512_BLOCK_SIZE;
1506
1507	return 0;
1508}
1509
1510static int safexcel_sha512_digest(struct ahash_request *areq)
1511{
1512	int ret = safexcel_sha512_init(areq);
1513
1514	if (ret)
1515		return ret;
1516
1517	return safexcel_ahash_finup(areq);
1518}
1519
1520struct safexcel_alg_template safexcel_alg_sha512 = {
1521	.type = SAFEXCEL_ALG_TYPE_AHASH,
1522	.algo_mask = SAFEXCEL_ALG_SHA2_512,
1523	.alg.ahash = {
1524		.init = safexcel_sha512_init,
1525		.update = safexcel_ahash_update,
1526		.final = safexcel_ahash_final,
1527		.finup = safexcel_ahash_finup,
1528		.digest = safexcel_sha512_digest,
1529		.export = safexcel_ahash_export,
1530		.import = safexcel_ahash_import,
1531		.halg = {
1532			.digestsize = SHA512_DIGEST_SIZE,
1533			.statesize = sizeof(struct safexcel_ahash_export_state),
1534			.base = {
1535				.cra_name = "sha512",
1536				.cra_driver_name = "safexcel-sha512",
1537				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1538				.cra_flags = CRYPTO_ALG_ASYNC |
1539					     CRYPTO_ALG_ALLOCATES_MEMORY |
1540					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1541				.cra_blocksize = SHA512_BLOCK_SIZE,
1542				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1543				.cra_init = safexcel_ahash_cra_init,
1544				.cra_exit = safexcel_ahash_cra_exit,
1545				.cra_module = THIS_MODULE,
1546			},
1547		},
1548	},
1549};
1550
1551static int safexcel_sha384_init(struct ahash_request *areq)
1552{
1553	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1554	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1555
1556	memset(req, 0, sizeof(*req));
1557
1558	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1559	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1560	req->state_sz = SHA512_DIGEST_SIZE;
1561	req->digest_sz = SHA512_DIGEST_SIZE;
1562	req->block_sz = SHA512_BLOCK_SIZE;
1563
1564	return 0;
1565}
1566
1567static int safexcel_sha384_digest(struct ahash_request *areq)
1568{
1569	int ret = safexcel_sha384_init(areq);
1570
1571	if (ret)
1572		return ret;
1573
1574	return safexcel_ahash_finup(areq);
1575}
1576
1577struct safexcel_alg_template safexcel_alg_sha384 = {
1578	.type = SAFEXCEL_ALG_TYPE_AHASH,
1579	.algo_mask = SAFEXCEL_ALG_SHA2_512,
1580	.alg.ahash = {
1581		.init = safexcel_sha384_init,
1582		.update = safexcel_ahash_update,
1583		.final = safexcel_ahash_final,
1584		.finup = safexcel_ahash_finup,
1585		.digest = safexcel_sha384_digest,
1586		.export = safexcel_ahash_export,
1587		.import = safexcel_ahash_import,
1588		.halg = {
1589			.digestsize = SHA384_DIGEST_SIZE,
1590			.statesize = sizeof(struct safexcel_ahash_export_state),
1591			.base = {
1592				.cra_name = "sha384",
1593				.cra_driver_name = "safexcel-sha384",
1594				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1595				.cra_flags = CRYPTO_ALG_ASYNC |
1596					     CRYPTO_ALG_ALLOCATES_MEMORY |
1597					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1598				.cra_blocksize = SHA384_BLOCK_SIZE,
1599				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1600				.cra_init = safexcel_ahash_cra_init,
1601				.cra_exit = safexcel_ahash_cra_exit,
1602				.cra_module = THIS_MODULE,
1603			},
1604		},
1605	},
1606};
1607
1608static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key,
1609				       unsigned int keylen)
1610{
1611	return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha512",
1612					SHA512_DIGEST_SIZE);
1613}
1614
1615static int safexcel_hmac_sha512_init(struct ahash_request *areq)
1616{
1617	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1618	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1619
1620	memset(req, 0, sizeof(*req));
1621
1622	/* Start from ipad precompute */
1623	memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
1624	/* Already processed the key^ipad part now! */
1625	req->len	= SHA512_BLOCK_SIZE;
1626	req->processed	= SHA512_BLOCK_SIZE;
1627
1628	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1629	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1630	req->state_sz = SHA512_DIGEST_SIZE;
1631	req->digest_sz = SHA512_DIGEST_SIZE;
1632	req->block_sz = SHA512_BLOCK_SIZE;
1633	req->hmac = true;
1634
1635	return 0;
1636}
1637
1638static int safexcel_hmac_sha512_digest(struct ahash_request *areq)
1639{
1640	int ret = safexcel_hmac_sha512_init(areq);
1641
1642	if (ret)
1643		return ret;
1644
1645	return safexcel_ahash_finup(areq);
1646}
1647
1648struct safexcel_alg_template safexcel_alg_hmac_sha512 = {
1649	.type = SAFEXCEL_ALG_TYPE_AHASH,
1650	.algo_mask = SAFEXCEL_ALG_SHA2_512,
1651	.alg.ahash = {
1652		.init = safexcel_hmac_sha512_init,
1653		.update = safexcel_ahash_update,
1654		.final = safexcel_ahash_final,
1655		.finup = safexcel_ahash_finup,
1656		.digest = safexcel_hmac_sha512_digest,
1657		.setkey = safexcel_hmac_sha512_setkey,
1658		.export = safexcel_ahash_export,
1659		.import = safexcel_ahash_import,
1660		.halg = {
1661			.digestsize = SHA512_DIGEST_SIZE,
1662			.statesize = sizeof(struct safexcel_ahash_export_state),
1663			.base = {
1664				.cra_name = "hmac(sha512)",
1665				.cra_driver_name = "safexcel-hmac-sha512",
1666				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1667				.cra_flags = CRYPTO_ALG_ASYNC |
1668					     CRYPTO_ALG_ALLOCATES_MEMORY |
1669					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1670				.cra_blocksize = SHA512_BLOCK_SIZE,
1671				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1672				.cra_init = safexcel_ahash_cra_init,
1673				.cra_exit = safexcel_ahash_cra_exit,
1674				.cra_module = THIS_MODULE,
1675			},
1676		},
1677	},
1678};
1679
1680static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key,
1681				       unsigned int keylen)
1682{
1683	return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha384",
1684					SHA512_DIGEST_SIZE);
1685}
1686
1687static int safexcel_hmac_sha384_init(struct ahash_request *areq)
1688{
1689	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1690	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1691
1692	memset(req, 0, sizeof(*req));
1693
1694	/* Start from ipad precompute */
1695	memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
1696	/* Already processed the key^ipad part now! */
1697	req->len	= SHA512_BLOCK_SIZE;
1698	req->processed	= SHA512_BLOCK_SIZE;
1699
1700	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1701	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1702	req->state_sz = SHA512_DIGEST_SIZE;
1703	req->digest_sz = SHA512_DIGEST_SIZE;
1704	req->block_sz = SHA512_BLOCK_SIZE;
1705	req->hmac = true;
1706
1707	return 0;
1708}
1709
1710static int safexcel_hmac_sha384_digest(struct ahash_request *areq)
1711{
1712	int ret = safexcel_hmac_sha384_init(areq);
1713
1714	if (ret)
1715		return ret;
1716
1717	return safexcel_ahash_finup(areq);
1718}
1719
1720struct safexcel_alg_template safexcel_alg_hmac_sha384 = {
1721	.type = SAFEXCEL_ALG_TYPE_AHASH,
1722	.algo_mask = SAFEXCEL_ALG_SHA2_512,
1723	.alg.ahash = {
1724		.init = safexcel_hmac_sha384_init,
1725		.update = safexcel_ahash_update,
1726		.final = safexcel_ahash_final,
1727		.finup = safexcel_ahash_finup,
1728		.digest = safexcel_hmac_sha384_digest,
1729		.setkey = safexcel_hmac_sha384_setkey,
1730		.export = safexcel_ahash_export,
1731		.import = safexcel_ahash_import,
1732		.halg = {
1733			.digestsize = SHA384_DIGEST_SIZE,
1734			.statesize = sizeof(struct safexcel_ahash_export_state),
1735			.base = {
1736				.cra_name = "hmac(sha384)",
1737				.cra_driver_name = "safexcel-hmac-sha384",
1738				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1739				.cra_flags = CRYPTO_ALG_ASYNC |
1740					     CRYPTO_ALG_ALLOCATES_MEMORY |
1741					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1742				.cra_blocksize = SHA384_BLOCK_SIZE,
1743				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1744				.cra_init = safexcel_ahash_cra_init,
1745				.cra_exit = safexcel_ahash_cra_exit,
1746				.cra_module = THIS_MODULE,
1747			},
1748		},
1749	},
1750};
1751
1752static int safexcel_md5_init(struct ahash_request *areq)
1753{
1754	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1755	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1756
1757	memset(req, 0, sizeof(*req));
1758
1759	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1760	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1761	req->state_sz = MD5_DIGEST_SIZE;
1762	req->digest_sz = MD5_DIGEST_SIZE;
1763	req->block_sz = MD5_HMAC_BLOCK_SIZE;
1764
1765	return 0;
1766}
1767
1768static int safexcel_md5_digest(struct ahash_request *areq)
1769{
1770	int ret = safexcel_md5_init(areq);
1771
1772	if (ret)
1773		return ret;
1774
1775	return safexcel_ahash_finup(areq);
1776}
1777
1778struct safexcel_alg_template safexcel_alg_md5 = {
1779	.type = SAFEXCEL_ALG_TYPE_AHASH,
1780	.algo_mask = SAFEXCEL_ALG_MD5,
1781	.alg.ahash = {
1782		.init = safexcel_md5_init,
1783		.update = safexcel_ahash_update,
1784		.final = safexcel_ahash_final,
1785		.finup = safexcel_ahash_finup,
1786		.digest = safexcel_md5_digest,
1787		.export = safexcel_ahash_export,
1788		.import = safexcel_ahash_import,
1789		.halg = {
1790			.digestsize = MD5_DIGEST_SIZE,
1791			.statesize = sizeof(struct safexcel_ahash_export_state),
1792			.base = {
1793				.cra_name = "md5",
1794				.cra_driver_name = "safexcel-md5",
1795				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1796				.cra_flags = CRYPTO_ALG_ASYNC |
1797					     CRYPTO_ALG_ALLOCATES_MEMORY |
1798					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1799				.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1800				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1801				.cra_init = safexcel_ahash_cra_init,
1802				.cra_exit = safexcel_ahash_cra_exit,
1803				.cra_module = THIS_MODULE,
1804			},
1805		},
1806	},
1807};
1808
1809static int safexcel_hmac_md5_init(struct ahash_request *areq)
1810{
1811	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1812	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1813
1814	memset(req, 0, sizeof(*req));
1815
1816	/* Start from ipad precompute */
1817	memcpy(req->state, &ctx->base.ipad, MD5_DIGEST_SIZE);
1818	/* Already processed the key^ipad part now! */
1819	req->len	= MD5_HMAC_BLOCK_SIZE;
1820	req->processed	= MD5_HMAC_BLOCK_SIZE;
1821
1822	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1823	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1824	req->state_sz = MD5_DIGEST_SIZE;
1825	req->digest_sz = MD5_DIGEST_SIZE;
1826	req->block_sz = MD5_HMAC_BLOCK_SIZE;
1827	req->len_is_le = true; /* MD5 is little endian! ... */
1828	req->hmac = true;
1829
1830	return 0;
1831}
1832
1833static int safexcel_hmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key,
1834				     unsigned int keylen)
1835{
1836	return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-md5",
1837					MD5_DIGEST_SIZE);
1838}
1839
1840static int safexcel_hmac_md5_digest(struct ahash_request *areq)
1841{
1842	int ret = safexcel_hmac_md5_init(areq);
1843
1844	if (ret)
1845		return ret;
1846
1847	return safexcel_ahash_finup(areq);
1848}
1849
1850struct safexcel_alg_template safexcel_alg_hmac_md5 = {
1851	.type = SAFEXCEL_ALG_TYPE_AHASH,
1852	.algo_mask = SAFEXCEL_ALG_MD5,
1853	.alg.ahash = {
1854		.init = safexcel_hmac_md5_init,
1855		.update = safexcel_ahash_update,
1856		.final = safexcel_ahash_final,
1857		.finup = safexcel_ahash_finup,
1858		.digest = safexcel_hmac_md5_digest,
1859		.setkey = safexcel_hmac_md5_setkey,
1860		.export = safexcel_ahash_export,
1861		.import = safexcel_ahash_import,
1862		.halg = {
1863			.digestsize = MD5_DIGEST_SIZE,
1864			.statesize = sizeof(struct safexcel_ahash_export_state),
1865			.base = {
1866				.cra_name = "hmac(md5)",
1867				.cra_driver_name = "safexcel-hmac-md5",
1868				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1869				.cra_flags = CRYPTO_ALG_ASYNC |
1870					     CRYPTO_ALG_ALLOCATES_MEMORY |
1871					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1872				.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1873				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1874				.cra_init = safexcel_ahash_cra_init,
1875				.cra_exit = safexcel_ahash_cra_exit,
1876				.cra_module = THIS_MODULE,
1877			},
1878		},
1879	},
1880};
1881
1882static int safexcel_cbcmac_init(struct ahash_request *areq)
1883{
1884	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1885	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1886
1887	memset(req, 0, sizeof(*req));
1888
1889	/* Start from loaded keys */
1890	memcpy(req->state, &ctx->base.ipad, ctx->key_sz);
1891	/* Set processed to non-zero to enable invalidation detection */
1892	req->len	= AES_BLOCK_SIZE;
1893	req->processed	= AES_BLOCK_SIZE;
1894
1895	req->digest   = CONTEXT_CONTROL_DIGEST_XCM;
1896	req->state_sz = ctx->key_sz;
1897	req->digest_sz = AES_BLOCK_SIZE;
1898	req->block_sz = AES_BLOCK_SIZE;
1899	req->xcbcmac  = true;
1900
1901	return 0;
1902}
1903
1904static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
1905				 unsigned int len)
1906{
1907	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1908	struct crypto_aes_ctx aes;
1909	int ret, i;
1910
1911	ret = aes_expandkey(&aes, key, len);
1912	if (ret)
1913		return ret;
1914
1915	memset(&ctx->base.ipad, 0, 2 * AES_BLOCK_SIZE);
1916	for (i = 0; i < len / sizeof(u32); i++)
1917		ctx->base.ipad.be[i + 8] = cpu_to_be32(aes.key_enc[i]);
1918
1919	if (len == AES_KEYSIZE_192) {
1920		ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
1921		ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
1922	} else if (len == AES_KEYSIZE_256) {
1923		ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
1924		ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
1925	} else {
1926		ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
1927		ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
1928	}
1929	ctx->cbcmac  = true;
1930
1931	memzero_explicit(&aes, sizeof(aes));
1932	return 0;
1933}
1934
1935static int safexcel_cbcmac_digest(struct ahash_request *areq)
1936{
1937	return safexcel_cbcmac_init(areq) ?: safexcel_ahash_finup(areq);
1938}
1939
1940struct safexcel_alg_template safexcel_alg_cbcmac = {
1941	.type = SAFEXCEL_ALG_TYPE_AHASH,
1942	.algo_mask = 0,
1943	.alg.ahash = {
1944		.init = safexcel_cbcmac_init,
1945		.update = safexcel_ahash_update,
1946		.final = safexcel_ahash_final,
1947		.finup = safexcel_ahash_finup,
1948		.digest = safexcel_cbcmac_digest,
1949		.setkey = safexcel_cbcmac_setkey,
1950		.export = safexcel_ahash_export,
1951		.import = safexcel_ahash_import,
1952		.halg = {
1953			.digestsize = AES_BLOCK_SIZE,
1954			.statesize = sizeof(struct safexcel_ahash_export_state),
1955			.base = {
1956				.cra_name = "cbcmac(aes)",
1957				.cra_driver_name = "safexcel-cbcmac-aes",
1958				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1959				.cra_flags = CRYPTO_ALG_ASYNC |
1960					     CRYPTO_ALG_ALLOCATES_MEMORY |
1961					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1962				.cra_blocksize = AES_BLOCK_SIZE,
1963				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1964				.cra_init = safexcel_ahash_cra_init,
1965				.cra_exit = safexcel_ahash_cra_exit,
1966				.cra_module = THIS_MODULE,
1967			},
1968		},
1969	},
1970};
1971
1972static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
1973				 unsigned int len)
1974{
1975	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1976	u32 key_tmp[3 * AES_BLOCK_SIZE / sizeof(u32)];
1977	int ret, i;
1978
1979	ret = aes_expandkey(ctx->aes, key, len);
1980	if (ret)
1981		return ret;
1982
1983	/* precompute the XCBC key material */
1984	aes_encrypt(ctx->aes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
1985		    "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1");
1986	aes_encrypt(ctx->aes, (u8 *)key_tmp,
1987		    "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2");
1988	aes_encrypt(ctx->aes, (u8 *)key_tmp + AES_BLOCK_SIZE,
1989		    "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3");
1990	for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++)
1991		ctx->base.ipad.word[i] = swab32(key_tmp[i]);
1992
1993	ret = aes_expandkey(ctx->aes,
1994			    (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
1995			    AES_MIN_KEY_SIZE);
1996	if (ret)
1997		return ret;
1998
1999	ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2000	ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2001	ctx->cbcmac = false;
2002
2003	return 0;
2004}
2005
2006static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm)
2007{
2008	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2009
2010	safexcel_ahash_cra_init(tfm);
2011	ctx->aes = kmalloc(sizeof(*ctx->aes), GFP_KERNEL);
2012	return ctx->aes == NULL ? -ENOMEM : 0;
2013}
2014
2015static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm)
2016{
2017	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2018
2019	kfree(ctx->aes);
2020	safexcel_ahash_cra_exit(tfm);
2021}
2022
2023struct safexcel_alg_template safexcel_alg_xcbcmac = {
2024	.type = SAFEXCEL_ALG_TYPE_AHASH,
2025	.algo_mask = 0,
2026	.alg.ahash = {
2027		.init = safexcel_cbcmac_init,
2028		.update = safexcel_ahash_update,
2029		.final = safexcel_ahash_final,
2030		.finup = safexcel_ahash_finup,
2031		.digest = safexcel_cbcmac_digest,
2032		.setkey = safexcel_xcbcmac_setkey,
2033		.export = safexcel_ahash_export,
2034		.import = safexcel_ahash_import,
2035		.halg = {
2036			.digestsize = AES_BLOCK_SIZE,
2037			.statesize = sizeof(struct safexcel_ahash_export_state),
2038			.base = {
2039				.cra_name = "xcbc(aes)",
2040				.cra_driver_name = "safexcel-xcbc-aes",
2041				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2042				.cra_flags = CRYPTO_ALG_ASYNC |
2043					     CRYPTO_ALG_ALLOCATES_MEMORY |
2044					     CRYPTO_ALG_KERN_DRIVER_ONLY,
2045				.cra_blocksize = AES_BLOCK_SIZE,
2046				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2047				.cra_init = safexcel_xcbcmac_cra_init,
2048				.cra_exit = safexcel_xcbcmac_cra_exit,
2049				.cra_module = THIS_MODULE,
2050			},
2051		},
2052	},
2053};
2054
2055static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2056				unsigned int len)
2057{
2058	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2059	__be64 consts[4];
2060	u64 _const[2];
2061	u8 msb_mask, gfmask;
2062	int ret, i;
2063
2064	/* precompute the CMAC key material */
2065	ret = aes_expandkey(ctx->aes, key, len);
2066	if (ret)
2067		return ret;
2068
2069	for (i = 0; i < len / sizeof(u32); i++)
2070		ctx->base.ipad.word[i + 8] = swab32(ctx->aes->key_enc[i]);
2071
2072	/* code below borrowed from crypto/cmac.c */
2073	/* encrypt the zero block */
2074	memset(consts, 0, AES_BLOCK_SIZE);
2075	aes_encrypt(ctx->aes, (u8 *)consts, (u8 *)consts);
2076
2077	gfmask = 0x87;
2078	_const[0] = be64_to_cpu(consts[1]);
2079	_const[1] = be64_to_cpu(consts[0]);
2080
2081	/* gf(2^128) multiply zero-ciphertext with u and u^2 */
2082	for (i = 0; i < 4; i += 2) {
2083		msb_mask = ((s64)_const[1] >> 63) & gfmask;
2084		_const[1] = (_const[1] << 1) | (_const[0] >> 63);
2085		_const[0] = (_const[0] << 1) ^ msb_mask;
2086
2087		consts[i + 0] = cpu_to_be64(_const[1]);
2088		consts[i + 1] = cpu_to_be64(_const[0]);
2089	}
2090	/* end of code borrowed from crypto/cmac.c */
2091
2092	for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++)
2093		ctx->base.ipad.be[i] = cpu_to_be32(((u32 *)consts)[i]);
2094
2095	if (len == AES_KEYSIZE_192) {
2096		ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2097		ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2098	} else if (len == AES_KEYSIZE_256) {
2099		ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2100		ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2101	} else {
2102		ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2103		ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2104	}
2105	ctx->cbcmac = false;
2106
2107	return 0;
2108}
2109
2110struct safexcel_alg_template safexcel_alg_cmac = {
2111	.type = SAFEXCEL_ALG_TYPE_AHASH,
2112	.algo_mask = 0,
2113	.alg.ahash = {
2114		.init = safexcel_cbcmac_init,
2115		.update = safexcel_ahash_update,
2116		.final = safexcel_ahash_final,
2117		.finup = safexcel_ahash_finup,
2118		.digest = safexcel_cbcmac_digest,
2119		.setkey = safexcel_cmac_setkey,
2120		.export = safexcel_ahash_export,
2121		.import = safexcel_ahash_import,
2122		.halg = {
2123			.digestsize = AES_BLOCK_SIZE,
2124			.statesize = sizeof(struct safexcel_ahash_export_state),
2125			.base = {
2126				.cra_name = "cmac(aes)",
2127				.cra_driver_name = "safexcel-cmac-aes",
2128				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2129				.cra_flags = CRYPTO_ALG_ASYNC |
2130					     CRYPTO_ALG_ALLOCATES_MEMORY |
2131					     CRYPTO_ALG_KERN_DRIVER_ONLY,
2132				.cra_blocksize = AES_BLOCK_SIZE,
2133				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2134				.cra_init = safexcel_xcbcmac_cra_init,
2135				.cra_exit = safexcel_xcbcmac_cra_exit,
2136				.cra_module = THIS_MODULE,
2137			},
2138		},
2139	},
2140};
2141
2142static int safexcel_sm3_init(struct ahash_request *areq)
2143{
2144	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2145	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2146
2147	memset(req, 0, sizeof(*req));
2148
2149	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2150	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2151	req->state_sz = SM3_DIGEST_SIZE;
2152	req->digest_sz = SM3_DIGEST_SIZE;
2153	req->block_sz = SM3_BLOCK_SIZE;
2154
2155	return 0;
2156}
2157
2158static int safexcel_sm3_digest(struct ahash_request *areq)
2159{
2160	int ret = safexcel_sm3_init(areq);
2161
2162	if (ret)
2163		return ret;
2164
2165	return safexcel_ahash_finup(areq);
2166}
2167
2168struct safexcel_alg_template safexcel_alg_sm3 = {
2169	.type = SAFEXCEL_ALG_TYPE_AHASH,
2170	.algo_mask = SAFEXCEL_ALG_SM3,
2171	.alg.ahash = {
2172		.init = safexcel_sm3_init,
2173		.update = safexcel_ahash_update,
2174		.final = safexcel_ahash_final,
2175		.finup = safexcel_ahash_finup,
2176		.digest = safexcel_sm3_digest,
2177		.export = safexcel_ahash_export,
2178		.import = safexcel_ahash_import,
2179		.halg = {
2180			.digestsize = SM3_DIGEST_SIZE,
2181			.statesize = sizeof(struct safexcel_ahash_export_state),
2182			.base = {
2183				.cra_name = "sm3",
2184				.cra_driver_name = "safexcel-sm3",
2185				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2186				.cra_flags = CRYPTO_ALG_ASYNC |
2187					     CRYPTO_ALG_ALLOCATES_MEMORY |
2188					     CRYPTO_ALG_KERN_DRIVER_ONLY,
2189				.cra_blocksize = SM3_BLOCK_SIZE,
2190				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2191				.cra_init = safexcel_ahash_cra_init,
2192				.cra_exit = safexcel_ahash_cra_exit,
2193				.cra_module = THIS_MODULE,
2194			},
2195		},
2196	},
2197};
2198
2199static int safexcel_hmac_sm3_setkey(struct crypto_ahash *tfm, const u8 *key,
2200				    unsigned int keylen)
2201{
2202	return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sm3",
2203					SM3_DIGEST_SIZE);
2204}
2205
2206static int safexcel_hmac_sm3_init(struct ahash_request *areq)
2207{
2208	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2209	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2210
2211	memset(req, 0, sizeof(*req));
2212
2213	/* Start from ipad precompute */
2214	memcpy(req->state, &ctx->base.ipad, SM3_DIGEST_SIZE);
2215	/* Already processed the key^ipad part now! */
2216	req->len	= SM3_BLOCK_SIZE;
2217	req->processed	= SM3_BLOCK_SIZE;
2218
2219	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2220	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2221	req->state_sz = SM3_DIGEST_SIZE;
2222	req->digest_sz = SM3_DIGEST_SIZE;
2223	req->block_sz = SM3_BLOCK_SIZE;
2224	req->hmac = true;
2225
2226	return 0;
2227}
2228
2229static int safexcel_hmac_sm3_digest(struct ahash_request *areq)
2230{
2231	int ret = safexcel_hmac_sm3_init(areq);
2232
2233	if (ret)
2234		return ret;
2235
2236	return safexcel_ahash_finup(areq);
2237}
2238
2239struct safexcel_alg_template safexcel_alg_hmac_sm3 = {
2240	.type = SAFEXCEL_ALG_TYPE_AHASH,
2241	.algo_mask = SAFEXCEL_ALG_SM3,
2242	.alg.ahash = {
2243		.init = safexcel_hmac_sm3_init,
2244		.update = safexcel_ahash_update,
2245		.final = safexcel_ahash_final,
2246		.finup = safexcel_ahash_finup,
2247		.digest = safexcel_hmac_sm3_digest,
2248		.setkey = safexcel_hmac_sm3_setkey,
2249		.export = safexcel_ahash_export,
2250		.import = safexcel_ahash_import,
2251		.halg = {
2252			.digestsize = SM3_DIGEST_SIZE,
2253			.statesize = sizeof(struct safexcel_ahash_export_state),
2254			.base = {
2255				.cra_name = "hmac(sm3)",
2256				.cra_driver_name = "safexcel-hmac-sm3",
2257				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2258				.cra_flags = CRYPTO_ALG_ASYNC |
2259					     CRYPTO_ALG_ALLOCATES_MEMORY |
2260					     CRYPTO_ALG_KERN_DRIVER_ONLY,
2261				.cra_blocksize = SM3_BLOCK_SIZE,
2262				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2263				.cra_init = safexcel_ahash_cra_init,
2264				.cra_exit = safexcel_ahash_cra_exit,
2265				.cra_module = THIS_MODULE,
2266			},
2267		},
2268	},
2269};
2270
2271static int safexcel_sha3_224_init(struct ahash_request *areq)
2272{
2273	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2274	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2275	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2276
2277	memset(req, 0, sizeof(*req));
2278
2279	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2280	req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2281	req->state_sz = SHA3_224_DIGEST_SIZE;
2282	req->digest_sz = SHA3_224_DIGEST_SIZE;
2283	req->block_sz = SHA3_224_BLOCK_SIZE;
2284	ctx->do_fallback = false;
2285	ctx->fb_init_done = false;
2286	return 0;
2287}
2288
2289static int safexcel_sha3_fbcheck(struct ahash_request *req)
2290{
2291	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2292	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2293	struct ahash_request *subreq = ahash_request_ctx_dma(req);
2294	int ret = 0;
2295
2296	if (ctx->do_fallback) {
2297		ahash_request_set_tfm(subreq, ctx->fback);
2298		ahash_request_set_callback(subreq, req->base.flags,
2299					   req->base.complete, req->base.data);
2300		ahash_request_set_crypt(subreq, req->src, req->result,
2301					req->nbytes);
2302		if (!ctx->fb_init_done) {
2303			if (ctx->fb_do_setkey) {
2304				/* Set fallback cipher HMAC key */
2305				u8 key[SHA3_224_BLOCK_SIZE];
2306
2307				memcpy(key, &ctx->base.ipad,
2308				       crypto_ahash_blocksize(ctx->fback) / 2);
2309				memcpy(key +
2310				       crypto_ahash_blocksize(ctx->fback) / 2,
2311				       &ctx->base.opad,
2312				       crypto_ahash_blocksize(ctx->fback) / 2);
2313				ret = crypto_ahash_setkey(ctx->fback, key,
2314					crypto_ahash_blocksize(ctx->fback));
2315				memzero_explicit(key,
2316					crypto_ahash_blocksize(ctx->fback));
2317				ctx->fb_do_setkey = false;
2318			}
2319			ret = ret ?: crypto_ahash_init(subreq);
2320			ctx->fb_init_done = true;
2321		}
2322	}
2323	return ret;
2324}
2325
2326static int safexcel_sha3_update(struct ahash_request *req)
2327{
2328	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2329	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2330	struct ahash_request *subreq = ahash_request_ctx_dma(req);
2331
2332	ctx->do_fallback = true;
2333	return safexcel_sha3_fbcheck(req) ?: crypto_ahash_update(subreq);
2334}
2335
2336static int safexcel_sha3_final(struct ahash_request *req)
2337{
2338	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2339	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2340	struct ahash_request *subreq = ahash_request_ctx_dma(req);
2341
2342	ctx->do_fallback = true;
2343	return safexcel_sha3_fbcheck(req) ?: crypto_ahash_final(subreq);
2344}
2345
2346static int safexcel_sha3_finup(struct ahash_request *req)
2347{
2348	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2349	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2350	struct ahash_request *subreq = ahash_request_ctx_dma(req);
2351
2352	ctx->do_fallback |= !req->nbytes;
2353	if (ctx->do_fallback)
2354		/* Update or ex/import happened or len 0, cannot use the HW */
2355		return safexcel_sha3_fbcheck(req) ?:
2356		       crypto_ahash_finup(subreq);
2357	else
2358		return safexcel_ahash_finup(req);
2359}
2360
2361static int safexcel_sha3_digest_fallback(struct ahash_request *req)
2362{
2363	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2364	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2365	struct ahash_request *subreq = ahash_request_ctx_dma(req);
2366
2367	ctx->do_fallback = true;
2368	ctx->fb_init_done = false;
2369	return safexcel_sha3_fbcheck(req) ?: crypto_ahash_finup(subreq);
2370}
2371
2372static int safexcel_sha3_224_digest(struct ahash_request *req)
2373{
2374	if (req->nbytes)
2375		return safexcel_sha3_224_init(req) ?: safexcel_ahash_finup(req);
2376
2377	/* HW cannot do zero length hash, use fallback instead */
2378	return safexcel_sha3_digest_fallback(req);
2379}
2380
2381static int safexcel_sha3_export(struct ahash_request *req, void *out)
2382{
2383	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2384	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2385	struct ahash_request *subreq = ahash_request_ctx_dma(req);
2386
2387	ctx->do_fallback = true;
2388	return safexcel_sha3_fbcheck(req) ?: crypto_ahash_export(subreq, out);
2389}
2390
2391static int safexcel_sha3_import(struct ahash_request *req, const void *in)
2392{
2393	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2394	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2395	struct ahash_request *subreq = ahash_request_ctx_dma(req);
2396
2397	ctx->do_fallback = true;
2398	return safexcel_sha3_fbcheck(req) ?: crypto_ahash_import(subreq, in);
2399	// return safexcel_ahash_import(req, in);
2400}
2401
2402static int safexcel_sha3_cra_init(struct crypto_tfm *tfm)
2403{
2404	struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
2405	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2406
2407	safexcel_ahash_cra_init(tfm);
2408
2409	/* Allocate fallback implementation */
2410	ctx->fback = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0,
2411					CRYPTO_ALG_ASYNC |
2412					CRYPTO_ALG_NEED_FALLBACK);
2413	if (IS_ERR(ctx->fback))
2414		return PTR_ERR(ctx->fback);
2415
2416	/* Update statesize from fallback algorithm! */
2417	crypto_hash_alg_common(ahash)->statesize =
2418		crypto_ahash_statesize(ctx->fback);
2419	crypto_ahash_set_reqsize_dma(
2420		ahash, max(sizeof(struct safexcel_ahash_req),
2421			   sizeof(struct ahash_request) +
2422			   crypto_ahash_reqsize(ctx->fback)));
2423	return 0;
2424}
2425
2426static void safexcel_sha3_cra_exit(struct crypto_tfm *tfm)
2427{
2428	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2429
2430	crypto_free_ahash(ctx->fback);
2431	safexcel_ahash_cra_exit(tfm);
2432}
2433
2434struct safexcel_alg_template safexcel_alg_sha3_224 = {
2435	.type = SAFEXCEL_ALG_TYPE_AHASH,
2436	.algo_mask = SAFEXCEL_ALG_SHA3,
2437	.alg.ahash = {
2438		.init = safexcel_sha3_224_init,
2439		.update = safexcel_sha3_update,
2440		.final = safexcel_sha3_final,
2441		.finup = safexcel_sha3_finup,
2442		.digest = safexcel_sha3_224_digest,
2443		.export = safexcel_sha3_export,
2444		.import = safexcel_sha3_import,
2445		.halg = {
2446			.digestsize = SHA3_224_DIGEST_SIZE,
2447			.statesize = sizeof(struct safexcel_ahash_export_state),
2448			.base = {
2449				.cra_name = "sha3-224",
2450				.cra_driver_name = "safexcel-sha3-224",
2451				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2452				.cra_flags = CRYPTO_ALG_ASYNC |
2453					     CRYPTO_ALG_KERN_DRIVER_ONLY |
2454					     CRYPTO_ALG_NEED_FALLBACK,
2455				.cra_blocksize = SHA3_224_BLOCK_SIZE,
2456				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2457				.cra_init = safexcel_sha3_cra_init,
2458				.cra_exit = safexcel_sha3_cra_exit,
2459				.cra_module = THIS_MODULE,
2460			},
2461		},
2462	},
2463};
2464
2465static int safexcel_sha3_256_init(struct ahash_request *areq)
2466{
2467	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2468	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2469	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2470
2471	memset(req, 0, sizeof(*req));
2472
2473	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2474	req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2475	req->state_sz = SHA3_256_DIGEST_SIZE;
2476	req->digest_sz = SHA3_256_DIGEST_SIZE;
2477	req->block_sz = SHA3_256_BLOCK_SIZE;
2478	ctx->do_fallback = false;
2479	ctx->fb_init_done = false;
2480	return 0;
2481}
2482
2483static int safexcel_sha3_256_digest(struct ahash_request *req)
2484{
2485	if (req->nbytes)
2486		return safexcel_sha3_256_init(req) ?: safexcel_ahash_finup(req);
2487
2488	/* HW cannot do zero length hash, use fallback instead */
2489	return safexcel_sha3_digest_fallback(req);
2490}
2491
2492struct safexcel_alg_template safexcel_alg_sha3_256 = {
2493	.type = SAFEXCEL_ALG_TYPE_AHASH,
2494	.algo_mask = SAFEXCEL_ALG_SHA3,
2495	.alg.ahash = {
2496		.init = safexcel_sha3_256_init,
2497		.update = safexcel_sha3_update,
2498		.final = safexcel_sha3_final,
2499		.finup = safexcel_sha3_finup,
2500		.digest = safexcel_sha3_256_digest,
2501		.export = safexcel_sha3_export,
2502		.import = safexcel_sha3_import,
2503		.halg = {
2504			.digestsize = SHA3_256_DIGEST_SIZE,
2505			.statesize = sizeof(struct safexcel_ahash_export_state),
2506			.base = {
2507				.cra_name = "sha3-256",
2508				.cra_driver_name = "safexcel-sha3-256",
2509				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2510				.cra_flags = CRYPTO_ALG_ASYNC |
2511					     CRYPTO_ALG_KERN_DRIVER_ONLY |
2512					     CRYPTO_ALG_NEED_FALLBACK,
2513				.cra_blocksize = SHA3_256_BLOCK_SIZE,
2514				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2515				.cra_init = safexcel_sha3_cra_init,
2516				.cra_exit = safexcel_sha3_cra_exit,
2517				.cra_module = THIS_MODULE,
2518			},
2519		},
2520	},
2521};
2522
2523static int safexcel_sha3_384_init(struct ahash_request *areq)
2524{
2525	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2526	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2527	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2528
2529	memset(req, 0, sizeof(*req));
2530
2531	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
2532	req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2533	req->state_sz = SHA3_384_DIGEST_SIZE;
2534	req->digest_sz = SHA3_384_DIGEST_SIZE;
2535	req->block_sz = SHA3_384_BLOCK_SIZE;
2536	ctx->do_fallback = false;
2537	ctx->fb_init_done = false;
2538	return 0;
2539}
2540
2541static int safexcel_sha3_384_digest(struct ahash_request *req)
2542{
2543	if (req->nbytes)
2544		return safexcel_sha3_384_init(req) ?: safexcel_ahash_finup(req);
2545
2546	/* HW cannot do zero length hash, use fallback instead */
2547	return safexcel_sha3_digest_fallback(req);
2548}
2549
2550struct safexcel_alg_template safexcel_alg_sha3_384 = {
2551	.type = SAFEXCEL_ALG_TYPE_AHASH,
2552	.algo_mask = SAFEXCEL_ALG_SHA3,
2553	.alg.ahash = {
2554		.init = safexcel_sha3_384_init,
2555		.update = safexcel_sha3_update,
2556		.final = safexcel_sha3_final,
2557		.finup = safexcel_sha3_finup,
2558		.digest = safexcel_sha3_384_digest,
2559		.export = safexcel_sha3_export,
2560		.import = safexcel_sha3_import,
2561		.halg = {
2562			.digestsize = SHA3_384_DIGEST_SIZE,
2563			.statesize = sizeof(struct safexcel_ahash_export_state),
2564			.base = {
2565				.cra_name = "sha3-384",
2566				.cra_driver_name = "safexcel-sha3-384",
2567				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2568				.cra_flags = CRYPTO_ALG_ASYNC |
2569					     CRYPTO_ALG_KERN_DRIVER_ONLY |
2570					     CRYPTO_ALG_NEED_FALLBACK,
2571				.cra_blocksize = SHA3_384_BLOCK_SIZE,
2572				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2573				.cra_init = safexcel_sha3_cra_init,
2574				.cra_exit = safexcel_sha3_cra_exit,
2575				.cra_module = THIS_MODULE,
2576			},
2577		},
2578	},
2579};
2580
2581static int safexcel_sha3_512_init(struct ahash_request *areq)
2582{
2583	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2584	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2585	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2586
2587	memset(req, 0, sizeof(*req));
2588
2589	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
2590	req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2591	req->state_sz = SHA3_512_DIGEST_SIZE;
2592	req->digest_sz = SHA3_512_DIGEST_SIZE;
2593	req->block_sz = SHA3_512_BLOCK_SIZE;
2594	ctx->do_fallback = false;
2595	ctx->fb_init_done = false;
2596	return 0;
2597}
2598
2599static int safexcel_sha3_512_digest(struct ahash_request *req)
2600{
2601	if (req->nbytes)
2602		return safexcel_sha3_512_init(req) ?: safexcel_ahash_finup(req);
2603
2604	/* HW cannot do zero length hash, use fallback instead */
2605	return safexcel_sha3_digest_fallback(req);
2606}
2607
2608struct safexcel_alg_template safexcel_alg_sha3_512 = {
2609	.type = SAFEXCEL_ALG_TYPE_AHASH,
2610	.algo_mask = SAFEXCEL_ALG_SHA3,
2611	.alg.ahash = {
2612		.init = safexcel_sha3_512_init,
2613		.update = safexcel_sha3_update,
2614		.final = safexcel_sha3_final,
2615		.finup = safexcel_sha3_finup,
2616		.digest = safexcel_sha3_512_digest,
2617		.export = safexcel_sha3_export,
2618		.import = safexcel_sha3_import,
2619		.halg = {
2620			.digestsize = SHA3_512_DIGEST_SIZE,
2621			.statesize = sizeof(struct safexcel_ahash_export_state),
2622			.base = {
2623				.cra_name = "sha3-512",
2624				.cra_driver_name = "safexcel-sha3-512",
2625				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2626				.cra_flags = CRYPTO_ALG_ASYNC |
2627					     CRYPTO_ALG_KERN_DRIVER_ONLY |
2628					     CRYPTO_ALG_NEED_FALLBACK,
2629				.cra_blocksize = SHA3_512_BLOCK_SIZE,
2630				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2631				.cra_init = safexcel_sha3_cra_init,
2632				.cra_exit = safexcel_sha3_cra_exit,
2633				.cra_module = THIS_MODULE,
2634			},
2635		},
2636	},
2637};
2638
2639static int safexcel_hmac_sha3_cra_init(struct crypto_tfm *tfm, const char *alg)
2640{
2641	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2642	int ret;
2643
2644	ret = safexcel_sha3_cra_init(tfm);
2645	if (ret)
2646		return ret;
2647
2648	/* Allocate precalc basic digest implementation */
2649	ctx->shpre = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
2650	if (IS_ERR(ctx->shpre))
2651		return PTR_ERR(ctx->shpre);
2652
2653	ctx->shdesc = kmalloc(sizeof(*ctx->shdesc) +
2654			      crypto_shash_descsize(ctx->shpre), GFP_KERNEL);
2655	if (!ctx->shdesc) {
2656		crypto_free_shash(ctx->shpre);
2657		return -ENOMEM;
2658	}
2659	ctx->shdesc->tfm = ctx->shpre;
2660	return 0;
2661}
2662
2663static void safexcel_hmac_sha3_cra_exit(struct crypto_tfm *tfm)
2664{
2665	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2666
2667	crypto_free_ahash(ctx->fback);
2668	crypto_free_shash(ctx->shpre);
2669	kfree(ctx->shdesc);
2670	safexcel_ahash_cra_exit(tfm);
2671}
2672
2673static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key,
2674				     unsigned int keylen)
2675{
2676	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2677	int ret = 0;
2678
2679	if (keylen > crypto_ahash_blocksize(tfm)) {
2680		/*
2681		 * If the key is larger than the blocksize, then hash it
2682		 * first using our fallback cipher
2683		 */
2684		ret = crypto_shash_digest(ctx->shdesc, key, keylen,
2685					  ctx->base.ipad.byte);
2686		keylen = crypto_shash_digestsize(ctx->shpre);
2687
2688		/*
2689		 * If the digest is larger than half the blocksize, we need to
2690		 * move the rest to opad due to the way our HMAC infra works.
2691		 */
2692		if (keylen > crypto_ahash_blocksize(tfm) / 2)
2693			/* Buffers overlap, need to use memmove iso memcpy! */
2694			memmove(&ctx->base.opad,
2695				ctx->base.ipad.byte +
2696					crypto_ahash_blocksize(tfm) / 2,
2697				keylen - crypto_ahash_blocksize(tfm) / 2);
2698	} else {
2699		/*
2700		 * Copy the key to our ipad & opad buffers
2701		 * Note that ipad and opad each contain one half of the key,
2702		 * to match the existing HMAC driver infrastructure.
2703		 */
2704		if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2705			memcpy(&ctx->base.ipad, key, keylen);
2706		} else {
2707			memcpy(&ctx->base.ipad, key,
2708			       crypto_ahash_blocksize(tfm) / 2);
2709			memcpy(&ctx->base.opad,
2710			       key + crypto_ahash_blocksize(tfm) / 2,
2711			       keylen - crypto_ahash_blocksize(tfm) / 2);
2712		}
2713	}
2714
2715	/* Pad key with zeroes */
2716	if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2717		memset(ctx->base.ipad.byte + keylen, 0,
2718		       crypto_ahash_blocksize(tfm) / 2 - keylen);
2719		memset(&ctx->base.opad, 0, crypto_ahash_blocksize(tfm) / 2);
2720	} else {
2721		memset(ctx->base.opad.byte + keylen -
2722		       crypto_ahash_blocksize(tfm) / 2, 0,
2723		       crypto_ahash_blocksize(tfm) - keylen);
2724	}
2725
2726	/* If doing fallback, still need to set the new key! */
2727	ctx->fb_do_setkey = true;
2728	return ret;
2729}
2730
2731static int safexcel_hmac_sha3_224_init(struct ahash_request *areq)
2732{
2733	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2734	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2735	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2736
2737	memset(req, 0, sizeof(*req));
2738
2739	/* Copy (half of) the key */
2740	memcpy(req->state, &ctx->base.ipad, SHA3_224_BLOCK_SIZE / 2);
2741	/* Start of HMAC should have len == processed == blocksize */
2742	req->len	= SHA3_224_BLOCK_SIZE;
2743	req->processed	= SHA3_224_BLOCK_SIZE;
2744	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2745	req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2746	req->state_sz = SHA3_224_BLOCK_SIZE / 2;
2747	req->digest_sz = SHA3_224_DIGEST_SIZE;
2748	req->block_sz = SHA3_224_BLOCK_SIZE;
2749	req->hmac = true;
2750	ctx->do_fallback = false;
2751	ctx->fb_init_done = false;
2752	return 0;
2753}
2754
2755static int safexcel_hmac_sha3_224_digest(struct ahash_request *req)
2756{
2757	if (req->nbytes)
2758		return safexcel_hmac_sha3_224_init(req) ?:
2759		       safexcel_ahash_finup(req);
2760
2761	/* HW cannot do zero length HMAC, use fallback instead */
2762	return safexcel_sha3_digest_fallback(req);
2763}
2764
2765static int safexcel_hmac_sha3_224_cra_init(struct crypto_tfm *tfm)
2766{
2767	return safexcel_hmac_sha3_cra_init(tfm, "sha3-224");
2768}
2769
2770struct safexcel_alg_template safexcel_alg_hmac_sha3_224 = {
2771	.type = SAFEXCEL_ALG_TYPE_AHASH,
2772	.algo_mask = SAFEXCEL_ALG_SHA3,
2773	.alg.ahash = {
2774		.init = safexcel_hmac_sha3_224_init,
2775		.update = safexcel_sha3_update,
2776		.final = safexcel_sha3_final,
2777		.finup = safexcel_sha3_finup,
2778		.digest = safexcel_hmac_sha3_224_digest,
2779		.setkey = safexcel_hmac_sha3_setkey,
2780		.export = safexcel_sha3_export,
2781		.import = safexcel_sha3_import,
2782		.halg = {
2783			.digestsize = SHA3_224_DIGEST_SIZE,
2784			.statesize = sizeof(struct safexcel_ahash_export_state),
2785			.base = {
2786				.cra_name = "hmac(sha3-224)",
2787				.cra_driver_name = "safexcel-hmac-sha3-224",
2788				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2789				.cra_flags = CRYPTO_ALG_ASYNC |
2790					     CRYPTO_ALG_KERN_DRIVER_ONLY |
2791					     CRYPTO_ALG_NEED_FALLBACK,
2792				.cra_blocksize = SHA3_224_BLOCK_SIZE,
2793				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2794				.cra_init = safexcel_hmac_sha3_224_cra_init,
2795				.cra_exit = safexcel_hmac_sha3_cra_exit,
2796				.cra_module = THIS_MODULE,
2797			},
2798		},
2799	},
2800};
2801
2802static int safexcel_hmac_sha3_256_init(struct ahash_request *areq)
2803{
2804	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2805	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2806	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2807
2808	memset(req, 0, sizeof(*req));
2809
2810	/* Copy (half of) the key */
2811	memcpy(req->state, &ctx->base.ipad, SHA3_256_BLOCK_SIZE / 2);
2812	/* Start of HMAC should have len == processed == blocksize */
2813	req->len	= SHA3_256_BLOCK_SIZE;
2814	req->processed	= SHA3_256_BLOCK_SIZE;
2815	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2816	req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2817	req->state_sz = SHA3_256_BLOCK_SIZE / 2;
2818	req->digest_sz = SHA3_256_DIGEST_SIZE;
2819	req->block_sz = SHA3_256_BLOCK_SIZE;
2820	req->hmac = true;
2821	ctx->do_fallback = false;
2822	ctx->fb_init_done = false;
2823	return 0;
2824}
2825
2826static int safexcel_hmac_sha3_256_digest(struct ahash_request *req)
2827{
2828	if (req->nbytes)
2829		return safexcel_hmac_sha3_256_init(req) ?:
2830		       safexcel_ahash_finup(req);
2831
2832	/* HW cannot do zero length HMAC, use fallback instead */
2833	return safexcel_sha3_digest_fallback(req);
2834}
2835
2836static int safexcel_hmac_sha3_256_cra_init(struct crypto_tfm *tfm)
2837{
2838	return safexcel_hmac_sha3_cra_init(tfm, "sha3-256");
2839}
2840
2841struct safexcel_alg_template safexcel_alg_hmac_sha3_256 = {
2842	.type = SAFEXCEL_ALG_TYPE_AHASH,
2843	.algo_mask = SAFEXCEL_ALG_SHA3,
2844	.alg.ahash = {
2845		.init = safexcel_hmac_sha3_256_init,
2846		.update = safexcel_sha3_update,
2847		.final = safexcel_sha3_final,
2848		.finup = safexcel_sha3_finup,
2849		.digest = safexcel_hmac_sha3_256_digest,
2850		.setkey = safexcel_hmac_sha3_setkey,
2851		.export = safexcel_sha3_export,
2852		.import = safexcel_sha3_import,
2853		.halg = {
2854			.digestsize = SHA3_256_DIGEST_SIZE,
2855			.statesize = sizeof(struct safexcel_ahash_export_state),
2856			.base = {
2857				.cra_name = "hmac(sha3-256)",
2858				.cra_driver_name = "safexcel-hmac-sha3-256",
2859				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2860				.cra_flags = CRYPTO_ALG_ASYNC |
2861					     CRYPTO_ALG_KERN_DRIVER_ONLY |
2862					     CRYPTO_ALG_NEED_FALLBACK,
2863				.cra_blocksize = SHA3_256_BLOCK_SIZE,
2864				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2865				.cra_init = safexcel_hmac_sha3_256_cra_init,
2866				.cra_exit = safexcel_hmac_sha3_cra_exit,
2867				.cra_module = THIS_MODULE,
2868			},
2869		},
2870	},
2871};
2872
2873static int safexcel_hmac_sha3_384_init(struct ahash_request *areq)
2874{
2875	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2876	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2877	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2878
2879	memset(req, 0, sizeof(*req));
2880
2881	/* Copy (half of) the key */
2882	memcpy(req->state, &ctx->base.ipad, SHA3_384_BLOCK_SIZE / 2);
2883	/* Start of HMAC should have len == processed == blocksize */
2884	req->len	= SHA3_384_BLOCK_SIZE;
2885	req->processed	= SHA3_384_BLOCK_SIZE;
2886	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
2887	req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2888	req->state_sz = SHA3_384_BLOCK_SIZE / 2;
2889	req->digest_sz = SHA3_384_DIGEST_SIZE;
2890	req->block_sz = SHA3_384_BLOCK_SIZE;
2891	req->hmac = true;
2892	ctx->do_fallback = false;
2893	ctx->fb_init_done = false;
2894	return 0;
2895}
2896
2897static int safexcel_hmac_sha3_384_digest(struct ahash_request *req)
2898{
2899	if (req->nbytes)
2900		return safexcel_hmac_sha3_384_init(req) ?:
2901		       safexcel_ahash_finup(req);
2902
2903	/* HW cannot do zero length HMAC, use fallback instead */
2904	return safexcel_sha3_digest_fallback(req);
2905}
2906
2907static int safexcel_hmac_sha3_384_cra_init(struct crypto_tfm *tfm)
2908{
2909	return safexcel_hmac_sha3_cra_init(tfm, "sha3-384");
2910}
2911
2912struct safexcel_alg_template safexcel_alg_hmac_sha3_384 = {
2913	.type = SAFEXCEL_ALG_TYPE_AHASH,
2914	.algo_mask = SAFEXCEL_ALG_SHA3,
2915	.alg.ahash = {
2916		.init = safexcel_hmac_sha3_384_init,
2917		.update = safexcel_sha3_update,
2918		.final = safexcel_sha3_final,
2919		.finup = safexcel_sha3_finup,
2920		.digest = safexcel_hmac_sha3_384_digest,
2921		.setkey = safexcel_hmac_sha3_setkey,
2922		.export = safexcel_sha3_export,
2923		.import = safexcel_sha3_import,
2924		.halg = {
2925			.digestsize = SHA3_384_DIGEST_SIZE,
2926			.statesize = sizeof(struct safexcel_ahash_export_state),
2927			.base = {
2928				.cra_name = "hmac(sha3-384)",
2929				.cra_driver_name = "safexcel-hmac-sha3-384",
2930				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2931				.cra_flags = CRYPTO_ALG_ASYNC |
2932					     CRYPTO_ALG_KERN_DRIVER_ONLY |
2933					     CRYPTO_ALG_NEED_FALLBACK,
2934				.cra_blocksize = SHA3_384_BLOCK_SIZE,
2935				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2936				.cra_init = safexcel_hmac_sha3_384_cra_init,
2937				.cra_exit = safexcel_hmac_sha3_cra_exit,
2938				.cra_module = THIS_MODULE,
2939			},
2940		},
2941	},
2942};
2943
2944static int safexcel_hmac_sha3_512_init(struct ahash_request *areq)
2945{
2946	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2947	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2948	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2949
2950	memset(req, 0, sizeof(*req));
2951
2952	/* Copy (half of) the key */
2953	memcpy(req->state, &ctx->base.ipad, SHA3_512_BLOCK_SIZE / 2);
2954	/* Start of HMAC should have len == processed == blocksize */
2955	req->len	= SHA3_512_BLOCK_SIZE;
2956	req->processed	= SHA3_512_BLOCK_SIZE;
2957	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
2958	req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2959	req->state_sz = SHA3_512_BLOCK_SIZE / 2;
2960	req->digest_sz = SHA3_512_DIGEST_SIZE;
2961	req->block_sz = SHA3_512_BLOCK_SIZE;
2962	req->hmac = true;
2963	ctx->do_fallback = false;
2964	ctx->fb_init_done = false;
2965	return 0;
2966}
2967
2968static int safexcel_hmac_sha3_512_digest(struct ahash_request *req)
2969{
2970	if (req->nbytes)
2971		return safexcel_hmac_sha3_512_init(req) ?:
2972		       safexcel_ahash_finup(req);
2973
2974	/* HW cannot do zero length HMAC, use fallback instead */
2975	return safexcel_sha3_digest_fallback(req);
2976}
2977
2978static int safexcel_hmac_sha3_512_cra_init(struct crypto_tfm *tfm)
2979{
2980	return safexcel_hmac_sha3_cra_init(tfm, "sha3-512");
2981}
2982struct safexcel_alg_template safexcel_alg_hmac_sha3_512 = {
2983	.type = SAFEXCEL_ALG_TYPE_AHASH,
2984	.algo_mask = SAFEXCEL_ALG_SHA3,
2985	.alg.ahash = {
2986		.init = safexcel_hmac_sha3_512_init,
2987		.update = safexcel_sha3_update,
2988		.final = safexcel_sha3_final,
2989		.finup = safexcel_sha3_finup,
2990		.digest = safexcel_hmac_sha3_512_digest,
2991		.setkey = safexcel_hmac_sha3_setkey,
2992		.export = safexcel_sha3_export,
2993		.import = safexcel_sha3_import,
2994		.halg = {
2995			.digestsize = SHA3_512_DIGEST_SIZE,
2996			.statesize = sizeof(struct safexcel_ahash_export_state),
2997			.base = {
2998				.cra_name = "hmac(sha3-512)",
2999				.cra_driver_name = "safexcel-hmac-sha3-512",
3000				.cra_priority = SAFEXCEL_CRA_PRIORITY,
3001				.cra_flags = CRYPTO_ALG_ASYNC |
3002					     CRYPTO_ALG_KERN_DRIVER_ONLY |
3003					     CRYPTO_ALG_NEED_FALLBACK,
3004				.cra_blocksize = SHA3_512_BLOCK_SIZE,
3005				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
3006				.cra_init = safexcel_hmac_sha3_512_cra_init,
3007				.cra_exit = safexcel_hmac_sha3_cra_exit,
3008				.cra_module = THIS_MODULE,
3009			},
3010		},
3011	},
3012};