jh7110-aes.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * StarFive AES acceleration driver
   4 *
   5 * Copyright (c) 2022 StarFive Technology
   6 */
   7
   8#include <crypto/engine.h>
   9#include <crypto/gcm.h>
  10#include <crypto/internal/aead.h>
  11#include <crypto/internal/skcipher.h>
  12#include <crypto/scatterwalk.h>
  13#include "jh7110-cryp.h"
  14#include <linux/err.h>
  15#include <linux/iopoll.h>
  16#include <linux/kernel.h>
  17#include <linux/slab.h>
  18#include <linux/string.h>
  19
  20#define STARFIVE_AES_REGS_OFFSET	0x100
  21#define STARFIVE_AES_AESDIO0R		(STARFIVE_AES_REGS_OFFSET + 0x0)
  22#define STARFIVE_AES_KEY0		(STARFIVE_AES_REGS_OFFSET + 0x4)
  23#define STARFIVE_AES_KEY1		(STARFIVE_AES_REGS_OFFSET + 0x8)
  24#define STARFIVE_AES_KEY2		(STARFIVE_AES_REGS_OFFSET + 0xC)
  25#define STARFIVE_AES_KEY3		(STARFIVE_AES_REGS_OFFSET + 0x10)
  26#define STARFIVE_AES_KEY4		(STARFIVE_AES_REGS_OFFSET + 0x14)
  27#define STARFIVE_AES_KEY5		(STARFIVE_AES_REGS_OFFSET + 0x18)
  28#define STARFIVE_AES_KEY6		(STARFIVE_AES_REGS_OFFSET + 0x1C)
  29#define STARFIVE_AES_KEY7		(STARFIVE_AES_REGS_OFFSET + 0x20)
  30#define STARFIVE_AES_CSR		(STARFIVE_AES_REGS_OFFSET + 0x24)
  31#define STARFIVE_AES_IV0		(STARFIVE_AES_REGS_OFFSET + 0x28)
  32#define STARFIVE_AES_IV1		(STARFIVE_AES_REGS_OFFSET + 0x2C)
  33#define STARFIVE_AES_IV2		(STARFIVE_AES_REGS_OFFSET + 0x30)
  34#define STARFIVE_AES_IV3		(STARFIVE_AES_REGS_OFFSET + 0x34)
  35#define STARFIVE_AES_NONCE0		(STARFIVE_AES_REGS_OFFSET + 0x3C)
  36#define STARFIVE_AES_NONCE1		(STARFIVE_AES_REGS_OFFSET + 0x40)
  37#define STARFIVE_AES_NONCE2		(STARFIVE_AES_REGS_OFFSET + 0x44)
  38#define STARFIVE_AES_NONCE3		(STARFIVE_AES_REGS_OFFSET + 0x48)
  39#define STARFIVE_AES_ALEN0		(STARFIVE_AES_REGS_OFFSET + 0x4C)
  40#define STARFIVE_AES_ALEN1		(STARFIVE_AES_REGS_OFFSET + 0x50)
  41#define STARFIVE_AES_MLEN0		(STARFIVE_AES_REGS_OFFSET + 0x54)
  42#define STARFIVE_AES_MLEN1		(STARFIVE_AES_REGS_OFFSET + 0x58)
  43#define STARFIVE_AES_IVLEN		(STARFIVE_AES_REGS_OFFSET + 0x5C)
  44
  45#define FLG_MODE_MASK			GENMASK(2, 0)
  46#define FLG_ENCRYPT			BIT(4)
  47
  48/* Misc */
  49#define CCM_B0_ADATA			0x40
  50#define AES_BLOCK_32			(AES_BLOCK_SIZE / sizeof(u32))
  51
  52static inline int starfive_aes_wait_busy(struct starfive_cryp_dev *cryp)
  53{
  54	u32 status;
  55
  56	return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,
  57					  !(status & STARFIVE_AES_BUSY), 10, 100000);
  58}
  59
  60static inline int starfive_aes_wait_keydone(struct starfive_cryp_dev *cryp)
  61{
  62	u32 status;
  63
  64	return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,
  65					  (status & STARFIVE_AES_KEY_DONE), 10, 100000);
  66}
  67
  68static inline int starfive_aes_wait_gcmdone(struct starfive_cryp_dev *cryp)
  69{
  70	u32 status;
  71
  72	return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,
  73					  (status & STARFIVE_AES_GCM_DONE), 10, 100000);
  74}
  75
  76static inline int is_gcm(struct starfive_cryp_dev *cryp)
  77{
  78	return (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM;
  79}
  80
  81static inline bool is_encrypt(struct starfive_cryp_dev *cryp)
  82{
  83	return cryp->flags & FLG_ENCRYPT;
  84}
  85
  86static void starfive_aes_aead_hw_start(struct starfive_cryp_ctx *ctx, u32 hw_mode)
  87{
  88	struct starfive_cryp_dev *cryp = ctx->cryp;
  89	unsigned int value;
  90
  91	switch (hw_mode) {
  92	case STARFIVE_AES_MODE_GCM:
  93		value = readl(ctx->cryp->base + STARFIVE_AES_CSR);
  94		value |= STARFIVE_AES_GCM_START;
  95		writel(value, cryp->base + STARFIVE_AES_CSR);
  96		starfive_aes_wait_gcmdone(cryp);
  97		break;
  98	case STARFIVE_AES_MODE_CCM:
  99		value = readl(ctx->cryp->base + STARFIVE_AES_CSR);
 100		value |= STARFIVE_AES_CCM_START;
 101		writel(value, cryp->base + STARFIVE_AES_CSR);
 102		break;
 103	}
 104}
 105
 106static inline void starfive_aes_set_alen(struct starfive_cryp_ctx *ctx)
 107{
 108	struct starfive_cryp_dev *cryp = ctx->cryp;
 109
 110	writel(upper_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN0);
 111	writel(lower_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN1);
 112}
 113
 114static inline void starfive_aes_set_mlen(struct starfive_cryp_ctx *ctx)
 115{
 116	struct starfive_cryp_dev *cryp = ctx->cryp;
 117
 118	writel(upper_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN0);
 119	writel(lower_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN1);
 120}
 121
 122static inline int starfive_aes_ccm_check_iv(const u8 *iv)
 123{
 124	/* 2 <= L <= 8, so 1 <= L' <= 7. */
 125	if (iv[0] < 1 || iv[0] > 7)
 126		return -EINVAL;
 127
 128	return 0;
 129}
 130
 131static int starfive_aes_write_iv(struct starfive_cryp_ctx *ctx, u32 *iv)
 132{
 133	struct starfive_cryp_dev *cryp = ctx->cryp;
 134
 135	writel(iv[0], cryp->base + STARFIVE_AES_IV0);
 136	writel(iv[1], cryp->base + STARFIVE_AES_IV1);
 137	writel(iv[2], cryp->base + STARFIVE_AES_IV2);
 138
 139	if (is_gcm(cryp)) {
 140		if (starfive_aes_wait_gcmdone(cryp))
 141			return -ETIMEDOUT;
 142
 143		return 0;
 144	}
 145
 146	writel(iv[3], cryp->base + STARFIVE_AES_IV3);
 147
 148	return 0;
 149}
 150
 151static inline void starfive_aes_get_iv(struct starfive_cryp_dev *cryp, u32 *iv)
 152{
 153	iv[0] = readl(cryp->base + STARFIVE_AES_IV0);
 154	iv[1] = readl(cryp->base + STARFIVE_AES_IV1);
 155	iv[2] = readl(cryp->base + STARFIVE_AES_IV2);
 156	iv[3] = readl(cryp->base + STARFIVE_AES_IV3);
 157}
 158
 159static inline void starfive_aes_write_nonce(struct starfive_cryp_ctx *ctx, u32 *nonce)
 160{
 161	struct starfive_cryp_dev *cryp = ctx->cryp;
 162
 163	writel(nonce[0], cryp->base + STARFIVE_AES_NONCE0);
 164	writel(nonce[1], cryp->base + STARFIVE_AES_NONCE1);
 165	writel(nonce[2], cryp->base + STARFIVE_AES_NONCE2);
 166	writel(nonce[3], cryp->base + STARFIVE_AES_NONCE3);
 167}
 168
 169static int starfive_aes_write_key(struct starfive_cryp_ctx *ctx)
 170{
 171	struct starfive_cryp_dev *cryp = ctx->cryp;
 172	u32 *key = (u32 *)ctx->key;
 173
 174	if (ctx->keylen >= AES_KEYSIZE_128) {
 175		writel(key[0], cryp->base + STARFIVE_AES_KEY0);
 176		writel(key[1], cryp->base + STARFIVE_AES_KEY1);
 177		writel(key[2], cryp->base + STARFIVE_AES_KEY2);
 178		writel(key[3], cryp->base + STARFIVE_AES_KEY3);
 179	}
 180
 181	if (ctx->keylen >= AES_KEYSIZE_192) {
 182		writel(key[4], cryp->base + STARFIVE_AES_KEY4);
 183		writel(key[5], cryp->base + STARFIVE_AES_KEY5);
 184	}
 185
 186	if (ctx->keylen >= AES_KEYSIZE_256) {
 187		writel(key[6], cryp->base + STARFIVE_AES_KEY6);
 188		writel(key[7], cryp->base + STARFIVE_AES_KEY7);
 189	}
 190
 191	if (starfive_aes_wait_keydone(cryp))
 192		return -ETIMEDOUT;
 193
 194	return 0;
 195}
 196
 197static int starfive_aes_ccm_init(struct starfive_cryp_ctx *ctx)
 198{
 199	struct starfive_cryp_dev *cryp = ctx->cryp;
 200	u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE];
 201	unsigned int textlen;
 202
 203	memcpy(iv, cryp->req.areq->iv, AES_BLOCK_SIZE);
 204	memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
 205
 206	/* Build B0 */
 207	memcpy(b0, iv, AES_BLOCK_SIZE);
 208
 209	b0[0] |= (8 * ((cryp->authsize - 2) / 2));
 210
 211	if (cryp->assoclen)
 212		b0[0] |= CCM_B0_ADATA;
 213
 214	textlen = cryp->total_in;
 215
 216	b0[AES_BLOCK_SIZE - 2] = textlen >> 8;
 217	b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF;
 218
 219	starfive_aes_write_nonce(ctx, (u32 *)b0);
 220
 221	return 0;
 222}
 223
 224static int starfive_aes_hw_init(struct starfive_cryp_ctx *ctx)
 225{
 226	struct starfive_cryp_request_ctx *rctx = ctx->rctx;
 227	struct starfive_cryp_dev *cryp = ctx->cryp;
 228	u32 hw_mode;
 229
 230	/* reset */
 231	rctx->csr.aes.v = 0;
 232	rctx->csr.aes.aesrst = 1;
 233	writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR);
 234
 235	/* csr setup */
 236	hw_mode = cryp->flags & FLG_MODE_MASK;
 237
 238	rctx->csr.aes.v = 0;
 239
 240	switch (ctx->keylen) {
 241	case AES_KEYSIZE_128:
 242		rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_128;
 243		break;
 244	case AES_KEYSIZE_192:
 245		rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_192;
 246		break;
 247	case AES_KEYSIZE_256:
 248		rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_256;
 249		break;
 250	}
 251
 252	rctx->csr.aes.mode  = hw_mode;
 253	rctx->csr.aes.cmode = !is_encrypt(cryp);
 254	rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_1;
 255
 256	if (cryp->side_chan) {
 257		rctx->csr.aes.delay_aes = 1;
 258		rctx->csr.aes.vaes_start = 1;
 259	}
 260
 261	writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR);
 262
 263	cryp->err = starfive_aes_write_key(ctx);
 264	if (cryp->err)
 265		return cryp->err;
 266
 267	switch (hw_mode) {
 268	case STARFIVE_AES_MODE_GCM:
 269		starfive_aes_set_alen(ctx);
 270		starfive_aes_set_mlen(ctx);
 271		writel(GCM_AES_IV_SIZE, cryp->base + STARFIVE_AES_IVLEN);
 272		starfive_aes_aead_hw_start(ctx, hw_mode);
 273		starfive_aes_write_iv(ctx, (void *)cryp->req.areq->iv);
 274		break;
 275	case STARFIVE_AES_MODE_CCM:
 276		starfive_aes_set_alen(ctx);
 277		starfive_aes_set_mlen(ctx);
 278		starfive_aes_ccm_init(ctx);
 279		starfive_aes_aead_hw_start(ctx, hw_mode);
 280		break;
 281	case STARFIVE_AES_MODE_CBC:
 282	case STARFIVE_AES_MODE_CTR:
 283		starfive_aes_write_iv(ctx, (void *)cryp->req.sreq->iv);
 284		break;
 285	default:
 286		break;
 287	}
 288
 289	return cryp->err;
 290}
 291
 292static int starfive_aes_read_authtag(struct starfive_cryp_ctx *ctx)
 293{
 294	struct starfive_cryp_dev *cryp = ctx->cryp;
 295	struct starfive_cryp_request_ctx *rctx = ctx->rctx;
 296	int i;
 297
 298	if (starfive_aes_wait_busy(cryp))
 299		return dev_err_probe(cryp->dev, -ETIMEDOUT,
 300				     "Timeout waiting for tag generation.");
 301
 302	if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM) {
 303		cryp->tag_out[0] = readl(cryp->base + STARFIVE_AES_NONCE0);
 304		cryp->tag_out[1] = readl(cryp->base + STARFIVE_AES_NONCE1);
 305		cryp->tag_out[2] = readl(cryp->base + STARFIVE_AES_NONCE2);
 306		cryp->tag_out[3] = readl(cryp->base + STARFIVE_AES_NONCE3);
 307	} else {
 308		for (i = 0; i < AES_BLOCK_32; i++)
 309			cryp->tag_out[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R);
 310	}
 311
 312	if (is_encrypt(cryp)) {
 313		scatterwalk_map_and_copy(cryp->tag_out, rctx->out_sg,
 314					 cryp->total_in, cryp->authsize, 1);
 315	} else {
 316		if (crypto_memneq(cryp->tag_in, cryp->tag_out, cryp->authsize))
 317			return -EBADMSG;
 318	}
 319
 320	return 0;
 321}
 322
 323static void starfive_aes_finish_req(struct starfive_cryp_ctx *ctx)
 324{
 325	struct starfive_cryp_dev *cryp = ctx->cryp;
 326	int err = cryp->err;
 327
 328	if (!err && cryp->authsize)
 329		err = starfive_aes_read_authtag(ctx);
 330
 331	if (!err && ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC ||
 332		     (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CTR))
 333		starfive_aes_get_iv(cryp, (void *)cryp->req.sreq->iv);
 334
 335	if (cryp->authsize)
 336		crypto_finalize_aead_request(cryp->engine, cryp->req.areq, err);
 337	else
 338		crypto_finalize_skcipher_request(cryp->engine, cryp->req.sreq,
 339						 err);
 340}
 341
 342static int starfive_aes_gcm_write_adata(struct starfive_cryp_ctx *ctx)
 343{
 344	struct starfive_cryp_dev *cryp = ctx->cryp;
 345	struct starfive_cryp_request_ctx *rctx = ctx->rctx;
 346	u32 *buffer;
 347	int total_len, loop;
 348
 349	total_len = ALIGN(cryp->assoclen, AES_BLOCK_SIZE) / sizeof(unsigned int);
 350	buffer = (u32 *)rctx->adata;
 351
 352	for (loop = 0; loop < total_len; loop += 4) {
 353		writel(*buffer, cryp->base + STARFIVE_AES_NONCE0);
 354		buffer++;
 355		writel(*buffer, cryp->base + STARFIVE_AES_NONCE1);
 356		buffer++;
 357		writel(*buffer, cryp->base + STARFIVE_AES_NONCE2);
 358		buffer++;
 359		writel(*buffer, cryp->base + STARFIVE_AES_NONCE3);
 360		buffer++;
 361	}
 362
 363	if (starfive_aes_wait_gcmdone(cryp))
 364		return dev_err_probe(cryp->dev, -ETIMEDOUT,
 365				     "Timeout processing gcm aad block");
 366
 367	return 0;
 368}
 369
 370static int starfive_aes_ccm_write_adata(struct starfive_cryp_ctx *ctx)
 371{
 372	struct starfive_cryp_dev *cryp = ctx->cryp;
 373	struct starfive_cryp_request_ctx *rctx = ctx->rctx;
 374	u32 *buffer;
 375	u8 *ci;
 376	int total_len, loop;
 377
 378	total_len = cryp->assoclen;
 379
 380	ci = rctx->adata;
 381	writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R);
 382	ci++;
 383	writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R);
 384	ci++;
 385	total_len -= 2;
 386	buffer = (u32 *)ci;
 387
 388	for (loop = 0; loop < 3; loop++, buffer++)
 389		writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R);
 390
 391	total_len -= 12;
 392
 393	while (total_len > 0) {
 394		for (loop = 0; loop < AES_BLOCK_32; loop++, buffer++)
 395			writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R);
 396
 397		total_len -= AES_BLOCK_SIZE;
 398	}
 399
 400	if (starfive_aes_wait_busy(cryp))
 401		return dev_err_probe(cryp->dev, -ETIMEDOUT,
 402				     "Timeout processing ccm aad block");
 403
 404	return 0;
 405}
 406
 407static void starfive_aes_dma_done(void *param)
 408{
 409	struct starfive_cryp_dev *cryp = param;
 410
 411	complete(&cryp->dma_done);
 412}
 413
 414static void starfive_aes_dma_init(struct starfive_cryp_dev *cryp)
 415{
 416	cryp->cfg_in.direction = DMA_MEM_TO_DEV;
 417	cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES;
 418	cryp->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 419	cryp->cfg_in.src_maxburst = cryp->dma_maxburst;
 420	cryp->cfg_in.dst_maxburst = cryp->dma_maxburst;
 421	cryp->cfg_in.dst_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET;
 422
 423	dmaengine_slave_config(cryp->tx, &cryp->cfg_in);
 424
 425	cryp->cfg_out.direction = DMA_DEV_TO_MEM;
 426	cryp->cfg_out.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 427	cryp->cfg_out.dst_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES;
 428	cryp->cfg_out.src_maxburst = 4;
 429	cryp->cfg_out.dst_maxburst = 4;
 430	cryp->cfg_out.src_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET;
 431
 432	dmaengine_slave_config(cryp->rx, &cryp->cfg_out);
 433
 434	init_completion(&cryp->dma_done);
 435}
 436
 437static int starfive_aes_dma_xfer(struct starfive_cryp_dev *cryp,
 438				 struct scatterlist *src,
 439				 struct scatterlist *dst,
 440				 int len)
 441{
 442	struct dma_async_tx_descriptor *in_desc, *out_desc;
 443	union starfive_alg_cr alg_cr;
 444	int ret = 0, in_save, out_save;
 445
 446	alg_cr.v = 0;
 447	alg_cr.start = 1;
 448	alg_cr.aes_dma_en = 1;
 449	writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
 450
 451	in_save = sg_dma_len(src);
 452	out_save = sg_dma_len(dst);
 453
 454	writel(ALIGN(len, AES_BLOCK_SIZE), cryp->base + STARFIVE_DMA_IN_LEN_OFFSET);
 455	writel(ALIGN(len, AES_BLOCK_SIZE), cryp->base + STARFIVE_DMA_OUT_LEN_OFFSET);
 456
 457	sg_dma_len(src) = ALIGN(len, AES_BLOCK_SIZE);
 458	sg_dma_len(dst) = ALIGN(len, AES_BLOCK_SIZE);
 459
 460	out_desc = dmaengine_prep_slave_sg(cryp->rx, dst, 1, DMA_DEV_TO_MEM,
 461					   DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 462	if (!out_desc) {
 463		ret = -EINVAL;
 464		goto dma_err;
 465	}
 466
 467	out_desc->callback = starfive_aes_dma_done;
 468	out_desc->callback_param = cryp;
 469
 470	reinit_completion(&cryp->dma_done);
 471	dmaengine_submit(out_desc);
 472	dma_async_issue_pending(cryp->rx);
 473
 474	in_desc = dmaengine_prep_slave_sg(cryp->tx, src, 1, DMA_MEM_TO_DEV,
 475					  DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 476	if (!in_desc) {
 477		ret = -EINVAL;
 478		goto dma_err;
 479	}
 480
 481	dmaengine_submit(in_desc);
 482	dma_async_issue_pending(cryp->tx);
 483
 484	if (!wait_for_completion_timeout(&cryp->dma_done,
 485					 msecs_to_jiffies(1000)))
 486		ret = -ETIMEDOUT;
 487
 488dma_err:
 489	sg_dma_len(src) = in_save;
 490	sg_dma_len(dst) = out_save;
 491
 492	alg_cr.v = 0;
 493	alg_cr.clear = 1;
 494	writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
 495
 496	return ret;
 497}
 498
 499static int starfive_aes_map_sg(struct starfive_cryp_dev *cryp,
 500			       struct scatterlist *src,
 501			       struct scatterlist *dst)
 502{
 503	struct scatterlist *stsg, *dtsg;
 504	struct scatterlist _src[2], _dst[2];
 505	unsigned int remain = cryp->total_in;
 506	unsigned int len, src_nents, dst_nents;
 507	int ret;
 508
 509	if (src == dst) {
 510		for (stsg = src, dtsg = dst; remain > 0;
 511		     stsg = sg_next(stsg), dtsg = sg_next(dtsg)) {
 512			src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL);
 513			if (src_nents == 0)
 514				return -ENOMEM;
 515
 516			dst_nents = src_nents;
 517			len = min(sg_dma_len(stsg), remain);
 518
 519			ret = starfive_aes_dma_xfer(cryp, stsg, dtsg, len);
 520			dma_unmap_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL);
 521			if (ret)
 522				return ret;
 523
 524			remain -= len;
 525		}
 526	} else {
 527		for (stsg = src, dtsg = dst;;) {
 528			src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE);
 529			if (src_nents == 0)
 530				return -ENOMEM;
 531
 532			dst_nents = dma_map_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE);
 533			if (dst_nents == 0)
 534				return -ENOMEM;
 535
 536			len = min(sg_dma_len(stsg), sg_dma_len(dtsg));
 537			len = min(len, remain);
 538
 539			ret = starfive_aes_dma_xfer(cryp, stsg, dtsg, len);
 540			dma_unmap_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE);
 541			dma_unmap_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE);
 542			if (ret)
 543				return ret;
 544
 545			remain -= len;
 546			if (remain == 0)
 547				break;
 548
 549			if (sg_dma_len(stsg) - len) {
 550				stsg = scatterwalk_ffwd(_src, stsg, len);
 551				dtsg = sg_next(dtsg);
 552			} else if (sg_dma_len(dtsg) - len) {
 553				dtsg = scatterwalk_ffwd(_dst, dtsg, len);
 554				stsg = sg_next(stsg);
 555			} else {
 556				stsg = sg_next(stsg);
 557				dtsg = sg_next(dtsg);
 558			}
 559		}
 560	}
 561
 562	return 0;
 563}
 564
 565static int starfive_aes_do_one_req(struct crypto_engine *engine, void *areq)
 566{
 567	struct skcipher_request *req =
 568		container_of(areq, struct skcipher_request, base);
 569	struct starfive_cryp_ctx *ctx =
 570		crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
 571	struct starfive_cryp_request_ctx *rctx = skcipher_request_ctx(req);
 572	struct starfive_cryp_dev *cryp = ctx->cryp;
 573	int ret;
 574
 575	cryp->req.sreq = req;
 576	cryp->total_in = req->cryptlen;
 577	cryp->total_out = req->cryptlen;
 578	cryp->assoclen = 0;
 579	cryp->authsize = 0;
 580
 581	rctx->in_sg = req->src;
 582	rctx->out_sg = req->dst;
 583
 584	ctx->rctx = rctx;
 585
 586	ret = starfive_aes_hw_init(ctx);
 587	if (ret)
 588		return ret;
 589
 590	if (!cryp->total_in)
 591		goto finish_req;
 592
 593	starfive_aes_dma_init(cryp);
 594
 595	ret = starfive_aes_map_sg(cryp, rctx->in_sg, rctx->out_sg);
 596	if (ret)
 597		return ret;
 598
 599finish_req:
 600	starfive_aes_finish_req(ctx);
 601
 602	return 0;
 603}
 604
 605static int starfive_aes_init_tfm(struct crypto_skcipher *tfm,
 606				 const char *alg_name)
 607{
 608	struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
 609
 610	ctx->cryp = starfive_cryp_find_dev(ctx);
 611	if (!ctx->cryp)
 612		return -ENODEV;
 613
 614	ctx->skcipher_fbk = crypto_alloc_skcipher(alg_name, 0,
 615						  CRYPTO_ALG_NEED_FALLBACK);
 616	if (IS_ERR(ctx->skcipher_fbk))
 617		return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->skcipher_fbk),
 618				     "%s() failed to allocate fallback for %s\n",
 619				     __func__, alg_name);
 620
 621	crypto_skcipher_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) +
 622				    crypto_skcipher_reqsize(ctx->skcipher_fbk));
 623
 624	return 0;
 625}
 626
 627static void starfive_aes_exit_tfm(struct crypto_skcipher *tfm)
 628{
 629	struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
 630
 631	crypto_free_skcipher(ctx->skcipher_fbk);
 632}
 633
 634static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq)
 635{
 636	struct aead_request *req =
 637		container_of(areq, struct aead_request, base);
 638	struct starfive_cryp_ctx *ctx =
 639		crypto_aead_ctx(crypto_aead_reqtfm(req));
 640	struct starfive_cryp_dev *cryp = ctx->cryp;
 641	struct starfive_cryp_request_ctx *rctx = aead_request_ctx(req);
 642	struct scatterlist _src[2], _dst[2];
 643	int ret;
 644
 645	cryp->req.areq = req;
 646	cryp->assoclen = req->assoclen;
 647	cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
 648
 649	rctx->in_sg = scatterwalk_ffwd(_src, req->src, cryp->assoclen);
 650	if (req->src == req->dst)
 651		rctx->out_sg = rctx->in_sg;
 652	else
 653		rctx->out_sg = scatterwalk_ffwd(_dst, req->dst, cryp->assoclen);
 654
 655	if (is_encrypt(cryp)) {
 656		cryp->total_in = req->cryptlen;
 657		cryp->total_out = req->cryptlen;
 658	} else {
 659		cryp->total_in = req->cryptlen - cryp->authsize;
 660		cryp->total_out = cryp->total_in;
 661		scatterwalk_map_and_copy(cryp->tag_in, req->src,
 662					 cryp->total_in + cryp->assoclen,
 663					 cryp->authsize, 0);
 664	}
 665
 666	if (cryp->assoclen) {
 667		rctx->adata = kzalloc(cryp->assoclen + AES_BLOCK_SIZE, GFP_KERNEL);
 668		if (!rctx->adata)
 669			return -ENOMEM;
 670
 671		if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, cryp->assoclen),
 672				      rctx->adata, cryp->assoclen) != cryp->assoclen) {
 673			kfree(rctx->adata);
 674			return -EINVAL;
 675		}
 676	}
 677
 678	if (cryp->total_in)
 679		sg_zero_buffer(rctx->in_sg, sg_nents(rctx->in_sg),
 680			       sg_dma_len(rctx->in_sg) - cryp->total_in,
 681			       cryp->total_in);
 682
 683	ctx->rctx = rctx;
 684
 685	ret = starfive_aes_hw_init(ctx);
 686	if (ret) {
 687		if (cryp->assoclen)
 688			kfree(rctx->adata);
 689		return ret;
 690	}
 691
 692	if (!cryp->assoclen)
 693		goto write_text;
 694
 695	if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM)
 696		ret = starfive_aes_ccm_write_adata(ctx);
 697	else
 698		ret = starfive_aes_gcm_write_adata(ctx);
 699
 700	kfree(rctx->adata);
 701
 702	if (ret)
 703		return ret;
 704
 705write_text:
 706	if (!cryp->total_in)
 707		goto finish_req;
 708
 709	starfive_aes_dma_init(cryp);
 710
 711	ret = starfive_aes_map_sg(cryp, rctx->in_sg, rctx->out_sg);
 712	if (ret)
 713		return ret;
 714
 715finish_req:
 716	starfive_aes_finish_req(ctx);
 717	return 0;
 718}
 719
 720static int starfive_aes_aead_init_tfm(struct crypto_aead *tfm,
 721				      const char *alg_name)
 722{
 723	struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
 724
 725	ctx->cryp = starfive_cryp_find_dev(ctx);
 726	if (!ctx->cryp)
 727		return -ENODEV;
 728
 729	ctx->aead_fbk = crypto_alloc_aead(alg_name, 0,
 730					  CRYPTO_ALG_NEED_FALLBACK);
 731	if (IS_ERR(ctx->aead_fbk))
 732		return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->aead_fbk),
 733				     "%s() failed to allocate fallback for %s\n",
 734				     __func__, alg_name);
 735
 736	crypto_aead_set_reqsize(tfm, sizeof(struct starfive_cryp_request_ctx) +
 737				crypto_aead_reqsize(ctx->aead_fbk));
 738
 739	return 0;
 740}
 741
 742static void starfive_aes_aead_exit_tfm(struct crypto_aead *tfm)
 743{
 744	struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
 745
 746	crypto_free_aead(ctx->aead_fbk);
 747}
 748
 749static bool starfive_aes_check_unaligned(struct starfive_cryp_dev *cryp,
 750					 struct scatterlist *src,
 751					 struct scatterlist *dst)
 752{
 753	struct scatterlist *tsg;
 754	int i;
 755
 756	for_each_sg(src, tsg, sg_nents(src), i)
 757		if (!IS_ALIGNED(tsg->offset, sizeof(u32)) ||
 758		    (!IS_ALIGNED(tsg->length, AES_BLOCK_SIZE) &&
 759		     !sg_is_last(tsg)))
 760			return true;
 761
 762	if (src != dst)
 763		for_each_sg(dst, tsg, sg_nents(dst), i)
 764			if (!IS_ALIGNED(tsg->offset, sizeof(u32)) ||
 765			    (!IS_ALIGNED(tsg->length, AES_BLOCK_SIZE) &&
 766			     !sg_is_last(tsg)))
 767				return true;
 768
 769	return false;
 770}
 771
 772static int starfive_aes_do_fallback(struct skcipher_request *req, bool enc)
 773{
 774	struct starfive_cryp_ctx *ctx =
 775		crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
 776	struct skcipher_request *subreq = skcipher_request_ctx(req);
 777
 778	skcipher_request_set_tfm(subreq, ctx->skcipher_fbk);
 779	skcipher_request_set_callback(subreq, req->base.flags,
 780				      req->base.complete,
 781				      req->base.data);
 782	skcipher_request_set_crypt(subreq, req->src, req->dst,
 783				   req->cryptlen, req->iv);
 784
 785	return enc ? crypto_skcipher_encrypt(subreq) :
 786		     crypto_skcipher_decrypt(subreq);
 787}
 788
 789static int starfive_aes_crypt(struct skcipher_request *req, unsigned long flags)
 790{
 791	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 792	struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
 793	struct starfive_cryp_dev *cryp = ctx->cryp;
 794	unsigned int blocksize_align = crypto_skcipher_blocksize(tfm) - 1;
 795
 796	cryp->flags = flags;
 797
 798	if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_ECB ||
 799	    (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC)
 800		if (req->cryptlen & blocksize_align)
 801			return -EINVAL;
 802
 803	if (starfive_aes_check_unaligned(cryp, req->src, req->dst))
 804		return starfive_aes_do_fallback(req, is_encrypt(cryp));
 805
 806	return crypto_transfer_skcipher_request_to_engine(cryp->engine, req);
 807}
 808
 809static int starfive_aes_aead_do_fallback(struct aead_request *req, bool enc)
 810{
 811	struct starfive_cryp_ctx *ctx =
 812		crypto_aead_ctx(crypto_aead_reqtfm(req));
 813	struct aead_request *subreq = aead_request_ctx(req);
 814
 815	aead_request_set_tfm(subreq, ctx->aead_fbk);
 816	aead_request_set_callback(subreq, req->base.flags,
 817				  req->base.complete,
 818				  req->base.data);
 819	aead_request_set_crypt(subreq, req->src, req->dst,
 820			       req->cryptlen, req->iv);
 821	aead_request_set_ad(subreq, req->assoclen);
 822
 823	return enc ? crypto_aead_encrypt(subreq) :
 824		     crypto_aead_decrypt(subreq);
 825}
 826
 827static int starfive_aes_aead_crypt(struct aead_request *req, unsigned long flags)
 828{
 829	struct starfive_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 830	struct starfive_cryp_dev *cryp = ctx->cryp;
 831	struct scatterlist *src, *dst, _src[2], _dst[2];
 832
 833	cryp->flags = flags;
 834
 835	/* aes-ccm does not support tag verification for non-aligned text,
 836	 * use fallback for ccm decryption instead.
 837	 */
 838	if (((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM) &&
 839	    !is_encrypt(cryp))
 840		return starfive_aes_aead_do_fallback(req, 0);
 841
 842	src = scatterwalk_ffwd(_src, req->src, req->assoclen);
 843
 844	if (req->src == req->dst)
 845		dst = src;
 846	else
 847		dst = scatterwalk_ffwd(_dst, req->dst, req->assoclen);
 848
 849	if (starfive_aes_check_unaligned(cryp, src, dst))
 850		return starfive_aes_aead_do_fallback(req, is_encrypt(cryp));
 851
 852	return crypto_transfer_aead_request_to_engine(cryp->engine, req);
 853}
 854
 855static int starfive_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
 856			       unsigned int keylen)
 857{
 858	struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
 859
 860	if (!key || !keylen)
 861		return -EINVAL;
 862
 863	if (keylen != AES_KEYSIZE_128 &&
 864	    keylen != AES_KEYSIZE_192 &&
 865	    keylen != AES_KEYSIZE_256)
 866		return -EINVAL;
 867
 868	memcpy(ctx->key, key, keylen);
 869	ctx->keylen = keylen;
 870
 871	return crypto_skcipher_setkey(ctx->skcipher_fbk, key, keylen);
 872}
 873
 874static int starfive_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key,
 875				    unsigned int keylen)
 876{
 877	struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
 878
 879	if (!key || !keylen)
 880		return -EINVAL;
 881
 882	if (keylen != AES_KEYSIZE_128 &&
 883	    keylen != AES_KEYSIZE_192 &&
 884	    keylen != AES_KEYSIZE_256)
 885		return -EINVAL;
 886
 887	memcpy(ctx->key, key, keylen);
 888	ctx->keylen = keylen;
 889
 890	return crypto_aead_setkey(ctx->aead_fbk, key, keylen);
 891}
 892
 893static int starfive_aes_gcm_setauthsize(struct crypto_aead *tfm,
 894					unsigned int authsize)
 895{
 896	struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
 897	int ret;
 898
 899	ret = crypto_gcm_check_authsize(authsize);
 900	if (ret)
 901		return ret;
 902
 903	return crypto_aead_setauthsize(ctx->aead_fbk, authsize);
 904}
 905
 906static int starfive_aes_ccm_setauthsize(struct crypto_aead *tfm,
 907					unsigned int authsize)
 908{
 909	struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
 910
 911	switch (authsize) {
 912	case 4:
 913	case 6:
 914	case 8:
 915	case 10:
 916	case 12:
 917	case 14:
 918	case 16:
 919		break;
 920	default:
 921		return -EINVAL;
 922	}
 923
 924	return crypto_aead_setauthsize(ctx->aead_fbk, authsize);
 925}
 926
 927static int starfive_aes_ecb_encrypt(struct skcipher_request *req)
 928{
 929	return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB | FLG_ENCRYPT);
 930}
 931
 932static int starfive_aes_ecb_decrypt(struct skcipher_request *req)
 933{
 934	return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB);
 935}
 936
 937static int starfive_aes_cbc_encrypt(struct skcipher_request *req)
 938{
 939	return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC | FLG_ENCRYPT);
 940}
 941
 942static int starfive_aes_cbc_decrypt(struct skcipher_request *req)
 943{
 944	return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC);
 945}
 946
 947static int starfive_aes_ctr_encrypt(struct skcipher_request *req)
 948{
 949	return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR | FLG_ENCRYPT);
 950}
 951
 952static int starfive_aes_ctr_decrypt(struct skcipher_request *req)
 953{
 954	return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR);
 955}
 956
 957static int starfive_aes_gcm_encrypt(struct aead_request *req)
 958{
 959	return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM | FLG_ENCRYPT);
 960}
 961
 962static int starfive_aes_gcm_decrypt(struct aead_request *req)
 963{
 964	return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM);
 965}
 966
 967static int starfive_aes_ccm_encrypt(struct aead_request *req)
 968{
 969	int ret;
 970
 971	ret = starfive_aes_ccm_check_iv(req->iv);
 972	if (ret)
 973		return ret;
 974
 975	return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM | FLG_ENCRYPT);
 976}
 977
 978static int starfive_aes_ccm_decrypt(struct aead_request *req)
 979{
 980	int ret;
 981
 982	ret = starfive_aes_ccm_check_iv(req->iv);
 983	if (ret)
 984		return ret;
 985
 986	return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM);
 987}
 988
 989static int starfive_aes_ecb_init_tfm(struct crypto_skcipher *tfm)
 990{
 991	return starfive_aes_init_tfm(tfm, "ecb(aes-generic)");
 992}
 993
 994static int starfive_aes_cbc_init_tfm(struct crypto_skcipher *tfm)
 995{
 996	return starfive_aes_init_tfm(tfm, "cbc(aes-generic)");
 997}
 998
 999static int starfive_aes_ctr_init_tfm(struct crypto_skcipher *tfm)
1000{
1001	return starfive_aes_init_tfm(tfm, "ctr(aes-generic)");
1002}
1003
1004static int starfive_aes_ccm_init_tfm(struct crypto_aead *tfm)
1005{
1006	return starfive_aes_aead_init_tfm(tfm, "ccm_base(ctr(aes-generic),cbcmac(aes-generic))");
1007}
1008
1009static int starfive_aes_gcm_init_tfm(struct crypto_aead *tfm)
1010{
1011	return starfive_aes_aead_init_tfm(tfm, "gcm_base(ctr(aes-generic),ghash-generic)");
1012}
1013
1014static struct skcipher_engine_alg skcipher_algs[] = {
1015{
1016	.base.init			= starfive_aes_ecb_init_tfm,
1017	.base.exit			= starfive_aes_exit_tfm,
1018	.base.setkey			= starfive_aes_setkey,
1019	.base.encrypt			= starfive_aes_ecb_encrypt,
1020	.base.decrypt			= starfive_aes_ecb_decrypt,
1021	.base.min_keysize		= AES_MIN_KEY_SIZE,
1022	.base.max_keysize		= AES_MAX_KEY_SIZE,
1023	.base.base = {
1024		.cra_name		= "ecb(aes)",
1025		.cra_driver_name	= "starfive-ecb-aes",
1026		.cra_priority		= 200,
1027		.cra_flags		= CRYPTO_ALG_ASYNC |
1028					  CRYPTO_ALG_NEED_FALLBACK,
1029		.cra_blocksize		= AES_BLOCK_SIZE,
1030		.cra_ctxsize		= sizeof(struct starfive_cryp_ctx),
1031		.cra_alignmask		= 0xf,
1032		.cra_module		= THIS_MODULE,
1033	},
1034	.op = {
1035		.do_one_request = starfive_aes_do_one_req,
1036	},
1037}, {
1038	.base.init			= starfive_aes_cbc_init_tfm,
1039	.base.exit			= starfive_aes_exit_tfm,
1040	.base.setkey			= starfive_aes_setkey,
1041	.base.encrypt			= starfive_aes_cbc_encrypt,
1042	.base.decrypt			= starfive_aes_cbc_decrypt,
1043	.base.min_keysize		= AES_MIN_KEY_SIZE,
1044	.base.max_keysize		= AES_MAX_KEY_SIZE,
1045	.base.ivsize			= AES_BLOCK_SIZE,
1046	.base.base = {
1047		.cra_name		= "cbc(aes)",
1048		.cra_driver_name	= "starfive-cbc-aes",
1049		.cra_priority		= 200,
1050		.cra_flags		= CRYPTO_ALG_ASYNC |
1051					  CRYPTO_ALG_NEED_FALLBACK,
1052		.cra_blocksize		= AES_BLOCK_SIZE,
1053		.cra_ctxsize		= sizeof(struct starfive_cryp_ctx),
1054		.cra_alignmask		= 0xf,
1055		.cra_module		= THIS_MODULE,
1056	},
1057	.op = {
1058		.do_one_request = starfive_aes_do_one_req,
1059	},
1060}, {
1061	.base.init			= starfive_aes_ctr_init_tfm,
1062	.base.exit			= starfive_aes_exit_tfm,
1063	.base.setkey			= starfive_aes_setkey,
1064	.base.encrypt			= starfive_aes_ctr_encrypt,
1065	.base.decrypt			= starfive_aes_ctr_decrypt,
1066	.base.min_keysize		= AES_MIN_KEY_SIZE,
1067	.base.max_keysize		= AES_MAX_KEY_SIZE,
1068	.base.ivsize			= AES_BLOCK_SIZE,
1069	.base.base = {
1070		.cra_name		= "ctr(aes)",
1071		.cra_driver_name	= "starfive-ctr-aes",
1072		.cra_priority		= 200,
1073		.cra_flags		= CRYPTO_ALG_ASYNC |
1074					  CRYPTO_ALG_NEED_FALLBACK,
1075		.cra_blocksize		= 1,
1076		.cra_ctxsize		= sizeof(struct starfive_cryp_ctx),
1077		.cra_alignmask		= 0xf,
1078		.cra_module		= THIS_MODULE,
1079	},
1080	.op = {
1081		.do_one_request = starfive_aes_do_one_req,
1082	},
1083},
1084};
1085
1086static struct aead_engine_alg aead_algs[] = {
1087{
1088	.base.setkey			= starfive_aes_aead_setkey,
1089	.base.setauthsize		= starfive_aes_gcm_setauthsize,
1090	.base.encrypt			= starfive_aes_gcm_encrypt,
1091	.base.decrypt			= starfive_aes_gcm_decrypt,
1092	.base.init			= starfive_aes_gcm_init_tfm,
1093	.base.exit			= starfive_aes_aead_exit_tfm,
1094	.base.ivsize			= GCM_AES_IV_SIZE,
1095	.base.maxauthsize		= AES_BLOCK_SIZE,
1096	.base.base = {
1097		.cra_name               = "gcm(aes)",
1098		.cra_driver_name        = "starfive-gcm-aes",
1099		.cra_priority           = 200,
1100		.cra_flags              = CRYPTO_ALG_ASYNC |
1101					  CRYPTO_ALG_NEED_FALLBACK,
1102		.cra_blocksize          = 1,
1103		.cra_ctxsize            = sizeof(struct starfive_cryp_ctx),
1104		.cra_alignmask          = 0xf,
1105		.cra_module             = THIS_MODULE,
1106	},
1107	.op = {
1108		.do_one_request = starfive_aes_aead_do_one_req,
1109	},
1110}, {
1111	.base.setkey			= starfive_aes_aead_setkey,
1112	.base.setauthsize		= starfive_aes_ccm_setauthsize,
1113	.base.encrypt			= starfive_aes_ccm_encrypt,
1114	.base.decrypt			= starfive_aes_ccm_decrypt,
1115	.base.init			= starfive_aes_ccm_init_tfm,
1116	.base.exit			= starfive_aes_aead_exit_tfm,
1117	.base.ivsize			= AES_BLOCK_SIZE,
1118	.base.maxauthsize		= AES_BLOCK_SIZE,
1119	.base.base = {
1120		.cra_name		= "ccm(aes)",
1121		.cra_driver_name	= "starfive-ccm-aes",
1122		.cra_priority		= 200,
1123		.cra_flags		= CRYPTO_ALG_ASYNC |
1124					  CRYPTO_ALG_NEED_FALLBACK,
1125		.cra_blocksize		= 1,
1126		.cra_ctxsize		= sizeof(struct starfive_cryp_ctx),
1127		.cra_alignmask		= 0xf,
1128		.cra_module		= THIS_MODULE,
1129	},
1130	.op = {
1131		.do_one_request = starfive_aes_aead_do_one_req,
1132	},
1133},
1134};
1135
1136int starfive_aes_register_algs(void)
1137{
1138	int ret;
1139
1140	ret = crypto_engine_register_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
1141	if (ret)
1142		return ret;
1143
1144	ret = crypto_engine_register_aeads(aead_algs, ARRAY_SIZE(aead_algs));
1145	if (ret)
1146		crypto_engine_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
1147
1148	return ret;
1149}
1150
1151void starfive_aes_unregister_algs(void)
1152{
1153	crypto_engine_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs));
1154	crypto_engine_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
1155}