caamalg_desc.c

   1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * Shared descriptors for aead, skcipher algorithms
   4 *
   5 * Copyright 2016-2019, 2025 NXP
   6 */
   7
   8#include "compat.h"
   9#include "desc_constr.h"
  10#include "caamalg_desc.h"
  11#include <soc/fsl/caam-blob.h>
  12
  13/*
  14 * For aead functions, read payload and write payload,
  15 * both of which are specified in req->src and req->dst
  16 */
  17static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
  18{
  19	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
  20	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
  21			     KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
  22}
  23
  24/* Set DK bit in class 1 operation if shared */
  25static inline void append_dec_op1(u32 *desc, u32 type)
  26{
  27	u32 *jump_cmd, *uncond_jump_cmd;
  28
  29	/* DK bit is valid only for AES */
  30	if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
  31		append_operation(desc, type | OP_ALG_AS_INITFINAL |
  32				 OP_ALG_DECRYPT);
  33		return;
  34	}
  35
  36	jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
  37	append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT);
  38	uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
  39	set_jump_tgt_here(desc, jump_cmd);
  40	append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT |
  41			 OP_ALG_AAI_DK);
  42	set_jump_tgt_here(desc, uncond_jump_cmd);
  43}
  44
  45/**
  46 * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
  47 *                               (non-protocol) with no (null) encryption.
  48 * @desc: pointer to buffer used for descriptor construction
  49 * @adata: pointer to authentication transform definitions.
  50 *         A split key is required for SEC Era < 6; the size of the split key
  51 *         is specified in this case. Valid algorithm values - one of
  52 *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
  53 *         with OP_ALG_AAI_HMAC_PRECOMP.
  54 * @icvsize: integrity check value (ICV) size (truncated or full)
  55 * @era: SEC Era
  56 */
  57void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
  58				 unsigned int icvsize, int era)
  59{
  60	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
  61
  62	init_sh_desc(desc, HDR_SHARE_SERIAL);
  63
  64	/* Skip if already shared */
  65	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
  66				   JUMP_COND_SHRD);
  67	if (era < 6) {
  68		if (adata->key_inline)
  69			append_key_as_imm(desc, adata->key_virt,
  70					  adata->keylen_pad, adata->keylen,
  71					  CLASS_2 | KEY_DEST_MDHA_SPLIT |
  72					  KEY_ENC);
  73		else
  74			append_key(desc, adata->key_dma, adata->keylen,
  75				   CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
  76	} else {
  77		append_proto_dkp(desc, adata);
  78	}
  79	set_jump_tgt_here(desc, key_jump_cmd);
  80
  81	/* assoclen + cryptlen = seqinlen */
  82	append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
  83
  84	/* Prepare to read and write cryptlen + assoclen bytes */
  85	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
  86	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
  87
  88	/*
  89	 * MOVE_LEN opcode is not available in all SEC HW revisions,
  90	 * thus need to do some magic, i.e. self-patch the descriptor
  91	 * buffer.
  92	 */
  93	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
  94				    MOVE_DEST_MATH3 |
  95				    (0x6 << MOVE_LEN_SHIFT));
  96	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
  97				     MOVE_DEST_DESCBUF |
  98				     MOVE_WAITCOMP |
  99				     (0x8 << MOVE_LEN_SHIFT));
 100
 101	/* Class 2 operation */
 102	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
 103			 OP_ALG_ENCRYPT);
 104
 105	/* Read and write cryptlen bytes */
 106	aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 107
 108	set_move_tgt_here(desc, read_move_cmd);
 109	set_move_tgt_here(desc, write_move_cmd);
 110	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 111	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
 112		    MOVE_AUX_LS);
 113
 114	/* Write ICV */
 115	append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
 116			 LDST_SRCDST_BYTE_CONTEXT);
 117
 118	print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ",
 119			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 120			     1);
 121}
 122EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
 123
 124/**
 125 * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
 126 *                               (non-protocol) with no (null) decryption.
 127 * @desc: pointer to buffer used for descriptor construction
 128 * @adata: pointer to authentication transform definitions.
 129 *         A split key is required for SEC Era < 6; the size of the split key
 130 *         is specified in this case. Valid algorithm values - one of
 131 *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
 132 *         with OP_ALG_AAI_HMAC_PRECOMP.
 133 * @icvsize: integrity check value (ICV) size (truncated or full)
 134 * @era: SEC Era
 135 */
 136void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
 137				 unsigned int icvsize, int era)
 138{
 139	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
 140
 141	init_sh_desc(desc, HDR_SHARE_SERIAL);
 142
 143	/* Skip if already shared */
 144	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 145				   JUMP_COND_SHRD);
 146	if (era < 6) {
 147		if (adata->key_inline)
 148			append_key_as_imm(desc, adata->key_virt,
 149					  adata->keylen_pad, adata->keylen,
 150					  CLASS_2 | KEY_DEST_MDHA_SPLIT |
 151					  KEY_ENC);
 152		else
 153			append_key(desc, adata->key_dma, adata->keylen,
 154				   CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
 155	} else {
 156		append_proto_dkp(desc, adata);
 157	}
 158	set_jump_tgt_here(desc, key_jump_cmd);
 159
 160	/* Class 2 operation */
 161	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
 162			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 163
 164	/* assoclen + cryptlen = seqoutlen */
 165	append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 166
 167	/* Prepare to read and write cryptlen + assoclen bytes */
 168	append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
 169	append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
 170
 171	/*
 172	 * MOVE_LEN opcode is not available in all SEC HW revisions,
 173	 * thus need to do some magic, i.e. self-patch the descriptor
 174	 * buffer.
 175	 */
 176	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
 177				    MOVE_DEST_MATH2 |
 178				    (0x6 << MOVE_LEN_SHIFT));
 179	write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
 180				     MOVE_DEST_DESCBUF |
 181				     MOVE_WAITCOMP |
 182				     (0x8 << MOVE_LEN_SHIFT));
 183
 184	/* Read and write cryptlen bytes */
 185	aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 186
 187	/*
 188	 * Insert a NOP here, since we need at least 4 instructions between
 189	 * code patching the descriptor buffer and the location being patched.
 190	 */
 191	jump_cmd = append_jump(desc, JUMP_TEST_ALL);
 192	set_jump_tgt_here(desc, jump_cmd);
 193
 194	set_move_tgt_here(desc, read_move_cmd);
 195	set_move_tgt_here(desc, write_move_cmd);
 196	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 197	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
 198		    MOVE_AUX_LS);
 199	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 200
 201	/* Load ICV */
 202	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
 203			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 204
 205	print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ",
 206			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 207			     1);
 208}
 209EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
 210
 211static void init_sh_desc_key_aead(u32 * const desc,
 212				  struct alginfo * const cdata,
 213				  struct alginfo * const adata,
 214				  const bool is_rfc3686, u32 *nonce, int era)
 215{
 216	u32 *key_jump_cmd;
 217	unsigned int enckeylen = cdata->keylen;
 218
 219	/* Note: Context registers are saved. */
 220	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
 221
 222	/* Skip if already shared */
 223	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 224				   JUMP_COND_SHRD);
 225
 226	/*
 227	 * RFC3686 specific:
 228	 *	| key = {AUTH_KEY, ENC_KEY, NONCE}
 229	 *	| enckeylen = encryption key size + nonce size
 230	 */
 231	if (is_rfc3686)
 232		enckeylen -= CTR_RFC3686_NONCE_SIZE;
 233
 234	if (era < 6) {
 235		if (adata->key_inline)
 236			append_key_as_imm(desc, adata->key_virt,
 237					  adata->keylen_pad, adata->keylen,
 238					  CLASS_2 | KEY_DEST_MDHA_SPLIT |
 239					  KEY_ENC);
 240		else
 241			append_key(desc, adata->key_dma, adata->keylen,
 242				   CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
 243	} else {
 244		append_proto_dkp(desc, adata);
 245	}
 246
 247	if (cdata->key_inline)
 248		append_key_as_imm(desc, cdata->key_virt, enckeylen,
 249				  enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 250	else
 251		append_key(desc, cdata->key_dma, enckeylen, CLASS_1 |
 252			   KEY_DEST_CLASS_REG);
 253
 254	/* Load Counter into CONTEXT1 reg */
 255	if (is_rfc3686) {
 256		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
 257				   LDST_CLASS_IND_CCB |
 258				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
 259		append_move(desc,
 260			    MOVE_SRC_OUTFIFO |
 261			    MOVE_DEST_CLASS1CTX |
 262			    (16 << MOVE_OFFSET_SHIFT) |
 263			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
 264	}
 265
 266	set_jump_tgt_here(desc, key_jump_cmd);
 267}
 268
 269/**
 270 * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor
 271 *                          (non-protocol).
 272 * @desc: pointer to buffer used for descriptor construction
 273 * @cdata: pointer to block cipher transform definitions
 274 *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
 275 *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
 276 * @adata: pointer to authentication transform definitions.
 277 *         A split key is required for SEC Era < 6; the size of the split key
 278 *         is specified in this case. Valid algorithm values - one of
 279 *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
 280 *         with OP_ALG_AAI_HMAC_PRECOMP.
 281 * @ivsize: initialization vector size
 282 * @icvsize: integrity check value (ICV) size (truncated or full)
 283 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
 284 * @nonce: pointer to rfc3686 nonce
 285 * @ctx1_iv_off: IV offset in CONTEXT1 register
 286 * @is_qi: true when called from caam/qi
 287 * @era: SEC Era
 288 */
 289void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
 290			    struct alginfo *adata, unsigned int ivsize,
 291			    unsigned int icvsize, const bool is_rfc3686,
 292			    u32 *nonce, const u32 ctx1_iv_off, const bool is_qi,
 293			    int era)
 294{
 295	/* Note: Context registers are saved. */
 296	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
 297
 298	/* Class 2 operation */
 299	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
 300			 OP_ALG_ENCRYPT);
 301
 302	if (is_qi) {
 303		u32 *wait_load_cmd;
 304
 305		/* REG3 = assoclen */
 306		append_seq_load(desc, 4, LDST_CLASS_DECO |
 307				LDST_SRCDST_WORD_DECO_MATH3 |
 308				(4 << LDST_OFFSET_SHIFT));
 309
 310		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 311					    JUMP_COND_CALM | JUMP_COND_NCP |
 312					    JUMP_COND_NOP | JUMP_COND_NIP |
 313					    JUMP_COND_NIFP);
 314		set_jump_tgt_here(desc, wait_load_cmd);
 315
 316		append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
 317				LDST_SRCDST_BYTE_CONTEXT |
 318				(ctx1_iv_off << LDST_OFFSET_SHIFT));
 319	}
 320
 321	/* Read and write assoclen bytes */
 322	if (is_qi || era < 3) {
 323		append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 324		append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 325	} else {
 326		append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
 327		append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
 328	}
 329
 330	/* Skip assoc data */
 331	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 332
 333	/* read assoc before reading payload */
 334	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 335				      FIFOLDST_VLF);
 336
 337	/* Load Counter into CONTEXT1 reg */
 338	if (is_rfc3686)
 339		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
 340				     LDST_SRCDST_BYTE_CONTEXT |
 341				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
 342				      LDST_OFFSET_SHIFT));
 343
 344	/* Class 1 operation */
 345	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 346			 OP_ALG_ENCRYPT);
 347
 348	/* Read and write cryptlen bytes */
 349	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 350	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 351	aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
 352
 353	/* Write ICV */
 354	append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
 355			 LDST_SRCDST_BYTE_CONTEXT);
 356
 357	print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ",
 358			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 359			     1);
 360}
 361EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
 362
 363/**
 364 * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor
 365 *                          (non-protocol).
 366 * @desc: pointer to buffer used for descriptor construction
 367 * @cdata: pointer to block cipher transform definitions
 368 *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
 369 *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
 370 * @adata: pointer to authentication transform definitions.
 371 *         A split key is required for SEC Era < 6; the size of the split key
 372 *         is specified in this case. Valid algorithm values - one of
 373 *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
 374 *         with OP_ALG_AAI_HMAC_PRECOMP.
 375 * @ivsize: initialization vector size
 376 * @icvsize: integrity check value (ICV) size (truncated or full)
 377 * @geniv: whether to generate Encrypted Chain IV
 378 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
 379 * @nonce: pointer to rfc3686 nonce
 380 * @ctx1_iv_off: IV offset in CONTEXT1 register
 381 * @is_qi: true when called from caam/qi
 382 * @era: SEC Era
 383 */
 384void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
 385			    struct alginfo *adata, unsigned int ivsize,
 386			    unsigned int icvsize, const bool geniv,
 387			    const bool is_rfc3686, u32 *nonce,
 388			    const u32 ctx1_iv_off, const bool is_qi, int era)
 389{
 390	/* Note: Context registers are saved. */
 391	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
 392
 393	/* Class 2 operation */
 394	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
 395			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 396
 397	if (is_qi) {
 398		u32 *wait_load_cmd;
 399
 400		/* REG3 = assoclen */
 401		append_seq_load(desc, 4, LDST_CLASS_DECO |
 402				LDST_SRCDST_WORD_DECO_MATH3 |
 403				(4 << LDST_OFFSET_SHIFT));
 404
 405		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 406					    JUMP_COND_CALM | JUMP_COND_NCP |
 407					    JUMP_COND_NOP | JUMP_COND_NIP |
 408					    JUMP_COND_NIFP);
 409		set_jump_tgt_here(desc, wait_load_cmd);
 410
 411		if (!geniv)
 412			append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
 413					LDST_SRCDST_BYTE_CONTEXT |
 414					(ctx1_iv_off << LDST_OFFSET_SHIFT));
 415	}
 416
 417	/* Read and write assoclen bytes */
 418	if (is_qi || era < 3) {
 419		append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 420		if (geniv)
 421			append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM,
 422						ivsize);
 423		else
 424			append_math_add(desc, VARSEQOUTLEN, ZERO, REG3,
 425					CAAM_CMD_SZ);
 426	} else {
 427		append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
 428		if (geniv)
 429			append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM,
 430						ivsize);
 431		else
 432			append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD,
 433					CAAM_CMD_SZ);
 434	}
 435
 436	/* Skip assoc data */
 437	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 438
 439	/* read assoc before reading payload */
 440	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 441			     KEY_VLF);
 442
 443	if (geniv) {
 444		append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
 445				LDST_SRCDST_BYTE_CONTEXT |
 446				(ctx1_iv_off << LDST_OFFSET_SHIFT));
 447		append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
 448			    (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
 449	}
 450
 451	/* Load Counter into CONTEXT1 reg */
 452	if (is_rfc3686)
 453		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
 454				     LDST_SRCDST_BYTE_CONTEXT |
 455				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
 456				      LDST_OFFSET_SHIFT));
 457
 458	/* Choose operation */
 459	if (ctx1_iv_off)
 460		append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 461				 OP_ALG_DECRYPT);
 462	else
 463		append_dec_op1(desc, cdata->algtype);
 464
 465	/* Read and write cryptlen bytes */
 466	append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 467	append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 468	aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
 469
 470	/* Load ICV */
 471	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
 472			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 473
 474	print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ",
 475			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 476			     1);
 477}
 478EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
 479
 480/**
 481 * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor
 482 *                             (non-protocol) with HW-generated initialization
 483 *                             vector.
 484 * @desc: pointer to buffer used for descriptor construction
 485 * @cdata: pointer to block cipher transform definitions
 486 *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
 487 *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
 488 * @adata: pointer to authentication transform definitions.
 489 *         A split key is required for SEC Era < 6; the size of the split key
 490 *         is specified in this case. Valid algorithm values - one of
 491 *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
 492 *         with OP_ALG_AAI_HMAC_PRECOMP.
 493 * @ivsize: initialization vector size
 494 * @icvsize: integrity check value (ICV) size (truncated or full)
 495 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
 496 * @nonce: pointer to rfc3686 nonce
 497 * @ctx1_iv_off: IV offset in CONTEXT1 register
 498 * @is_qi: true when called from caam/qi
 499 * @era: SEC Era
 500 */
 501void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
 502			       struct alginfo *adata, unsigned int ivsize,
 503			       unsigned int icvsize, const bool is_rfc3686,
 504			       u32 *nonce, const u32 ctx1_iv_off,
 505			       const bool is_qi, int era)
 506{
 507	u32 geniv, moveiv;
 508	u32 *wait_cmd;
 509
 510	/* Note: Context registers are saved. */
 511	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
 512
 513	if (is_qi) {
 514		u32 *wait_load_cmd;
 515
 516		/* REG3 = assoclen */
 517		append_seq_load(desc, 4, LDST_CLASS_DECO |
 518				LDST_SRCDST_WORD_DECO_MATH3 |
 519				(4 << LDST_OFFSET_SHIFT));
 520
 521		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 522					    JUMP_COND_CALM | JUMP_COND_NCP |
 523					    JUMP_COND_NOP | JUMP_COND_NIP |
 524					    JUMP_COND_NIFP);
 525		set_jump_tgt_here(desc, wait_load_cmd);
 526	}
 527
 528	if (is_rfc3686) {
 529		if (is_qi)
 530			append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
 531					LDST_SRCDST_BYTE_CONTEXT |
 532					(ctx1_iv_off << LDST_OFFSET_SHIFT));
 533
 534		goto copy_iv;
 535	}
 536
 537	/* Generate IV */
 538	geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
 539		NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
 540		NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
 541	append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
 542			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 543	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 544	append_move(desc, MOVE_WAITCOMP |
 545		    MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
 546		    (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
 547		    (ivsize << MOVE_LEN_SHIFT));
 548	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 549
 550copy_iv:
 551	/* Copy IV to class 1 context */
 552	append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
 553		    (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
 554		    (ivsize << MOVE_LEN_SHIFT));
 555
 556	/* Return to encryption */
 557	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
 558			 OP_ALG_ENCRYPT);
 559
 560	/* Read and write assoclen bytes */
 561	if (is_qi || era < 3) {
 562		append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 563		append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 564	} else {
 565		append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
 566		append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
 567	}
 568
 569	/* Skip assoc data */
 570	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 571
 572	/* read assoc before reading payload */
 573	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 574			     KEY_VLF);
 575
 576	/* Copy iv from outfifo to class 2 fifo */
 577	moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
 578		 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
 579	append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
 580			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 581	append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
 582			    LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
 583
 584	/* Load Counter into CONTEXT1 reg */
 585	if (is_rfc3686)
 586		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
 587				     LDST_SRCDST_BYTE_CONTEXT |
 588				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
 589				      LDST_OFFSET_SHIFT));
 590
 591	/* Class 1 operation */
 592	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 593			 OP_ALG_ENCRYPT);
 594
 595	/* Will write ivsize + cryptlen */
 596	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 597
 598	/* Not need to reload iv */
 599	append_seq_fifo_load(desc, ivsize,
 600			     FIFOLD_CLASS_SKIP);
 601
 602	/* Will read cryptlen */
 603	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 604
 605	/*
 606	 * Wait for IV transfer (ofifo -> class2) to finish before starting
 607	 * ciphertext transfer (ofifo -> external memory).
 608	 */
 609	wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP);
 610	set_jump_tgt_here(desc, wait_cmd);
 611
 612	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
 613			     FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
 614	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
 615
 616	/* Write ICV */
 617	append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
 618			 LDST_SRCDST_BYTE_CONTEXT);
 619
 620	print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ",
 621			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 622			     1);
 623}
 624EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
 625
 626/**
 627 * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
 628 * @desc: pointer to buffer used for descriptor construction
 629 * @cdata: pointer to block cipher transform definitions
 630 *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
 631 * @ivsize: initialization vector size
 632 * @icvsize: integrity check value (ICV) size (truncated or full)
 633 * @is_qi: true when called from caam/qi
 634 */
 635void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
 636			   unsigned int ivsize, unsigned int icvsize,
 637			   const bool is_qi)
 638{
 639	u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
 640	    *zero_assoc_jump_cmd2;
 641
 642	init_sh_desc(desc, HDR_SHARE_SERIAL);
 643
 644	/* skip key loading if they are loaded due to sharing */
 645	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 646				   JUMP_COND_SHRD);
 647	if (cdata->key_inline)
 648		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
 649				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
 650	else
 651		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
 652			   KEY_DEST_CLASS_REG);
 653	set_jump_tgt_here(desc, key_jump_cmd);
 654
 655	/* class 1 operation */
 656	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 657			 OP_ALG_ENCRYPT);
 658
 659	if (is_qi) {
 660		u32 *wait_load_cmd;
 661
 662		/* REG3 = assoclen */
 663		append_seq_load(desc, 4, LDST_CLASS_DECO |
 664				LDST_SRCDST_WORD_DECO_MATH3 |
 665				(4 << LDST_OFFSET_SHIFT));
 666
 667		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 668					    JUMP_COND_CALM | JUMP_COND_NCP |
 669					    JUMP_COND_NOP | JUMP_COND_NIP |
 670					    JUMP_COND_NIFP);
 671		set_jump_tgt_here(desc, wait_load_cmd);
 672
 673		append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM,
 674					ivsize);
 675	} else {
 676		append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0,
 677				CAAM_CMD_SZ);
 678	}
 679
 680	/* if assoclen + cryptlen is ZERO, skip to ICV write */
 681	zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
 682						 JUMP_COND_MATH_Z);
 683
 684	if (is_qi)
 685		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
 686				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
 687
 688	/* if assoclen is ZERO, skip reading the assoc data */
 689	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 690	zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
 691					   JUMP_COND_MATH_Z);
 692
 693	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 694
 695	/* skip assoc data */
 696	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 697
 698	/* cryptlen = seqinlen - assoclen */
 699	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
 700
 701	/* if cryptlen is ZERO jump to zero-payload commands */
 702	zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
 703					    JUMP_COND_MATH_Z);
 704
 705	/* read assoc data */
 706	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 707			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
 708	set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
 709
 710	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 711
 712	/* write encrypted data */
 713	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
 714
 715	/* read payload data */
 716	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 717			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
 718
 719	/* jump to ICV writing */
 720	if (is_qi)
 721		append_jump(desc, JUMP_TEST_ALL | 4);
 722	else
 723		append_jump(desc, JUMP_TEST_ALL | 2);
 724
 725	/* zero-payload commands */
 726	set_jump_tgt_here(desc, zero_payload_jump_cmd);
 727
 728	/* read assoc data */
 729	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 730			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
 731	if (is_qi)
 732		/* jump to ICV writing */
 733		append_jump(desc, JUMP_TEST_ALL | 2);
 734
 735	/* There is no input data */
 736	set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
 737
 738	if (is_qi)
 739		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
 740				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
 741				     FIFOLD_TYPE_LAST1);
 742
 743	/* write ICV */
 744	append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
 745			 LDST_SRCDST_BYTE_CONTEXT);
 746
 747	print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ",
 748			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 749			     1);
 750}
 751EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
 752
 753/**
 754 * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor
 755 * @desc: pointer to buffer used for descriptor construction
 756 * @cdata: pointer to block cipher transform definitions
 757 *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
 758 * @ivsize: initialization vector size
 759 * @icvsize: integrity check value (ICV) size (truncated or full)
 760 * @is_qi: true when called from caam/qi
 761 */
 762void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
 763			   unsigned int ivsize, unsigned int icvsize,
 764			   const bool is_qi)
 765{
 766	u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
 767
 768	init_sh_desc(desc, HDR_SHARE_SERIAL);
 769
 770	/* skip key loading if they are loaded due to sharing */
 771	key_jump_cmd = append_jump(desc, JUMP_JSL |
 772				   JUMP_TEST_ALL | JUMP_COND_SHRD);
 773	if (cdata->key_inline)
 774		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
 775				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
 776	else
 777		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
 778			   KEY_DEST_CLASS_REG);
 779	set_jump_tgt_here(desc, key_jump_cmd);
 780
 781	/* class 1 operation */
 782	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 783			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 784
 785	if (is_qi) {
 786		u32 *wait_load_cmd;
 787
 788		/* REG3 = assoclen */
 789		append_seq_load(desc, 4, LDST_CLASS_DECO |
 790				LDST_SRCDST_WORD_DECO_MATH3 |
 791				(4 << LDST_OFFSET_SHIFT));
 792
 793		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 794					    JUMP_COND_CALM | JUMP_COND_NCP |
 795					    JUMP_COND_NOP | JUMP_COND_NIP |
 796					    JUMP_COND_NIFP);
 797		set_jump_tgt_here(desc, wait_load_cmd);
 798
 799		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
 800				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
 801	}
 802
 803	/* if assoclen is ZERO, skip reading the assoc data */
 804	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 805	zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
 806						 JUMP_COND_MATH_Z);
 807
 808	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 809
 810	/* skip assoc data */
 811	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 812
 813	/* read assoc data */
 814	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 815			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
 816
 817	set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
 818
 819	/* cryptlen = seqoutlen - assoclen */
 820	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 821
 822	/* jump to zero-payload command if cryptlen is zero */
 823	zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
 824					    JUMP_COND_MATH_Z);
 825
 826	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 827
 828	/* store encrypted data */
 829	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
 830
 831	/* read payload data */
 832	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 833			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 834
 835	/* zero-payload command */
 836	set_jump_tgt_here(desc, zero_payload_jump_cmd);
 837
 838	/* read ICV */
 839	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
 840			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
 841
 842	print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ",
 843			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 844			     1);
 845}
 846EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
 847
 848/**
 849 * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor
 850 *                             (non-protocol).
 851 * @desc: pointer to buffer used for descriptor construction
 852 * @cdata: pointer to block cipher transform definitions
 853 *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
 854 * @ivsize: initialization vector size
 855 * @icvsize: integrity check value (ICV) size (truncated or full)
 856 * @is_qi: true when called from caam/qi
 857 *
 858 * Input sequence: AAD | PTXT
 859 * Output sequence: AAD | CTXT | ICV
 860 * AAD length (assoclen), which includes the IV length, is available in Math3.
 861 */
 862void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
 863			       unsigned int ivsize, unsigned int icvsize,
 864			       const bool is_qi)
 865{
 866	u32 *key_jump_cmd, *zero_cryptlen_jump_cmd, *skip_instructions;
 867	init_sh_desc(desc, HDR_SHARE_SERIAL);
 868
 869	/* Skip key loading if it is loaded due to sharing */
 870	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 871				   JUMP_COND_SHRD);
 872	if (cdata->key_inline)
 873		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
 874				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
 875	else
 876		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
 877			   KEY_DEST_CLASS_REG);
 878	set_jump_tgt_here(desc, key_jump_cmd);
 879
 880	/* Class 1 operation */
 881	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 882			 OP_ALG_ENCRYPT);
 883
 884	if (is_qi) {
 885		u32 *wait_load_cmd;
 886
 887		/* REG3 = assoclen */
 888		append_seq_load(desc, 4, LDST_CLASS_DECO |
 889				LDST_SRCDST_WORD_DECO_MATH3 |
 890				(4 << LDST_OFFSET_SHIFT));
 891
 892		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 893					    JUMP_COND_CALM | JUMP_COND_NCP |
 894					    JUMP_COND_NOP | JUMP_COND_NIP |
 895					    JUMP_COND_NIFP);
 896		set_jump_tgt_here(desc, wait_load_cmd);
 897
 898		/* Read salt and IV */
 899		append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
 900					cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
 901					FIFOLD_TYPE_IV);
 902		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
 903				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
 904	}
 905
 906	append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
 907	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 908
 909	/* Skip AAD */
 910	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 911
 912	/* Read cryptlen and set this value into VARSEQOUTLEN */
 913	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
 914
 915	/* If cryptlen is ZERO jump to AAD command */
 916	zero_cryptlen_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
 917					    JUMP_COND_MATH_Z);
 918
 919	/* Read AAD data */
 920	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 921			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
 922
 923	/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
 924	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA);
 925
 926	/* Skip IV */
 927	append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
 928	append_math_add(desc, VARSEQINLEN, VARSEQOUTLEN, REG0, CAAM_CMD_SZ);
 929
 930	/* Write encrypted data */
 931	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
 932
 933	/* Read payload data */
 934	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 935			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
 936
 937	/* Jump instructions to avoid double reading of AAD */
 938	skip_instructions = append_jump(desc, JUMP_TEST_ALL);
 939
 940	/* There is no input data, cryptlen = 0 */
 941	set_jump_tgt_here(desc, zero_cryptlen_jump_cmd);
 942
 943	/* Read AAD */
 944	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
 945			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
 946
 947	set_jump_tgt_here(desc, skip_instructions);
 948
 949	/* Write ICV */
 950	append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
 951			 LDST_SRCDST_BYTE_CONTEXT);
 952
 953	print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ",
 954			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
 955			     1);
 956}
 957EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
 958
 959/**
 960 * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor
 961 *                             (non-protocol).
 962 * @desc: pointer to buffer used for descriptor construction
 963 * @cdata: pointer to block cipher transform definitions
 964 *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
 965 * @ivsize: initialization vector size
 966 * @icvsize: integrity check value (ICV) size (truncated or full)
 967 * @is_qi: true when called from caam/qi
 968 */
 969void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
 970			       unsigned int ivsize, unsigned int icvsize,
 971			       const bool is_qi)
 972{
 973	u32 *key_jump_cmd;
 974
 975	init_sh_desc(desc, HDR_SHARE_SERIAL);
 976
 977	/* Skip key loading if it is loaded due to sharing */
 978	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 979				   JUMP_COND_SHRD);
 980	if (cdata->key_inline)
 981		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
 982				  cdata->keylen, CLASS_1 |
 983				  KEY_DEST_CLASS_REG);
 984	else
 985		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
 986			   KEY_DEST_CLASS_REG);
 987	set_jump_tgt_here(desc, key_jump_cmd);
 988
 989	/* Class 1 operation */
 990	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
 991			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 992
 993	if (is_qi) {
 994		u32 *wait_load_cmd;
 995
 996		/* REG3 = assoclen */
 997		append_seq_load(desc, 4, LDST_CLASS_DECO |
 998				LDST_SRCDST_WORD_DECO_MATH3 |
 999				(4 << LDST_OFFSET_SHIFT));
1000
1001		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1002					    JUMP_COND_CALM | JUMP_COND_NCP |
1003					    JUMP_COND_NOP | JUMP_COND_NIP |
1004					    JUMP_COND_NIFP);
1005		set_jump_tgt_here(desc, wait_load_cmd);
1006
1007		/* Read salt and IV */
1008		append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1009					cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1010					FIFOLD_TYPE_IV);
1011		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1012				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1013	}
1014
1015	append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
1016	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1017
1018	/* Read assoc data */
1019	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1020			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1021
1022	/* Skip IV */
1023	append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
1024
1025	/* Will read cryptlen bytes */
1026	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1027
1028	/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1029	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1030
1031	/* Skip assoc data */
1032	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1033
1034	/* Will write cryptlen bytes */
1035	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1036
1037	/* Store payload data */
1038	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1039
1040	/* Read encrypted data */
1041	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1042			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1043
1044	/* Read ICV */
1045	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1046			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1047
1048	print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ",
1049			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1050			     1);
1051}
1052EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
1053
1054/**
1055 * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor
1056 *                             (non-protocol).
1057 * @desc: pointer to buffer used for descriptor construction
1058 * @cdata: pointer to block cipher transform definitions
1059 *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1060 * @ivsize: initialization vector size
1061 * @icvsize: integrity check value (ICV) size (truncated or full)
1062 * @is_qi: true when called from caam/qi
1063 */
1064void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
1065			       unsigned int ivsize, unsigned int icvsize,
1066			       const bool is_qi)
1067{
1068	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1069
1070	init_sh_desc(desc, HDR_SHARE_SERIAL);
1071
1072	/* Skip key loading if it is loaded due to sharing */
1073	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1074				   JUMP_COND_SHRD);
1075	if (cdata->key_inline)
1076		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1077				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1078	else
1079		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1080			   KEY_DEST_CLASS_REG);
1081	set_jump_tgt_here(desc, key_jump_cmd);
1082
1083	/* Class 1 operation */
1084	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1085			 OP_ALG_ENCRYPT);
1086
1087	if (is_qi) {
1088		/* assoclen is not needed, skip it */
1089		append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1090
1091		/* Read salt and IV */
1092		append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1093					cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1094					FIFOLD_TYPE_IV);
1095		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1096				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1097	}
1098
1099	/* assoclen + cryptlen = seqinlen */
1100	append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1101
1102	/*
1103	 * MOVE_LEN opcode is not available in all SEC HW revisions,
1104	 * thus need to do some magic, i.e. self-patch the descriptor
1105	 * buffer.
1106	 */
1107	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1108				    (0x6 << MOVE_LEN_SHIFT));
1109	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1110				     (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1111
1112	/* Will read assoclen + cryptlen bytes */
1113	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1114
1115	/* Will write assoclen + cryptlen bytes */
1116	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1117
1118	/* Read and write assoclen + cryptlen bytes */
1119	aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1120
1121	set_move_tgt_here(desc, read_move_cmd);
1122	set_move_tgt_here(desc, write_move_cmd);
1123	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1124	/* Move payload data to OFIFO */
1125	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1126
1127	/* Write ICV */
1128	append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
1129			 LDST_SRCDST_BYTE_CONTEXT);
1130
1131	print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ",
1132			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1133			     1);
1134}
1135EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
1136
1137/**
1138 * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor
1139 *                             (non-protocol).
1140 * @desc: pointer to buffer used for descriptor construction
1141 * @cdata: pointer to block cipher transform definitions
1142 *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1143 * @ivsize: initialization vector size
1144 * @icvsize: integrity check value (ICV) size (truncated or full)
1145 * @is_qi: true when called from caam/qi
1146 */
1147void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
1148			       unsigned int ivsize, unsigned int icvsize,
1149			       const bool is_qi)
1150{
1151	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1152
1153	init_sh_desc(desc, HDR_SHARE_SERIAL);
1154
1155	/* Skip key loading if it is loaded due to sharing */
1156	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1157				   JUMP_COND_SHRD);
1158	if (cdata->key_inline)
1159		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1160				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1161	else
1162		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1163			   KEY_DEST_CLASS_REG);
1164	set_jump_tgt_here(desc, key_jump_cmd);
1165
1166	/* Class 1 operation */
1167	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1168			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1169
1170	if (is_qi) {
1171		/* assoclen is not needed, skip it */
1172		append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1173
1174		/* Read salt and IV */
1175		append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1176					cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1177					FIFOLD_TYPE_IV);
1178		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1179				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1180	}
1181
1182	/* assoclen + cryptlen = seqoutlen */
1183	append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1184
1185	/*
1186	 * MOVE_LEN opcode is not available in all SEC HW revisions,
1187	 * thus need to do some magic, i.e. self-patch the descriptor
1188	 * buffer.
1189	 */
1190	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1191				    (0x6 << MOVE_LEN_SHIFT));
1192	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1193				     (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1194
1195	/* Will read assoclen + cryptlen bytes */
1196	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1197
1198	/* Will write assoclen + cryptlen bytes */
1199	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1200
1201	/* Store payload data */
1202	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1203
1204	/* In-snoop assoclen + cryptlen data */
1205	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1206			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1207
1208	set_move_tgt_here(desc, read_move_cmd);
1209	set_move_tgt_here(desc, write_move_cmd);
1210	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1211	/* Move payload data to OFIFO */
1212	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1213	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1214
1215	/* Read ICV */
1216	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1217			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1218
1219	print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1220			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1221			     1);
1222}
1223EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
1224
1225/**
1226 * cnstr_shdsc_chachapoly - Chacha20 + Poly1305 generic AEAD (rfc7539) and
1227 *                          IPsec ESP (rfc7634, a.k.a. rfc7539esp) shared
1228 *                          descriptor (non-protocol).
1229 * @desc: pointer to buffer used for descriptor construction
1230 * @cdata: pointer to block cipher transform definitions
1231 *         Valid algorithm values - OP_ALG_ALGSEL_CHACHA20 ANDed with
1232 *         OP_ALG_AAI_AEAD.
1233 * @adata: pointer to authentication transform definitions
1234 *         Valid algorithm values - OP_ALG_ALGSEL_POLY1305 ANDed with
1235 *         OP_ALG_AAI_AEAD.
1236 * @ivsize: initialization vector size
1237 * @icvsize: integrity check value (ICV) size (truncated or full)
1238 * @encap: true if encapsulation, false if decapsulation
1239 * @is_qi: true when called from caam/qi
1240 */
1241void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata,
1242			    struct alginfo *adata, unsigned int ivsize,
1243			    unsigned int icvsize, const bool encap,
1244			    const bool is_qi)
1245{
1246	u32 *key_jump_cmd, *wait_cmd;
1247	u32 nfifo;
1248	const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE);
1249
1250	/* Note: Context registers are saved. */
1251	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1252
1253	/* skip key loading if they are loaded due to sharing */
1254	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1255				   JUMP_COND_SHRD);
1256
1257	append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen,
1258			  CLASS_1 | KEY_DEST_CLASS_REG);
1259
1260	/* For IPsec load the salt from keymat in the context register */
1261	if (is_ipsec)
1262		append_load_as_imm(desc, cdata->key_virt + cdata->keylen, 4,
1263				   LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT |
1264				   4 << LDST_OFFSET_SHIFT);
1265
1266	set_jump_tgt_here(desc, key_jump_cmd);
1267
1268	/* Class 2 and 1 operations: Poly & ChaCha */
1269	if (encap) {
1270		append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1271				 OP_ALG_ENCRYPT);
1272		append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1273				 OP_ALG_ENCRYPT);
1274	} else {
1275		append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1276				 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1277		append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1278				 OP_ALG_DECRYPT);
1279	}
1280
1281	if (is_qi) {
1282		u32 *wait_load_cmd;
1283		u32 ctx1_iv_off = is_ipsec ? 8 : 4;
1284
1285		/* REG3 = assoclen */
1286		append_seq_load(desc, 4, LDST_CLASS_DECO |
1287				LDST_SRCDST_WORD_DECO_MATH3 |
1288				4 << LDST_OFFSET_SHIFT);
1289
1290		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1291					    JUMP_COND_CALM | JUMP_COND_NCP |
1292					    JUMP_COND_NOP | JUMP_COND_NIP |
1293					    JUMP_COND_NIFP);
1294		set_jump_tgt_here(desc, wait_load_cmd);
1295
1296		append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
1297				LDST_SRCDST_BYTE_CONTEXT |
1298				ctx1_iv_off << LDST_OFFSET_SHIFT);
1299	}
1300
1301	/*
1302	 * MAGIC with NFIFO
1303	 * Read associated data from the input and send them to class1 and
1304	 * class2 alignment blocks. From class1 send data to output fifo and
1305	 * then write it to memory since we don't need to encrypt AD.
1306	 */
1307	nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 |
1308		NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND;
1309	append_load_imm_u32(desc, nfifo, LDST_CLASS_IND_CCB |
1310			    LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3);
1311
1312	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1313	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1314	append_seq_fifo_load(desc, 0, FIFOLD_TYPE_NOINFOFIFO |
1315			     FIFOLD_CLASS_CLASS1 | LDST_VLF);
1316	append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK |
1317			MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3);
1318	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF);
1319
1320	/* IPsec - copy IV at the output */
1321	if (is_ipsec)
1322		append_seq_fifo_store(desc, ivsize, FIFOST_TYPE_METADATA |
1323				      0x2 << 25);
1324
1325	wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL |
1326			       JUMP_COND_NOP | JUMP_TEST_ALL);
1327	set_jump_tgt_here(desc, wait_cmd);
1328
1329	if (encap) {
1330		/* Read and write cryptlen bytes */
1331		append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1332		append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0,
1333				CAAM_CMD_SZ);
1334		aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
1335
1336		/* Write ICV */
1337		append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
1338				 LDST_SRCDST_BYTE_CONTEXT);
1339	} else {
1340		/* Read and write cryptlen bytes */
1341		append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0,
1342				CAAM_CMD_SZ);
1343		append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0,
1344				CAAM_CMD_SZ);
1345		aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
1346
1347		/* Load ICV for verification */
1348		append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
1349				     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
1350	}
1351
1352	print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": ",
1353			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1354			     1);
1355}
1356EXPORT_SYMBOL(cnstr_shdsc_chachapoly);
1357
1358/* For skcipher encrypt and decrypt, read from req->src and write to req->dst */
1359static inline void skcipher_append_src_dst(u32 *desc)
1360{
1361	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1362	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1363	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
1364			     KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1365	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
1366}
1367
1368void cnstr_desc_skcipher_enc_dec(u32 * const desc, struct alginfo *cdata,
1369				 dma_addr_t src, dma_addr_t dst, unsigned int data_sz,
1370				 unsigned int in_options, unsigned int out_options,
1371				 unsigned int ivsize, const bool encrypt)
1372{
1373	u32 options = cdata->algtype | OP_ALG_AS_INIT;
1374
1375	if (encrypt)
1376		options |= OP_ALG_ENCRYPT;
1377	else
1378		options |= OP_ALG_DECRYPT;
1379
1380	init_job_desc(desc, 0);
1381
1382	append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL |
1383			       JUMP_COND_NOP | JUMP_TEST_ALL | 1);
1384
1385	append_key(desc, cdata->protected_key_dma, cdata->plain_keylen,
1386		   CLASS_1 | KEY_DEST_CLASS_REG | cdata->key_cmd_opt);
1387
1388	append_seq_in_ptr(desc, src, data_sz, in_options);
1389
1390	append_seq_out_ptr(desc, dst, data_sz, out_options);
1391
1392	/* Load IV, if there is one */
1393	if (ivsize)
1394		append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1395				LDST_CLASS_1_CCB);
1396
1397	append_operation(desc, options);
1398
1399	skcipher_append_src_dst(desc);
1400
1401	/* Store IV */
1402	if (ivsize)
1403		append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1404				 LDST_CLASS_1_CCB);
1405
1406	print_hex_dump_debug("skcipher_enc_dec job desc@" __stringify(__LINE__)": ",
1407			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1408			     1);
1409}
1410EXPORT_SYMBOL(cnstr_desc_skcipher_enc_dec);
1411
1412void cnstr_desc_protected_blob_decap(u32 * const desc, struct alginfo *cdata,
1413				     dma_addr_t next_desc_addr)
1414{
1415	u32 protected_store;
1416
1417	init_job_desc(desc, 0);
1418
1419	/* Load key modifier */
1420	append_load_as_imm(desc, KEYMOD, sizeof(KEYMOD) - 1,
1421			   LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_KEY);
1422
1423	append_seq_in_ptr_intlen(desc, cdata->key_dma,
1424				 cdata->plain_keylen + CAAM_BLOB_OVERHEAD, 0);
1425
1426	append_seq_out_ptr_intlen(desc, cdata->protected_key_dma,
1427				  cdata->plain_keylen, 0);
1428
1429	protected_store = OP_PCLID_BLOB | OP_PCL_BLOB_BLACK;
1430	if ((cdata->key_cmd_opt >> KEY_EKT_OFFSET) & 1)
1431		protected_store |= OP_PCL_BLOB_EKT;
1432
1433	append_operation(desc, OP_TYPE_DECAP_PROTOCOL | protected_store);
1434
1435	if (next_desc_addr) {
1436		append_jump(desc, JUMP_TYPE_NONLOCAL | JUMP_TEST_ALL);
1437		append_ptr(desc, next_desc_addr);
1438	}
1439
1440	print_hex_dump_debug("protected blob decap job desc@" __stringify(__LINE__) ":",
1441			     DUMP_PREFIX_ADDRESS, 16, 4, desc,
1442			     desc_bytes(desc), 1);
1443}
1444EXPORT_SYMBOL(cnstr_desc_protected_blob_decap);
1445
1446/**
1447 * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor
1448 * @desc: pointer to buffer used for descriptor construction
1449 * @cdata: pointer to block cipher transform definitions
1450 *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1451 *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1452 *                                - OP_ALG_ALGSEL_CHACHA20
1453 * @ivsize: initialization vector size
1454 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1455 * @ctx1_iv_off: IV offset in CONTEXT1 register
1456 */
1457void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
1458				unsigned int ivsize, const bool is_rfc3686,
1459				const u32 ctx1_iv_off)
1460{
1461	u32 *key_jump_cmd;
1462	u32 options = cdata->algtype | OP_ALG_AS_INIT | OP_ALG_ENCRYPT;
1463	bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) ==
1464			    OP_ALG_ALGSEL_CHACHA20);
1465
1466	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1467	/* Skip if already shared */
1468	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1469				   JUMP_COND_SHRD);
1470
1471	/* Load class1 key only */
1472	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1473			  cdata->plain_keylen, CLASS_1 | KEY_DEST_CLASS_REG
1474					       | cdata->key_cmd_opt);
1475
1476	/* Load nonce into CONTEXT1 reg */
1477	if (is_rfc3686) {
1478		const u8 *nonce = cdata->key_virt + cdata->keylen;
1479
1480		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1481				   LDST_CLASS_IND_CCB |
1482				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1483		append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1484			    MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1485			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1486	}
1487
1488	set_jump_tgt_here(desc, key_jump_cmd);
1489
1490	/* Load IV, if there is one */
1491	if (ivsize)
1492		append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1493				LDST_CLASS_1_CCB | (ctx1_iv_off <<
1494				LDST_OFFSET_SHIFT));
1495
1496	/* Load counter into CONTEXT1 reg */
1497	if (is_rfc3686)
1498		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1499				     LDST_SRCDST_BYTE_CONTEXT |
1500				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1501				      LDST_OFFSET_SHIFT));
1502
1503	/* Load operation */
1504	if (is_chacha20)
1505		options |= OP_ALG_AS_FINALIZE;
1506	append_operation(desc, options);
1507
1508	/* Perform operation */
1509	skcipher_append_src_dst(desc);
1510
1511	/* Store IV */
1512	if (!is_chacha20 && ivsize)
1513		append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1514				 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1515				 LDST_OFFSET_SHIFT));
1516
1517	print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ",
1518			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1519			     1);
1520}
1521EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
1522
1523/**
1524 * cnstr_shdsc_skcipher_decap - skcipher decapsulation shared descriptor
1525 * @desc: pointer to buffer used for descriptor construction
1526 * @cdata: pointer to block cipher transform definitions
1527 *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1528 *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1529 *                                - OP_ALG_ALGSEL_CHACHA20
1530 * @ivsize: initialization vector size
1531 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1532 * @ctx1_iv_off: IV offset in CONTEXT1 register
1533 */
1534void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
1535				unsigned int ivsize, const bool is_rfc3686,
1536				const u32 ctx1_iv_off)
1537{
1538	u32 *key_jump_cmd;
1539	bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) ==
1540			    OP_ALG_ALGSEL_CHACHA20);
1541
1542	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1543	/* Skip if already shared */
1544	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1545				   JUMP_COND_SHRD);
1546
1547	/* Load class1 key only */
1548	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1549			  cdata->plain_keylen, CLASS_1 | KEY_DEST_CLASS_REG
1550					       | cdata->key_cmd_opt);
1551
1552	/* Load nonce into CONTEXT1 reg */
1553	if (is_rfc3686) {
1554		const u8 *nonce = cdata->key_virt + cdata->keylen;
1555
1556		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1557				   LDST_CLASS_IND_CCB |
1558				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1559		append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1560			    MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1561			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1562	}
1563
1564	set_jump_tgt_here(desc, key_jump_cmd);
1565
1566	/* Load IV, if there is one */
1567	if (ivsize)
1568		append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1569				LDST_CLASS_1_CCB | (ctx1_iv_off <<
1570				LDST_OFFSET_SHIFT));
1571
1572	/* Load counter into CONTEXT1 reg */
1573	if (is_rfc3686)
1574		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1575				     LDST_SRCDST_BYTE_CONTEXT |
1576				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1577				      LDST_OFFSET_SHIFT));
1578
1579	/* Choose operation */
1580	if (ctx1_iv_off)
1581		append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
1582				 OP_ALG_DECRYPT);
1583	else
1584		append_dec_op1(desc, cdata->algtype);
1585
1586	/* Perform operation */
1587	skcipher_append_src_dst(desc);
1588
1589	/* Store IV */
1590	if (!is_chacha20 && ivsize)
1591		append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1592				 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1593				 LDST_OFFSET_SHIFT));
1594
1595	print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ",
1596			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1597			     1);
1598}
1599EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
1600
1601/**
1602 * cnstr_shdsc_xts_skcipher_encap - xts skcipher encapsulation shared descriptor
1603 * @desc: pointer to buffer used for descriptor construction
1604 * @cdata: pointer to block cipher transform definitions
1605 *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1606 */
1607void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
1608{
1609	/*
1610	 * Set sector size to a big value, practically disabling
1611	 * sector size segmentation in xts implementation. We cannot
1612	 * take full advantage of this HW feature with existing
1613	 * crypto API / dm-crypt SW architecture.
1614	 */
1615	__be64 sector_size = cpu_to_be64(BIT(15));
1616	u32 *key_jump_cmd;
1617
1618	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1619	/* Skip if already shared */
1620	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1621				   JUMP_COND_SHRD);
1622
1623	/* Load class1 keys only */
1624	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1625			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1626
1627	/* Load sector size with index 40 bytes (0x28) */
1628	append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1629			   LDST_SRCDST_BYTE_CONTEXT |
1630			   (0x28 << LDST_OFFSET_SHIFT));
1631
1632	set_jump_tgt_here(desc, key_jump_cmd);
1633
1634	/*
1635	 * create sequence for loading the sector index / 16B tweak value
1636	 * Lower 8B of IV - sector index / tweak lower half
1637	 * Upper 8B of IV - upper half of 16B tweak
1638	 */
1639	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1640			(0x20 << LDST_OFFSET_SHIFT));
1641	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1642			(0x30 << LDST_OFFSET_SHIFT));
1643
1644	/* Load operation */
1645	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1646			 OP_ALG_ENCRYPT);
1647
1648	/* Perform operation */
1649	skcipher_append_src_dst(desc);
1650
1651	/* Store lower 8B and upper 8B of IV */
1652	append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1653			 (0x20 << LDST_OFFSET_SHIFT));
1654	append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1655			 (0x30 << LDST_OFFSET_SHIFT));
1656
1657	print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__)
1658			     ": ", DUMP_PREFIX_ADDRESS, 16, 4,
1659			     desc, desc_bytes(desc), 1);
1660}
1661EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
1662
1663/**
1664 * cnstr_shdsc_xts_skcipher_decap - xts skcipher decapsulation shared descriptor
1665 * @desc: pointer to buffer used for descriptor construction
1666 * @cdata: pointer to block cipher transform definitions
1667 *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1668 */
1669void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
1670{
1671	/*
1672	 * Set sector size to a big value, practically disabling
1673	 * sector size segmentation in xts implementation. We cannot
1674	 * take full advantage of this HW feature with existing
1675	 * crypto API / dm-crypt SW architecture.
1676	 */
1677	__be64 sector_size = cpu_to_be64(BIT(15));
1678	u32 *key_jump_cmd;
1679
1680	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1681	/* Skip if already shared */
1682	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1683				   JUMP_COND_SHRD);
1684
1685	/* Load class1 key only */
1686	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1687			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1688
1689	/* Load sector size with index 40 bytes (0x28) */
1690	append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1691			   LDST_SRCDST_BYTE_CONTEXT |
1692			   (0x28 << LDST_OFFSET_SHIFT));
1693
1694	set_jump_tgt_here(desc, key_jump_cmd);
1695
1696	/*
1697	 * create sequence for loading the sector index / 16B tweak value
1698	 * Lower 8B of IV - sector index / tweak lower half
1699	 * Upper 8B of IV - upper half of 16B tweak
1700	 */
1701	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1702			(0x20 << LDST_OFFSET_SHIFT));
1703	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1704			(0x30 << LDST_OFFSET_SHIFT));
1705	/* Load operation */
1706	append_dec_op1(desc, cdata->algtype);
1707
1708	/* Perform operation */
1709	skcipher_append_src_dst(desc);
1710
1711	/* Store lower 8B and upper 8B of IV */
1712	append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1713			 (0x20 << LDST_OFFSET_SHIFT));
1714	append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1715			 (0x30 << LDST_OFFSET_SHIFT));
1716
1717	print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__)
1718			     ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
1719			     desc_bytes(desc), 1);
1720}
1721EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);
1722
1723MODULE_LICENSE("GPL");
1724MODULE_DESCRIPTION("FSL CAAM descriptor support");
1725MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");