ocs-aes.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Intel Keem Bay OCS AES Crypto Driver.
   4 *
   5 * Copyright (C) 2018-2020 Intel Corporation
   6 */
   7
   8#include <linux/dma-mapping.h>
   9#include <linux/interrupt.h>
  10#include <linux/kernel.h>
  11#include <linux/platform_device.h>
  12#include <linux/slab.h>
  13#include <linux/swab.h>
  14
  15#include <asm/byteorder.h>
  16#include <asm/errno.h>
  17
  18#include <crypto/aes.h>
  19#include <crypto/gcm.h>
  20
  21#include "ocs-aes.h"
  22
  23#define AES_COMMAND_OFFSET			0x0000
  24#define AES_KEY_0_OFFSET			0x0004
  25#define AES_KEY_1_OFFSET			0x0008
  26#define AES_KEY_2_OFFSET			0x000C
  27#define AES_KEY_3_OFFSET			0x0010
  28#define AES_KEY_4_OFFSET			0x0014
  29#define AES_KEY_5_OFFSET			0x0018
  30#define AES_KEY_6_OFFSET			0x001C
  31#define AES_KEY_7_OFFSET			0x0020
  32#define AES_IV_0_OFFSET				0x0024
  33#define AES_IV_1_OFFSET				0x0028
  34#define AES_IV_2_OFFSET				0x002C
  35#define AES_IV_3_OFFSET				0x0030
  36#define AES_ACTIVE_OFFSET			0x0034
  37#define AES_STATUS_OFFSET			0x0038
  38#define AES_KEY_SIZE_OFFSET			0x0044
  39#define AES_IER_OFFSET				0x0048
  40#define AES_ISR_OFFSET				0x005C
  41#define AES_MULTIPURPOSE1_0_OFFSET		0x0200
  42#define AES_MULTIPURPOSE1_1_OFFSET		0x0204
  43#define AES_MULTIPURPOSE1_2_OFFSET		0x0208
  44#define AES_MULTIPURPOSE1_3_OFFSET		0x020C
  45#define AES_MULTIPURPOSE2_0_OFFSET		0x0220
  46#define AES_MULTIPURPOSE2_1_OFFSET		0x0224
  47#define AES_MULTIPURPOSE2_2_OFFSET		0x0228
  48#define AES_MULTIPURPOSE2_3_OFFSET		0x022C
  49#define AES_BYTE_ORDER_CFG_OFFSET		0x02C0
  50#define AES_TLEN_OFFSET				0x0300
  51#define AES_T_MAC_0_OFFSET			0x0304
  52#define AES_T_MAC_1_OFFSET			0x0308
  53#define AES_T_MAC_2_OFFSET			0x030C
  54#define AES_T_MAC_3_OFFSET			0x0310
  55#define AES_PLEN_OFFSET				0x0314
  56#define AES_A_DMA_SRC_ADDR_OFFSET		0x0400
  57#define AES_A_DMA_DST_ADDR_OFFSET		0x0404
  58#define AES_A_DMA_SRC_SIZE_OFFSET		0x0408
  59#define AES_A_DMA_DST_SIZE_OFFSET		0x040C
  60#define AES_A_DMA_DMA_MODE_OFFSET		0x0410
  61#define AES_A_DMA_NEXT_SRC_DESCR_OFFSET		0x0418
  62#define AES_A_DMA_NEXT_DST_DESCR_OFFSET		0x041C
  63#define AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET	0x0420
  64#define AES_A_DMA_LOG_OFFSET			0x0424
  65#define AES_A_DMA_STATUS_OFFSET			0x0428
  66#define AES_A_DMA_PERF_CNTR_OFFSET		0x042C
  67#define AES_A_DMA_MSI_ISR_OFFSET		0x0480
  68#define AES_A_DMA_MSI_IER_OFFSET		0x0484
  69#define AES_A_DMA_MSI_MASK_OFFSET		0x0488
  70#define AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET	0x0600
  71#define AES_A_DMA_OUTBUFFER_READ_FIFO_OFFSET	0x0700
  72
  73/*
  74 * AES_A_DMA_DMA_MODE register.
  75 * Default: 0x00000000.
  76 * bit[31]	ACTIVE
  77 *		This bit activates the DMA. When the DMA finishes, it resets
  78 *		this bit to zero.
  79 * bit[30:26]	Unused by this driver.
  80 * bit[25]	SRC_LINK_LIST_EN
  81 *		Source link list enable bit. When the linked list is terminated
  82 *		this bit is reset by the DMA.
  83 * bit[24]	DST_LINK_LIST_EN
  84 *		Destination link list enable bit. When the linked list is
  85 *		terminated this bit is reset by the DMA.
  86 * bit[23:0]	Unused by this driver.
  87 */
  88#define AES_A_DMA_DMA_MODE_ACTIVE		BIT(31)
  89#define AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN	BIT(25)
  90#define AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN	BIT(24)
  91
  92/*
  93 * AES_ACTIVE register
  94 * default 0x00000000
  95 * bit[31:10]	Reserved
  96 * bit[9]	LAST_ADATA
  97 * bit[8]	LAST_GCX
  98 * bit[7:2]	Reserved
  99 * bit[1]	TERMINATION
 100 * bit[0]	TRIGGER
 101 */
 102#define AES_ACTIVE_LAST_ADATA			BIT(9)
 103#define AES_ACTIVE_LAST_CCM_GCM			BIT(8)
 104#define AES_ACTIVE_TERMINATION			BIT(1)
 105#define AES_ACTIVE_TRIGGER			BIT(0)
 106
 107#define AES_DISABLE_INT				0x00000000
 108#define AES_DMA_CPD_ERR_INT			BIT(8)
 109#define AES_DMA_OUTBUF_RD_ERR_INT		BIT(7)
 110#define AES_DMA_OUTBUF_WR_ERR_INT		BIT(6)
 111#define AES_DMA_INBUF_RD_ERR_INT		BIT(5)
 112#define AES_DMA_INBUF_WR_ERR_INT		BIT(4)
 113#define AES_DMA_BAD_COMP_INT			BIT(3)
 114#define AES_DMA_SAI_INT				BIT(2)
 115#define AES_DMA_SRC_DONE_INT			BIT(0)
 116#define AES_COMPLETE_INT			BIT(1)
 117
 118#define AES_DMA_MSI_MASK_CLEAR			BIT(0)
 119
 120#define AES_128_BIT_KEY				0x00000000
 121#define AES_256_BIT_KEY				BIT(0)
 122
 123#define AES_DEACTIVATE_PERF_CNTR		0x00000000
 124#define AES_ACTIVATE_PERF_CNTR			BIT(0)
 125
 126#define AES_MAX_TAG_SIZE_U32			4
 127
 128#define OCS_LL_DMA_FLAG_TERMINATE		BIT(31)
 129
 130/*
 131 * There is an inconsistency in the documentation. This is documented as a
 132 * 11-bit value, but it is actually 10-bits.
 133 */
 134#define AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK	0x3FF
 135
 136/*
 137 * During CCM decrypt, the OCS block needs to finish processing the ciphertext
 138 * before the tag is written. For 128-bit mode this required delay is 28 OCS
 139 * clock cycles. For 256-bit mode it is 36 OCS clock cycles.
 140 */
 141#define CCM_DECRYPT_DELAY_TAG_CLK_COUNT		36UL
 142
 143/*
 144 * During CCM decrypt there must be a delay of at least 42 OCS clock cycles
 145 * between setting the TRIGGER bit in AES_ACTIVE and setting the LAST_CCM_GCM
 146 * bit in the same register (as stated in the OCS databook)
 147 */
 148#define CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT	42UL
 149
 150/* See RFC3610 section 2.2 */
 151#define L_PRIME_MIN (1)
 152#define L_PRIME_MAX (7)
 153/*
 154 * CCM IV format from RFC 3610 section 2.3
 155 *
 156 *   Octet Number   Contents
 157 *   ------------   ---------
 158 *   0              Flags
 159 *   1 ... 15-L     Nonce N
 160 *   16-L ... 15    Counter i
 161 *
 162 * Flags = L' = L - 1
 163 */
 164#define L_PRIME_IDX		0
 165#define COUNTER_START(lprime)	(16 - ((lprime) + 1))
 166#define COUNTER_LEN(lprime)	((lprime) + 1)
 167
 168enum aes_counter_mode {
 169	AES_CTR_M_NO_INC = 0,
 170	AES_CTR_M_32_INC = 1,
 171	AES_CTR_M_64_INC = 2,
 172	AES_CTR_M_128_INC = 3,
 173};
 174
 175/**
 176 * struct ocs_dma_linked_list - OCS DMA linked list entry.
 177 * @src_addr:   Source address of the data.
 178 * @src_len:    Length of data to be fetched.
 179 * @next:	Next dma_list to fetch.
 180 * @ll_flags:   Flags (Freeze @ terminate) for the DMA engine.
 181 */
 182struct ocs_dma_linked_list {
 183	u32 src_addr;
 184	u32 src_len;
 185	u32 next;
 186	u32 ll_flags;
 187} __packed;
 188
 189/*
 190 * Set endianness of inputs and outputs
 191 * AES_BYTE_ORDER_CFG
 192 * default 0x00000000
 193 * bit [10] - KEY_HI_LO_SWAP
 194 * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD
 195 * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD
 196 * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD
 197 * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD
 198 * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD
 199 * bit [4] - IV_SWAP_BYTES_IN_DWORD
 200 * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD
 201 * bit [2] - DOUT_SWAP_BYTES_IN_DWORD
 202 * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD
 203 * bit [0] - DOUT_SWAP_BYTES_IN_DWORD
 204 */
 205static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev)
 206{
 207	iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET);
 208}
 209
 210/* Trigger AES process start. */
 211static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev)
 212{
 213	iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET);
 214}
 215
 216/* Indicate last bulk of data. */
 217static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev)
 218{
 219	iowrite32(AES_ACTIVE_TERMINATION,
 220		  aes_dev->base_reg + AES_ACTIVE_OFFSET);
 221}
 222
 223/*
 224 * Set LAST_CCM_GCM in AES_ACTIVE register and clear all other bits.
 225 *
 226 * Called when DMA is programmed to fetch the last batch of data.
 227 * - For AES-CCM it is called for the last batch of Payload data and Ciphertext
 228 *   data.
 229 * - For AES-GCM, it is called for the last batch of Plaintext data and
 230 *   Ciphertext data.
 231 */
 232static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev)
 233{
 234	iowrite32(AES_ACTIVE_LAST_CCM_GCM,
 235		  aes_dev->base_reg + AES_ACTIVE_OFFSET);
 236}
 237
 238/* Wait for LAST_CCM_GCM bit to be unset. */
 239static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev)
 240{
 241	u32 aes_active_reg;
 242
 243	do {
 244		aes_active_reg = ioread32(aes_dev->base_reg +
 245					  AES_ACTIVE_OFFSET);
 246	} while (aes_active_reg & AES_ACTIVE_LAST_CCM_GCM);
 247}
 248
 249/* Wait for 10 bits of input occupancy. */
 250static void aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev *aes_dev)
 251{
 252	u32 reg;
 253
 254	do {
 255		reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET);
 256	} while (reg & AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK);
 257}
 258
 259 /*
 260  * Set LAST_CCM_GCM and LAST_ADATA bits in AES_ACTIVE register (and clear all
 261  * other bits).
 262  *
 263  * Called when DMA is programmed to fetch the last batch of Associated Data
 264  * (CCM case) or Additional Authenticated Data (GCM case).
 265  */
 266static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev)
 267{
 268	iowrite32(AES_ACTIVE_LAST_ADATA | AES_ACTIVE_LAST_CCM_GCM,
 269		  aes_dev->base_reg + AES_ACTIVE_OFFSET);
 270}
 271
 272/* Set DMA src and dst transfer size to 0 */
 273static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev)
 274{
 275	iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET);
 276	iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET);
 277}
 278
 279/* Activate DMA for zero-byte transfer case. */
 280static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev)
 281{
 282	iowrite32(AES_A_DMA_DMA_MODE_ACTIVE,
 283		  aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
 284}
 285
 286/* Activate DMA and enable src linked list */
 287static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev)
 288{
 289	iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
 290		  AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN,
 291		  aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
 292}
 293
 294/* Activate DMA and enable dst linked list */
 295static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev)
 296{
 297	iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
 298		  AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN,
 299		  aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
 300}
 301
 302/* Activate DMA and enable src and dst linked lists */
 303static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev)
 304{
 305	iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
 306		  AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN |
 307		  AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN,
 308		  aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
 309}
 310
 311/* Reset PERF_CNTR to 0 and activate it */
 312static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev)
 313{
 314	iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET);
 315	iowrite32(AES_ACTIVATE_PERF_CNTR,
 316		  aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET);
 317}
 318
 319/* Wait until PERF_CNTR is > delay, then deactivate it */
 320static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev,
 321							   int delay)
 322{
 323	while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay)
 324		;
 325	iowrite32(AES_DEACTIVATE_PERF_CNTR,
 326		  aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET);
 327}
 328
 329/* Disable AES and DMA IRQ. */
 330static void aes_irq_disable(struct ocs_aes_dev *aes_dev)
 331{
 332	u32 isr_val = 0;
 333
 334	/* Disable interrupts */
 335	iowrite32(AES_DISABLE_INT,
 336		  aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
 337	iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET);
 338
 339	/* Clear any pending interrupt */
 340	isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
 341	if (isr_val)
 342		iowrite32(isr_val,
 343			  aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
 344
 345	isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET);
 346	if (isr_val)
 347		iowrite32(isr_val,
 348			  aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET);
 349
 350	isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET);
 351	if (isr_val)
 352		iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET);
 353}
 354
 355/* Enable AES or DMA IRQ.  IRQ is disabled once fired. */
 356static void aes_irq_enable(struct ocs_aes_dev *aes_dev, u8 irq)
 357{
 358	if (irq == AES_COMPLETE_INT) {
 359		/* Ensure DMA error interrupts are enabled */
 360		iowrite32(AES_DMA_CPD_ERR_INT |
 361			  AES_DMA_OUTBUF_RD_ERR_INT |
 362			  AES_DMA_OUTBUF_WR_ERR_INT |
 363			  AES_DMA_INBUF_RD_ERR_INT |
 364			  AES_DMA_INBUF_WR_ERR_INT |
 365			  AES_DMA_BAD_COMP_INT |
 366			  AES_DMA_SAI_INT,
 367			  aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
 368		/*
 369		 * AES_IER
 370		 * default 0x00000000
 371		 * bits [31:3] - reserved
 372		 * bit [2] - EN_SKS_ERR
 373		 * bit [1] - EN_AES_COMPLETE
 374		 * bit [0] - reserved
 375		 */
 376		iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET);
 377		return;
 378	}
 379	if (irq == AES_DMA_SRC_DONE_INT) {
 380		/* Ensure AES interrupts are disabled */
 381		iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET);
 382		/*
 383		 * DMA_MSI_IER
 384		 * default 0x00000000
 385		 * bits [31:9] - reserved
 386		 * bit [8] - CPD_ERR_INT_EN
 387		 * bit [7] - OUTBUF_RD_ERR_INT_EN
 388		 * bit [6] - OUTBUF_WR_ERR_INT_EN
 389		 * bit [5] - INBUF_RD_ERR_INT_EN
 390		 * bit [4] - INBUF_WR_ERR_INT_EN
 391		 * bit [3] - BAD_COMP_INT_EN
 392		 * bit [2] - SAI_INT_EN
 393		 * bit [1] - DST_DONE_INT_EN
 394		 * bit [0] - SRC_DONE_INT_EN
 395		 */
 396		iowrite32(AES_DMA_CPD_ERR_INT |
 397			  AES_DMA_OUTBUF_RD_ERR_INT |
 398			  AES_DMA_OUTBUF_WR_ERR_INT |
 399			  AES_DMA_INBUF_RD_ERR_INT |
 400			  AES_DMA_INBUF_WR_ERR_INT |
 401			  AES_DMA_BAD_COMP_INT |
 402			  AES_DMA_SAI_INT |
 403			  AES_DMA_SRC_DONE_INT,
 404			  aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
 405	}
 406}
 407
 408/* Enable and wait for IRQ (either from OCS AES engine or DMA) */
 409static int ocs_aes_irq_enable_and_wait(struct ocs_aes_dev *aes_dev, u8 irq)
 410{
 411	int rc;
 412
 413	reinit_completion(&aes_dev->irq_completion);
 414	aes_irq_enable(aes_dev, irq);
 415	rc = wait_for_completion_interruptible(&aes_dev->irq_completion);
 416	if (rc)
 417		return rc;
 418
 419	return aes_dev->dma_err_mask ? -EIO : 0;
 420}
 421
 422/* Configure DMA to OCS, linked list mode */
 423static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev,
 424				     dma_addr_t dma_list)
 425{
 426	iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET);
 427	iowrite32(dma_list,
 428		  aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET);
 429}
 430
 431/* Configure DMA from OCS, linked list mode */
 432static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev,
 433				       dma_addr_t dma_list)
 434{
 435	iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET);
 436	iowrite32(dma_list,
 437		  aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET);
 438}
 439
 440irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id)
 441{
 442	struct ocs_aes_dev *aes_dev = dev_id;
 443	u32 aes_dma_isr;
 444
 445	/* Read DMA ISR status. */
 446	aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
 447
 448	/* Disable and clear interrupts. */
 449	aes_irq_disable(aes_dev);
 450
 451	/* Save DMA error status. */
 452	aes_dev->dma_err_mask = aes_dma_isr &
 453				(AES_DMA_CPD_ERR_INT |
 454				 AES_DMA_OUTBUF_RD_ERR_INT |
 455				 AES_DMA_OUTBUF_WR_ERR_INT |
 456				 AES_DMA_INBUF_RD_ERR_INT |
 457				 AES_DMA_INBUF_WR_ERR_INT |
 458				 AES_DMA_BAD_COMP_INT |
 459				 AES_DMA_SAI_INT);
 460
 461	/* Signal IRQ completion. */
 462	complete(&aes_dev->irq_completion);
 463
 464	return IRQ_HANDLED;
 465}
 466
 467/**
 468 * ocs_aes_set_key() - Write key into OCS AES hardware.
 469 * @aes_dev:	The OCS AES device to write the key to.
 470 * @key_size:	The size of the key (in bytes).
 471 * @key:	The key to write.
 472 * @cipher:	The cipher the key is for.
 473 *
 474 * For AES @key_size must be either 16 or 32. For SM4 @key_size must be 16.
 475 *
 476 * Return:	0 on success, negative error code otherwise.
 477 */
 478int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, u32 key_size, const u8 *key,
 479		    enum ocs_cipher cipher)
 480{
 481	const u32 *key_u32;
 482	u32 val;
 483	int i;
 484
 485	/* OCS AES supports 128-bit and 256-bit keys only. */
 486	if (cipher == OCS_AES && !(key_size == 32 || key_size == 16)) {
 487		dev_err(aes_dev->dev,
 488			"%d-bit keys not supported by AES cipher\n",
 489			key_size * 8);
 490		return -EINVAL;
 491	}
 492	/* OCS SM4 supports 128-bit keys only. */
 493	if (cipher == OCS_SM4 && key_size != 16) {
 494		dev_err(aes_dev->dev,
 495			"%d-bit keys not supported for SM4 cipher\n",
 496			key_size * 8);
 497		return -EINVAL;
 498	}
 499
 500	if (!key)
 501		return -EINVAL;
 502
 503	key_u32 = (const u32 *)key;
 504
 505	/* Write key to AES_KEY[0-7] registers */
 506	for (i = 0; i < (key_size / sizeof(u32)); i++) {
 507		iowrite32(key_u32[i],
 508			  aes_dev->base_reg + AES_KEY_0_OFFSET +
 509			  (i * sizeof(u32)));
 510	}
 511	/*
 512	 * Write key size
 513	 * bits [31:1] - reserved
 514	 * bit [0] - AES_KEY_SIZE
 515	 *           0 - 128 bit key
 516	 *           1 - 256 bit key
 517	 */
 518	val = (key_size == 16) ? AES_128_BIT_KEY : AES_256_BIT_KEY;
 519	iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET);
 520
 521	return 0;
 522}
 523
 524/* Write AES_COMMAND */
 525static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev,
 526				       enum ocs_cipher cipher,
 527				       enum ocs_mode mode,
 528				       enum ocs_instruction instruction)
 529{
 530	u32 val;
 531
 532	/* AES_COMMAND
 533	 * default 0x000000CC
 534	 * bit [14] - CIPHER_SELECT
 535	 *            0 - AES
 536	 *            1 - SM4
 537	 * bits [11:8] - OCS_AES_MODE
 538	 *               0000 - ECB
 539	 *               0001 - CBC
 540	 *               0010 - CTR
 541	 *               0110 - CCM
 542	 *               0111 - GCM
 543	 *               1001 - CTS
 544	 * bits [7:6] - AES_INSTRUCTION
 545	 *              00 - ENCRYPT
 546	 *              01 - DECRYPT
 547	 *              10 - EXPAND
 548	 *              11 - BYPASS
 549	 * bits [3:2] - CTR_M_BITS
 550	 *              00 - No increment
 551	 *              01 - Least significant 32 bits are incremented
 552	 *              10 - Least significant 64 bits are incremented
 553	 *              11 - Full 128 bits are incremented
 554	 */
 555	val = (cipher << 14) | (mode << 8) | (instruction << 6) |
 556	      (AES_CTR_M_128_INC << 2);
 557	iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET);
 558}
 559
 560static void ocs_aes_init(struct ocs_aes_dev *aes_dev,
 561			 enum ocs_mode mode,
 562			 enum ocs_cipher cipher,
 563			 enum ocs_instruction instruction)
 564{
 565	/* Ensure interrupts are disabled and pending interrupts cleared. */
 566	aes_irq_disable(aes_dev);
 567
 568	/* Set endianness recommended by data-sheet. */
 569	aes_a_set_endianness(aes_dev);
 570
 571	/* Set AES_COMMAND register. */
 572	set_ocs_aes_command(aes_dev, cipher, mode, instruction);
 573}
 574
 575/*
 576 * Write the byte length of the last AES/SM4 block of Payload data (without
 577 * zero padding and without the length of the MAC) in register AES_PLEN.
 578 */
 579static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev,
 580						   u32 size)
 581{
 582	u32 val;
 583
 584	if (size == 0) {
 585		val = 0;
 586		goto exit;
 587	}
 588
 589	val = size % AES_BLOCK_SIZE;
 590	if (val == 0)
 591		val = AES_BLOCK_SIZE;
 592
 593exit:
 594	iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET);
 595}
 596
 597/*
 598 * Validate inputs according to mode.
 599 * If OK return 0; else return -EINVAL.
 600 */
 601static int ocs_aes_validate_inputs(dma_addr_t src_dma_list, u32 src_size,
 602				   const u8 *iv, u32 iv_size,
 603				   dma_addr_t aad_dma_list, u32 aad_size,
 604				   const u8 *tag, u32 tag_size,
 605				   enum ocs_cipher cipher, enum ocs_mode mode,
 606				   enum ocs_instruction instruction,
 607				   dma_addr_t dst_dma_list)
 608{
 609	/* Ensure cipher, mode and instruction are valid. */
 610	if (!(cipher == OCS_AES || cipher == OCS_SM4))
 611		return -EINVAL;
 612
 613	if (mode != OCS_MODE_ECB && mode != OCS_MODE_CBC &&
 614	    mode != OCS_MODE_CTR && mode != OCS_MODE_CCM &&
 615	    mode != OCS_MODE_GCM && mode != OCS_MODE_CTS)
 616		return -EINVAL;
 617
 618	if (instruction != OCS_ENCRYPT && instruction != OCS_DECRYPT &&
 619	    instruction != OCS_EXPAND  && instruction != OCS_BYPASS)
 620		return -EINVAL;
 621
 622	/*
 623	 * When instruction is OCS_BYPASS, OCS simply copies data from source
 624	 * to destination using DMA.
 625	 *
 626	 * AES mode is irrelevant, but both source and destination DMA
 627	 * linked-list must be defined.
 628	 */
 629	if (instruction == OCS_BYPASS) {
 630		if (src_dma_list == DMA_MAPPING_ERROR ||
 631		    dst_dma_list == DMA_MAPPING_ERROR)
 632			return -EINVAL;
 633
 634		return 0;
 635	}
 636
 637	/*
 638	 * For performance reasons switch based on mode to limit unnecessary
 639	 * conditionals for each mode
 640	 */
 641	switch (mode) {
 642	case OCS_MODE_ECB:
 643		/* Ensure input length is multiple of block size */
 644		if (src_size % AES_BLOCK_SIZE != 0)
 645			return -EINVAL;
 646
 647		/* Ensure source and destination linked lists are created */
 648		if (src_dma_list == DMA_MAPPING_ERROR ||
 649		    dst_dma_list == DMA_MAPPING_ERROR)
 650			return -EINVAL;
 651
 652		return 0;
 653
 654	case OCS_MODE_CBC:
 655		/* Ensure input length is multiple of block size */
 656		if (src_size % AES_BLOCK_SIZE != 0)
 657			return -EINVAL;
 658
 659		/* Ensure source and destination linked lists are created */
 660		if (src_dma_list == DMA_MAPPING_ERROR ||
 661		    dst_dma_list == DMA_MAPPING_ERROR)
 662			return -EINVAL;
 663
 664		/* Ensure IV is present and block size in length */
 665		if (!iv || iv_size != AES_BLOCK_SIZE)
 666			return -EINVAL;
 667
 668		return 0;
 669
 670	case OCS_MODE_CTR:
 671		/* Ensure input length of 1 byte or greater */
 672		if (src_size == 0)
 673			return -EINVAL;
 674
 675		/* Ensure source and destination linked lists are created */
 676		if (src_dma_list == DMA_MAPPING_ERROR ||
 677		    dst_dma_list == DMA_MAPPING_ERROR)
 678			return -EINVAL;
 679
 680		/* Ensure IV is present and block size in length */
 681		if (!iv || iv_size != AES_BLOCK_SIZE)
 682			return -EINVAL;
 683
 684		return 0;
 685
 686	case OCS_MODE_CTS:
 687		/* Ensure input length >= block size */
 688		if (src_size < AES_BLOCK_SIZE)
 689			return -EINVAL;
 690
 691		/* Ensure source and destination linked lists are created */
 692		if (src_dma_list == DMA_MAPPING_ERROR ||
 693		    dst_dma_list == DMA_MAPPING_ERROR)
 694			return -EINVAL;
 695
 696		/* Ensure IV is present and block size in length */
 697		if (!iv || iv_size != AES_BLOCK_SIZE)
 698			return -EINVAL;
 699
 700		return 0;
 701
 702	case OCS_MODE_GCM:
 703		/* Ensure IV is present and GCM_AES_IV_SIZE in length */
 704		if (!iv || iv_size != GCM_AES_IV_SIZE)
 705			return -EINVAL;
 706
 707		/*
 708		 * If input data present ensure source and destination linked
 709		 * lists are created
 710		 */
 711		if (src_size && (src_dma_list == DMA_MAPPING_ERROR ||
 712				 dst_dma_list == DMA_MAPPING_ERROR))
 713			return -EINVAL;
 714
 715		/* If aad present ensure aad linked list is created */
 716		if (aad_size && aad_dma_list == DMA_MAPPING_ERROR)
 717			return -EINVAL;
 718
 719		/* Ensure tag destination is set */
 720		if (!tag)
 721			return -EINVAL;
 722
 723		/* Just ensure that tag_size doesn't cause overflows. */
 724		if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32)))
 725			return -EINVAL;
 726
 727		return 0;
 728
 729	case OCS_MODE_CCM:
 730		/* Ensure IV is present and block size in length */
 731		if (!iv || iv_size != AES_BLOCK_SIZE)
 732			return -EINVAL;
 733
 734		/* 2 <= L <= 8, so 1 <= L' <= 7 */
 735		if (iv[L_PRIME_IDX] < L_PRIME_MIN ||
 736		    iv[L_PRIME_IDX] > L_PRIME_MAX)
 737			return -EINVAL;
 738
 739		/* If aad present ensure aad linked list is created */
 740		if (aad_size && aad_dma_list == DMA_MAPPING_ERROR)
 741			return -EINVAL;
 742
 743		/* Just ensure that tag_size doesn't cause overflows. */
 744		if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32)))
 745			return -EINVAL;
 746
 747		if (instruction == OCS_DECRYPT) {
 748			/*
 749			 * If input data present ensure source and destination
 750			 * linked lists are created
 751			 */
 752			if (src_size && (src_dma_list == DMA_MAPPING_ERROR ||
 753					 dst_dma_list == DMA_MAPPING_ERROR))
 754				return -EINVAL;
 755
 756			/* Ensure input tag is present */
 757			if (!tag)
 758				return -EINVAL;
 759
 760			return 0;
 761		}
 762
 763		/* Instruction == OCS_ENCRYPT */
 764
 765		/*
 766		 * Destination linked list always required (for tag even if no
 767		 * input data)
 768		 */
 769		if (dst_dma_list == DMA_MAPPING_ERROR)
 770			return -EINVAL;
 771
 772		/* If input data present ensure src linked list is created */
 773		if (src_size && src_dma_list == DMA_MAPPING_ERROR)
 774			return -EINVAL;
 775
 776		return 0;
 777
 778	default:
 779		return -EINVAL;
 780	}
 781}
 782
 783/**
 784 * ocs_aes_op() - Perform AES/SM4 operation.
 785 * @aes_dev:		The OCS AES device to use.
 786 * @mode:		The mode to use (ECB, CBC, CTR, or CTS).
 787 * @cipher:		The cipher to use (AES or SM4).
 788 * @instruction:	The instruction to perform (encrypt or decrypt).
 789 * @dst_dma_list:	The OCS DMA list mapping output memory.
 790 * @src_dma_list:	The OCS DMA list mapping input payload data.
 791 * @src_size:		The amount of data mapped by @src_dma_list.
 792 * @iv:			The IV vector.
 793 * @iv_size:		The size (in bytes) of @iv.
 794 *
 795 * Return: 0 on success, negative error code otherwise.
 796 */
 797int ocs_aes_op(struct ocs_aes_dev *aes_dev,
 798	       enum ocs_mode mode,
 799	       enum ocs_cipher cipher,
 800	       enum ocs_instruction instruction,
 801	       dma_addr_t dst_dma_list,
 802	       dma_addr_t src_dma_list,
 803	       u32 src_size,
 804	       u8 *iv,
 805	       u32 iv_size)
 806{
 807	u32 *iv32;
 808	int rc;
 809
 810	rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, iv_size, 0, 0,
 811				     NULL, 0, cipher, mode, instruction,
 812				     dst_dma_list);
 813	if (rc)
 814		return rc;
 815	/*
 816	 * ocs_aes_validate_inputs() is a generic check, now ensure mode is not
 817	 * GCM or CCM.
 818	 */
 819	if (mode == OCS_MODE_GCM || mode == OCS_MODE_CCM)
 820		return -EINVAL;
 821
 822	/* Cast IV to u32 array. */
 823	iv32 = (u32 *)iv;
 824
 825	ocs_aes_init(aes_dev, mode, cipher, instruction);
 826
 827	if (mode == OCS_MODE_CTS) {
 828		/* Write the byte length of the last data block to engine. */
 829		ocs_aes_write_last_data_blk_len(aes_dev, src_size);
 830	}
 831
 832	/* ECB is the only mode that doesn't use IV. */
 833	if (mode != OCS_MODE_ECB) {
 834		iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET);
 835		iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET);
 836		iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET);
 837		iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET);
 838	}
 839
 840	/* Set AES_ACTIVE.TRIGGER to start the operation. */
 841	aes_a_op_trigger(aes_dev);
 842
 843	/* Configure and activate input / output DMA. */
 844	dma_to_ocs_aes_ll(aes_dev, src_dma_list);
 845	dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
 846	aes_a_dma_active_src_dst_ll_en(aes_dev);
 847
 848	if (mode == OCS_MODE_CTS) {
 849		/*
 850		 * For CTS mode, instruct engine to activate ciphertext
 851		 * stealing if last block of data is incomplete.
 852		 */
 853		aes_a_set_last_gcx(aes_dev);
 854	} else {
 855		/* For all other modes, just write the 'termination' bit. */
 856		aes_a_op_termination(aes_dev);
 857	}
 858
 859	/* Wait for engine to complete processing. */
 860	rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
 861	if (rc)
 862		return rc;
 863
 864	if (mode == OCS_MODE_CTR) {
 865		/* Read back IV for streaming mode */
 866		iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET);
 867		iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET);
 868		iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET);
 869		iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET);
 870	}
 871
 872	return 0;
 873}
 874
 875/* Compute and write J0 to engine registers. */
 876static void ocs_aes_gcm_write_j0(const struct ocs_aes_dev *aes_dev,
 877				 const u8 *iv)
 878{
 879	const u32 *j0 = (u32 *)iv;
 880
 881	/*
 882	 * IV must be 12 bytes; Other sizes not supported as Linux crypto API
 883	 * does only expects/allows 12 byte IV for GCM
 884	 */
 885	iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET);
 886	iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET);
 887	iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET);
 888	iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET);
 889}
 890
 891/* Read GCM tag from engine registers. */
 892static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev,
 893					u8 *tag, u32 tag_size)
 894{
 895	u32 tag_u32[AES_MAX_TAG_SIZE_U32];
 896
 897	/*
 898	 * The Authentication Tag T is stored in Little Endian order in the
 899	 * registers with the most significant bytes stored from AES_T_MAC[3]
 900	 * downward.
 901	 */
 902	tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET));
 903	tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET));
 904	tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET));
 905	tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET));
 906
 907	memcpy(tag, tag_u32, tag_size);
 908}
 909
 910/**
 911 * ocs_aes_gcm_op() - Perform GCM operation.
 912 * @aes_dev:		The OCS AES device to use.
 913 * @cipher:		The Cipher to use (AES or SM4).
 914 * @instruction:	The instruction to perform (encrypt or decrypt).
 915 * @dst_dma_list:	The OCS DMA list mapping output memory.
 916 * @src_dma_list:	The OCS DMA list mapping input payload data.
 917 * @src_size:		The amount of data mapped by @src_dma_list.
 918 * @iv:			The input IV vector.
 919 * @aad_dma_list:	The OCS DMA list mapping input AAD data.
 920 * @aad_size:		The amount of data mapped by @aad_dma_list.
 921 * @out_tag:		Where to store computed tag.
 922 * @tag_size:		The size (in bytes) of @out_tag.
 923 *
 924 * Return: 0 on success, negative error code otherwise.
 925 */
 926int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev,
 927		   enum ocs_cipher cipher,
 928		   enum ocs_instruction instruction,
 929		   dma_addr_t dst_dma_list,
 930		   dma_addr_t src_dma_list,
 931		   u32 src_size,
 932		   const u8 *iv,
 933		   dma_addr_t aad_dma_list,
 934		   u32 aad_size,
 935		   u8 *out_tag,
 936		   u32 tag_size)
 937{
 938	u64 bit_len;
 939	u32 val;
 940	int rc;
 941
 942	rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv,
 943				     GCM_AES_IV_SIZE, aad_dma_list,
 944				     aad_size, out_tag, tag_size, cipher,
 945				     OCS_MODE_GCM, instruction,
 946				     dst_dma_list);
 947	if (rc)
 948		return rc;
 949
 950	ocs_aes_init(aes_dev, OCS_MODE_GCM, cipher, instruction);
 951
 952	/* Compute and write J0 to OCS HW. */
 953	ocs_aes_gcm_write_j0(aes_dev, iv);
 954
 955	/* Write out_tag byte length */
 956	iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET);
 957
 958	/* Write the byte length of the last plaintext / ciphertext block. */
 959	ocs_aes_write_last_data_blk_len(aes_dev, src_size);
 960
 961	/* Write ciphertext bit length */
 962	bit_len = (u64)src_size * 8;
 963	val = bit_len & 0xFFFFFFFF;
 964	iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET);
 965	val = bit_len >> 32;
 966	iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET);
 967
 968	/* Write aad bit length */
 969	bit_len = (u64)aad_size * 8;
 970	val = bit_len & 0xFFFFFFFF;
 971	iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET);
 972	val = bit_len >> 32;
 973	iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET);
 974
 975	/* Set AES_ACTIVE.TRIGGER to start the operation. */
 976	aes_a_op_trigger(aes_dev);
 977
 978	/* Process AAD. */
 979	if (aad_size) {
 980		/* If aad present, configure DMA to feed it to the engine. */
 981		dma_to_ocs_aes_ll(aes_dev, aad_dma_list);
 982		aes_a_dma_active_src_ll_en(aes_dev);
 983
 984		/* Instructs engine to pad last block of aad, if needed. */
 985		aes_a_set_last_gcx_and_adata(aes_dev);
 986
 987		/* Wait for DMA transfer to complete. */
 988		rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
 989		if (rc)
 990			return rc;
 991	} else {
 992		aes_a_set_last_gcx_and_adata(aes_dev);
 993	}
 994
 995	/* Wait until adata (if present) has been processed. */
 996	aes_a_wait_last_gcx(aes_dev);
 997	aes_a_dma_wait_input_buffer_occupancy(aes_dev);
 998
 999	/* Now process payload. */
1000	if (src_size) {
1001		/* Configure and activate DMA for both input and output data. */
1002		dma_to_ocs_aes_ll(aes_dev, src_dma_list);
1003		dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
1004		aes_a_dma_active_src_dst_ll_en(aes_dev);
1005	} else {
1006		aes_a_dma_set_xfer_size_zero(aes_dev);
1007		aes_a_dma_active(aes_dev);
1008	}
1009
1010	/* Instruct AES/SMA4 engine payload processing is over. */
1011	aes_a_set_last_gcx(aes_dev);
1012
1013	/* Wait for OCS AES engine to complete processing. */
1014	rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
1015	if (rc)
1016		return rc;
1017
1018	ocs_aes_gcm_read_tag(aes_dev, out_tag, tag_size);
1019
1020	return 0;
1021}
1022
1023/* Write encrypted tag to AES/SM4 engine. */
1024static void ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev *aes_dev,
1025					    const u8 *in_tag, u32 tag_size)
1026{
1027	int i;
1028
1029	/* Ensure DMA input buffer is empty */
1030	aes_a_dma_wait_input_buffer_occupancy(aes_dev);
1031
1032	/*
1033	 * During CCM decrypt, the OCS block needs to finish processing the
1034	 * ciphertext before the tag is written.  So delay needed after DMA has
1035	 * completed writing the ciphertext
1036	 */
1037	aes_a_dma_reset_and_activate_perf_cntr(aes_dev);
1038	aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev,
1039						CCM_DECRYPT_DELAY_TAG_CLK_COUNT);
1040
1041	/* Write encrypted tag to AES/SM4 engine. */
1042	for (i = 0; i < tag_size; i++) {
1043		iowrite8(in_tag[i], aes_dev->base_reg +
1044				    AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
1045	}
1046}
1047
1048/*
1049 * Write B0 CCM block to OCS AES HW.
1050 *
1051 * Note: B0 format is documented in NIST Special Publication 800-38C
1052 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
1053 * (see Section A.2.1)
1054 */
1055static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev,
1056				const u8 *iv, u32 adata_size, u32 tag_size,
1057				u32 cryptlen)
1058{
1059	u8 b0[16]; /* CCM B0 block is 16 bytes long. */
1060	int i, q;
1061
1062	/* Initialize B0 to 0. */
1063	memset(b0, 0, sizeof(b0));
1064
1065	/*
1066	 * B0[0] is the 'Flags Octet' and has the following structure:
1067	 *   bit 7: Reserved
1068	 *   bit 6: Adata flag
1069	 *   bit 5-3: t value encoded as (t-2)/2
1070	 *   bit 2-0: q value encoded as q - 1
1071	 */
1072	/* If there is AAD data, set the Adata flag. */
1073	if (adata_size)
1074		b0[0] |= BIT(6);
1075	/*
1076	 * t denotes the octet length of T.
1077	 * t can only be an element of { 4, 6, 8, 10, 12, 14, 16} and is
1078	 * encoded as (t - 2) / 2
1079	 */
1080	b0[0] |= (((tag_size - 2) / 2) & 0x7)  << 3;
1081	/*
1082	 * q is the octet length of Q.
1083	 * q can only be an element of {2, 3, 4, 5, 6, 7, 8} and is encoded as
1084	 * q - 1 == iv[0] & 0x7;
1085	 */
1086	b0[0] |= iv[0] & 0x7;
1087	/*
1088	 * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q]
1089	 * and must be copied to b0[1]..b0[15-q].
1090	 * q == (iv[0] & 0x7) + 1
1091	 */
1092	q = (iv[0] & 0x7) + 1;
1093	for (i = 1; i <= 15 - q; i++)
1094		b0[i] = iv[i];
1095	/*
1096	 * The rest of B0 must contain Q, i.e., the message length.
1097	 * Q is encoded in q octets, in big-endian order, so to write it, we
1098	 * start from the end of B0 and we move backward.
1099	 */
1100	i = sizeof(b0) - 1;
1101	while (q) {
1102		b0[i] = cryptlen & 0xff;
1103		cryptlen >>= 8;
1104		i--;
1105		q--;
1106	}
1107	/*
1108	 * If cryptlen is not zero at this point, it means that its original
1109	 * value was too big.
1110	 */
1111	if (cryptlen)
1112		return -EOVERFLOW;
1113	/* Now write B0 to OCS AES input buffer. */
1114	for (i = 0; i < sizeof(b0); i++)
1115		iowrite8(b0[i], aes_dev->base_reg +
1116				AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
1117	return 0;
1118}
1119
1120/*
1121 * Write adata length to OCS AES HW.
1122 *
1123 * Note: adata len encoding is documented in NIST Special Publication 800-38C
1124 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
1125 * (see Section A.2.2)
1126 */
1127static void ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev *aes_dev,
1128					u64 adata_len)
1129{
1130	u8 enc_a[10]; /* Maximum encoded size: 10 octets. */
1131	int i, len;
1132
1133	/*
1134	 * adata_len ('a') is encoded as follows:
1135	 * If 0 < a < 2^16 - 2^8    ==> 'a' encoded as [a]16, i.e., two octets
1136	 *				(big endian).
1137	 * If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32,
1138	 *				i.e., six octets (big endian).
1139	 * If 2^32 ≤ a < 2^64       ==> 'a' encoded as 0xff || 0xff || [a]64,
1140	 *				i.e., ten octets (big endian).
1141	 */
1142	if (adata_len < 65280) {
1143		len = 2;
1144		*(__be16 *)enc_a = cpu_to_be16(adata_len);
1145	} else if (adata_len <= 0xFFFFFFFF) {
1146		len = 6;
1147		*(__be16 *)enc_a = cpu_to_be16(0xfffe);
1148		*(__be32 *)&enc_a[2] = cpu_to_be32(adata_len);
1149	} else { /* adata_len >= 2^32 */
1150		len = 10;
1151		*(__be16 *)enc_a = cpu_to_be16(0xffff);
1152		*(__be64 *)&enc_a[2] = cpu_to_be64(adata_len);
1153	}
1154	for (i = 0; i < len; i++)
1155		iowrite8(enc_a[i],
1156			 aes_dev->base_reg +
1157			 AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
1158}
1159
1160static int ocs_aes_ccm_do_adata(struct ocs_aes_dev *aes_dev,
1161				dma_addr_t adata_dma_list, u32 adata_size)
1162{
1163	int rc;
1164
1165	if (!adata_size) {
1166		/* Since no aad the LAST_GCX bit can be set now */
1167		aes_a_set_last_gcx_and_adata(aes_dev);
1168		goto exit;
1169	}
1170
1171	/* Adata case. */
1172
1173	/*
1174	 * Form the encoding of the Associated data length and write it
1175	 * to the AES/SM4 input buffer.
1176	 */
1177	ocs_aes_ccm_write_adata_len(aes_dev, adata_size);
1178
1179	/* Configure the AES/SM4 DMA to fetch the Associated Data */
1180	dma_to_ocs_aes_ll(aes_dev, adata_dma_list);
1181
1182	/* Activate DMA to fetch Associated data. */
1183	aes_a_dma_active_src_ll_en(aes_dev);
1184
1185	/* Set LAST_GCX and LAST_ADATA in AES ACTIVE register. */
1186	aes_a_set_last_gcx_and_adata(aes_dev);
1187
1188	/* Wait for DMA transfer to complete. */
1189	rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
1190	if (rc)
1191		return rc;
1192
1193exit:
1194	/* Wait until adata (if present) has been processed. */
1195	aes_a_wait_last_gcx(aes_dev);
1196	aes_a_dma_wait_input_buffer_occupancy(aes_dev);
1197
1198	return 0;
1199}
1200
1201static int ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev *aes_dev,
1202					  dma_addr_t dst_dma_list,
1203					  dma_addr_t src_dma_list,
1204					  u32 src_size)
1205{
1206	if (src_size) {
1207		/*
1208		 * Configure and activate DMA for both input and output
1209		 * data.
1210		 */
1211		dma_to_ocs_aes_ll(aes_dev, src_dma_list);
1212		dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
1213		aes_a_dma_active_src_dst_ll_en(aes_dev);
1214	} else {
1215		/* Configure and activate DMA for output data only. */
1216		dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
1217		aes_a_dma_active_dst_ll_en(aes_dev);
1218	}
1219
1220	/*
1221	 * Set the LAST GCX bit in AES_ACTIVE Register to instruct
1222	 * AES/SM4 engine to pad the last block of data.
1223	 */
1224	aes_a_set_last_gcx(aes_dev);
1225
1226	/* We are done, wait for IRQ and return. */
1227	return ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
1228}
1229
1230static int ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev *aes_dev,
1231					  dma_addr_t dst_dma_list,
1232					  dma_addr_t src_dma_list,
1233					  u32 src_size)
1234{
1235	if (!src_size) {
1236		/* Let engine process 0-length input. */
1237		aes_a_dma_set_xfer_size_zero(aes_dev);
1238		aes_a_dma_active(aes_dev);
1239		aes_a_set_last_gcx(aes_dev);
1240
1241		return 0;
1242	}
1243
1244	/*
1245	 * Configure and activate DMA for both input and output
1246	 * data.
1247	 */
1248	dma_to_ocs_aes_ll(aes_dev, src_dma_list);
1249	dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
1250	aes_a_dma_active_src_dst_ll_en(aes_dev);
1251	/*
1252	 * Set the LAST GCX bit in AES_ACTIVE Register; this allows the
1253	 * AES/SM4 engine to differentiate between encrypted data and
1254	 * encrypted MAC.
1255	 */
1256	aes_a_set_last_gcx(aes_dev);
1257	 /*
1258	  * Enable DMA DONE interrupt; once DMA transfer is over,
1259	  * interrupt handler will process the MAC/tag.
1260	  */
1261	return ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
1262}
1263
1264/*
1265 * Compare Tag to Yr.
1266 *
1267 * Only used at the end of CCM decrypt. If tag == yr, message authentication
1268 * has succeeded.
1269 */
1270static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev,
1271					u8 tag_size_bytes)
1272{
1273	u32 tag[AES_MAX_TAG_SIZE_U32];
1274	u32 yr[AES_MAX_TAG_SIZE_U32];
1275	u8 i;
1276
1277	/* Read Tag and Yr from AES registers. */
1278	for (i = 0; i < AES_MAX_TAG_SIZE_U32; i++) {
1279		tag[i] = ioread32(aes_dev->base_reg +
1280				  AES_T_MAC_0_OFFSET + (i * sizeof(u32)));
1281		yr[i] = ioread32(aes_dev->base_reg +
1282				 AES_MULTIPURPOSE2_0_OFFSET +
1283				 (i * sizeof(u32)));
1284	}
1285
1286	return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0;
1287}
1288
1289/**
1290 * ocs_aes_ccm_op() - Perform CCM operation.
1291 * @aes_dev:		The OCS AES device to use.
1292 * @cipher:		The Cipher to use (AES or SM4).
1293 * @instruction:	The instruction to perform (encrypt or decrypt).
1294 * @dst_dma_list:	The OCS DMA list mapping output memory.
1295 * @src_dma_list:	The OCS DMA list mapping input payload data.
1296 * @src_size:		The amount of data mapped by @src_dma_list.
1297 * @iv:			The input IV vector.
1298 * @adata_dma_list:	The OCS DMA list mapping input A-data.
1299 * @adata_size:		The amount of data mapped by @adata_dma_list.
1300 * @in_tag:		Input tag.
1301 * @tag_size:		The size (in bytes) of @in_tag.
1302 *
1303 * Note: for encrypt the tag is appended to the ciphertext (in the memory
1304 *	 mapped by @dst_dma_list).
1305 *
1306 * Return: 0 on success, negative error code otherwise.
1307 */
1308int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev,
1309		   enum ocs_cipher cipher,
1310		   enum ocs_instruction instruction,
1311		   dma_addr_t dst_dma_list,
1312		   dma_addr_t src_dma_list,
1313		   u32 src_size,
1314		   u8 *iv,
1315		   dma_addr_t adata_dma_list,
1316		   u32 adata_size,
1317		   u8 *in_tag,
1318		   u32 tag_size)
1319{
1320	u32 *iv_32;
1321	u8 lprime;
1322	int rc;
1323
1324	rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv,
1325				     AES_BLOCK_SIZE, adata_dma_list, adata_size,
1326				     in_tag, tag_size, cipher, OCS_MODE_CCM,
1327				     instruction, dst_dma_list);
1328	if (rc)
1329		return rc;
1330
1331	ocs_aes_init(aes_dev, OCS_MODE_CCM, cipher, instruction);
1332
1333	/*
1334	 * Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt
1335	 * auth tag so ensure this is the case
1336	 */
1337	lprime = iv[L_PRIME_IDX];
1338	memset(&iv[COUNTER_START(lprime)], 0, COUNTER_LEN(lprime));
1339
1340	/*
1341	 * Nonce is already converted to ctr0 before being passed into this
1342	 * function as iv.
1343	 */
1344	iv_32 = (u32 *)iv;
1345	iowrite32(__swab32(iv_32[0]),
1346		  aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET);
1347	iowrite32(__swab32(iv_32[1]),
1348		  aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET);
1349	iowrite32(__swab32(iv_32[2]),
1350		  aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET);
1351	iowrite32(__swab32(iv_32[3]),
1352		  aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET);
1353
1354	/* Write MAC/tag length in register AES_TLEN */
1355	iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET);
1356	/*
1357	 * Write the byte length of the last AES/SM4 block of Payload data
1358	 * (without zero padding and without the length of the MAC) in register
1359	 * AES_PLEN.
1360	 */
1361	ocs_aes_write_last_data_blk_len(aes_dev, src_size);
1362
1363	/* Set AES_ACTIVE.TRIGGER to start the operation. */
1364	aes_a_op_trigger(aes_dev);
1365
1366	aes_a_dma_reset_and_activate_perf_cntr(aes_dev);
1367
1368	/* Form block B0 and write it to the AES/SM4 input buffer. */
1369	rc = ocs_aes_ccm_write_b0(aes_dev, iv, adata_size, tag_size, src_size);
1370	if (rc)
1371		return rc;
1372	/*
1373	 * Ensure there has been at least CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT
1374	 * clock cycles since TRIGGER bit was set
1375	 */
1376	aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev,
1377						CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT);
1378
1379	/* Process Adata. */
1380	ocs_aes_ccm_do_adata(aes_dev, adata_dma_list, adata_size);
1381
1382	/* For Encrypt case we just process the payload and return. */
1383	if (instruction == OCS_ENCRYPT) {
1384		return ocs_aes_ccm_encrypt_do_payload(aes_dev, dst_dma_list,
1385						      src_dma_list, src_size);
1386	}
1387	/* For Decypt we need to process the payload and then the tag. */
1388	rc = ocs_aes_ccm_decrypt_do_payload(aes_dev, dst_dma_list,
1389					    src_dma_list, src_size);
1390	if (rc)
1391		return rc;
1392
1393	/* Process MAC/tag directly: feed tag to engine and wait for IRQ. */
1394	ocs_aes_ccm_write_encrypted_tag(aes_dev, in_tag, tag_size);
1395	rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
1396	if (rc)
1397		return rc;
1398
1399	return ccm_compare_tag_to_yr(aes_dev, tag_size);
1400}
1401
1402/**
1403 * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list.
1404 * @aes_dev:	  The OCS AES device the list will be created for.
1405 * @sg:		  The SG list OCS DMA linked list will be created from. When
1406 *		  passed to this function, @sg must have been already mapped
1407 *		  with dma_map_sg().
1408 * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the
1409 *		  value returned by dma_map_sg() when @sg was mapped.
1410 * @dll_desc:	  The OCS DMA dma_list to use to store information about the
1411 *		  created linked list.
1412 * @data_size:	  The size of the data (from the SG list) to be mapped into the
1413 *		  OCS DMA linked list.
1414 * @data_offset:  The offset (within the SG list) of the data to be mapped.
1415 *
1416 * Return:	0 on success, negative error code otherwise.
1417 */
1418int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev,
1419				   struct scatterlist *sg,
1420				   int sg_dma_count,
1421				   struct ocs_dll_desc *dll_desc,
1422				   size_t data_size, size_t data_offset)
1423{
1424	struct ocs_dma_linked_list *ll = NULL;
1425	struct scatterlist *sg_tmp;
1426	unsigned int tmp;
1427	int dma_nents;
1428	int i;
1429
1430	if (!dll_desc || !sg || !aes_dev)
1431		return -EINVAL;
1432
1433	/* Default values for when no ddl_desc is created. */
1434	dll_desc->vaddr = NULL;
1435	dll_desc->dma_addr = DMA_MAPPING_ERROR;
1436	dll_desc->size = 0;
1437
1438	if (data_size == 0)
1439		return 0;
1440
1441	/* Loop over sg_list until we reach entry at specified offset. */
1442	while (data_offset >= sg_dma_len(sg)) {
1443		data_offset -= sg_dma_len(sg);
1444		sg_dma_count--;
1445		sg = sg_next(sg);
1446		/* If we reach the end of the list, offset was invalid. */
1447		if (!sg || sg_dma_count == 0)
1448			return -EINVAL;
1449	}
1450
1451	/* Compute number of DMA-mapped SG entries to add into OCS DMA list. */
1452	dma_nents = 0;
1453	tmp = 0;
1454	sg_tmp = sg;
1455	while (tmp < data_offset + data_size) {
1456		/* If we reach the end of the list, data_size was invalid. */
1457		if (!sg_tmp)
1458			return -EINVAL;
1459		tmp += sg_dma_len(sg_tmp);
1460		dma_nents++;
1461		sg_tmp = sg_next(sg_tmp);
1462	}
1463	if (dma_nents > sg_dma_count)
1464		return -EINVAL;
1465
1466	/* Allocate the DMA list, one entry for each SG entry. */
1467	dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents;
1468	dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size,
1469					     &dll_desc->dma_addr, GFP_KERNEL);
1470	if (!dll_desc->vaddr)
1471		return -ENOMEM;
1472
1473	/* Populate DMA linked list entries. */
1474	ll = dll_desc->vaddr;
1475	for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) {
1476		ll[i].src_addr = sg_dma_address(sg) + data_offset;
1477		ll[i].src_len = min(sg_dma_len(sg) - data_offset, data_size);
1478		data_offset = 0;
1479		data_size -= ll[i].src_len;
1480		/* Current element points to the DMA address of the next one. */
1481		ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1));
1482		ll[i].ll_flags = 0;
1483	}
1484	/* Terminate last element. */
1485	ll[i - 1].next = 0;
1486	ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE;
1487
1488	return 0;
1489}