qat_hal.c

   1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
   2/* Copyright(c) 2014 - 2020 Intel Corporation */
   3#include <linux/slab.h>
   4#include <linux/delay.h>
   5#include <linux/pci_ids.h>
   6
   7#include "adf_accel_devices.h"
   8#include "adf_common_drv.h"
   9#include "icp_qat_hal.h"
  10#include "icp_qat_uclo.h"
  11
  12#define BAD_REGADDR	       0xffff
  13#define MAX_RETRY_TIMES	   10000
  14#define INIT_CTX_ARB_VALUE	0x0
  15#define INIT_CTX_ENABLE_VALUE     0x0
  16#define INIT_PC_VALUE	     0x0
  17#define INIT_WAKEUP_EVENTS_VALUE  0x1
  18#define INIT_SIG_EVENTS_VALUE     0x1
  19#define INIT_CCENABLE_VALUE       0x2000
  20#define RST_CSR_QAT_LSB	   20
  21#define RST_CSR_AE_LSB		  0
  22#define MC_TIMESTAMP_ENABLE       (0x1 << 7)
  23
  24#define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \
  25	(~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \
  26	(~(1 << CE_REG_PAR_ERR_BITPOS)))
  27#define INSERT_IMMED_GPRA_CONST(inst, const_val) \
  28	(inst = ((inst & 0xFFFF00C03FFull) | \
  29		((((const_val) << 12) & 0x0FF00000ull) | \
  30		(((const_val) << 10) & 0x0003FC00ull))))
  31#define INSERT_IMMED_GPRB_CONST(inst, const_val) \
  32	(inst = ((inst & 0xFFFF00FFF00ull) | \
  33		((((const_val) << 12) & 0x0FF00000ull) | \
  34		(((const_val) <<  0) & 0x000000FFull))))
  35
  36#define AE(handle, ae) ((handle)->hal_handle->aes[ae])
  37
  38static const u64 inst_4b[] = {
  39	0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull,
  40	0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  41	0x0A021000000ull
  42};
  43
  44static const u64 inst[] = {
  45	0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull,
  46	0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  47	0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull,
  48	0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  49	0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull,
  50	0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull,
  51	0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull,
  52	0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull,
  53	0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull,
  54	0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull,
  55	0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull,
  56	0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull,
  57	0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull,
  58	0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull,
  59	0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull,
  60	0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull,
  61	0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull,
  62	0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull,
  63	0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull,
  64	0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull,
  65	0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull,
  66	0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,
  67};
  68
  69void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
  70			  unsigned char ae, unsigned int ctx_mask)
  71{
  72	AE(handle, ae).live_ctx_mask = ctx_mask;
  73}
  74
  75#define CSR_RETRY_TIMES 500
  76static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,
  77			     unsigned char ae, unsigned int csr)
  78{
  79	unsigned int iterations = CSR_RETRY_TIMES;
  80	int value;
  81
  82	do {
  83		value = GET_AE_CSR(handle, ae, csr);
  84		if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
  85			return value;
  86	} while (iterations--);
  87
  88	pr_err("QAT: Read CSR timeout\n");
  89	return 0;
  90}
  91
  92static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,
  93			     unsigned char ae, unsigned int csr,
  94			     unsigned int value)
  95{
  96	unsigned int iterations = CSR_RETRY_TIMES;
  97
  98	do {
  99		SET_AE_CSR(handle, ae, csr, value);
 100		if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
 101			return 0;
 102	} while (iterations--);
 103
 104	pr_err("QAT: Write CSR Timeout\n");
 105	return -EFAULT;
 106}
 107
 108static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,
 109				     unsigned char ae, unsigned char ctx,
 110				     unsigned int *events)
 111{
 112	unsigned int cur_ctx;
 113
 114	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 115	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 116	*events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT);
 117	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 118}
 119
 120static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,
 121			       unsigned char ae, unsigned int cycles,
 122			       int chk_inactive)
 123{
 124	unsigned int base_cnt = 0, cur_cnt = 0;
 125	unsigned int csr = (1 << ACS_ABO_BITPOS);
 126	int times = MAX_RETRY_TIMES;
 127	int elapsed_cycles = 0;
 128
 129	base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 130	base_cnt &= 0xffff;
 131	while ((int)cycles > elapsed_cycles && times--) {
 132		if (chk_inactive)
 133			csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 134
 135		cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 136		cur_cnt &= 0xffff;
 137		elapsed_cycles = cur_cnt - base_cnt;
 138
 139		if (elapsed_cycles < 0)
 140			elapsed_cycles += 0x10000;
 141
 142		/* ensure at least 8 time cycles elapsed in wait_cycles */
 143		if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS)))
 144			return 0;
 145	}
 146	if (times < 0) {
 147		pr_err("QAT: wait_num_cycles time out\n");
 148		return -EFAULT;
 149	}
 150	return 0;
 151}
 152
 153#define CLR_BIT(wrd, bit) ((wrd) & ~(1 << (bit)))
 154#define SET_BIT(wrd, bit) ((wrd) | 1 << (bit))
 155
 156int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
 157			    unsigned char ae, unsigned char mode)
 158{
 159	unsigned int csr, new_csr;
 160
 161	if (mode != 4 && mode != 8) {
 162		pr_err("QAT: bad ctx mode=%d\n", mode);
 163		return -EINVAL;
 164	}
 165
 166	/* Sets the acceleration engine context mode to either four or eight */
 167	csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 168	csr = IGNORE_W1C_MASK & csr;
 169	new_csr = (mode == 4) ?
 170		SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :
 171		CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS);
 172	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 173	return 0;
 174}
 175
 176int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
 177			   unsigned char ae, unsigned char mode)
 178{
 179	unsigned int csr, new_csr;
 180
 181	csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 182	csr &= IGNORE_W1C_MASK;
 183
 184	new_csr = (mode) ?
 185		SET_BIT(csr, CE_NN_MODE_BITPOS) :
 186		CLR_BIT(csr, CE_NN_MODE_BITPOS);
 187
 188	if (new_csr != csr)
 189		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 190
 191	return 0;
 192}
 193
 194int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
 195			   unsigned char ae, enum icp_qat_uof_regtype lm_type,
 196			   unsigned char mode)
 197{
 198	unsigned int csr, new_csr;
 199
 200	csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 201	csr &= IGNORE_W1C_MASK;
 202	switch (lm_type) {
 203	case ICP_LMEM0:
 204		new_csr = (mode) ?
 205			SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) :
 206			CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS);
 207		break;
 208	case ICP_LMEM1:
 209		new_csr = (mode) ?
 210			SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) :
 211			CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS);
 212		break;
 213	case ICP_LMEM2:
 214		new_csr = (mode) ?
 215			SET_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS) :
 216			CLR_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS);
 217		break;
 218	case ICP_LMEM3:
 219		new_csr = (mode) ?
 220			SET_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS) :
 221			CLR_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS);
 222		break;
 223	default:
 224		pr_err("QAT: lmType = 0x%x\n", lm_type);
 225		return -EINVAL;
 226	}
 227
 228	if (new_csr != csr)
 229		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 230	return 0;
 231}
 232
 233void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle,
 234				unsigned char ae, unsigned char mode)
 235{
 236	unsigned int csr, new_csr;
 237
 238	csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 239	csr &= IGNORE_W1C_MASK;
 240	new_csr = (mode) ?
 241		  SET_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS) :
 242		  CLR_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS);
 243	if (new_csr != csr)
 244		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 245}
 246
 247static unsigned short qat_hal_get_reg_addr(unsigned int type,
 248					   unsigned short reg_num)
 249{
 250	unsigned short reg_addr;
 251
 252	switch (type) {
 253	case ICP_GPA_ABS:
 254	case ICP_GPB_ABS:
 255		reg_addr = 0x80 | (reg_num & 0x7f);
 256		break;
 257	case ICP_GPA_REL:
 258	case ICP_GPB_REL:
 259		reg_addr = reg_num & 0x1f;
 260		break;
 261	case ICP_SR_RD_REL:
 262	case ICP_SR_WR_REL:
 263	case ICP_SR_REL:
 264		reg_addr = 0x180 | (reg_num & 0x1f);
 265		break;
 266	case ICP_SR_ABS:
 267		reg_addr = 0x140 | ((reg_num & 0x3) << 1);
 268		break;
 269	case ICP_DR_RD_REL:
 270	case ICP_DR_WR_REL:
 271	case ICP_DR_REL:
 272		reg_addr = 0x1c0 | (reg_num & 0x1f);
 273		break;
 274	case ICP_DR_ABS:
 275		reg_addr = 0x100 | ((reg_num & 0x3) << 1);
 276		break;
 277	case ICP_NEIGH_REL:
 278		reg_addr = 0x280 | (reg_num & 0x1f);
 279		break;
 280	case ICP_LMEM0:
 281		reg_addr = 0x200;
 282		break;
 283	case ICP_LMEM1:
 284		reg_addr = 0x220;
 285		break;
 286	case ICP_LMEM2:
 287		reg_addr = 0x2c0;
 288		break;
 289	case ICP_LMEM3:
 290		reg_addr = 0x2e0;
 291		break;
 292	case ICP_NO_DEST:
 293		reg_addr = 0x300 | (reg_num & 0xff);
 294		break;
 295	default:
 296		reg_addr = BAD_REGADDR;
 297		break;
 298	}
 299	return reg_addr;
 300}
 301
 302void qat_hal_reset(struct icp_qat_fw_loader_handle *handle)
 303{
 304	unsigned int reset_mask = handle->chip_info->icp_rst_mask;
 305	unsigned int reset_csr = handle->chip_info->icp_rst_csr;
 306	unsigned int csr_val;
 307
 308	csr_val = GET_CAP_CSR(handle, reset_csr);
 309	csr_val |= reset_mask;
 310	SET_CAP_CSR(handle, reset_csr, csr_val);
 311}
 312
 313static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
 314				unsigned char ae, unsigned int ctx_mask,
 315				unsigned int ae_csr, unsigned int csr_val)
 316{
 317	unsigned int ctx, cur_ctx;
 318
 319	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 320
 321	for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 322		if (!(ctx_mask & (1 << ctx)))
 323			continue;
 324		qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 325		qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val);
 326	}
 327
 328	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 329}
 330
 331static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
 332				unsigned char ae, unsigned char ctx,
 333				unsigned int ae_csr)
 334{
 335	unsigned int cur_ctx, csr_val;
 336
 337	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 338	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 339	csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr);
 340	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 341
 342	return csr_val;
 343}
 344
 345static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
 346				  unsigned char ae, unsigned int ctx_mask,
 347				  unsigned int events)
 348{
 349	unsigned int ctx, cur_ctx;
 350
 351	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 352	for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 353		if (!(ctx_mask & (1 << ctx)))
 354			continue;
 355		qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 356		qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events);
 357	}
 358	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 359}
 360
 361static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,
 362				     unsigned char ae, unsigned int ctx_mask,
 363				     unsigned int events)
 364{
 365	unsigned int ctx, cur_ctx;
 366
 367	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 368	for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 369		if (!(ctx_mask & (1 << ctx)))
 370			continue;
 371		qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 372		qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT,
 373				  events);
 374	}
 375	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 376}
 377
 378static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
 379{
 380	unsigned long ae_mask = handle->hal_handle->ae_mask;
 381	unsigned int base_cnt, cur_cnt;
 382	unsigned char ae;
 383	int times = MAX_RETRY_TIMES;
 384
 385	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 386		base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 387		base_cnt &= 0xffff;
 388
 389		do {
 390			cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 391			cur_cnt &= 0xffff;
 392		} while (times-- && (cur_cnt == base_cnt));
 393
 394		if (times < 0) {
 395			pr_err("QAT: AE%d is inactive!!\n", ae);
 396			return -EFAULT;
 397		}
 398	}
 399
 400	return 0;
 401}
 402
 403int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle,
 404			    unsigned int ae)
 405{
 406	unsigned int enable = 0, active = 0;
 407
 408	enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 409	active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 410	if ((enable & (0xff << CE_ENABLE_BITPOS)) ||
 411	    (active & (1 << ACS_ABO_BITPOS)))
 412		return 1;
 413	else
 414		return 0;
 415}
 416
 417static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)
 418{
 419	unsigned long ae_mask = handle->hal_handle->ae_mask;
 420	unsigned int misc_ctl_csr, misc_ctl;
 421	unsigned char ae;
 422
 423	misc_ctl_csr = handle->chip_info->misc_ctl_csr;
 424	/* stop the timestamp timers */
 425	misc_ctl = GET_CAP_CSR(handle, misc_ctl_csr);
 426	if (misc_ctl & MC_TIMESTAMP_ENABLE)
 427		SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl &
 428			    (~MC_TIMESTAMP_ENABLE));
 429
 430	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 431		qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0);
 432		qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0);
 433	}
 434	/* start timestamp timers */
 435	SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE);
 436}
 437
 438#define ESRAM_AUTO_TINIT	BIT(2)
 439#define ESRAM_AUTO_TINIT_DONE	BIT(3)
 440#define ESRAM_AUTO_INIT_USED_CYCLES (1640)
 441#define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C
 442static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
 443{
 444	void __iomem *csr_addr =
 445			(void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v +
 446			ESRAM_AUTO_INIT_CSR_OFFSET);
 447	unsigned int csr_val;
 448	int times = 30;
 449
 450	if (handle->pci_dev->device != PCI_DEVICE_ID_INTEL_QAT_DH895XCC)
 451		return 0;
 452
 453	csr_val = ADF_CSR_RD(csr_addr, 0);
 454	if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))
 455		return 0;
 456
 457	csr_val = ADF_CSR_RD(csr_addr, 0);
 458	csr_val |= ESRAM_AUTO_TINIT;
 459	ADF_CSR_WR(csr_addr, 0, csr_val);
 460
 461	do {
 462		qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);
 463		csr_val = ADF_CSR_RD(csr_addr, 0);
 464	} while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);
 465	if (times < 0) {
 466		pr_err("QAT: Fail to init eSram!\n");
 467		return -EFAULT;
 468	}
 469	return 0;
 470}
 471
 472#define SHRAM_INIT_CYCLES 2060
 473int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)
 474{
 475	unsigned int clk_csr = handle->chip_info->glb_clk_enable_csr;
 476	unsigned int reset_mask = handle->chip_info->icp_rst_mask;
 477	unsigned int reset_csr = handle->chip_info->icp_rst_csr;
 478	unsigned long ae_mask = handle->hal_handle->ae_mask;
 479	unsigned char ae = 0;
 480	unsigned int times = 100;
 481	unsigned int csr_val;
 482
 483	/* write to the reset csr */
 484	csr_val = GET_CAP_CSR(handle, reset_csr);
 485	csr_val &= ~reset_mask;
 486	do {
 487		SET_CAP_CSR(handle, reset_csr, csr_val);
 488		if (!(times--))
 489			goto out_err;
 490		csr_val = GET_CAP_CSR(handle, reset_csr);
 491		csr_val &= reset_mask;
 492	} while (csr_val);
 493	/* enable clock */
 494	csr_val = GET_CAP_CSR(handle, clk_csr);
 495	csr_val |= reset_mask;
 496	SET_CAP_CSR(handle, clk_csr, csr_val);
 497	if (qat_hal_check_ae_alive(handle))
 498		goto out_err;
 499
 500	/* Set undefined power-up/reset states to reasonable default values */
 501	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 502		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
 503				  INIT_CTX_ENABLE_VALUE);
 504		qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX,
 505				    CTX_STS_INDIRECT,
 506				    handle->hal_handle->upc_mask &
 507				    INIT_PC_VALUE);
 508		qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
 509		qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
 510		qat_hal_put_wakeup_event(handle, ae,
 511					 ICP_QAT_UCLO_AE_ALL_CTX,
 512					 INIT_WAKEUP_EVENTS_VALUE);
 513		qat_hal_put_sig_event(handle, ae,
 514				      ICP_QAT_UCLO_AE_ALL_CTX,
 515				      INIT_SIG_EVENTS_VALUE);
 516	}
 517	if (qat_hal_init_esram(handle))
 518		goto out_err;
 519	if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0))
 520		goto out_err;
 521	qat_hal_reset_timestamp(handle);
 522
 523	return 0;
 524out_err:
 525	pr_err("QAT: failed to get device out of reset\n");
 526	return -EFAULT;
 527}
 528
 529static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,
 530				unsigned char ae, unsigned int ctx_mask)
 531{
 532	unsigned int ctx;
 533
 534	ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 535	ctx &= IGNORE_W1C_MASK &
 536		(~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));
 537	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
 538}
 539
 540static u64 qat_hal_parity_64bit(u64 word)
 541{
 542	word ^= word >> 1;
 543	word ^= word >> 2;
 544	word ^= word >> 4;
 545	word ^= word >> 8;
 546	word ^= word >> 16;
 547	word ^= word >> 32;
 548	return word & 1;
 549}
 550
 551static u64 qat_hal_set_uword_ecc(u64 uword)
 552{
 553	u64 bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL,
 554		bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL,
 555		bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL,
 556		bit6_mask = 0xdaf69a46910ULL;
 557
 558	/* clear the ecc bits */
 559	uword &= ~(0x7fULL << 0x2C);
 560	uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C;
 561	uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D;
 562	uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E;
 563	uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F;
 564	uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30;
 565	uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31;
 566	uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32;
 567	return uword;
 568}
 569
 570void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
 571		       unsigned char ae, unsigned int uaddr,
 572		       unsigned int words_num, u64 *uword)
 573{
 574	unsigned int ustore_addr;
 575	unsigned int i;
 576
 577	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 578	uaddr |= UA_ECS;
 579	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 580	for (i = 0; i < words_num; i++) {
 581		unsigned int uwrd_lo, uwrd_hi;
 582		u64 tmp;
 583
 584		tmp = qat_hal_set_uword_ecc(uword[i]);
 585		uwrd_lo = (unsigned int)(tmp & 0xffffffff);
 586		uwrd_hi = (unsigned int)(tmp >> 0x20);
 587		qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
 588		qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
 589	}
 590	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 591}
 592
 593static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,
 594			       unsigned char ae, unsigned int ctx_mask)
 595{
 596	unsigned int ctx;
 597
 598	ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 599	ctx &= IGNORE_W1C_MASK;
 600	ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;
 601	ctx |= (ctx_mask << CE_ENABLE_BITPOS);
 602	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
 603}
 604
 605static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle)
 606{
 607	unsigned long ae_mask = handle->hal_handle->ae_mask;
 608	unsigned char ae;
 609	unsigned short reg;
 610
 611	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 612		for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) {
 613			qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS,
 614					     reg, 0);
 615			qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS,
 616					     reg, 0);
 617		}
 618	}
 619}
 620
 621static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
 622{
 623	unsigned long ae_mask = handle->hal_handle->ae_mask;
 624	unsigned char ae;
 625	unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX;
 626	int times = MAX_RETRY_TIMES;
 627	unsigned int csr_val = 0;
 628	unsigned int savctx = 0;
 629	int ret = 0;
 630
 631	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 632		csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
 633		csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);
 634		qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
 635		csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 636		csr_val &= IGNORE_W1C_MASK;
 637		if (handle->chip_info->nn)
 638			csr_val |= CE_NN_MODE;
 639
 640		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);
 641		qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst),
 642				  (u64 *)inst);
 643		qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 644				    handle->hal_handle->upc_mask &
 645				    INIT_PC_VALUE);
 646		savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 647		qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);
 648		qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);
 649		qat_hal_wr_indr_csr(handle, ae, ctx_mask,
 650				    CTX_SIG_EVENTS_INDIRECT, 0);
 651		qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
 652		qat_hal_enable_ctx(handle, ae, ctx_mask);
 653	}
 654	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 655		/* wait for AE to finish */
 656		do {
 657			ret = qat_hal_wait_cycles(handle, ae, 20, 1);
 658		} while (ret && times--);
 659
 660		if (times < 0) {
 661			pr_err("QAT: clear GPR of AE %d failed", ae);
 662			return -EINVAL;
 663		}
 664		qat_hal_disable_ctx(handle, ae, ctx_mask);
 665		qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
 666				  savctx & ACS_ACNO);
 667		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
 668				  INIT_CTX_ENABLE_VALUE);
 669		qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 670				    handle->hal_handle->upc_mask &
 671				    INIT_PC_VALUE);
 672		qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
 673		qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
 674		qat_hal_put_wakeup_event(handle, ae, ctx_mask,
 675					 INIT_WAKEUP_EVENTS_VALUE);
 676		qat_hal_put_sig_event(handle, ae, ctx_mask,
 677				      INIT_SIG_EVENTS_VALUE);
 678	}
 679	return 0;
 680}
 681
 682static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
 683			     struct adf_accel_dev *accel_dev)
 684{
 685	struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
 686	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 687	void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
 688	unsigned int max_en_ae_id = 0;
 689	struct adf_bar *sram_bar;
 690	unsigned int csr_val = 0;
 691	unsigned long ae_mask;
 692	unsigned char ae = 0;
 693	int ret = 0;
 694
 695	handle->pci_dev = pci_info->pci_dev;
 696	switch (handle->pci_dev->device) {
 697	case PCI_DEVICE_ID_INTEL_QAT_4XXX:
 698	case PCI_DEVICE_ID_INTEL_QAT_401XX:
 699	case PCI_DEVICE_ID_INTEL_QAT_402XX:
 700	case PCI_DEVICE_ID_INTEL_QAT_420XX:
 701	case PCI_DEVICE_ID_INTEL_QAT_6XXX:
 702		handle->chip_info->mmp_sram_size = 0;
 703		handle->chip_info->nn = false;
 704		handle->chip_info->lm2lm3 = true;
 705		handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X;
 706		handle->chip_info->icp_rst_csr = ICP_RESET_CPP0;
 707		if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_420XX)
 708			handle->chip_info->icp_rst_mask = 0x100155;
 709		else
 710			handle->chip_info->icp_rst_mask = 0x100015;
 711		handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE_CPP0;
 712		handle->chip_info->misc_ctl_csr = MISC_CONTROL_C4XXX;
 713		handle->chip_info->wakeup_event_val = 0x80000000;
 714		handle->chip_info->fw_auth = true;
 715		handle->chip_info->css_3k = true;
 716		if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_6XXX)
 717			handle->chip_info->dual_sign = true;
 718		handle->chip_info->tgroup_share_ustore = true;
 719		handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX;
 720		handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX;
 721		handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI_4XXX;
 722		handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO_4XXX;
 723		handle->chip_info->fcu_loaded_ae_csr = FCU_AE_LOADED_4XXX;
 724		handle->chip_info->fcu_loaded_ae_pos = 0;
 725
 726		handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET_4XXX;
 727		handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET_4XXX;
 728		handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET_4XXX;
 729		handle->hal_cap_ae_local_csr_addr_v =
 730			(void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
 731			+ LOCAL_TO_XFER_REG_OFFSET);
 732		break;
 733	case PCI_DEVICE_ID_INTEL_QAT_C62X:
 734	case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
 735		handle->chip_info->mmp_sram_size = 0;
 736		handle->chip_info->nn = true;
 737		handle->chip_info->lm2lm3 = false;
 738		handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG;
 739		handle->chip_info->icp_rst_csr = ICP_RESET;
 740		handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) |
 741						  (hw_data->accel_mask << RST_CSR_QAT_LSB);
 742		handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE;
 743		handle->chip_info->misc_ctl_csr = MISC_CONTROL;
 744		handle->chip_info->wakeup_event_val = WAKEUP_EVENT;
 745		handle->chip_info->fw_auth = true;
 746		handle->chip_info->css_3k = false;
 747		handle->chip_info->tgroup_share_ustore = false;
 748		handle->chip_info->fcu_ctl_csr = FCU_CONTROL;
 749		handle->chip_info->fcu_sts_csr = FCU_STATUS;
 750		handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI;
 751		handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO;
 752		handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS;
 753		handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS;
 754		handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET;
 755		handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET;
 756		handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET;
 757		handle->hal_cap_ae_local_csr_addr_v =
 758			(void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
 759			+ LOCAL_TO_XFER_REG_OFFSET);
 760		break;
 761	case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:
 762		handle->chip_info->mmp_sram_size = 0x40000;
 763		handle->chip_info->nn = true;
 764		handle->chip_info->lm2lm3 = false;
 765		handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG;
 766		handle->chip_info->icp_rst_csr = ICP_RESET;
 767		handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) |
 768						  (hw_data->accel_mask << RST_CSR_QAT_LSB);
 769		handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE;
 770		handle->chip_info->misc_ctl_csr = MISC_CONTROL;
 771		handle->chip_info->wakeup_event_val = WAKEUP_EVENT;
 772		handle->chip_info->fw_auth = false;
 773		handle->chip_info->css_3k = false;
 774		handle->chip_info->tgroup_share_ustore = false;
 775		handle->chip_info->fcu_ctl_csr = 0;
 776		handle->chip_info->fcu_sts_csr = 0;
 777		handle->chip_info->fcu_dram_addr_hi = 0;
 778		handle->chip_info->fcu_dram_addr_lo = 0;
 779		handle->chip_info->fcu_loaded_ae_csr = 0;
 780		handle->chip_info->fcu_loaded_ae_pos = 0;
 781		handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET;
 782		handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET;
 783		handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET;
 784		handle->hal_cap_ae_local_csr_addr_v =
 785			(void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
 786			+ LOCAL_TO_XFER_REG_OFFSET);
 787		break;
 788	default:
 789		ret = -EINVAL;
 790		goto out_err;
 791	}
 792
 793	if (handle->chip_info->mmp_sram_size > 0) {
 794		sram_bar =
 795			&pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];
 796		handle->hal_sram_addr_v = sram_bar->virt_addr;
 797	}
 798	handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
 799	handle->hal_handle->ae_mask = hw_data->ae_mask;
 800	handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask;
 801	handle->hal_handle->slice_mask = hw_data->accel_mask;
 802	handle->cfg_ae_mask = ALL_AE_MASK;
 803	/* create AE objects */
 804	handle->hal_handle->upc_mask = 0x1ffff;
 805	handle->hal_handle->max_ustore = 0x4000;
 806
 807	ae_mask = handle->hal_handle->ae_mask;
 808	for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) {
 809		handle->hal_handle->aes[ae].free_addr = 0;
 810		handle->hal_handle->aes[ae].free_size =
 811		    handle->hal_handle->max_ustore;
 812		handle->hal_handle->aes[ae].ustore_size =
 813		    handle->hal_handle->max_ustore;
 814		handle->hal_handle->aes[ae].live_ctx_mask =
 815						ICP_QAT_UCLO_AE_ALL_CTX;
 816		max_en_ae_id = ae;
 817	}
 818	handle->hal_handle->ae_max_num = max_en_ae_id + 1;
 819
 820	/* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
 821	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 822		csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE);
 823		csr_val |= 0x1;
 824		qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);
 825	}
 826out_err:
 827	return ret;
 828}
 829
 830int qat_hal_init(struct adf_accel_dev *accel_dev)
 831{
 832	struct icp_qat_fw_loader_handle *handle;
 833	int ret = 0;
 834
 835	handle = kzalloc(sizeof(*handle), GFP_KERNEL);
 836	if (!handle)
 837		return -ENOMEM;
 838
 839	handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);
 840	if (!handle->hal_handle) {
 841		ret = -ENOMEM;
 842		goto out_hal_handle;
 843	}
 844
 845	handle->chip_info = kzalloc(sizeof(*handle->chip_info), GFP_KERNEL);
 846	if (!handle->chip_info) {
 847		ret = -ENOMEM;
 848		goto out_chip_info;
 849	}
 850
 851	ret = qat_hal_chip_init(handle, accel_dev);
 852	if (ret) {
 853		dev_err(&GET_DEV(accel_dev), "qat_hal_chip_init error\n");
 854		goto out_err;
 855	}
 856
 857	/* take all AEs out of reset */
 858	ret = qat_hal_clr_reset(handle);
 859	if (ret) {
 860		dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n");
 861		goto out_err;
 862	}
 863
 864	qat_hal_clear_xfer(handle);
 865	if (!handle->chip_info->fw_auth) {
 866		ret = qat_hal_clear_gpr(handle);
 867		if (ret)
 868			goto out_err;
 869	}
 870
 871	accel_dev->fw_loader->fw_loader = handle;
 872	return 0;
 873
 874out_err:
 875	kfree(handle->chip_info);
 876out_chip_info:
 877	kfree(handle->hal_handle);
 878out_hal_handle:
 879	kfree(handle);
 880	return ret;
 881}
 882
 883void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)
 884{
 885	if (!handle)
 886		return;
 887	kfree(handle->chip_info);
 888	kfree(handle->hal_handle);
 889	kfree(handle);
 890}
 891
 892int qat_hal_start(struct icp_qat_fw_loader_handle *handle)
 893{
 894	unsigned long ae_mask = handle->hal_handle->ae_mask;
 895	u32 wakeup_val = handle->chip_info->wakeup_event_val;
 896	u32 fcu_ctl_csr, fcu_sts_csr;
 897	unsigned int fcu_sts;
 898	unsigned char ae;
 899	u32 ae_ctr = 0;
 900	int retry = 0;
 901
 902	if (handle->chip_info->fw_auth) {
 903		fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;
 904		fcu_sts_csr = handle->chip_info->fcu_sts_csr;
 905		ae_ctr = hweight32(ae_mask);
 906		SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START);
 907		do {
 908			msleep(FW_AUTH_WAIT_PERIOD);
 909			fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);
 910			if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1))
 911				return ae_ctr;
 912		} while (retry++ < FW_AUTH_MAX_RETRY);
 913		pr_err("QAT: start error (FCU_STS = 0x%x)\n", fcu_sts);
 914		return 0;
 915	} else {
 916		for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 917			qat_hal_put_wakeup_event(handle, ae, 0, wakeup_val);
 918			qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX);
 919			ae_ctr++;
 920		}
 921		return ae_ctr;
 922	}
 923}
 924
 925void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
 926		  unsigned int ctx_mask)
 927{
 928	if (!handle->chip_info->fw_auth)
 929		qat_hal_disable_ctx(handle, ae, ctx_mask);
 930}
 931
 932void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,
 933		    unsigned char ae, unsigned int ctx_mask, unsigned int upc)
 934{
 935	qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 936			    handle->hal_handle->upc_mask & upc);
 937}
 938
 939static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,
 940			       unsigned char ae, unsigned int uaddr,
 941			       unsigned int words_num, u64 *uword)
 942{
 943	unsigned int i, uwrd_lo, uwrd_hi;
 944	unsigned int ustore_addr, misc_control;
 945
 946	misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
 947	qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL,
 948			  misc_control & 0xfffffffb);
 949	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 950	uaddr |= UA_ECS;
 951	for (i = 0; i < words_num; i++) {
 952		qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 953		uaddr++;
 954		uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER);
 955		uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER);
 956		uword[i] = uwrd_hi;
 957		uword[i] = (uword[i] << 0x20) | uwrd_lo;
 958	}
 959	qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control);
 960	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 961}
 962
 963void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
 964		     unsigned char ae, unsigned int uaddr,
 965		     unsigned int words_num, unsigned int *data)
 966{
 967	unsigned int i, ustore_addr;
 968
 969	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 970	uaddr |= UA_ECS;
 971	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 972	for (i = 0; i < words_num; i++) {
 973		unsigned int uwrd_lo, uwrd_hi, tmp;
 974
 975		uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) |
 976			  ((data[i] & 0xff00) << 2) |
 977			  (0x3 << 8) | (data[i] & 0xff);
 978		uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28);
 979		uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8;
 980		tmp = ((data[i] >> 0x10) & 0xffff);
 981		uwrd_hi |= (hweight32(tmp) & 0x1) << 9;
 982		qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
 983		qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
 984	}
 985	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 986}
 987
 988#define MAX_EXEC_INST 100
 989static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
 990				   unsigned char ae, unsigned char ctx,
 991				   u64 *micro_inst, unsigned int inst_num,
 992				   int code_off, unsigned int max_cycle,
 993				   unsigned int *endpc)
 994{
 995	unsigned int ind_lm_addr_byte0 = 0, ind_lm_addr_byte1 = 0;
 996	unsigned int ind_lm_addr_byte2 = 0, ind_lm_addr_byte3 = 0;
 997	unsigned int ind_t_index = 0, ind_t_index_byte = 0;
 998	unsigned int ind_lm_addr0 = 0, ind_lm_addr1 = 0;
 999	unsigned int ind_lm_addr2 = 0, ind_lm_addr3 = 0;
1000	u64 savuwords[MAX_EXEC_INST];
1001	unsigned int ind_cnt_sig;
1002	unsigned int ind_sig, act_sig;
1003	unsigned int csr_val = 0, newcsr_val;
1004	unsigned int savctx;
1005	unsigned int savcc, wakeup_events, savpc;
1006	unsigned int ctxarb_ctl, ctx_enables;
1007
1008	if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) {
1009		pr_err("QAT: invalid instruction num %d\n", inst_num);
1010		return -EINVAL;
1011	}
1012	/* save current context */
1013	ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT);
1014	ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT);
1015	ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx,
1016						INDIRECT_LM_ADDR_0_BYTE_INDEX);
1017	ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx,
1018						INDIRECT_LM_ADDR_1_BYTE_INDEX);
1019	if (handle->chip_info->lm2lm3) {
1020		ind_lm_addr2 = qat_hal_rd_indr_csr(handle, ae, ctx,
1021						   LM_ADDR_2_INDIRECT);
1022		ind_lm_addr3 = qat_hal_rd_indr_csr(handle, ae, ctx,
1023						   LM_ADDR_3_INDIRECT);
1024		ind_lm_addr_byte2 = qat_hal_rd_indr_csr(handle, ae, ctx,
1025							INDIRECT_LM_ADDR_2_BYTE_INDEX);
1026		ind_lm_addr_byte3 = qat_hal_rd_indr_csr(handle, ae, ctx,
1027							INDIRECT_LM_ADDR_3_BYTE_INDEX);
1028		ind_t_index = qat_hal_rd_indr_csr(handle, ae, ctx,
1029						  INDIRECT_T_INDEX);
1030		ind_t_index_byte = qat_hal_rd_indr_csr(handle, ae, ctx,
1031						       INDIRECT_T_INDEX_BYTE_INDEX);
1032	}
1033	if (inst_num <= MAX_EXEC_INST)
1034		qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);
1035	qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);
1036	savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT);
1037	savpc = (savpc & handle->hal_handle->upc_mask) >> 0;
1038	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1039	ctx_enables &= IGNORE_W1C_MASK;
1040	savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE);
1041	savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
1042	ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
1043	ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
1044					  FUTURE_COUNT_SIGNAL_INDIRECT);
1045	ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
1046				      CTX_SIG_EVENTS_INDIRECT);
1047	act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE);
1048	/* execute micro codes */
1049	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1050	qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);
1051	qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0);
1052	qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO);
1053	if (code_off)
1054		qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff);
1055	qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY);
1056	qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0);
1057	qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
1058	qat_hal_enable_ctx(handle, ae, (1 << ctx));
1059	/* wait for micro codes to finish */
1060	if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0)
1061		return -EFAULT;
1062	if (endpc) {
1063		unsigned int ctx_status;
1064
1065		ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx,
1066						 CTX_STS_INDIRECT);
1067		*endpc = ctx_status & handle->hal_handle->upc_mask;
1068	}
1069	/* retore to saved context */
1070	qat_hal_disable_ctx(handle, ae, (1 << ctx));
1071	if (inst_num <= MAX_EXEC_INST)
1072		qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords);
1073	qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);
1074	qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT,
1075			    handle->hal_handle->upc_mask & savpc);
1076	csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
1077	newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
1078	qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
1079	qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);
1080	qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO);
1081	qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl);
1082	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1083			    LM_ADDR_0_INDIRECT, ind_lm_addr0);
1084	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1085			    LM_ADDR_1_INDIRECT, ind_lm_addr1);
1086	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1087			    INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0);
1088	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1089			    INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1);
1090	if (handle->chip_info->lm2lm3) {
1091		qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_2_INDIRECT,
1092				    ind_lm_addr2);
1093		qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_3_INDIRECT,
1094				    ind_lm_addr3);
1095		qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1096				    INDIRECT_LM_ADDR_2_BYTE_INDEX,
1097				    ind_lm_addr_byte2);
1098		qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1099				    INDIRECT_LM_ADDR_3_BYTE_INDEX,
1100				    ind_lm_addr_byte3);
1101		qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1102				    INDIRECT_T_INDEX, ind_t_index);
1103		qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1104				    INDIRECT_T_INDEX_BYTE_INDEX,
1105				    ind_t_index_byte);
1106	}
1107	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1108			    FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig);
1109	qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1110			    CTX_SIG_EVENTS_INDIRECT, ind_sig);
1111	qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);
1112	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1113
1114	return 0;
1115}
1116
1117static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
1118			      unsigned char ae, unsigned char ctx,
1119			      enum icp_qat_uof_regtype reg_type,
1120			      unsigned short reg_num, unsigned int *data)
1121{
1122	unsigned int savctx, uaddr, uwrd_lo, uwrd_hi;
1123	unsigned int ctxarb_cntl, ustore_addr, ctx_enables;
1124	unsigned short reg_addr;
1125	int status = 0;
1126	u64 insts, savuword;
1127
1128	reg_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1129	if (reg_addr == BAD_REGADDR) {
1130		pr_err("QAT: bad regaddr=0x%x\n", reg_addr);
1131		return -EINVAL;
1132	}
1133	switch (reg_type) {
1134	case ICP_GPA_REL:
1135		insts = 0xA070000000ull | (reg_addr & 0x3ff);
1136		break;
1137	default:
1138		insts = (u64)0xA030000000ull | ((reg_addr & 0x3ff) << 10);
1139		break;
1140	}
1141	savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
1142	ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
1143	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1144	ctx_enables &= IGNORE_W1C_MASK;
1145	if (ctx != (savctx & ACS_ACNO))
1146		qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
1147				  ctx & ACS_ACNO);
1148	qat_hal_get_uwords(handle, ae, 0, 1, &savuword);
1149	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1150	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
1151	uaddr = UA_ECS;
1152	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1153	insts = qat_hal_set_uword_ecc(insts);
1154	uwrd_lo = (unsigned int)(insts & 0xffffffff);
1155	uwrd_hi = (unsigned int)(insts >> 0x20);
1156	qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
1157	qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
1158	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1159	/* delay for at least 8 cycles */
1160	qat_hal_wait_cycles(handle, ae, 0x8, 0);
1161	/*
1162	 * read ALU output
1163	 * the instruction should have been executed
1164	 * prior to clearing the ECS in putUwords
1165	 */
1166	*data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT);
1167	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
1168	qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);
1169	if (ctx != (savctx & ACS_ACNO))
1170		qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
1171				  savctx & ACS_ACNO);
1172	qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl);
1173	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1174
1175	return status;
1176}
1177
1178static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,
1179			      unsigned char ae, unsigned char ctx,
1180			      enum icp_qat_uof_regtype reg_type,
1181			      unsigned short reg_num, unsigned int data)
1182{
1183	unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo;
1184	u64 insts[] = {
1185		0x0F440000000ull,
1186		0x0F040000000ull,
1187		0x0F0000C0300ull,
1188		0x0E000010000ull
1189	};
1190	const int num_inst = ARRAY_SIZE(insts), code_off = 1;
1191	const int imm_w1 = 0, imm_w0 = 1;
1192
1193	dest_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1194	if (dest_addr == BAD_REGADDR) {
1195		pr_err("QAT: bad destAddr=0x%x\n", dest_addr);
1196		return -EINVAL;
1197	}
1198
1199	data16lo = 0xffff & data;
1200	data16hi = 0xffff & (data >> 0x10);
1201	src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1202					  (0xff & data16hi));
1203	src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1204					   (0xff & data16lo));
1205	switch (reg_type) {
1206	case ICP_GPA_REL:
1207		insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1208		    ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1209		insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1210		    ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1211		break;
1212	default:
1213		insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1214		    ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1215
1216		insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1217		    ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1218		break;
1219	}
1220
1221	return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst,
1222				       code_off, num_inst * 0x5, NULL);
1223}
1224
1225int qat_hal_get_ins_num(void)
1226{
1227	return ARRAY_SIZE(inst_4b);
1228}
1229
1230static int qat_hal_concat_micro_code(u64 *micro_inst,
1231				     unsigned int inst_num, unsigned int size,
1232				     unsigned int addr, unsigned int *value)
1233{
1234	int i;
1235	unsigned int cur_value;
1236	const u64 *inst_arr;
1237	int fixup_offset;
1238	int usize = 0;
1239	int orig_num;
1240
1241	orig_num = inst_num;
1242	cur_value = value[0];
1243	inst_arr = inst_4b;
1244	usize = ARRAY_SIZE(inst_4b);
1245	fixup_offset = inst_num;
1246	for (i = 0; i < usize; i++)
1247		micro_inst[inst_num++] = inst_arr[i];
1248	INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr));
1249	fixup_offset++;
1250	INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0);
1251	fixup_offset++;
1252	INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0));
1253	fixup_offset++;
1254	INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10));
1255
1256	return inst_num - orig_num;
1257}
1258
1259static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,
1260				      unsigned char ae, unsigned char ctx,
1261				      int *pfirst_exec, u64 *micro_inst,
1262				      unsigned int inst_num)
1263{
1264	int stat = 0;
1265	unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0;
1266	unsigned int gprb0 = 0, gprb1 = 0;
1267
1268	if (*pfirst_exec) {
1269		qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0);
1270		qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1);
1271		qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2);
1272		qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0);
1273		qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1);
1274		*pfirst_exec = 0;
1275	}
1276	stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1,
1277				       inst_num * 0x5, NULL);
1278	if (stat != 0)
1279		return -EFAULT;
1280	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0);
1281	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1);
1282	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2);
1283	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0);
1284	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1);
1285
1286	return 0;
1287}
1288
1289int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,
1290			unsigned char ae,
1291			struct icp_qat_uof_batch_init *lm_init_header)
1292{
1293	struct icp_qat_uof_batch_init *plm_init;
1294	u64 *micro_inst_arry;
1295	int micro_inst_num;
1296	int alloc_inst_size;
1297	int first_exec = 1;
1298	int stat = 0;
1299
1300	plm_init = lm_init_header->next;
1301	alloc_inst_size = lm_init_header->size;
1302	if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)
1303		alloc_inst_size = handle->hal_handle->max_ustore;
1304	micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(u64),
1305					GFP_KERNEL);
1306	if (!micro_inst_arry)
1307		return -ENOMEM;
1308	micro_inst_num = 0;
1309	while (plm_init) {
1310		unsigned int addr, *value, size;
1311
1312		ae = plm_init->ae;
1313		addr = plm_init->addr;
1314		value = plm_init->value;
1315		size = plm_init->size;
1316		micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry,
1317							    micro_inst_num,
1318							    size, addr, value);
1319		plm_init = plm_init->next;
1320	}
1321	/* exec micro codes */
1322	if (micro_inst_arry && micro_inst_num > 0) {
1323		micro_inst_arry[micro_inst_num++] = 0x0E000010000ull;
1324		stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec,
1325						  micro_inst_arry,
1326						  micro_inst_num);
1327	}
1328	kfree(micro_inst_arry);
1329	return stat;
1330}
1331
1332static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1333				   unsigned char ae, unsigned char ctx,
1334				   enum icp_qat_uof_regtype reg_type,
1335				   unsigned short reg_num, unsigned int val)
1336{
1337	int status = 0;
1338	unsigned int reg_addr;
1339	unsigned int ctx_enables;
1340	unsigned short mask;
1341	unsigned short dr_offset = 0x10;
1342
1343	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1344	if (CE_INUSE_CONTEXTS & ctx_enables) {
1345		if (ctx & 0x1) {
1346			pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);
1347			return -EINVAL;
1348		}
1349		mask = 0x1f;
1350		dr_offset = 0x20;
1351	} else {
1352		mask = 0x0f;
1353	}
1354	if (reg_num & ~mask)
1355		return -EINVAL;
1356	reg_addr = reg_num + (ctx << 0x5);
1357	switch (reg_type) {
1358	case ICP_SR_RD_REL:
1359	case ICP_SR_REL:
1360		SET_AE_XFER(handle, ae, reg_addr, val);
1361		break;
1362	case ICP_DR_RD_REL:
1363	case ICP_DR_REL:
1364		SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);
1365		break;
1366	default:
1367		status = -EINVAL;
1368		break;
1369	}
1370	return status;
1371}
1372
1373static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1374				   unsigned char ae, unsigned char ctx,
1375				   enum icp_qat_uof_regtype reg_type,
1376				   unsigned short reg_num, unsigned int data)
1377{
1378	unsigned int gprval, ctx_enables;
1379	unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi,
1380	    data16low;
1381	unsigned short reg_mask;
1382	int status = 0;
1383	u64 micro_inst[] = {
1384		0x0F440000000ull,
1385		0x0F040000000ull,
1386		0x0A000000000ull,
1387		0x0F0000C0300ull,
1388		0x0E000010000ull
1389	};
1390	const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;
1391	const unsigned short gprnum = 0, dly = num_inst * 0x5;
1392
1393	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1394	if (CE_INUSE_CONTEXTS & ctx_enables) {
1395		if (ctx & 0x1) {
1396			pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);
1397			return -EINVAL;
1398		}
1399		reg_mask = (unsigned short)~0x1f;
1400	} else {
1401		reg_mask = (unsigned short)~0xf;
1402	}
1403	if (reg_num & reg_mask)
1404		return -EINVAL;
1405	xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1406	if (xfr_addr == BAD_REGADDR) {
1407		pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);
1408		return -EINVAL;
1409	}
1410	status = qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);
1411	if (status) {
1412		pr_err("QAT: failed to read register");
1413		return status;
1414	}
1415	gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum);
1416	data16low = 0xffff & data;
1417	data16hi = 0xffff & (data >> 0x10);
1418	src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1419					  (unsigned short)(0xff & data16hi));
1420	src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1421					   (unsigned short)(0xff & data16low));
1422	micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) |
1423	    ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1424	micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) |
1425	    ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1426	micro_inst[0x2] = micro_inst[0x2] |
1427	    ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10);
1428	status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst,
1429					 code_off, dly, NULL);
1430	qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval);
1431	return status;
1432}
1433
1434static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,
1435			      unsigned char ae, unsigned char ctx,
1436			      unsigned short nn, unsigned int val)
1437{
1438	unsigned int ctx_enables;
1439	int stat = 0;
1440
1441	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1442	ctx_enables &= IGNORE_W1C_MASK;
1443	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);
1444
1445	stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val);
1446	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1447	return stat;
1448}
1449
1450static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle
1451				      *handle, unsigned char ae,
1452				      unsigned short absreg_num,
1453				      unsigned short *relreg,
1454				      unsigned char *ctx)
1455{
1456	unsigned int ctx_enables;
1457
1458	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1459	if (ctx_enables & CE_INUSE_CONTEXTS) {
1460		/* 4-ctx mode */
1461		*relreg = absreg_num & 0x1F;
1462		*ctx = (absreg_num >> 0x4) & 0x6;
1463	} else {
1464		/* 8-ctx mode */
1465		*relreg = absreg_num & 0x0F;
1466		*ctx = (absreg_num >> 0x4) & 0x7;
1467	}
1468	return 0;
1469}
1470
1471int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,
1472		     unsigned char ae, unsigned long ctx_mask,
1473		     enum icp_qat_uof_regtype reg_type,
1474		     unsigned short reg_num, unsigned int regdata)
1475{
1476	int stat = 0;
1477	unsigned short reg;
1478	unsigned char ctx = 0;
1479	enum icp_qat_uof_regtype type;
1480
1481	if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG)
1482		return -EINVAL;
1483
1484	do {
1485		if (ctx_mask == 0) {
1486			qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1487						   &ctx);
1488			type = reg_type - 1;
1489		} else {
1490			reg = reg_num;
1491			type = reg_type;
1492			if (!test_bit(ctx, &ctx_mask))
1493				continue;
1494		}
1495		stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata);
1496		if (stat) {
1497			pr_err("QAT: write gpr fail\n");
1498			return -EINVAL;
1499		}
1500	} while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1501
1502	return 0;
1503}
1504
1505int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1506			 unsigned char ae, unsigned long ctx_mask,
1507			 enum icp_qat_uof_regtype reg_type,
1508			 unsigned short reg_num, unsigned int regdata)
1509{
1510	int stat = 0;
1511	unsigned short reg;
1512	unsigned char ctx = 0;
1513	enum icp_qat_uof_regtype type;
1514
1515	if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1516		return -EINVAL;
1517
1518	do {
1519		if (ctx_mask == 0) {
1520			qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1521						   &ctx);
1522			type = reg_type - 3;
1523		} else {
1524			reg = reg_num;
1525			type = reg_type;
1526			if (!test_bit(ctx, &ctx_mask))
1527				continue;
1528		}
1529		stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg,
1530					       regdata);
1531		if (stat) {
1532			pr_err("QAT: write wr xfer fail\n");
1533			return -EINVAL;
1534		}
1535	} while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1536
1537	return 0;
1538}
1539
1540int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1541			 unsigned char ae, unsigned long ctx_mask,
1542			 enum icp_qat_uof_regtype reg_type,
1543			 unsigned short reg_num, unsigned int regdata)
1544{
1545	int stat = 0;
1546	unsigned short reg;
1547	unsigned char ctx = 0;
1548	enum icp_qat_uof_regtype type;
1549
1550	if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1551		return -EINVAL;
1552
1553	do {
1554		if (ctx_mask == 0) {
1555			qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1556						   &ctx);
1557			type = reg_type - 3;
1558		} else {
1559			reg = reg_num;
1560			type = reg_type;
1561			if (!test_bit(ctx, &ctx_mask))
1562				continue;
1563		}
1564		stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg,
1565					       regdata);
1566		if (stat) {
1567			pr_err("QAT: write rd xfer fail\n");
1568			return -EINVAL;
1569		}
1570	} while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1571
1572	return 0;
1573}
1574
1575int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,
1576		    unsigned char ae, unsigned long ctx_mask,
1577		    unsigned short reg_num, unsigned int regdata)
1578{
1579	int stat = 0;
1580	unsigned char ctx;
1581	if (!handle->chip_info->nn) {
1582		dev_err(&handle->pci_dev->dev, "QAT: No next neigh in 0x%x\n",
1583			handle->pci_dev->device);
1584		return -EINVAL;
1585	}
1586
1587	if (ctx_mask == 0)
1588		return -EINVAL;
1589
1590	for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
1591		if (!test_bit(ctx, &ctx_mask))
1592			continue;
1593		stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata);
1594		if (stat) {
1595			pr_err("QAT: write neigh error\n");
1596			return -EINVAL;
1597		}
1598	}
1599
1600	return 0;
1601}