adf_6xxx_hw_data.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright(c) 2025 Intel Corporation */
  3#include <linux/array_size.h>
  4#include <linux/bitfield.h>
  5#include <linux/bitops.h>
  6#include <linux/bits.h>
  7#include <linux/iopoll.h>
  8#include <linux/pci.h>
  9#include <linux/types.h>
 10
 11#include <adf_accel_devices.h>
 12#include <adf_admin.h>
 13#include <adf_bank_state.h>
 14#include <adf_cfg.h>
 15#include <adf_cfg_services.h>
 16#include <adf_clock.h>
 17#include <adf_common_drv.h>
 18#include <adf_fw_config.h>
 19#include <adf_gen6_pm.h>
 20#include <adf_gen6_ras.h>
 21#include <adf_gen6_shared.h>
 22#include <adf_gen6_tl.h>
 23#include <adf_timer.h>
 24#include "adf_6xxx_hw_data.h"
 25#include "icp_qat_fw_comp.h"
 26#include "icp_qat_hw_51_comp.h"
 27
 28#define RP_GROUP_0_MASK		(BIT(0) | BIT(2))
 29#define RP_GROUP_1_MASK		(BIT(1) | BIT(3))
 30#define RP_GROUP_ALL_MASK	(RP_GROUP_0_MASK | RP_GROUP_1_MASK)
 31
 32#define ADF_AE_GROUP_0		GENMASK(3, 0)
 33#define ADF_AE_GROUP_1		GENMASK(7, 4)
 34#define ADF_AE_GROUP_2		BIT(8)
 35
 36struct adf_ring_config {
 37	u32 ring_mask;
 38	enum adf_cfg_service_type ring_type;
 39	const unsigned long *thrd_mask;
 40};
 41
 42static u32 rmask_two_services[] = {
 43	RP_GROUP_0_MASK,
 44	RP_GROUP_1_MASK,
 45};
 46
 47enum adf_gen6_rps {
 48	RP0 = 0,
 49	RP1 = 1,
 50	RP2 = 2,
 51	RP3 = 3,
 52	RP_MAX = RP3
 53};
 54
 55/*
 56 * thrd_mask_[sym|asym|cpr|dcc]: these static arrays define the thread
 57 * configuration for handling requests of specific services across the
 58 * accelerator engines. Each element in an array corresponds to an
 59 * accelerator engine, with the value being a bitmask that specifies which
 60 * threads within that engine are capable of processing the particular service.
 61 *
 62 * For example, a value of 0x0C means that threads 2 and 3 are enabled for the
 63 * service in the respective accelerator engine.
 64 */
 65static const unsigned long thrd_mask_sym[ADF_6XXX_MAX_ACCELENGINES] = {
 66	0x0C, 0x0C, 0x0C, 0x0C, 0x1C, 0x1C, 0x1C, 0x1C, 0x00
 67};
 68
 69static const unsigned long thrd_mask_asym[ADF_6XXX_MAX_ACCELENGINES] = {
 70	0x70, 0x70, 0x70, 0x70, 0x60, 0x60, 0x60, 0x60, 0x00
 71};
 72
 73static const unsigned long thrd_mask_cpr[ADF_6XXX_MAX_ACCELENGINES] = {
 74	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00
 75};
 76
 77static const unsigned long thrd_mask_dcc[ADF_6XXX_MAX_ACCELENGINES] = {
 78	0x00, 0x00, 0x00, 0x00, 0x07, 0x07, 0x03, 0x03, 0x00
 79};
 80
 81static const unsigned long thrd_mask_dcpr[ADF_6XXX_MAX_ACCELENGINES] = {
 82	0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x00
 83};
 84
 85static const char *const adf_6xxx_fw_objs[] = {
 86	[ADF_FW_CY_OBJ] = ADF_6XXX_CY_OBJ,
 87	[ADF_FW_DC_OBJ] = ADF_6XXX_DC_OBJ,
 88	[ADF_FW_ADMIN_OBJ] = ADF_6XXX_ADMIN_OBJ,
 89};
 90
 91static const struct adf_fw_config adf_default_fw_config[] = {
 92	{ ADF_AE_GROUP_1, ADF_FW_DC_OBJ },
 93	{ ADF_AE_GROUP_0, ADF_FW_CY_OBJ },
 94	{ ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ },
 95};
 96
 97static struct adf_hw_device_class adf_6xxx_class = {
 98	.name = ADF_6XXX_DEVICE_NAME,
 99	.type = DEV_6XXX,
100};
101
102static bool services_supported(unsigned long mask)
103{
104	int num_svc;
105
106	if (mask >= BIT(SVC_COUNT))
107		return false;
108
109	num_svc = hweight_long(mask);
110	switch (num_svc) {
111	case ADF_ONE_SERVICE:
112		return true;
113	case ADF_TWO_SERVICES:
114	case ADF_THREE_SERVICES:
115		return !test_bit(SVC_DCC, &mask);
116	default:
117		return false;
118	}
119}
120
121static int get_service(unsigned long *mask)
122{
123	if (test_and_clear_bit(SVC_ASYM, mask))
124		return SVC_ASYM;
125
126	if (test_and_clear_bit(SVC_SYM, mask))
127		return SVC_SYM;
128
129	if (test_and_clear_bit(SVC_DC, mask))
130		return SVC_DC;
131
132	if (test_and_clear_bit(SVC_DCC, mask))
133		return SVC_DCC;
134
135	if (test_and_clear_bit(SVC_DECOMP, mask))
136		return SVC_DECOMP;
137
138	return -EINVAL;
139}
140
141static enum adf_cfg_service_type get_ring_type(unsigned int service)
142{
143	switch (service) {
144	case SVC_SYM:
145		return SYM;
146	case SVC_ASYM:
147		return ASYM;
148	case SVC_DC:
149	case SVC_DCC:
150		return COMP;
151	case SVC_DECOMP:
152		return DECOMP;
153	default:
154		return UNUSED;
155	}
156}
157
158static const unsigned long *get_thrd_mask(unsigned int service)
159{
160	switch (service) {
161	case SVC_SYM:
162		return thrd_mask_sym;
163	case SVC_ASYM:
164		return thrd_mask_asym;
165	case SVC_DC:
166		return thrd_mask_cpr;
167	case SVC_DCC:
168		return thrd_mask_dcc;
169	case SVC_DECOMP:
170		return thrd_mask_dcpr;
171	default:
172		return NULL;
173	}
174}
175
176static int get_rp_config(struct adf_accel_dev *accel_dev, struct adf_ring_config *rp_config,
177			 unsigned int *num_services)
178{
179	unsigned int i, nservices;
180	unsigned long mask;
181	int ret, service;
182
183	ret = adf_get_service_mask(accel_dev, &mask);
184	if (ret)
185		return ret;
186
187	nservices = hweight_long(mask);
188	if (nservices > MAX_NUM_CONCURR_SVC)
189		return -EINVAL;
190
191	for (i = 0; i < nservices; i++) {
192		service = get_service(&mask);
193		if (service < 0)
194			return service;
195
196		rp_config[i].ring_type = get_ring_type(service);
197		rp_config[i].thrd_mask = get_thrd_mask(service);
198
199		/*
200		 * If there is only one service enabled, use all ring pairs for
201		 * that service.
202		 * If there are two services enabled, use ring pairs 0 and 2 for
203		 * one service and ring pairs 1 and 3 for the other service.
204		 */
205		switch (nservices) {
206		case ADF_ONE_SERVICE:
207			rp_config[i].ring_mask = RP_GROUP_ALL_MASK;
208			break;
209		case ADF_TWO_SERVICES:
210			rp_config[i].ring_mask = rmask_two_services[i];
211			break;
212		case ADF_THREE_SERVICES:
213			rp_config[i].ring_mask = BIT(i);
214
215			/* If ASYM is enabled, use additional ring pair */
216			if (service == SVC_ASYM)
217				rp_config[i].ring_mask |= BIT(RP3);
218
219			break;
220		default:
221			return -EINVAL;
222		}
223	}
224
225	*num_services = nservices;
226
227	return 0;
228}
229
230static u32 adf_gen6_get_arb_mask(struct adf_accel_dev *accel_dev, unsigned int ae)
231{
232	struct adf_ring_config rp_config[MAX_NUM_CONCURR_SVC];
233	unsigned int num_services, i, thrd;
234	u32 ring_mask, thd2arb_mask = 0;
235	const unsigned long *p_mask;
236
237	if (get_rp_config(accel_dev, rp_config, &num_services))
238		return 0;
239
240	/*
241	 * The thd2arb_mask maps ring pairs to threads within an accelerator engine.
242	 * It ensures that jobs submitted to ring pairs are scheduled on threads capable
243	 * of handling the specified service type.
244	 *
245	 * Each group of 4 bits in the mask corresponds to a thread, with each bit
246	 * indicating whether a job from a ring pair can be scheduled on that thread.
247	 * The use of 4 bits is due to the organization of ring pairs into groups of
248	 * four, where each group shares the same configuration.
249	 */
250	for (i = 0; i < num_services; i++) {
251		p_mask = &rp_config[i].thrd_mask[ae];
252		ring_mask = rp_config[i].ring_mask;
253
254		for_each_set_bit(thrd, p_mask, ADF_NUM_THREADS_PER_AE)
255			thd2arb_mask |= ring_mask << (thrd * 4);
256	}
257
258	return thd2arb_mask;
259}
260
261static u16 get_ring_to_svc_map(struct adf_accel_dev *accel_dev)
262{
263	enum adf_cfg_service_type rps[ADF_GEN6_NUM_BANKS_PER_VF] = { };
264	struct adf_ring_config rp_config[MAX_NUM_CONCURR_SVC];
265	unsigned int num_services, rp_num, i;
266	unsigned long cfg_mask;
267	u16 ring_to_svc_map;
268
269	if (get_rp_config(accel_dev, rp_config, &num_services))
270		return 0;
271
272	/*
273	 * Loop through the configured services and populate the `rps` array that
274	 * contains what service that particular ring pair can handle (i.e. symmetric
275	 * crypto, asymmetric crypto, data compression or compression chaining).
276	 */
277	for (i = 0; i < num_services; i++) {
278		cfg_mask = rp_config[i].ring_mask;
279		for_each_set_bit(rp_num, &cfg_mask, ADF_GEN6_NUM_BANKS_PER_VF)
280			rps[rp_num] = rp_config[i].ring_type;
281	}
282
283	/*
284	 * The ring_mask is structured into segments of 3 bits, with each
285	 * segment representing the service configuration for a specific ring pair.
286	 * Since ring pairs are organized into groups of 4, the ring_mask contains 4
287	 * such 3-bit segments, each corresponding to one ring pair.
288	 *
289	 * The device has 64 ring pairs, which are organized in groups of 4, namely
290	 * 16 groups. Each group has the same configuration, represented here by
291	 * `ring_to_svc_map`.
292	 */
293	ring_to_svc_map = rps[RP0] << ADF_CFG_SERV_RING_PAIR_0_SHIFT |
294			  rps[RP1] << ADF_CFG_SERV_RING_PAIR_1_SHIFT |
295			  rps[RP2] << ADF_CFG_SERV_RING_PAIR_2_SHIFT |
296			  rps[RP3] << ADF_CFG_SERV_RING_PAIR_3_SHIFT;
297
298	return ring_to_svc_map;
299}
300
301static u32 get_accel_mask(struct adf_hw_device_data *self)
302{
303	return ADF_GEN6_ACCELERATORS_MASK;
304}
305
306static u32 get_num_accels(struct adf_hw_device_data *self)
307{
308	return ADF_GEN6_MAX_ACCELERATORS;
309}
310
311static u32 get_num_aes(struct adf_hw_device_data *self)
312{
313	return self ? hweight32(self->ae_mask) : 0;
314}
315
316static u32 get_misc_bar_id(struct adf_hw_device_data *self)
317{
318	return ADF_GEN6_PMISC_BAR;
319}
320
321static u32 get_etr_bar_id(struct adf_hw_device_data *self)
322{
323	return ADF_GEN6_ETR_BAR;
324}
325
326static u32 get_sram_bar_id(struct adf_hw_device_data *self)
327{
328	return ADF_GEN6_SRAM_BAR;
329}
330
331static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
332{
333	return DEV_SKU_1;
334}
335
336static void get_arb_info(struct arb_info *arb_info)
337{
338	arb_info->arb_cfg = ADF_GEN6_ARB_CONFIG;
339	arb_info->arb_offset = ADF_GEN6_ARB_OFFSET;
340	arb_info->wt2sam_offset = ADF_GEN6_ARB_WRK_2_SER_MAP_OFFSET;
341}
342
343static void get_admin_info(struct admin_info *admin_csrs_info)
344{
345	admin_csrs_info->mailbox_offset = ADF_GEN6_MAILBOX_BASE_OFFSET;
346	admin_csrs_info->admin_msg_ur = ADF_GEN6_ADMINMSGUR_OFFSET;
347	admin_csrs_info->admin_msg_lr = ADF_GEN6_ADMINMSGLR_OFFSET;
348}
349
350static u32 get_heartbeat_clock(struct adf_hw_device_data *self)
351{
352	return ADF_GEN6_COUNTER_FREQ;
353}
354
355static void enable_error_correction(struct adf_accel_dev *accel_dev)
356{
357	void __iomem *csr = adf_get_pmisc_base(accel_dev);
358
359	/*
360	 * Enable all error notification bits in errsou3 except VFLR
361	 * notification on host.
362	 */
363	ADF_CSR_WR(csr, ADF_GEN6_ERRMSK3, ADF_GEN6_VFLNOTIFY);
364}
365
366static void enable_ints(struct adf_accel_dev *accel_dev)
367{
368	void __iomem *addr = adf_get_pmisc_base(accel_dev);
369
370	/* Enable bundle interrupts */
371	ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_RP_X0_MASK_OFFSET, 0);
372	ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_RP_X1_MASK_OFFSET, 0);
373
374	/* Enable misc interrupts */
375	ADF_CSR_WR(addr, ADF_GEN6_SMIAPF_MASK_OFFSET, 0);
376}
377
378static void set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
379{
380	void __iomem *addr = adf_get_pmisc_base(accel_dev);
381	u64 val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE;
382	u64 val = ADF_SSM_WDT_DEFAULT_VALUE;
383
384	/* Enable watchdog timer for sym and dc */
385	ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTATHL_OFFSET, ADF_SSMWDTATHH_OFFSET, val);
386	ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTCNVL_OFFSET, ADF_SSMWDTCNVH_OFFSET, val);
387	ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTUCSL_OFFSET, ADF_SSMWDTUCSH_OFFSET, val);
388	ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTDCPRL_OFFSET, ADF_SSMWDTDCPRH_OFFSET, val);
389
390	/* Enable watchdog timer for pke */
391	ADF_CSR_WR64_LO_HI(addr, ADF_SSMWDTPKEL_OFFSET, ADF_SSMWDTPKEH_OFFSET, val_pke);
392}
393
394/*
395 * The vector routing table is used to select the MSI-X entry to use for each
396 * interrupt source.
397 * The first ADF_GEN6_ETR_MAX_BANKS entries correspond to ring interrupts.
398 * The final entry corresponds to VF2PF or error interrupts.
399 * This vector table could be used to configure one MSI-X entry to be shared
400 * between multiple interrupt sources.
401 *
402 * The default routing is set to have a one to one correspondence between the
403 * interrupt source and the MSI-X entry used.
404 */
405static void set_msix_default_rttable(struct adf_accel_dev *accel_dev)
406{
407	void __iomem *csr = adf_get_pmisc_base(accel_dev);
408	unsigned int i;
409
410	for (i = 0; i <= ADF_GEN6_ETR_MAX_BANKS; i++)
411		ADF_CSR_WR(csr, ADF_GEN6_MSIX_RTTABLE_OFFSET(i), i);
412}
413
414static int reset_ring_pair(void __iomem *csr, u32 bank_number)
415{
416	u32 status;
417	int ret;
418
419	/*
420	 * Write rpresetctl register BIT(0) as 1.
421	 * Since rpresetctl registers have no RW fields, no need to preserve
422	 * values for other bits. Just write directly.
423	 */
424	ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number),
425		   ADF_WQM_CSR_RPRESETCTL_RESET);
426
427	/* Read rpresetsts register and wait for rp reset to complete */
428	ret = read_poll_timeout(ADF_CSR_RD, status,
429				status & ADF_WQM_CSR_RPRESETSTS_STATUS,
430				ADF_RPRESET_POLL_DELAY_US,
431				ADF_RPRESET_POLL_TIMEOUT_US, true,
432				csr, ADF_WQM_CSR_RPRESETSTS(bank_number));
433	if (ret)
434		return ret;
435
436	/* When ring pair reset is done, clear rpresetsts */
437	ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number), ADF_WQM_CSR_RPRESETSTS_STATUS);
438
439	return 0;
440}
441
442static int ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number)
443{
444	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
445	void __iomem *csr = adf_get_etr_base(accel_dev);
446	int ret;
447
448	if (bank_number >= hw_data->num_banks)
449		return -EINVAL;
450
451	dev_dbg(&GET_DEV(accel_dev), "ring pair reset for bank:%d\n", bank_number);
452
453	ret = reset_ring_pair(csr, bank_number);
454	if (ret)
455		dev_err(&GET_DEV(accel_dev), "ring pair reset failed (timeout)\n");
456	else
457		dev_dbg(&GET_DEV(accel_dev), "ring pair reset successful\n");
458
459	return ret;
460}
461
462static int build_comp_block(void *ctx, enum adf_dc_algo algo)
463{
464	struct icp_qat_fw_comp_req *req_tmpl = ctx;
465	struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
466	struct icp_qat_hw_comp_51_config_csr_lower hw_comp_lower_csr = { };
467	struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
468	u32 lower_val;
469
470	switch (algo) {
471	case QAT_DEFLATE:
472		header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DYNAMIC;
473	break;
474	default:
475		return -EINVAL;
476	}
477
478	hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_51_LLLBD_CTRL_LLLBD_DISABLED;
479	hw_comp_lower_csr.sd = ICP_QAT_HW_COMP_51_SEARCH_DEPTH_LEVEL_1;
480	lower_val = ICP_QAT_FW_COMP_51_BUILD_CONFIG_LOWER(hw_comp_lower_csr);
481	cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val;
482	cd_pars->u.sl.comp_slice_cfg_word[1] = 0;
483
484	return 0;
485}
486
487static int build_decomp_block(void *ctx, enum adf_dc_algo algo)
488{
489	struct icp_qat_fw_comp_req *req_tmpl = ctx;
490	struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
491	struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
492
493	switch (algo) {
494	case QAT_DEFLATE:
495		header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS;
496	break;
497	default:
498		return -EINVAL;
499	}
500
501	cd_pars->u.sl.comp_slice_cfg_word[0] = 0;
502	cd_pars->u.sl.comp_slice_cfg_word[1] = 0;
503
504	return 0;
505}
506
507static void adf_gen6_init_dc_ops(struct adf_dc_ops *dc_ops)
508{
509	dc_ops->build_comp_block = build_comp_block;
510	dc_ops->build_decomp_block = build_decomp_block;
511}
512
513static int adf_gen6_init_thd2arb_map(struct adf_accel_dev *accel_dev)
514{
515	struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
516	u32 *thd2arb_map = hw_data->thd_to_arb_map;
517	unsigned int i;
518
519	for (i = 0; i < hw_data->num_engines; i++) {
520		thd2arb_map[i] = adf_gen6_get_arb_mask(accel_dev, i);
521		dev_dbg(&GET_DEV(accel_dev), "ME:%d arb_mask:%#x\n", i, thd2arb_map[i]);
522	}
523
524	return 0;
525}
526
527static void init_num_svc_aes(struct adf_rl_hw_data *device_data)
528{
529	enum adf_fw_objs obj_type, obj_iter;
530	unsigned int svc, i, num_grp;
531	u32 ae_mask;
532
533	for (svc = 0; svc < SVC_BASE_COUNT; svc++) {
534		switch (svc) {
535		case SVC_SYM:
536		case SVC_ASYM:
537			obj_type = ADF_FW_CY_OBJ;
538			break;
539		case SVC_DC:
540		case SVC_DECOMP:
541			obj_type = ADF_FW_DC_OBJ;
542			break;
543		}
544
545		num_grp = ARRAY_SIZE(adf_default_fw_config);
546		for (i = 0; i < num_grp; i++) {
547			obj_iter = adf_default_fw_config[i].obj;
548			if (obj_iter == obj_type) {
549				ae_mask = adf_default_fw_config[i].ae_mask;
550				device_data->svc_ae_mask[svc] = hweight32(ae_mask);
551				break;
552			}
553		}
554	}
555}
556
557static u32 adf_gen6_get_svc_slice_cnt(struct adf_accel_dev *accel_dev,
558				      enum adf_base_services svc)
559{
560	struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data;
561
562	switch (svc) {
563	case SVC_SYM:
564		return device_data->slices.cph_cnt;
565	case SVC_ASYM:
566		return device_data->slices.pke_cnt;
567	case SVC_DC:
568		return device_data->slices.cpr_cnt + device_data->slices.dcpr_cnt;
569	case SVC_DECOMP:
570		return device_data->slices.dcpr_cnt;
571	default:
572		return 0;
573	}
574}
575
576static void set_vc_csr_for_bank(void __iomem *csr, u32 bank_number)
577{
578	u32 value;
579
580	/*
581	 * After each PF FLR, for each of the 64 ring pairs in the PF, the
582	 * driver must program the ringmodectl CSRs.
583	 */
584	value = ADF_CSR_RD(csr, ADF_GEN6_CSR_RINGMODECTL(bank_number));
585	FIELD_MODIFY(ADF_GEN6_RINGMODECTL_TC_MASK, &value, ADF_GEN6_RINGMODECTL_TC_DEFAULT);
586	FIELD_MODIFY(ADF_GEN6_RINGMODECTL_TC_EN_MASK, &value, ADF_GEN6_RINGMODECTL_TC_EN_OP1);
587	ADF_CSR_WR(csr, ADF_GEN6_CSR_RINGMODECTL(bank_number), value);
588}
589
590static int set_vc_config(struct adf_accel_dev *accel_dev)
591{
592	struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
593	u32 value;
594	int err;
595
596	/*
597	 * After each PF FLR, the driver must program the Port Virtual Channel (VC)
598	 * Control Registers.
599	 * Read PVC0CTL then write the masked values.
600	 */
601	pci_read_config_dword(pdev, ADF_GEN6_PVC0CTL_OFFSET, &value);
602	FIELD_MODIFY(ADF_GEN6_PVC0CTL_TCVCMAP_MASK, &value, ADF_GEN6_PVC0CTL_TCVCMAP_DEFAULT);
603	err = pci_write_config_dword(pdev, ADF_GEN6_PVC0CTL_OFFSET, value);
604	if (err) {
605		dev_err(&GET_DEV(accel_dev), "pci write to PVC0CTL failed\n");
606		return pcibios_err_to_errno(err);
607	}
608
609	/* Read PVC1CTL then write masked values */
610	pci_read_config_dword(pdev, ADF_GEN6_PVC1CTL_OFFSET, &value);
611	FIELD_MODIFY(ADF_GEN6_PVC1CTL_TCVCMAP_MASK, &value, ADF_GEN6_PVC1CTL_TCVCMAP_DEFAULT);
612	FIELD_MODIFY(ADF_GEN6_PVC1CTL_VCEN_MASK, &value, ADF_GEN6_PVC1CTL_VCEN_ON);
613	err = pci_write_config_dword(pdev, ADF_GEN6_PVC1CTL_OFFSET, value);
614	if (err)
615		dev_err(&GET_DEV(accel_dev), "pci write to PVC1CTL failed\n");
616
617	return pcibios_err_to_errno(err);
618}
619
620static int adf_gen6_set_vc(struct adf_accel_dev *accel_dev)
621{
622	struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
623	void __iomem *csr = adf_get_etr_base(accel_dev);
624	u32 i;
625
626	for (i = 0; i < hw_data->num_banks; i++) {
627		dev_dbg(&GET_DEV(accel_dev), "set virtual channels for bank:%d\n", i);
628		set_vc_csr_for_bank(csr, i);
629	}
630
631	return set_vc_config(accel_dev);
632}
633
634static u32 get_ae_mask(struct adf_hw_device_data *self)
635{
636	unsigned long fuses = self->fuses[ADF_FUSECTL4];
637	u32 mask = ADF_6XXX_ACCELENGINES_MASK;
638
639	/*
640	 * If bit 0 is set in the fuses, the first 4 engines are disabled.
641	 * If bit 4 is set, the second group of 4 engines are disabled.
642	 * If bit 8 is set, the admin engine (bit 8) is disabled.
643	 */
644	if (test_bit(0, &fuses))
645		mask &= ~ADF_AE_GROUP_0;
646
647	if (test_bit(4, &fuses))
648		mask &= ~ADF_AE_GROUP_1;
649
650	if (test_bit(8, &fuses))
651		mask &= ~ADF_AE_GROUP_2;
652
653	return mask;
654}
655
656static u32 get_accel_cap(struct adf_accel_dev *accel_dev)
657{
658	u32 capabilities_sym, capabilities_asym;
659	u32 capabilities_dc;
660	unsigned long mask;
661	u32 caps = 0;
662	u32 fusectl1;
663
664	fusectl1 = GET_HW_DATA(accel_dev)->fuses[ADF_FUSECTL1];
665
666	/* Read accelerator capabilities mask */
667	capabilities_sym = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
668			  ICP_ACCEL_CAPABILITIES_CIPHER |
669			  ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
670			  ICP_ACCEL_CAPABILITIES_SHA3 |
671			  ICP_ACCEL_CAPABILITIES_SHA3_EXT |
672			  ICP_ACCEL_CAPABILITIES_CHACHA_POLY |
673			  ICP_ACCEL_CAPABILITIES_AESGCM_SPC |
674			  ICP_ACCEL_CAPABILITIES_AES_V2;
675
676	/* A set bit in fusectl1 means the corresponding feature is OFF in this SKU */
677	if (fusectl1 & ICP_ACCEL_GEN6_MASK_UCS_SLICE) {
678		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
679		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
680		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY;
681		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC;
682		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AES_V2;
683	}
684	if (fusectl1 & ICP_ACCEL_GEN6_MASK_AUTH_SLICE) {
685		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
686		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3;
687		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT;
688		capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
689	}
690
691	capabilities_asym = ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
692			    ICP_ACCEL_CAPABILITIES_SM2 |
693			    ICP_ACCEL_CAPABILITIES_ECEDMONT;
694
695	if (fusectl1 & ICP_ACCEL_GEN6_MASK_PKE_SLICE) {
696		capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
697		capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_SM2;
698		capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT;
699	}
700
701	capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION |
702			  ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION |
703			  ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION |
704			  ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
705
706	if (fusectl1 & ICP_ACCEL_GEN6_MASK_CPR_SLICE) {
707		capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
708		capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION;
709		capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION;
710		capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
711	}
712
713	if (adf_get_service_mask(accel_dev, &mask))
714		return 0;
715
716	if (test_bit(SVC_ASYM, &mask))
717		caps |= capabilities_asym;
718	if (test_bit(SVC_SYM, &mask))
719		caps |= capabilities_sym;
720	if (test_bit(SVC_DC, &mask) || test_bit(SVC_DECOMP, &mask))
721		caps |= capabilities_dc;
722	if (test_bit(SVC_DCC, &mask)) {
723		/*
724		 * Sym capabilities are available for chaining operations,
725		 * but sym crypto instances cannot be supported
726		 */
727		caps = capabilities_dc | capabilities_sym;
728		caps &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
729	}
730
731	return caps;
732}
733
734static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev)
735{
736	return ARRAY_SIZE(adf_default_fw_config);
737}
738
739static const char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num)
740{
741	int num_fw_objs = ARRAY_SIZE(adf_6xxx_fw_objs);
742	int id;
743
744	id = adf_default_fw_config[obj_num].obj;
745	if (id >= num_fw_objs)
746		return NULL;
747
748	return adf_6xxx_fw_objs[id];
749}
750
751static const char *uof_get_name_6xxx(struct adf_accel_dev *accel_dev, u32 obj_num)
752{
753	return uof_get_name(accel_dev, obj_num);
754}
755
756static int uof_get_obj_type(struct adf_accel_dev *accel_dev, u32 obj_num)
757{
758	if (obj_num >= uof_get_num_objs(accel_dev))
759		return -EINVAL;
760
761	return adf_default_fw_config[obj_num].obj;
762}
763
764static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num)
765{
766	return adf_default_fw_config[obj_num].ae_mask;
767}
768
769static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev)
770{
771	if (adf_gen6_init_thd2arb_map(accel_dev))
772		dev_warn(&GET_DEV(accel_dev),
773			 "Failed to generate thread to arbiter mapping");
774
775	return GET_HW_DATA(accel_dev)->thd_to_arb_map;
776}
777
778static int adf_init_device(struct adf_accel_dev *accel_dev)
779{
780	void __iomem *addr = adf_get_pmisc_base(accel_dev);
781	u32 status;
782	u32 csr;
783	int ret;
784
785	/* Temporarily mask PM interrupt */
786	csr = ADF_CSR_RD(addr, ADF_GEN6_ERRMSK2);
787	csr |= ADF_GEN6_PM_SOU;
788	ADF_CSR_WR(addr, ADF_GEN6_ERRMSK2, csr);
789
790	/* Set DRV_ACTIVE bit to power up the device */
791	ADF_CSR_WR(addr, ADF_GEN6_PM_INTERRUPT, ADF_GEN6_PM_DRV_ACTIVE);
792
793	/* Poll status register to make sure the device is powered up */
794	ret = read_poll_timeout(ADF_CSR_RD, status,
795				status & ADF_GEN6_PM_INIT_STATE,
796				ADF_GEN6_PM_POLL_DELAY_US,
797				ADF_GEN6_PM_POLL_TIMEOUT_US, true, addr,
798				ADF_GEN6_PM_STATUS);
799	if (ret) {
800		dev_err(&GET_DEV(accel_dev), "Failed to power up the device\n");
801		return ret;
802	}
803
804	dev_dbg(&GET_DEV(accel_dev), "Setting virtual channels for device qat_dev%d\n",
805		accel_dev->accel_id);
806
807	ret = adf_gen6_set_vc(accel_dev);
808	if (ret)
809		dev_err(&GET_DEV(accel_dev), "Failed to set virtual channels\n");
810
811	return ret;
812}
813
814static int enable_pm(struct adf_accel_dev *accel_dev)
815{
816	int ret;
817
818	ret = adf_init_admin_pm(accel_dev, ADF_GEN6_PM_DEFAULT_IDLE_FILTER);
819	if (ret)
820		return ret;
821
822	/* Initialize PM internal data */
823	adf_gen6_init_dev_pm_data(accel_dev);
824
825	return 0;
826}
827
828static int dev_config(struct adf_accel_dev *accel_dev)
829{
830	int ret;
831
832	ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC);
833	if (ret)
834		return ret;
835
836	ret = adf_cfg_section_add(accel_dev, "Accelerator0");
837	if (ret)
838		return ret;
839
840	switch (adf_get_service_enabled(accel_dev)) {
841	case SVC_DC:
842	case SVC_DCC:
843		ret = adf_gen6_comp_dev_config(accel_dev);
844		break;
845	default:
846		ret = adf_gen6_no_dev_config(accel_dev);
847		break;
848	}
849	if (ret)
850		return ret;
851
852	__set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
853
854	return ret;
855}
856
857static void adf_gen6_init_rl_data(struct adf_rl_hw_data *rl_data)
858{
859	rl_data->pciout_tb_offset = ADF_GEN6_RL_TOKEN_PCIEOUT_BUCKET_OFFSET;
860	rl_data->pciin_tb_offset = ADF_GEN6_RL_TOKEN_PCIEIN_BUCKET_OFFSET;
861	rl_data->r2l_offset = ADF_GEN6_RL_R2L_OFFSET;
862	rl_data->l2c_offset = ADF_GEN6_RL_L2C_OFFSET;
863	rl_data->c2s_offset = ADF_GEN6_RL_C2S_OFFSET;
864	rl_data->pcie_scale_div = ADF_6XXX_RL_PCIE_SCALE_FACTOR_DIV;
865	rl_data->pcie_scale_mul = ADF_6XXX_RL_PCIE_SCALE_FACTOR_MUL;
866	rl_data->max_tp[SVC_ASYM] = ADF_6XXX_RL_MAX_TP_ASYM;
867	rl_data->max_tp[SVC_SYM] = ADF_6XXX_RL_MAX_TP_SYM;
868	rl_data->max_tp[SVC_DC] = ADF_6XXX_RL_MAX_TP_DC;
869	rl_data->max_tp[SVC_DECOMP] = ADF_6XXX_RL_MAX_TP_DECOMP;
870	rl_data->scan_interval = ADF_6XXX_RL_SCANS_PER_SEC;
871	rl_data->scale_ref = ADF_6XXX_RL_SLICE_REF;
872
873	init_num_svc_aes(rl_data);
874}
875
876void adf_init_hw_data_6xxx(struct adf_hw_device_data *hw_data)
877{
878	hw_data->dev_class = &adf_6xxx_class;
879	hw_data->instance_id = adf_6xxx_class.instances++;
880	hw_data->num_banks = ADF_GEN6_ETR_MAX_BANKS;
881	hw_data->num_banks_per_vf = ADF_GEN6_NUM_BANKS_PER_VF;
882	hw_data->num_rings_per_bank = ADF_GEN6_NUM_RINGS_PER_BANK;
883	hw_data->num_accel = ADF_GEN6_MAX_ACCELERATORS;
884	hw_data->num_engines = ADF_6XXX_MAX_ACCELENGINES;
885	hw_data->num_logical_accel = 1;
886	hw_data->tx_rx_gap = ADF_GEN6_RX_RINGS_OFFSET;
887	hw_data->tx_rings_mask = ADF_GEN6_TX_RINGS_MASK;
888	hw_data->ring_to_svc_map = 0;
889	hw_data->alloc_irq = adf_isr_resource_alloc;
890	hw_data->free_irq = adf_isr_resource_free;
891	hw_data->enable_error_correction = enable_error_correction;
892	hw_data->get_accel_mask = get_accel_mask;
893	hw_data->get_ae_mask = get_ae_mask;
894	hw_data->get_num_accels = get_num_accels;
895	hw_data->get_num_aes = get_num_aes;
896	hw_data->get_sram_bar_id = get_sram_bar_id;
897	hw_data->get_etr_bar_id = get_etr_bar_id;
898	hw_data->get_misc_bar_id = get_misc_bar_id;
899	hw_data->get_arb_info = get_arb_info;
900	hw_data->get_admin_info = get_admin_info;
901	hw_data->get_accel_cap = get_accel_cap;
902	hw_data->get_sku = get_sku;
903	hw_data->init_admin_comms = adf_init_admin_comms;
904	hw_data->exit_admin_comms = adf_exit_admin_comms;
905	hw_data->send_admin_init = adf_send_admin_init;
906	hw_data->init_arb = adf_init_arb;
907	hw_data->exit_arb = adf_exit_arb;
908	hw_data->get_arb_mapping = adf_get_arbiter_mapping;
909	hw_data->enable_ints = enable_ints;
910	hw_data->reset_device = adf_reset_flr;
911	hw_data->admin_ae_mask = ADF_6XXX_ADMIN_AE_MASK;
912	hw_data->fw_name = ADF_6XXX_FW;
913	hw_data->fw_mmp_name = ADF_6XXX_MMP;
914	hw_data->uof_get_name = uof_get_name_6xxx;
915	hw_data->uof_get_num_objs = uof_get_num_objs;
916	hw_data->uof_get_obj_type = uof_get_obj_type;
917	hw_data->uof_get_ae_mask = uof_get_ae_mask;
918	hw_data->set_msix_rttable = set_msix_default_rttable;
919	hw_data->set_ssm_wdtimer = set_ssm_wdtimer;
920	hw_data->get_ring_to_svc_map = get_ring_to_svc_map;
921	hw_data->disable_iov = adf_disable_sriov;
922	hw_data->ring_pair_reset = ring_pair_reset;
923	hw_data->dev_config = dev_config;
924	hw_data->bank_state_save = adf_bank_state_save;
925	hw_data->bank_state_restore = adf_bank_state_restore;
926	hw_data->get_hb_clock = get_heartbeat_clock;
927	hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE;
928	hw_data->start_timer = adf_timer_start;
929	hw_data->stop_timer = adf_timer_stop;
930	hw_data->init_device = adf_init_device;
931	hw_data->enable_pm = enable_pm;
932	hw_data->services_supported = services_supported;
933	hw_data->num_rps = ADF_GEN6_ETR_MAX_BANKS;
934	hw_data->clock_frequency = ADF_6XXX_AE_FREQ;
935	hw_data->get_svc_slice_cnt = adf_gen6_get_svc_slice_cnt;
936
937	adf_gen6_init_hw_csr_ops(&hw_data->csr_ops);
938	adf_gen6_init_pf_pfvf_ops(&hw_data->pfvf_ops);
939	adf_gen6_init_dc_ops(&hw_data->dc_ops);
940	adf_gen6_init_vf_mig_ops(&hw_data->vfmig_ops);
941	adf_gen6_init_ras_ops(&hw_data->ras_ops);
942	adf_gen6_init_tl_data(&hw_data->tl_data);
943	adf_gen6_init_rl_data(&hw_data->rl_data);
944}
945
946void adf_clean_hw_data_6xxx(struct adf_hw_device_data *hw_data)
947{
948	if (hw_data->dev_class->instances)
949		hw_data->dev_class->instances--;
950}