qed_hw.c

  1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
  2/* QLogic qed NIC Driver
  3 * Copyright (c) 2015-2017  QLogic Corporation
  4 * Copyright (c) 2019-2020 Marvell International Ltd.
  5 */
  6
  7#include <linux/types.h>
  8#include <linux/io.h>
  9#include <linux/delay.h>
 10#include <linux/dma-mapping.h>
 11#include <linux/errno.h>
 12#include <linux/kernel.h>
 13#include <linux/list.h>
 14#include <linux/mutex.h>
 15#include <linux/pci.h>
 16#include <linux/slab.h>
 17#include <linux/spinlock.h>
 18#include <linux/string.h>
 19#include <linux/qed/qed_chain.h>
 20#include "qed.h"
 21#include "qed_hsi.h"
 22#include "qed_hw.h"
 23#include "qed_reg_addr.h"
 24#include "qed_sriov.h"
 25
 26#define QED_BAR_ACQUIRE_TIMEOUT_USLEEP_CNT	1000
 27#define QED_BAR_ACQUIRE_TIMEOUT_USLEEP		1000
 28#define QED_BAR_ACQUIRE_TIMEOUT_UDELAY_CNT	100000
 29#define QED_BAR_ACQUIRE_TIMEOUT_UDELAY		10
 30
 31/* Invalid values */
 32#define QED_BAR_INVALID_OFFSET          (cpu_to_le32(-1))
 33
 34struct qed_ptt {
 35	struct list_head	list_entry;
 36	unsigned int		idx;
 37	struct pxp_ptt_entry	pxp;
 38	u8			hwfn_id;
 39};
 40
 41struct qed_ptt_pool {
 42	struct list_head	free_list;
 43	spinlock_t		lock; /* ptt synchronized access */
 44	struct qed_ptt		ptts[PXP_EXTERNAL_BAR_PF_WINDOW_NUM];
 45};
 46
 47int qed_ptt_pool_alloc(struct qed_hwfn *p_hwfn)
 48{
 49	struct qed_ptt_pool *p_pool = kmalloc(sizeof(*p_pool), GFP_KERNEL);
 50	int i;
 51
 52	if (!p_pool)
 53		return -ENOMEM;
 54
 55	INIT_LIST_HEAD(&p_pool->free_list);
 56	for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) {
 57		p_pool->ptts[i].idx = i;
 58		p_pool->ptts[i].pxp.offset = QED_BAR_INVALID_OFFSET;
 59		p_pool->ptts[i].pxp.pretend.control = 0;
 60		p_pool->ptts[i].hwfn_id = p_hwfn->my_id;
 61		if (i >= RESERVED_PTT_MAX)
 62			list_add(&p_pool->ptts[i].list_entry,
 63				 &p_pool->free_list);
 64	}
 65
 66	p_hwfn->p_ptt_pool = p_pool;
 67	spin_lock_init(&p_pool->lock);
 68
 69	return 0;
 70}
 71
 72void qed_ptt_pool_free(struct qed_hwfn *p_hwfn)
 73{
 74	kfree(p_hwfn->p_ptt_pool);
 75	p_hwfn->p_ptt_pool = NULL;
 76}
 77
 78struct qed_ptt *qed_ptt_acquire(struct qed_hwfn *p_hwfn)
 79{
 80	return qed_ptt_acquire_context(p_hwfn, false);
 81}
 82
 83struct qed_ptt *qed_ptt_acquire_context(struct qed_hwfn *p_hwfn, bool is_atomic)
 84{
 85	struct qed_ptt *p_ptt;
 86	unsigned int i, count;
 87
 88	if (is_atomic)
 89		count = QED_BAR_ACQUIRE_TIMEOUT_UDELAY_CNT;
 90	else
 91		count = QED_BAR_ACQUIRE_TIMEOUT_USLEEP_CNT;
 92
 93	/* Take the free PTT from the list */
 94	for (i = 0; i < count; i++) {
 95		spin_lock_bh(&p_hwfn->p_ptt_pool->lock);
 96
 97		if (!list_empty(&p_hwfn->p_ptt_pool->free_list)) {
 98			p_ptt = list_first_entry(&p_hwfn->p_ptt_pool->free_list,
 99						 struct qed_ptt, list_entry);
100			list_del(&p_ptt->list_entry);
101
102			spin_unlock_bh(&p_hwfn->p_ptt_pool->lock);
103
104			DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
105				   "allocated ptt %d\n", p_ptt->idx);
106			return p_ptt;
107		}
108
109		spin_unlock_bh(&p_hwfn->p_ptt_pool->lock);
110
111		if (is_atomic)
112			udelay(QED_BAR_ACQUIRE_TIMEOUT_UDELAY);
113		else
114			usleep_range(QED_BAR_ACQUIRE_TIMEOUT_USLEEP,
115				     QED_BAR_ACQUIRE_TIMEOUT_USLEEP * 2);
116	}
117
118	DP_NOTICE(p_hwfn, "PTT acquire timeout - failed to allocate PTT\n");
119	return NULL;
120}
121
122void qed_ptt_release(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
123{
124	spin_lock_bh(&p_hwfn->p_ptt_pool->lock);
125	list_add(&p_ptt->list_entry, &p_hwfn->p_ptt_pool->free_list);
126	spin_unlock_bh(&p_hwfn->p_ptt_pool->lock);
127}
128
129u32 qed_ptt_get_hw_addr(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
130{
131	/* The HW is using DWORDS and we need to translate it to Bytes */
132	return le32_to_cpu(p_ptt->pxp.offset) << 2;
133}
134
135static u32 qed_ptt_config_addr(struct qed_ptt *p_ptt)
136{
137	return PXP_PF_WINDOW_ADMIN_PER_PF_START +
138	       p_ptt->idx * sizeof(struct pxp_ptt_entry);
139}
140
141u32 qed_ptt_get_bar_addr(struct qed_ptt *p_ptt)
142{
143	return PXP_EXTERNAL_BAR_PF_WINDOW_START +
144	       p_ptt->idx * PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE;
145}
146
147void qed_ptt_set_win(struct qed_hwfn *p_hwfn,
148		     struct qed_ptt *p_ptt, u32 new_hw_addr)
149{
150	u32 prev_hw_addr;
151
152	prev_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt);
153
154	if (new_hw_addr == prev_hw_addr)
155		return;
156
157	/* Update PTT entery in admin window */
158	DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
159		   "Updating PTT entry %d to offset 0x%x\n",
160		   p_ptt->idx, new_hw_addr);
161
162	/* The HW is using DWORDS and the address is in Bytes */
163	p_ptt->pxp.offset = cpu_to_le32(new_hw_addr >> 2);
164
165	REG_WR(p_hwfn,
166	       qed_ptt_config_addr(p_ptt) +
167	       offsetof(struct pxp_ptt_entry, offset),
168	       le32_to_cpu(p_ptt->pxp.offset));
169}
170
171static u32 qed_set_ptt(struct qed_hwfn *p_hwfn,
172		       struct qed_ptt *p_ptt, u32 hw_addr)
173{
174	u32 win_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt);
175	u32 offset;
176
177	offset = hw_addr - win_hw_addr;
178
179	if (p_ptt->hwfn_id != p_hwfn->my_id)
180		DP_NOTICE(p_hwfn,
181			  "ptt[%d] of hwfn[%02x] is used by hwfn[%02x]!\n",
182			  p_ptt->idx, p_ptt->hwfn_id, p_hwfn->my_id);
183
184	/* Verify the address is within the window */
185	if (hw_addr < win_hw_addr ||
186	    offset >= PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE) {
187		qed_ptt_set_win(p_hwfn, p_ptt, hw_addr);
188		offset = 0;
189	}
190
191	return qed_ptt_get_bar_addr(p_ptt) + offset;
192}
193
194struct qed_ptt *qed_get_reserved_ptt(struct qed_hwfn *p_hwfn,
195				     enum reserved_ptts ptt_idx)
196{
197	if (ptt_idx >= RESERVED_PTT_MAX) {
198		DP_NOTICE(p_hwfn,
199			  "Requested PTT %d is out of range\n", ptt_idx);
200		return NULL;
201	}
202
203	return &p_hwfn->p_ptt_pool->ptts[ptt_idx];
204}
205
206void qed_wr(struct qed_hwfn *p_hwfn,
207	    struct qed_ptt *p_ptt,
208	    u32 hw_addr, u32 val)
209{
210	u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr);
211
212	REG_WR(p_hwfn, bar_addr, val);
213	DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
214		   "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n",
215		   bar_addr, hw_addr, val);
216}
217
218u32 qed_rd(struct qed_hwfn *p_hwfn,
219	   struct qed_ptt *p_ptt,
220	   u32 hw_addr)
221{
222	u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr);
223	u32 val = REG_RD(p_hwfn, bar_addr);
224
225	DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
226		   "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n",
227		   bar_addr, hw_addr, val);
228
229	return val;
230}
231
232static void qed_memcpy_hw(struct qed_hwfn *p_hwfn,
233			  struct qed_ptt *p_ptt,
234			  void *addr, u32 hw_addr, size_t n, bool to_device)
235{
236	u32 dw_count, *host_addr, hw_offset;
237	size_t quota, done = 0;
238	u32 __iomem *reg_addr;
239
240	while (done < n) {
241		quota = min_t(size_t, n - done,
242			      PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE);
243
244		if (IS_PF(p_hwfn->cdev)) {
245			qed_ptt_set_win(p_hwfn, p_ptt, hw_addr + done);
246			hw_offset = qed_ptt_get_bar_addr(p_ptt);
247		} else {
248			hw_offset = hw_addr + done;
249		}
250
251		dw_count = quota / 4;
252		host_addr = (u32 *)((u8 *)addr + done);
253		reg_addr = (u32 __iomem *)REG_ADDR(p_hwfn, hw_offset);
254		if (to_device)
255			while (dw_count--)
256				DIRECT_REG_WR(reg_addr++, *host_addr++);
257		else
258			while (dw_count--)
259				*host_addr++ = DIRECT_REG_RD(reg_addr++);
260
261		done += quota;
262	}
263}
264
265void qed_memcpy_from(struct qed_hwfn *p_hwfn,
266		     struct qed_ptt *p_ptt, void *dest, u32 hw_addr, size_t n)
267{
268	DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
269		   "hw_addr 0x%x, dest %p hw_addr 0x%x, size %lu\n",
270		   hw_addr, dest, hw_addr, (unsigned long)n);
271
272	qed_memcpy_hw(p_hwfn, p_ptt, dest, hw_addr, n, false);
273}
274
275void qed_memcpy_to(struct qed_hwfn *p_hwfn,
276		   struct qed_ptt *p_ptt, u32 hw_addr, void *src, size_t n)
277{
278	DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
279		   "hw_addr 0x%x, hw_addr 0x%x, src %p size %lu\n",
280		   hw_addr, hw_addr, src, (unsigned long)n);
281
282	qed_memcpy_hw(p_hwfn, p_ptt, src, hw_addr, n, true);
283}
284
285void qed_fid_pretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 fid)
286{
287	u16 control = 0;
288
289	SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1);
290	SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1);
291
292	/* Every pretend undos previous pretends, including
293	 * previous port pretend.
294	 */
295	SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0);
296	SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0);
297	SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
298
299	if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID))
300		fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID);
301
302	p_ptt->pxp.pretend.control = cpu_to_le16(control);
303	p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid);
304
305	REG_WR(p_hwfn,
306	       qed_ptt_config_addr(p_ptt) +
307	       offsetof(struct pxp_ptt_entry, pretend),
308	       *(u32 *)&p_ptt->pxp.pretend);
309}
310
311void qed_port_pretend(struct qed_hwfn *p_hwfn,
312		      struct qed_ptt *p_ptt, u8 port_id)
313{
314	u16 control = 0;
315
316	SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id);
317	SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1);
318	SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
319
320	p_ptt->pxp.pretend.control = cpu_to_le16(control);
321
322	REG_WR(p_hwfn,
323	       qed_ptt_config_addr(p_ptt) +
324	       offsetof(struct pxp_ptt_entry, pretend),
325	       *(u32 *)&p_ptt->pxp.pretend);
326}
327
328void qed_port_unpretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
329{
330	u16 control = 0;
331
332	SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0);
333	SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0);
334	SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
335
336	p_ptt->pxp.pretend.control = cpu_to_le16(control);
337
338	REG_WR(p_hwfn,
339	       qed_ptt_config_addr(p_ptt) +
340	       offsetof(struct pxp_ptt_entry, pretend),
341	       *(u32 *)&p_ptt->pxp.pretend);
342}
343
344void qed_port_fid_pretend(struct qed_hwfn *p_hwfn,
345			  struct qed_ptt *p_ptt, u8 port_id, u16 fid)
346{
347	u16 control = 0;
348
349	SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id);
350	SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1);
351	SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
352	SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1);
353	SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1);
354	if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID))
355		fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID);
356	p_ptt->pxp.pretend.control = cpu_to_le16(control);
357	p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid);
358	REG_WR(p_hwfn,
359	       qed_ptt_config_addr(p_ptt) +
360	       offsetof(struct pxp_ptt_entry, pretend),
361	       *(u32 *)&p_ptt->pxp.pretend);
362}
363
364u32 qed_vfid_to_concrete(struct qed_hwfn *p_hwfn, u8 vfid)
365{
366	u32 concrete_fid = 0;
367
368	SET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID, p_hwfn->rel_pf_id);
369	SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID, vfid);
370	SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFVALID, 1);
371
372	return concrete_fid;
373}
374
375/* DMAE */
376#define QED_DMAE_FLAGS_IS_SET(params, flag) \
377	((params) != NULL && GET_FIELD((params)->flags, QED_DMAE_PARAMS_##flag))
378
379static void qed_dmae_opcode(struct qed_hwfn *p_hwfn,
380			    const u8 is_src_type_grc,
381			    const u8 is_dst_type_grc,
382			    struct qed_dmae_params *p_params)
383{
384	u8 src_pfid, dst_pfid, port_id;
385	u16 opcode_b = 0;
386	u32 opcode = 0;
387
388	/* Whether the source is the PCIe or the GRC.
389	 * 0- The source is the PCIe
390	 * 1- The source is the GRC.
391	 */
392	SET_FIELD(opcode, DMAE_CMD_SRC,
393		  (is_src_type_grc ? dmae_cmd_src_grc : dmae_cmd_src_pcie));
394	src_pfid = QED_DMAE_FLAGS_IS_SET(p_params, SRC_PF_VALID) ?
395	    p_params->src_pfid : p_hwfn->rel_pf_id;
396	SET_FIELD(opcode, DMAE_CMD_SRC_PF_ID, src_pfid);
397
398	/* The destination of the DMA can be: 0-None 1-PCIe 2-GRC 3-None */
399	SET_FIELD(opcode, DMAE_CMD_DST,
400		  (is_dst_type_grc ? dmae_cmd_dst_grc : dmae_cmd_dst_pcie));
401	dst_pfid = QED_DMAE_FLAGS_IS_SET(p_params, DST_PF_VALID) ?
402	    p_params->dst_pfid : p_hwfn->rel_pf_id;
403	SET_FIELD(opcode, DMAE_CMD_DST_PF_ID, dst_pfid);
404
405
406	/* Whether to write a completion word to the completion destination:
407	 * 0-Do not write a completion word
408	 * 1-Write the completion word
409	 */
410	SET_FIELD(opcode, DMAE_CMD_COMP_WORD_EN, 1);
411	SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1);
412
413	if (QED_DMAE_FLAGS_IS_SET(p_params, COMPLETION_DST))
414		SET_FIELD(opcode, DMAE_CMD_COMP_FUNC, 1);
415
416	/* swapping mode 3 - big endian */
417	SET_FIELD(opcode, DMAE_CMD_ENDIANITY_MODE, DMAE_CMD_ENDIANITY);
418
419	port_id = (QED_DMAE_FLAGS_IS_SET(p_params, PORT_VALID)) ?
420	    p_params->port_id : p_hwfn->port_id;
421	SET_FIELD(opcode, DMAE_CMD_PORT_ID, port_id);
422
423	/* reset source address in next go */
424	SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1);
425
426	/* reset dest address in next go */
427	SET_FIELD(opcode, DMAE_CMD_DST_ADDR_RESET, 1);
428
429	/* SRC/DST VFID: all 1's - pf, otherwise VF id */
430	if (QED_DMAE_FLAGS_IS_SET(p_params, SRC_VF_VALID)) {
431		SET_FIELD(opcode, DMAE_CMD_SRC_VF_ID_VALID, 1);
432		SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, p_params->src_vfid);
433	} else {
434		SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, 0xFF);
435	}
436	if (QED_DMAE_FLAGS_IS_SET(p_params, DST_VF_VALID)) {
437		SET_FIELD(opcode, DMAE_CMD_DST_VF_ID_VALID, 1);
438		SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, p_params->dst_vfid);
439	} else {
440		SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, 0xFF);
441	}
442
443	p_hwfn->dmae_info.p_dmae_cmd->opcode = cpu_to_le32(opcode);
444	p_hwfn->dmae_info.p_dmae_cmd->opcode_b = cpu_to_le16(opcode_b);
445}
446
447u32 qed_dmae_idx_to_go_cmd(u8 idx)
448{
449	/* All the DMAE 'go' registers form an array in internal memory */
450	return DMAE_REG_GO_C0 + (idx << 2);
451}
452
453static int qed_dmae_post_command(struct qed_hwfn *p_hwfn,
454				 struct qed_ptt *p_ptt)
455{
456	struct dmae_cmd *p_command = p_hwfn->dmae_info.p_dmae_cmd;
457	u8 idx_cmd = p_hwfn->dmae_info.channel, i;
458	int qed_status = 0;
459
460	/* verify address is not NULL */
461	if ((((!p_command->dst_addr_lo) && (!p_command->dst_addr_hi)) ||
462	     ((!p_command->src_addr_lo) && (!p_command->src_addr_hi)))) {
463		DP_NOTICE(p_hwfn,
464			  "source or destination address 0 idx_cmd=%d\n"
465			  "opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n",
466			  idx_cmd,
467			  le32_to_cpu(p_command->opcode),
468			  le16_to_cpu(p_command->opcode_b),
469			  le16_to_cpu(p_command->length_dw),
470			  le32_to_cpu(p_command->src_addr_hi),
471			  le32_to_cpu(p_command->src_addr_lo),
472			  le32_to_cpu(p_command->dst_addr_hi),
473			  le32_to_cpu(p_command->dst_addr_lo));
474
475		return -EINVAL;
476	}
477
478	DP_VERBOSE(p_hwfn,
479		   NETIF_MSG_HW,
480		   "Posting DMAE command [idx %d]: opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n",
481		   idx_cmd,
482		   le32_to_cpu(p_command->opcode),
483		   le16_to_cpu(p_command->opcode_b),
484		   le16_to_cpu(p_command->length_dw),
485		   le32_to_cpu(p_command->src_addr_hi),
486		   le32_to_cpu(p_command->src_addr_lo),
487		   le32_to_cpu(p_command->dst_addr_hi),
488		   le32_to_cpu(p_command->dst_addr_lo));
489
490	/* Copy the command to DMAE - need to do it before every call
491	 * for source/dest address no reset.
492	 * The first 9 DWs are the command registers, the 10 DW is the
493	 * GO register, and the rest are result registers
494	 * (which are read only by the client).
495	 */
496	for (i = 0; i < DMAE_CMD_SIZE; i++) {
497		u32 data = (i < DMAE_CMD_SIZE_TO_FILL) ?
498			   *(((u32 *)p_command) + i) : 0;
499
500		qed_wr(p_hwfn, p_ptt,
501		       DMAE_REG_CMD_MEM +
502		       (idx_cmd * DMAE_CMD_SIZE * sizeof(u32)) +
503		       (i * sizeof(u32)), data);
504	}
505
506	qed_wr(p_hwfn, p_ptt, qed_dmae_idx_to_go_cmd(idx_cmd), DMAE_GO_VALUE);
507
508	return qed_status;
509}
510
511int qed_dmae_info_alloc(struct qed_hwfn *p_hwfn)
512{
513	dma_addr_t *p_addr = &p_hwfn->dmae_info.completion_word_phys_addr;
514	struct dmae_cmd **p_cmd = &p_hwfn->dmae_info.p_dmae_cmd;
515	u32 **p_buff = &p_hwfn->dmae_info.p_intermediate_buffer;
516	u32 **p_comp = &p_hwfn->dmae_info.p_completion_word;
517
518	*p_comp = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
519				     sizeof(u32), p_addr, GFP_KERNEL);
520	if (!*p_comp)
521		goto err;
522
523	p_addr = &p_hwfn->dmae_info.dmae_cmd_phys_addr;
524	*p_cmd = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
525				    sizeof(struct dmae_cmd),
526				    p_addr, GFP_KERNEL);
527	if (!*p_cmd)
528		goto err;
529
530	p_addr = &p_hwfn->dmae_info.intermediate_buffer_phys_addr;
531	*p_buff = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
532				     sizeof(u32) * DMAE_MAX_RW_SIZE,
533				     p_addr, GFP_KERNEL);
534	if (!*p_buff)
535		goto err;
536
537	p_hwfn->dmae_info.channel = p_hwfn->rel_pf_id;
538
539	return 0;
540err:
541	qed_dmae_info_free(p_hwfn);
542	return -ENOMEM;
543}
544
545void qed_dmae_info_free(struct qed_hwfn *p_hwfn)
546{
547	dma_addr_t p_phys;
548
549	/* Just make sure no one is in the middle */
550	mutex_lock(&p_hwfn->dmae_info.mutex);
551
552	if (p_hwfn->dmae_info.p_completion_word) {
553		p_phys = p_hwfn->dmae_info.completion_word_phys_addr;
554		dma_free_coherent(&p_hwfn->cdev->pdev->dev,
555				  sizeof(u32),
556				  p_hwfn->dmae_info.p_completion_word, p_phys);
557		p_hwfn->dmae_info.p_completion_word = NULL;
558	}
559
560	if (p_hwfn->dmae_info.p_dmae_cmd) {
561		p_phys = p_hwfn->dmae_info.dmae_cmd_phys_addr;
562		dma_free_coherent(&p_hwfn->cdev->pdev->dev,
563				  sizeof(struct dmae_cmd),
564				  p_hwfn->dmae_info.p_dmae_cmd, p_phys);
565		p_hwfn->dmae_info.p_dmae_cmd = NULL;
566	}
567
568	if (p_hwfn->dmae_info.p_intermediate_buffer) {
569		p_phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr;
570		dma_free_coherent(&p_hwfn->cdev->pdev->dev,
571				  sizeof(u32) * DMAE_MAX_RW_SIZE,
572				  p_hwfn->dmae_info.p_intermediate_buffer,
573				  p_phys);
574		p_hwfn->dmae_info.p_intermediate_buffer = NULL;
575	}
576
577	mutex_unlock(&p_hwfn->dmae_info.mutex);
578}
579
580static int qed_dmae_operation_wait(struct qed_hwfn *p_hwfn)
581{
582	u32 wait_cnt_limit = 10000, wait_cnt = 0;
583	int qed_status = 0;
584
585	barrier();
586	while (*p_hwfn->dmae_info.p_completion_word != DMAE_COMPLETION_VAL) {
587		udelay(DMAE_MIN_WAIT_TIME);
588		cond_resched();
589		if (++wait_cnt > wait_cnt_limit) {
590			DP_NOTICE(p_hwfn->cdev,
591				  "Timed-out waiting for operation to complete. Completion word is 0x%08x expected 0x%08x.\n",
592				  *p_hwfn->dmae_info.p_completion_word,
593				 DMAE_COMPLETION_VAL);
594			qed_status = -EBUSY;
595			break;
596		}
597
598		/* to sync the completion_word since we are not
599		 * using the volatile keyword for p_completion_word
600		 */
601		barrier();
602	}
603
604	if (qed_status == 0)
605		*p_hwfn->dmae_info.p_completion_word = 0;
606
607	return qed_status;
608}
609
610static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn,
611					  struct qed_ptt *p_ptt,
612					  u64 src_addr,
613					  u64 dst_addr,
614					  u8 src_type,
615					  u8 dst_type,
616					  u32 length_dw)
617{
618	dma_addr_t phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr;
619	struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd;
620	int qed_status = 0;
621
622	switch (src_type) {
623	case QED_DMAE_ADDRESS_GRC:
624	case QED_DMAE_ADDRESS_HOST_PHYS:
625		cmd->src_addr_hi = cpu_to_le32(upper_32_bits(src_addr));
626		cmd->src_addr_lo = cpu_to_le32(lower_32_bits(src_addr));
627		break;
628	/* for virtual source addresses we use the intermediate buffer. */
629	case QED_DMAE_ADDRESS_HOST_VIRT:
630		cmd->src_addr_hi = cpu_to_le32(upper_32_bits(phys));
631		cmd->src_addr_lo = cpu_to_le32(lower_32_bits(phys));
632		memcpy(&p_hwfn->dmae_info.p_intermediate_buffer[0],
633		       (void *)(uintptr_t)src_addr,
634		       length_dw * sizeof(u32));
635		break;
636	default:
637		return -EINVAL;
638	}
639
640	switch (dst_type) {
641	case QED_DMAE_ADDRESS_GRC:
642	case QED_DMAE_ADDRESS_HOST_PHYS:
643		cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(dst_addr));
644		cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(dst_addr));
645		break;
646	/* for virtual source addresses we use the intermediate buffer. */
647	case QED_DMAE_ADDRESS_HOST_VIRT:
648		cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(phys));
649		cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(phys));
650		break;
651	default:
652		return -EINVAL;
653	}
654
655	cmd->length_dw = cpu_to_le16((u16)length_dw);
656
657	qed_dmae_post_command(p_hwfn, p_ptt);
658
659	qed_status = qed_dmae_operation_wait(p_hwfn);
660
661	if (qed_status) {
662		DP_NOTICE(p_hwfn,
663			  "qed_dmae_host2grc: Wait Failed. source_addr 0x%llx, grc_addr 0x%llx, size_in_dwords 0x%x\n",
664			  src_addr, dst_addr, length_dw);
665		return qed_status;
666	}
667
668	if (dst_type == QED_DMAE_ADDRESS_HOST_VIRT)
669		memcpy((void *)(uintptr_t)(dst_addr),
670		       &p_hwfn->dmae_info.p_intermediate_buffer[0],
671		       length_dw * sizeof(u32));
672
673	return 0;
674}
675
676static int qed_dmae_execute_command(struct qed_hwfn *p_hwfn,
677				    struct qed_ptt *p_ptt,
678				    u64 src_addr, u64 dst_addr,
679				    u8 src_type, u8 dst_type,
680				    u32 size_in_dwords,
681				    struct qed_dmae_params *p_params)
682{
683	dma_addr_t phys = p_hwfn->dmae_info.completion_word_phys_addr;
684	u16 length_cur = 0, i = 0, cnt_split = 0, length_mod = 0;
685	struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd;
686	u64 src_addr_split = 0, dst_addr_split = 0;
687	u16 length_limit = DMAE_MAX_RW_SIZE;
688	int qed_status = 0;
689	u32 offset = 0;
690
691	if (p_hwfn->cdev->recov_in_prog) {
692		DP_VERBOSE(p_hwfn,
693			   NETIF_MSG_HW,
694			   "Recovery is in progress. Avoid DMAE transaction [{src: addr 0x%llx, type %d}, {dst: addr 0x%llx, type %d}, size %d].\n",
695			   src_addr, src_type, dst_addr, dst_type,
696			   size_in_dwords);
697
698		/* Let the flow complete w/o any error handling */
699		return 0;
700	}
701
702	qed_dmae_opcode(p_hwfn,
703			(src_type == QED_DMAE_ADDRESS_GRC),
704			(dst_type == QED_DMAE_ADDRESS_GRC),
705			p_params);
706
707	cmd->comp_addr_lo = cpu_to_le32(lower_32_bits(phys));
708	cmd->comp_addr_hi = cpu_to_le32(upper_32_bits(phys));
709	cmd->comp_val = cpu_to_le32(DMAE_COMPLETION_VAL);
710
711	/* Check if the grc_addr is valid like < MAX_GRC_OFFSET */
712	cnt_split = size_in_dwords / length_limit;
713	length_mod = size_in_dwords % length_limit;
714
715	src_addr_split = src_addr;
716	dst_addr_split = dst_addr;
717
718	for (i = 0; i <= cnt_split; i++) {
719		offset = length_limit * i;
720
721		if (!QED_DMAE_FLAGS_IS_SET(p_params, RW_REPL_SRC)) {
722			if (src_type == QED_DMAE_ADDRESS_GRC)
723				src_addr_split = src_addr + offset;
724			else
725				src_addr_split = src_addr + (offset * 4);
726		}
727
728		if (dst_type == QED_DMAE_ADDRESS_GRC)
729			dst_addr_split = dst_addr + offset;
730		else
731			dst_addr_split = dst_addr + (offset * 4);
732
733		length_cur = (cnt_split == i) ? length_mod : length_limit;
734
735		/* might be zero on last iteration */
736		if (!length_cur)
737			continue;
738
739		qed_status = qed_dmae_execute_sub_operation(p_hwfn,
740							    p_ptt,
741							    src_addr_split,
742							    dst_addr_split,
743							    src_type,
744							    dst_type,
745							    length_cur);
746		if (qed_status) {
747			qed_hw_err_notify(p_hwfn, p_ptt, QED_HW_ERR_DMAE_FAIL,
748					  "qed_dmae_execute_sub_operation Failed with error 0x%x. source_addr 0x%llx, destination addr 0x%llx, size_in_dwords 0x%x\n",
749					  qed_status, src_addr,
750					  dst_addr, length_cur);
751			break;
752		}
753	}
754
755	return qed_status;
756}
757
758int qed_dmae_host2grc(struct qed_hwfn *p_hwfn,
759		      struct qed_ptt *p_ptt,
760		      u64 source_addr, u32 grc_addr, u32 size_in_dwords,
761		      struct qed_dmae_params *p_params)
762{
763	u32 grc_addr_in_dw = grc_addr / sizeof(u32);
764	int rc;
765
766
767	mutex_lock(&p_hwfn->dmae_info.mutex);
768
769	rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr,
770				      grc_addr_in_dw,
771				      QED_DMAE_ADDRESS_HOST_VIRT,
772				      QED_DMAE_ADDRESS_GRC,
773				      size_in_dwords, p_params);
774
775	mutex_unlock(&p_hwfn->dmae_info.mutex);
776
777	return rc;
778}
779
780int qed_dmae_grc2host(struct qed_hwfn *p_hwfn,
781		      struct qed_ptt *p_ptt,
782		      u32 grc_addr,
783		      dma_addr_t dest_addr, u32 size_in_dwords,
784		      struct qed_dmae_params *p_params)
785{
786	u32 grc_addr_in_dw = grc_addr / sizeof(u32);
787	int rc;
788
789
790	mutex_lock(&p_hwfn->dmae_info.mutex);
791
792	rc = qed_dmae_execute_command(p_hwfn, p_ptt, grc_addr_in_dw,
793				      dest_addr, QED_DMAE_ADDRESS_GRC,
794				      QED_DMAE_ADDRESS_HOST_VIRT,
795				      size_in_dwords, p_params);
796
797	mutex_unlock(&p_hwfn->dmae_info.mutex);
798
799	return rc;
800}
801
802int qed_dmae_host2host(struct qed_hwfn *p_hwfn,
803		       struct qed_ptt *p_ptt,
804		       dma_addr_t source_addr,
805		       dma_addr_t dest_addr,
806		       u32 size_in_dwords, struct qed_dmae_params *p_params)
807{
808	int rc;
809
810	mutex_lock(&(p_hwfn->dmae_info.mutex));
811
812	rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr,
813				      dest_addr,
814				      QED_DMAE_ADDRESS_HOST_PHYS,
815				      QED_DMAE_ADDRESS_HOST_PHYS,
816				      size_in_dwords, p_params);
817
818	mutex_unlock(&(p_hwfn->dmae_info.mutex));
819
820	return rc;
821}
822
823void qed_hw_err_notify(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
824		       enum qed_hw_err_type err_type, const char *fmt, ...)
825{
826	char buf[QED_HW_ERR_MAX_STR_SIZE];
827	va_list vl;
828	int len;
829
830	if (fmt) {
831		va_start(vl, fmt);
832		len = vsnprintf(buf, QED_HW_ERR_MAX_STR_SIZE, fmt, vl);
833		va_end(vl);
834
835		if (len > QED_HW_ERR_MAX_STR_SIZE - 1)
836			len = QED_HW_ERR_MAX_STR_SIZE - 1;
837
838		DP_NOTICE(p_hwfn, "%s", buf);
839	}
840
841	/* Fan failure cannot be masked by handling of another HW error */
842	if (p_hwfn->cdev->recov_in_prog &&
843	    err_type != QED_HW_ERR_FAN_FAIL) {
844		DP_VERBOSE(p_hwfn,
845			   NETIF_MSG_DRV,
846			   "Recovery is in progress. Avoid notifying about HW error %d.\n",
847			   err_type);
848		return;
849	}
850
851	qed_hw_error_occurred(p_hwfn, err_type);
852
853	if (fmt)
854		qed_mcp_send_raw_debug_data(p_hwfn, p_ptt, buf, len);
855}
856
857int qed_dmae_sanity(struct qed_hwfn *p_hwfn,
858		    struct qed_ptt *p_ptt, const char *phase)
859{
860	u32 size = PAGE_SIZE / 2, val;
861	int rc = 0;
862	dma_addr_t p_phys;
863	void *p_virt;
864	u32 *p_tmp;
865
866	p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
867				    2 * size, &p_phys, GFP_KERNEL);
868	if (!p_virt) {
869		DP_NOTICE(p_hwfn,
870			  "DMAE sanity [%s]: failed to allocate memory\n",
871			  phase);
872		return -ENOMEM;
873	}
874
875	/* Fill the bottom half of the allocated memory with a known pattern */
876	for (p_tmp = (u32 *)p_virt;
877	     p_tmp < (u32 *)((u8 *)p_virt + size); p_tmp++) {
878		/* Save the address itself as the value */
879		val = (u32)(uintptr_t)p_tmp;
880		*p_tmp = val;
881	}
882
883	/* Zero the top half of the allocated memory */
884	memset((u8 *)p_virt + size, 0, size);
885
886	DP_VERBOSE(p_hwfn,
887		   QED_MSG_SP,
888		   "DMAE sanity [%s]: src_addr={phys 0x%llx, virt %p}, dst_addr={phys 0x%llx, virt %p}, size 0x%x\n",
889		   phase,
890		   (u64)p_phys,
891		   p_virt, (u64)(p_phys + size), (u8 *)p_virt + size, size);
892
893	rc = qed_dmae_host2host(p_hwfn, p_ptt, p_phys, p_phys + size,
894				size / 4, NULL);
895	if (rc) {
896		DP_NOTICE(p_hwfn,
897			  "DMAE sanity [%s]: qed_dmae_host2host() failed. rc = %d.\n",
898			  phase, rc);
899		goto out;
900	}
901
902	/* Verify that the top half of the allocated memory has the pattern */
903	for (p_tmp = (u32 *)((u8 *)p_virt + size);
904	     p_tmp < (u32 *)((u8 *)p_virt + (2 * size)); p_tmp++) {
905		/* The corresponding address in the bottom half */
906		val = (u32)(uintptr_t)p_tmp - size;
907
908		if (*p_tmp != val) {
909			DP_NOTICE(p_hwfn,
910				  "DMAE sanity [%s]: addr={phys 0x%llx, virt %p}, read_val 0x%08x, expected_val 0x%08x\n",
911				  phase,
912				  (u64)p_phys + ((u8 *)p_tmp - (u8 *)p_virt),
913				  p_tmp, *p_tmp, val);
914			rc = -EINVAL;
915			goto out;
916		}
917	}
918
919out:
920	dma_free_coherent(&p_hwfn->cdev->pdev->dev, 2 * size, p_virt, p_phys);
921	return rc;
922}