bnge_hwrm.c

  1// SPDX-License-Identifier: GPL-2.0
  2// Copyright (c) 2025 Broadcom.
  3
  4#include <asm/byteorder.h>
  5#include <linux/dma-mapping.h>
  6#include <linux/dmapool.h>
  7#include <linux/delay.h>
  8#include <linux/errno.h>
  9#include <linux/kernel.h>
 10#include <linux/list.h>
 11#include <linux/pci.h>
 12
 13#include "bnge.h"
 14#include "bnge_hwrm.h"
 15
 16static u64 bnge_cal_sentinel(struct bnge_hwrm_ctx *ctx, u16 req_type)
 17{
 18	return (((uintptr_t)ctx) + req_type) ^ BNGE_HWRM_SENTINEL;
 19}
 20
 21int bnge_hwrm_req_create(struct bnge_dev *bd, void **req, u16 req_type,
 22			 u32 req_len)
 23{
 24	struct bnge_hwrm_ctx *ctx;
 25	dma_addr_t dma_handle;
 26	u8 *req_addr;
 27
 28	if (req_len > BNGE_HWRM_CTX_OFFSET)
 29		return -E2BIG;
 30
 31	req_addr = dma_pool_alloc(bd->hwrm_dma_pool, GFP_KERNEL | __GFP_ZERO,
 32				  &dma_handle);
 33	if (!req_addr)
 34		return -ENOMEM;
 35
 36	ctx = (struct bnge_hwrm_ctx *)(req_addr + BNGE_HWRM_CTX_OFFSET);
 37	/* safety first, sentinel used to check for invalid requests */
 38	ctx->sentinel = bnge_cal_sentinel(ctx, req_type);
 39	ctx->req_len = req_len;
 40	ctx->req = (struct input *)req_addr;
 41	ctx->resp = (struct output *)(req_addr + BNGE_HWRM_RESP_OFFSET);
 42	ctx->dma_handle = dma_handle;
 43	ctx->flags = 0; /* __GFP_ZERO, but be explicit regarding ownership */
 44	ctx->timeout = bd->hwrm_cmd_timeout ?: BNGE_DFLT_HWRM_CMD_TIMEOUT;
 45	ctx->allocated = BNGE_HWRM_DMA_SIZE - BNGE_HWRM_CTX_OFFSET;
 46	ctx->gfp = GFP_KERNEL;
 47	ctx->slice_addr = NULL;
 48
 49	/* initialize common request fields */
 50	ctx->req->req_type = cpu_to_le16(req_type);
 51	ctx->req->resp_addr = cpu_to_le64(dma_handle + BNGE_HWRM_RESP_OFFSET);
 52	ctx->req->cmpl_ring = cpu_to_le16(BNGE_HWRM_NO_CMPL_RING);
 53	ctx->req->target_id = cpu_to_le16(BNGE_HWRM_TARGET);
 54	*req = ctx->req;
 55
 56	return 0;
 57}
 58
 59static struct bnge_hwrm_ctx *__hwrm_ctx_get(struct bnge_dev *bd, u8 *req_addr)
 60{
 61	void *ctx_addr = req_addr + BNGE_HWRM_CTX_OFFSET;
 62	struct input *req = (struct input *)req_addr;
 63	struct bnge_hwrm_ctx *ctx = ctx_addr;
 64	u64 sentinel;
 65
 66	if (!req) {
 67		dev_err(bd->dev, "null HWRM request");
 68		dump_stack();
 69		return NULL;
 70	}
 71
 72	/* HWRM API has no type safety, verify sentinel to validate address */
 73	sentinel = bnge_cal_sentinel(ctx, le16_to_cpu(req->req_type));
 74	if (ctx->sentinel != sentinel) {
 75		dev_err(bd->dev, "HWRM sentinel mismatch, req_type = %u\n",
 76			(u32)le16_to_cpu(req->req_type));
 77		dump_stack();
 78		return NULL;
 79	}
 80
 81	return ctx;
 82}
 83
 84void bnge_hwrm_req_timeout(struct bnge_dev *bd,
 85			   void *req, unsigned int timeout)
 86{
 87	struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req);
 88
 89	if (ctx)
 90		ctx->timeout = timeout;
 91}
 92
 93void bnge_hwrm_req_alloc_flags(struct bnge_dev *bd, void *req, gfp_t gfp)
 94{
 95	struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req);
 96
 97	if (ctx)
 98		ctx->gfp = gfp;
 99}
100
101int bnge_hwrm_req_replace(struct bnge_dev *bd, void *req, void *new_req,
102			  u32 len)
103{
104	struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req);
105	struct input *internal_req = req;
106	u16 req_type;
107
108	if (!ctx)
109		return -EINVAL;
110
111	if (len > BNGE_HWRM_CTX_OFFSET)
112		return -E2BIG;
113
114	/* free any existing slices */
115	ctx->allocated = BNGE_HWRM_DMA_SIZE - BNGE_HWRM_CTX_OFFSET;
116	if (ctx->slice_addr) {
117		dma_free_coherent(bd->dev, ctx->slice_size,
118				  ctx->slice_addr, ctx->slice_handle);
119		ctx->slice_addr = NULL;
120	}
121	ctx->gfp = GFP_KERNEL;
122
123	if ((bd->fw_cap & BNGE_FW_CAP_SHORT_CMD) || len > BNGE_HWRM_MAX_REQ_LEN) {
124		memcpy(internal_req, new_req, len);
125	} else {
126		internal_req->req_type = ((struct input *)new_req)->req_type;
127		ctx->req = new_req;
128	}
129
130	ctx->req_len = len;
131	ctx->req->resp_addr = cpu_to_le64(ctx->dma_handle +
132					  BNGE_HWRM_RESP_OFFSET);
133
134	/* update sentinel for potentially new request type */
135	req_type = le16_to_cpu(internal_req->req_type);
136	ctx->sentinel = bnge_cal_sentinel(ctx, req_type);
137
138	return 0;
139}
140
141void bnge_hwrm_req_flags(struct bnge_dev *bd, void *req,
142			 enum bnge_hwrm_ctx_flags flags)
143{
144	struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req);
145
146	if (ctx)
147		ctx->flags |= (flags & BNGE_HWRM_API_FLAGS);
148}
149
150void *bnge_hwrm_req_hold(struct bnge_dev *bd, void *req)
151{
152	struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req);
153	struct input *input = (struct input *)req;
154
155	if (!ctx)
156		return NULL;
157
158	if (ctx->flags & BNGE_HWRM_INTERNAL_CTX_OWNED) {
159		dev_err(bd->dev, "HWRM context already owned, req_type = %u\n",
160			(u32)le16_to_cpu(input->req_type));
161		dump_stack();
162		return NULL;
163	}
164
165	ctx->flags |= BNGE_HWRM_INTERNAL_CTX_OWNED;
166	return ((u8 *)req) + BNGE_HWRM_RESP_OFFSET;
167}
168
169static void __hwrm_ctx_invalidate(struct bnge_dev *bd,
170				  struct bnge_hwrm_ctx *ctx)
171{
172	void *addr = ((u8 *)ctx) - BNGE_HWRM_CTX_OFFSET;
173	dma_addr_t dma_handle = ctx->dma_handle; /* save before invalidate */
174
175	/* unmap any auxiliary DMA slice */
176	if (ctx->slice_addr)
177		dma_free_coherent(bd->dev, ctx->slice_size,
178				  ctx->slice_addr, ctx->slice_handle);
179
180	/* invalidate, ensure ownership, sentinel and dma_handle are cleared */
181	memset(ctx, 0, sizeof(struct bnge_hwrm_ctx));
182
183	/* return the buffer to the DMA pool */
184	if (dma_handle)
185		dma_pool_free(bd->hwrm_dma_pool, addr, dma_handle);
186}
187
188void bnge_hwrm_req_drop(struct bnge_dev *bd, void *req)
189{
190	struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req);
191
192	if (ctx)
193		__hwrm_ctx_invalidate(bd, ctx);
194}
195
196static int bnge_map_hwrm_error(u32 hwrm_err)
197{
198	switch (hwrm_err) {
199	case HWRM_ERR_CODE_SUCCESS:
200		return 0;
201	case HWRM_ERR_CODE_RESOURCE_LOCKED:
202		return -EROFS;
203	case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED:
204		return -EACCES;
205	case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR:
206		return -ENOSPC;
207	case HWRM_ERR_CODE_INVALID_PARAMS:
208	case HWRM_ERR_CODE_INVALID_FLAGS:
209	case HWRM_ERR_CODE_INVALID_ENABLES:
210	case HWRM_ERR_CODE_UNSUPPORTED_TLV:
211	case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR:
212		return -EINVAL;
213	case HWRM_ERR_CODE_NO_BUFFER:
214		return -ENOMEM;
215	case HWRM_ERR_CODE_HOT_RESET_PROGRESS:
216	case HWRM_ERR_CODE_BUSY:
217		return -EAGAIN;
218	case HWRM_ERR_CODE_CMD_NOT_SUPPORTED:
219		return -EOPNOTSUPP;
220	case HWRM_ERR_CODE_PF_UNAVAILABLE:
221		return -ENODEV;
222	default:
223		return -EIO;
224	}
225}
226
227static struct bnge_hwrm_wait_token *
228bnge_hwrm_create_token(struct bnge_dev *bd, enum bnge_hwrm_chnl dst)
229{
230	struct bnge_hwrm_wait_token *token;
231
232	token = kzalloc(sizeof(*token), GFP_KERNEL);
233	if (!token)
234		return NULL;
235
236	mutex_lock(&bd->hwrm_cmd_lock);
237
238	token->dst = dst;
239	token->state = BNGE_HWRM_PENDING;
240	if (dst == BNGE_HWRM_CHNL_CHIMP) {
241		token->seq_id = bd->hwrm_cmd_seq++;
242		hlist_add_head_rcu(&token->node, &bd->hwrm_pending_list);
243	} else {
244		token->seq_id = bd->hwrm_cmd_kong_seq++;
245	}
246
247	return token;
248}
249
250static void
251bnge_hwrm_destroy_token(struct bnge_dev *bd, struct bnge_hwrm_wait_token *token)
252{
253	if (token->dst == BNGE_HWRM_CHNL_CHIMP) {
254		hlist_del_rcu(&token->node);
255		kfree_rcu(token, rcu);
256	} else {
257		kfree(token);
258	}
259	mutex_unlock(&bd->hwrm_cmd_lock);
260}
261
262static void bnge_hwrm_req_dbg(struct bnge_dev *bd, struct input *req)
263{
264	u32 ring = le16_to_cpu(req->cmpl_ring);
265	u32 type = le16_to_cpu(req->req_type);
266	u32 tgt = le16_to_cpu(req->target_id);
267	u32 seq = le16_to_cpu(req->seq_id);
268	char opt[32] = "\n";
269
270	if (unlikely(ring != (u16)BNGE_HWRM_NO_CMPL_RING))
271		snprintf(opt, 16, " ring %d\n", ring);
272
273	if (unlikely(tgt != BNGE_HWRM_TARGET))
274		snprintf(opt + strlen(opt) - 1, 16, " tgt 0x%x\n", tgt);
275
276	dev_dbg(bd->dev, "sent hwrm req_type 0x%x seq id 0x%x%s",
277		type, seq, opt);
278}
279
280#define bnge_hwrm_err(bd, ctx, fmt, ...)		\
281	do {							       \
282		if ((ctx)->flags & BNGE_HWRM_CTX_SILENT)	       \
283			dev_dbg((bd)->dev, fmt, __VA_ARGS__);       \
284		else						       \
285			dev_err((bd)->dev, fmt, __VA_ARGS__);       \
286	} while (0)
287
288static int __hwrm_send_ctx(struct bnge_dev *bd, struct bnge_hwrm_ctx *ctx)
289{
290	u32 doorbell_offset = BNGE_GRCPF_REG_CHIMP_COMM_TRIGGER;
291	enum bnge_hwrm_chnl dst = BNGE_HWRM_CHNL_CHIMP;
292	u32 bar_offset = BNGE_GRCPF_REG_CHIMP_COMM;
293	struct bnge_hwrm_wait_token *token = NULL;
294	u16 max_req_len = BNGE_HWRM_MAX_REQ_LEN;
295	unsigned int i, timeout, tmo_count;
296	u32 *data = (u32 *)ctx->req;
297	u32 msg_len = ctx->req_len;
298	int rc = -EBUSY;
299	u32 req_type;
300	u16 len = 0;
301	u8 *valid;
302
303	if (ctx->flags & BNGE_HWRM_INTERNAL_RESP_DIRTY)
304		memset(ctx->resp, 0, PAGE_SIZE);
305
306	req_type = le16_to_cpu(ctx->req->req_type);
307
308	if (msg_len > BNGE_HWRM_MAX_REQ_LEN &&
309	    msg_len > bd->hwrm_max_ext_req_len) {
310		dev_warn(bd->dev, "oversized hwrm request, req_type 0x%x",
311			 req_type);
312		rc = -E2BIG;
313		goto exit;
314	}
315
316	token = bnge_hwrm_create_token(bd, dst);
317	if (!token) {
318		rc = -ENOMEM;
319		goto exit;
320	}
321	ctx->req->seq_id = cpu_to_le16(token->seq_id);
322
323	/* Ensure any associated DMA buffers are written before doorbell */
324	wmb();
325
326	/* Write request msg to hwrm channel */
327	__iowrite32_copy(bd->bar0 + bar_offset, data, msg_len / 4);
328
329	for (i = msg_len; i < max_req_len; i += 4)
330		writel(0, bd->bar0 + bar_offset + i);
331
332	/* Ring channel doorbell */
333	writel(1, bd->bar0 + doorbell_offset);
334
335	bnge_hwrm_req_dbg(bd, ctx->req);
336
337	/* Limit timeout to an upper limit */
338	timeout = min(ctx->timeout,
339		      bd->hwrm_cmd_max_timeout ?: BNGE_HWRM_CMD_MAX_TIMEOUT);
340	/* convert timeout to usec */
341	timeout *= 1000;
342
343	i = 0;
344	/* Short timeout for the first few iterations:
345	 * number of loops = number of loops for short timeout +
346	 * number of loops for standard timeout.
347	 */
348	tmo_count = BNGE_HWRM_SHORT_TIMEOUT_COUNTER;
349	timeout = timeout - BNGE_HWRM_SHORT_MIN_TIMEOUT *
350			BNGE_HWRM_SHORT_TIMEOUT_COUNTER;
351	tmo_count += DIV_ROUND_UP(timeout, BNGE_HWRM_MIN_TIMEOUT);
352
353	if (le16_to_cpu(ctx->req->cmpl_ring) != INVALID_HW_RING_ID) {
354		/* Wait until hwrm response cmpl interrupt is processed */
355		while (READ_ONCE(token->state) < BNGE_HWRM_COMPLETE &&
356		       i++ < tmo_count) {
357			/* on first few passes, just barely sleep */
358			if (i < BNGE_HWRM_SHORT_TIMEOUT_COUNTER) {
359				usleep_range(BNGE_HWRM_SHORT_MIN_TIMEOUT,
360					     BNGE_HWRM_SHORT_MAX_TIMEOUT);
361			} else {
362				usleep_range(BNGE_HWRM_MIN_TIMEOUT,
363					     BNGE_HWRM_MAX_TIMEOUT);
364			}
365		}
366
367		if (READ_ONCE(token->state) != BNGE_HWRM_COMPLETE) {
368			bnge_hwrm_err(bd, ctx, "No hwrm cmpl received: 0x%x\n",
369				      req_type);
370			goto exit;
371		}
372		len = le16_to_cpu(READ_ONCE(ctx->resp->resp_len));
373		valid = ((u8 *)ctx->resp) + len - 1;
374	} else {
375		__le16 seen_out_of_seq = ctx->req->seq_id; /* will never see */
376		int j;
377
378		/* Check if response len is updated */
379		for (i = 0; i < tmo_count; i++) {
380			if (token &&
381			    READ_ONCE(token->state) == BNGE_HWRM_DEFERRED) {
382				bnge_hwrm_destroy_token(bd, token);
383				token = NULL;
384			}
385
386			len = le16_to_cpu(READ_ONCE(ctx->resp->resp_len));
387			if (len) {
388				__le16 resp_seq = READ_ONCE(ctx->resp->seq_id);
389
390				if (resp_seq == ctx->req->seq_id)
391					break;
392				if (resp_seq != seen_out_of_seq) {
393					dev_warn(bd->dev, "Discarding out of seq response: 0x%x for msg {0x%x 0x%x}\n",
394						 le16_to_cpu(resp_seq), req_type, le16_to_cpu(ctx->req->seq_id));
395					seen_out_of_seq = resp_seq;
396				}
397			}
398
399			/* on first few passes, just barely sleep */
400			if (i < BNGE_HWRM_SHORT_TIMEOUT_COUNTER) {
401				usleep_range(BNGE_HWRM_SHORT_MIN_TIMEOUT,
402					     BNGE_HWRM_SHORT_MAX_TIMEOUT);
403			} else {
404				usleep_range(BNGE_HWRM_MIN_TIMEOUT,
405					     BNGE_HWRM_MAX_TIMEOUT);
406			}
407		}
408
409		if (i >= tmo_count) {
410			bnge_hwrm_err(bd, ctx,
411				      "Error (timeout: %u) msg {0x%x 0x%x} len:%d\n",
412				      bnge_hwrm_timeout(i), req_type,
413				      le16_to_cpu(ctx->req->seq_id), len);
414			goto exit;
415		}
416
417		/* Last byte of resp contains valid bit */
418		valid = ((u8 *)ctx->resp) + len - 1;
419		for (j = 0; j < BNGE_HWRM_FIN_WAIT_USEC; ) {
420			/* make sure we read from updated DMA memory */
421			dma_rmb();
422			if (*valid)
423				break;
424			if (j < 10) {
425				udelay(1);
426				j++;
427			} else {
428				usleep_range(20, 30);
429				j += 20;
430			}
431		}
432
433		if (j >= BNGE_HWRM_FIN_WAIT_USEC) {
434			bnge_hwrm_err(bd, ctx, "Error (timeout: %u) msg {0x%x 0x%x} len:%d v:%d\n",
435				      bnge_hwrm_timeout(i) + j, req_type,
436				      le16_to_cpu(ctx->req->seq_id), len, *valid);
437			goto exit;
438		}
439	}
440
441	/* Zero valid bit for compatibility.  Valid bit in an older spec
442	 * may become a new field in a newer spec.  We must make sure that
443	 * a new field not implemented by old spec will read zero.
444	 */
445	*valid = 0;
446	rc = le16_to_cpu(ctx->resp->error_code);
447	if (rc == HWRM_ERR_CODE_BUSY && !(ctx->flags & BNGE_HWRM_CTX_SILENT))
448		dev_warn(bd->dev, "FW returned busy, hwrm req_type 0x%x\n",
449			 req_type);
450	else if (rc && rc != HWRM_ERR_CODE_PF_UNAVAILABLE)
451		bnge_hwrm_err(bd, ctx, "hwrm req_type 0x%x seq id 0x%x error %d\n",
452			      req_type, le16_to_cpu(ctx->req->seq_id), rc);
453	rc = bnge_map_hwrm_error(rc);
454
455exit:
456	if (token)
457		bnge_hwrm_destroy_token(bd, token);
458	if (ctx->flags & BNGE_HWRM_INTERNAL_CTX_OWNED)
459		ctx->flags |= BNGE_HWRM_INTERNAL_RESP_DIRTY;
460	else
461		__hwrm_ctx_invalidate(bd, ctx);
462	return rc;
463}
464
465int bnge_hwrm_req_send(struct bnge_dev *bd, void *req)
466{
467	struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req);
468
469	if (!ctx)
470		return -EINVAL;
471
472	return __hwrm_send_ctx(bd, ctx);
473}
474
475int bnge_hwrm_req_send_silent(struct bnge_dev *bd, void *req)
476{
477	bnge_hwrm_req_flags(bd, req, BNGE_HWRM_CTX_SILENT);
478	return bnge_hwrm_req_send(bd, req);
479}
480
481void *
482bnge_hwrm_req_dma_slice(struct bnge_dev *bd, void *req, u32 size,
483			dma_addr_t *dma_handle)
484{
485	struct bnge_hwrm_ctx *ctx = __hwrm_ctx_get(bd, req);
486	u8 *end = ((u8 *)req) + BNGE_HWRM_DMA_SIZE;
487	struct input *input = req;
488	u8 *addr, *req_addr = req;
489	u32 max_offset, offset;
490
491	if (!ctx)
492		return NULL;
493
494	max_offset = BNGE_HWRM_DMA_SIZE - ctx->allocated;
495	offset = max_offset - size;
496	offset = ALIGN_DOWN(offset, BNGE_HWRM_DMA_ALIGN);
497	addr = req_addr + offset;
498
499	if (addr < req_addr + max_offset && req_addr + ctx->req_len <= addr) {
500		ctx->allocated = end - addr;
501		*dma_handle = ctx->dma_handle + offset;
502		return addr;
503	}
504
505	if (ctx->slice_addr) {
506		dev_err(bd->dev, "HWRM refusing to reallocate DMA slice, req_type = %u\n",
507			(u32)le16_to_cpu(input->req_type));
508		dump_stack();
509		return NULL;
510	}
511
512	addr = dma_alloc_coherent(bd->dev, size, dma_handle, ctx->gfp);
513	if (!addr)
514		return NULL;
515
516	ctx->slice_addr = addr;
517	ctx->slice_size = size;
518	ctx->slice_handle = *dma_handle;
519
520	return addr;
521}
522
523void bnge_cleanup_hwrm_resources(struct bnge_dev *bd)
524{
525	struct bnge_hwrm_wait_token *token;
526
527	dma_pool_destroy(bd->hwrm_dma_pool);
528	bd->hwrm_dma_pool = NULL;
529
530	rcu_read_lock();
531	hlist_for_each_entry_rcu(token, &bd->hwrm_pending_list, node)
532		WRITE_ONCE(token->state, BNGE_HWRM_CANCELLED);
533	rcu_read_unlock();
534}
535
536int bnge_init_hwrm_resources(struct bnge_dev *bd)
537{
538	bd->hwrm_dma_pool = dma_pool_create("bnge_hwrm", bd->dev,
539					    BNGE_HWRM_DMA_SIZE,
540					    BNGE_HWRM_DMA_ALIGN, 0);
541	if (!bd->hwrm_dma_pool)
542		return -ENOMEM;
543
544	INIT_HLIST_HEAD(&bd->hwrm_pending_list);
545	mutex_init(&bd->hwrm_cmd_lock);
546
547	return 0;
548}