hinic3_hwif.c

  1// SPDX-License-Identifier: GPL-2.0
  2// Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved.
  3
  4#include <linux/bitfield.h>
  5#include <linux/device.h>
  6#include <linux/io.h>
  7
  8#include "hinic3_common.h"
  9#include "hinic3_csr.h"
 10#include "hinic3_hwdev.h"
 11#include "hinic3_hwif.h"
 12
 13#define HINIC3_HWIF_READY_TIMEOUT          10000
 14#define HINIC3_DB_AND_OUTBOUND_EN_TIMEOUT  60000
 15#define HINIC3_PCIE_LINK_DOWN              0xFFFFFFFF
 16
 17/* config BAR4/5 4MB, DB & DWQE both 2MB */
 18#define HINIC3_DB_DWQE_SIZE    0x00400000
 19
 20/* db/dwqe page size: 4K */
 21#define HINIC3_DB_PAGE_SIZE    0x00001000
 22#define HINIC3_DWQE_OFFSET     0x00000800
 23#define HINIC3_DB_MAX_AREAS    (HINIC3_DB_DWQE_SIZE / HINIC3_DB_PAGE_SIZE)
 24
 25#define HINIC3_MAX_MSIX_ENTRY  2048
 26
 27#define HINIC3_AF0_FUNC_GLOBAL_IDX_MASK  GENMASK(11, 0)
 28#define HINIC3_AF0_P2P_IDX_MASK          GENMASK(16, 12)
 29#define HINIC3_AF0_PCI_INTF_IDX_MASK     GENMASK(19, 17)
 30#define HINIC3_AF0_FUNC_TYPE_MASK        BIT(28)
 31#define HINIC3_AF0_GET(val, member) \
 32	FIELD_GET(HINIC3_AF0_##member##_MASK, val)
 33
 34#define HINIC3_AF1_AEQS_PER_FUNC_MASK     GENMASK(9, 8)
 35#define HINIC3_AF1_MGMT_INIT_STATUS_MASK  BIT(30)
 36#define HINIC3_AF1_GET(val, member) \
 37	FIELD_GET(HINIC3_AF1_##member##_MASK, val)
 38
 39#define HINIC3_AF2_CEQS_PER_FUNC_MASK      GENMASK(8, 0)
 40#define HINIC3_AF2_IRQS_PER_FUNC_MASK      GENMASK(26, 16)
 41#define HINIC3_AF2_GET(val, member) \
 42	FIELD_GET(HINIC3_AF2_##member##_MASK, val)
 43
 44#define HINIC3_AF4_DOORBELL_CTRL_MASK  BIT(0)
 45#define HINIC3_AF4_GET(val, member) \
 46	FIELD_GET(HINIC3_AF4_##member##_MASK, val)
 47#define HINIC3_AF4_SET(val, member) \
 48	FIELD_PREP(HINIC3_AF4_##member##_MASK, val)
 49
 50#define HINIC3_AF5_OUTBOUND_CTRL_MASK  BIT(0)
 51#define HINIC3_AF5_GET(val, member) \
 52	FIELD_GET(HINIC3_AF5_##member##_MASK, val)
 53
 54#define HINIC3_AF6_PF_STATUS_MASK     GENMASK(15, 0)
 55#define HINIC3_AF6_FUNC_MAX_SQ_MASK   GENMASK(31, 23)
 56#define HINIC3_AF6_MSIX_FLEX_EN_MASK  BIT(22)
 57#define HINIC3_AF6_GET(val, member) \
 58	FIELD_GET(HINIC3_AF6_##member##_MASK, val)
 59
 60#define HINIC3_GET_REG_ADDR(reg)  ((reg) & (HINIC3_REGS_FLAG_MASK))
 61
 62static void __iomem *hinic3_reg_addr(struct hinic3_hwif *hwif, u32 reg)
 63{
 64	return hwif->cfg_regs_base + HINIC3_GET_REG_ADDR(reg);
 65}
 66
 67u32 hinic3_hwif_read_reg(struct hinic3_hwif *hwif, u32 reg)
 68{
 69	void __iomem *addr = hinic3_reg_addr(hwif, reg);
 70
 71	return ioread32be(addr);
 72}
 73
 74void hinic3_hwif_write_reg(struct hinic3_hwif *hwif, u32 reg, u32 val)
 75{
 76	void __iomem *addr = hinic3_reg_addr(hwif, reg);
 77
 78	iowrite32be(val, addr);
 79}
 80
 81static enum hinic3_wait_return check_hwif_ready_handler(void *priv_data)
 82{
 83	struct hinic3_hwdev *hwdev = priv_data;
 84	u32 attr1;
 85
 86	attr1 = hinic3_hwif_read_reg(hwdev->hwif, HINIC3_CSR_FUNC_ATTR1_ADDR);
 87
 88	return HINIC3_AF1_GET(attr1, MGMT_INIT_STATUS) ?
 89	       HINIC3_WAIT_PROCESS_CPL : HINIC3_WAIT_PROCESS_WAITING;
 90}
 91
 92static int wait_hwif_ready(struct hinic3_hwdev *hwdev)
 93{
 94	return hinic3_wait_for_timeout(hwdev, check_hwif_ready_handler,
 95				       HINIC3_HWIF_READY_TIMEOUT,
 96				       USEC_PER_MSEC);
 97}
 98
 99/* Set attr struct from HW attr values. */
100static void set_hwif_attr(struct hinic3_func_attr *attr, u32 attr0, u32 attr1,
101			  u32 attr2, u32 attr3, u32 attr6)
102{
103	attr->func_global_idx = HINIC3_AF0_GET(attr0, FUNC_GLOBAL_IDX);
104	attr->port_to_port_idx = HINIC3_AF0_GET(attr0, P2P_IDX);
105	attr->pci_intf_idx = HINIC3_AF0_GET(attr0, PCI_INTF_IDX);
106	attr->func_type = HINIC3_AF0_GET(attr0, FUNC_TYPE);
107
108	attr->num_aeqs = BIT(HINIC3_AF1_GET(attr1, AEQS_PER_FUNC));
109	attr->num_ceqs = HINIC3_AF2_GET(attr2, CEQS_PER_FUNC);
110	attr->num_irqs = HINIC3_AF2_GET(attr2, IRQS_PER_FUNC);
111	if (attr->num_irqs > HINIC3_MAX_MSIX_ENTRY)
112		attr->num_irqs = HINIC3_MAX_MSIX_ENTRY;
113
114	attr->num_sq = HINIC3_AF6_GET(attr6, FUNC_MAX_SQ);
115	attr->msix_flex_en = HINIC3_AF6_GET(attr6, MSIX_FLEX_EN);
116}
117
118/* Read attributes from HW and set attribute struct. */
119static int init_hwif_attr(struct hinic3_hwdev *hwdev)
120{
121	u32 attr0, attr1, attr2, attr3, attr6;
122	struct hinic3_hwif *hwif;
123
124	hwif = hwdev->hwif;
125	attr0 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR0_ADDR);
126	if (attr0 == HINIC3_PCIE_LINK_DOWN)
127		return -EFAULT;
128
129	attr1 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR1_ADDR);
130	if (attr1 == HINIC3_PCIE_LINK_DOWN)
131		return -EFAULT;
132
133	attr2 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR2_ADDR);
134	if (attr2 == HINIC3_PCIE_LINK_DOWN)
135		return -EFAULT;
136
137	attr3 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR3_ADDR);
138	if (attr3 == HINIC3_PCIE_LINK_DOWN)
139		return -EFAULT;
140
141	attr6 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR6_ADDR);
142	if (attr6 == HINIC3_PCIE_LINK_DOWN)
143		return -EFAULT;
144
145	set_hwif_attr(&hwif->attr, attr0, attr1, attr2, attr3, attr6);
146
147	if (!hwif->attr.num_ceqs) {
148		dev_err(hwdev->dev, "Ceq num cfg in fw is zero\n");
149		return -EFAULT;
150	}
151
152	if (!hwif->attr.num_irqs) {
153		dev_err(hwdev->dev,
154			"Irq num cfg in fw is zero, msix_flex_en %d\n",
155			hwif->attr.msix_flex_en);
156		return -EFAULT;
157	}
158
159	return 0;
160}
161
162static enum hinic3_doorbell_ctrl hinic3_get_doorbell_ctrl_status(struct hinic3_hwif *hwif)
163{
164	u32 attr4 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR4_ADDR);
165
166	return HINIC3_AF4_GET(attr4, DOORBELL_CTRL);
167}
168
169static enum hinic3_outbound_ctrl hinic3_get_outbound_ctrl_status(struct hinic3_hwif *hwif)
170{
171	u32 attr5 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR5_ADDR);
172
173	return HINIC3_AF5_GET(attr5, OUTBOUND_CTRL);
174}
175
176void hinic3_toggle_doorbell(struct hinic3_hwif *hwif,
177			    enum hinic3_doorbell_ctrl flag)
178{
179	u32 addr, attr4;
180
181	addr = HINIC3_CSR_FUNC_ATTR4_ADDR;
182	attr4 = hinic3_hwif_read_reg(hwif, addr);
183
184	attr4 &= ~HINIC3_AF4_DOORBELL_CTRL_MASK;
185	attr4 |= HINIC3_AF4_SET(flag, DOORBELL_CTRL);
186
187	hinic3_hwif_write_reg(hwif, addr, attr4);
188}
189
190static int db_area_idx_init(struct hinic3_hwif *hwif, u64 db_base_phy,
191			    u8 __iomem *db_base, u64 db_dwqe_len)
192{
193	struct hinic3_db_area *db_area = &hwif->db_area;
194	u32 db_max_areas;
195
196	hwif->db_base_phy = db_base_phy;
197	hwif->db_base = db_base;
198	hwif->db_dwqe_len = db_dwqe_len;
199
200	db_max_areas = db_dwqe_len > HINIC3_DB_DWQE_SIZE ?
201		       HINIC3_DB_MAX_AREAS : db_dwqe_len / HINIC3_DB_PAGE_SIZE;
202	db_area->db_bitmap_array = bitmap_zalloc(db_max_areas, GFP_KERNEL);
203	if (!db_area->db_bitmap_array)
204		return -ENOMEM;
205
206	db_area->db_max_areas = db_max_areas;
207	spin_lock_init(&db_area->idx_lock);
208
209	return 0;
210}
211
212static void db_area_idx_free(struct hinic3_db_area *db_area)
213{
214	bitmap_free(db_area->db_bitmap_array);
215}
216
217static int get_db_idx(struct hinic3_hwif *hwif, u32 *idx)
218{
219	struct hinic3_db_area *db_area = &hwif->db_area;
220	u32 pg_idx;
221
222	spin_lock(&db_area->idx_lock);
223	pg_idx = find_first_zero_bit(db_area->db_bitmap_array,
224				     db_area->db_max_areas);
225	if (pg_idx == db_area->db_max_areas) {
226		spin_unlock(&db_area->idx_lock);
227		return -ENOMEM;
228	}
229	set_bit(pg_idx, db_area->db_bitmap_array);
230	spin_unlock(&db_area->idx_lock);
231
232	*idx = pg_idx;
233
234	return 0;
235}
236
237static void free_db_idx(struct hinic3_hwif *hwif, u32 idx)
238{
239	struct hinic3_db_area *db_area = &hwif->db_area;
240
241	spin_lock(&db_area->idx_lock);
242	clear_bit(idx, db_area->db_bitmap_array);
243	spin_unlock(&db_area->idx_lock);
244}
245
246void hinic3_free_db_addr(struct hinic3_hwdev *hwdev, const u8 __iomem *db_base)
247{
248	struct hinic3_hwif *hwif;
249	uintptr_t distance;
250	u32 idx;
251
252	hwif = hwdev->hwif;
253	distance = db_base - hwif->db_base;
254	idx = distance / HINIC3_DB_PAGE_SIZE;
255
256	free_db_idx(hwif, idx);
257}
258
259int hinic3_alloc_db_addr(struct hinic3_hwdev *hwdev, void __iomem **db_base,
260			 void __iomem **dwqe_base)
261{
262	struct hinic3_hwif *hwif;
263	u8 __iomem *addr;
264	u32 idx;
265	int err;
266
267	hwif = hwdev->hwif;
268
269	err = get_db_idx(hwif, &idx);
270	if (err)
271		return err;
272
273	addr = hwif->db_base + idx * HINIC3_DB_PAGE_SIZE;
274	*db_base = addr;
275
276	if (dwqe_base)
277		*dwqe_base = addr + HINIC3_DWQE_OFFSET;
278
279	return 0;
280}
281
282void hinic3_set_msix_state(struct hinic3_hwdev *hwdev, u16 msix_idx,
283			   enum hinic3_msix_state flag)
284{
285	struct hinic3_hwif *hwif;
286	u8 int_msk = 1;
287	u32 mask_bits;
288	u32 addr;
289
290	hwif = hwdev->hwif;
291
292	if (flag)
293		mask_bits = HINIC3_MSI_CLR_INDIR_SET(int_msk, INT_MSK_SET);
294	else
295		mask_bits = HINIC3_MSI_CLR_INDIR_SET(int_msk, INT_MSK_CLR);
296	mask_bits = mask_bits |
297		    HINIC3_MSI_CLR_INDIR_SET(msix_idx, SIMPLE_INDIR_IDX);
298
299	addr = HINIC3_CSR_FUNC_MSI_CLR_WR_ADDR;
300	hinic3_hwif_write_reg(hwif, addr, mask_bits);
301}
302
303static void disable_all_msix(struct hinic3_hwdev *hwdev)
304{
305	u16 num_irqs = hwdev->hwif->attr.num_irqs;
306	u16 i;
307
308	for (i = 0; i < num_irqs; i++)
309		hinic3_set_msix_state(hwdev, i, HINIC3_MSIX_DISABLE);
310}
311
312void hinic3_msix_intr_clear_resend_bit(struct hinic3_hwdev *hwdev, u16 msix_idx,
313				       u8 clear_resend_en)
314{
315	struct hinic3_hwif *hwif;
316	u32 msix_ctrl, addr;
317
318	hwif = hwdev->hwif;
319
320	msix_ctrl = HINIC3_MSI_CLR_INDIR_SET(msix_idx, SIMPLE_INDIR_IDX) |
321		    HINIC3_MSI_CLR_INDIR_SET(clear_resend_en, RESEND_TIMER_CLR);
322
323	addr = HINIC3_CSR_FUNC_MSI_CLR_WR_ADDR;
324	hinic3_hwif_write_reg(hwif, addr, msix_ctrl);
325}
326
327void hinic3_set_msix_auto_mask_state(struct hinic3_hwdev *hwdev, u16 msix_idx,
328				     enum hinic3_msix_auto_mask flag)
329{
330	struct hinic3_hwif *hwif;
331	u32 mask_bits;
332	u32 addr;
333
334	hwif = hwdev->hwif;
335
336	if (flag)
337		mask_bits = HINIC3_MSI_CLR_INDIR_SET(1, AUTO_MSK_SET);
338	else
339		mask_bits = HINIC3_MSI_CLR_INDIR_SET(1, AUTO_MSK_CLR);
340
341	mask_bits = mask_bits |
342		    HINIC3_MSI_CLR_INDIR_SET(msix_idx, SIMPLE_INDIR_IDX);
343
344	addr = HINIC3_CSR_FUNC_MSI_CLR_WR_ADDR;
345	hinic3_hwif_write_reg(hwif, addr, mask_bits);
346}
347
348static enum hinic3_wait_return check_db_outbound_enable_handler(void *priv_data)
349{
350	enum hinic3_outbound_ctrl outbound_ctrl;
351	struct hinic3_hwif *hwif = priv_data;
352	enum hinic3_doorbell_ctrl db_ctrl;
353
354	db_ctrl = hinic3_get_doorbell_ctrl_status(hwif);
355	outbound_ctrl = hinic3_get_outbound_ctrl_status(hwif);
356	if (outbound_ctrl == ENABLE_OUTBOUND && db_ctrl == ENABLE_DOORBELL)
357		return HINIC3_WAIT_PROCESS_CPL;
358
359	return HINIC3_WAIT_PROCESS_WAITING;
360}
361
362static int wait_until_doorbell_and_outbound_enabled(struct hinic3_hwif *hwif)
363{
364	return hinic3_wait_for_timeout(hwif, check_db_outbound_enable_handler,
365				       HINIC3_DB_AND_OUTBOUND_EN_TIMEOUT,
366				       USEC_PER_MSEC);
367}
368
369int hinic3_init_hwif(struct hinic3_hwdev *hwdev)
370{
371	struct hinic3_pcidev *pci_adapter = hwdev->adapter;
372	struct hinic3_hwif *hwif;
373	u32 attr1, attr4, attr5;
374	int err;
375
376	hwif = kzalloc(sizeof(*hwif), GFP_KERNEL);
377	if (!hwif)
378		return -ENOMEM;
379
380	hwdev->hwif = hwif;
381	hwif->cfg_regs_base = (u8 __iomem *)pci_adapter->cfg_reg_base +
382			      HINIC3_VF_CFG_REG_OFFSET;
383
384	err = db_area_idx_init(hwif, pci_adapter->db_base_phy,
385			       pci_adapter->db_base,
386			       pci_adapter->db_dwqe_len);
387	if (err) {
388		dev_err(hwdev->dev, "Failed to init db area.\n");
389		goto err_free_hwif;
390	}
391
392	err = wait_hwif_ready(hwdev);
393	if (err) {
394		attr1 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR1_ADDR);
395		dev_err(hwdev->dev, "Chip status is not ready, attr1:0x%x\n",
396			attr1);
397		goto err_free_db_area_idx;
398	}
399
400	err = init_hwif_attr(hwdev);
401	if (err) {
402		dev_err(hwdev->dev, "Init hwif attr failed\n");
403		goto err_free_db_area_idx;
404	}
405
406	err = wait_until_doorbell_and_outbound_enabled(hwif);
407	if (err) {
408		attr4 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR4_ADDR);
409		attr5 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR5_ADDR);
410		dev_err(hwdev->dev, "HW doorbell/outbound is disabled, attr4 0x%x attr5 0x%x\n",
411			attr4, attr5);
412		goto err_free_db_area_idx;
413	}
414
415	disable_all_msix(hwdev);
416
417	return 0;
418
419err_free_db_area_idx:
420	db_area_idx_free(&hwif->db_area);
421err_free_hwif:
422	kfree(hwif);
423
424	return err;
425}
426
427void hinic3_free_hwif(struct hinic3_hwdev *hwdev)
428{
429	db_area_idx_free(&hwdev->hwif->db_area);
430	kfree(hwdev->hwif);
431}
432
433u16 hinic3_global_func_id(struct hinic3_hwdev *hwdev)
434{
435	return hwdev->hwif->attr.func_global_idx;
436}