iwl-io.c

  1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
  2/*
  3 * Copyright (C) 2003-2014, 2018-2022, 2024-2025 Intel Corporation
  4 * Copyright (C) 2015-2016 Intel Deutschland GmbH
  5 */
  6#include <linux/device.h>
  7#include <linux/export.h>
  8
  9#include "iwl-drv.h"
 10#include "iwl-io.h"
 11#include "iwl-csr.h"
 12#include "iwl-debug.h"
 13#include "iwl-prph.h"
 14#include "iwl-fh.h"
 15#include "pcie/gen1_2/internal.h"
 16
 17void iwl_write8(struct iwl_trans *trans, u32 ofs, u8 val)
 18{
 19	trace_iwlwifi_dev_iowrite8(trans->dev, ofs, val);
 20	iwl_trans_write8(trans, ofs, val);
 21}
 22IWL_EXPORT_SYMBOL(iwl_write8);
 23
 24void iwl_write32(struct iwl_trans *trans, u32 ofs, u32 val)
 25{
 26	trace_iwlwifi_dev_iowrite32(trans->dev, ofs, val);
 27	iwl_trans_write32(trans, ofs, val);
 28}
 29IWL_EXPORT_SYMBOL(iwl_write32);
 30
 31void iwl_write64(struct iwl_trans *trans, u64 ofs, u64 val)
 32{
 33	trace_iwlwifi_dev_iowrite64(trans->dev, ofs, val);
 34	iwl_trans_write32(trans, ofs, lower_32_bits(val));
 35	iwl_trans_write32(trans, ofs + 4, upper_32_bits(val));
 36}
 37IWL_EXPORT_SYMBOL(iwl_write64);
 38
 39u32 iwl_read32(struct iwl_trans *trans, u32 ofs)
 40{
 41	u32 val = iwl_trans_read32(trans, ofs);
 42
 43	trace_iwlwifi_dev_ioread32(trans->dev, ofs, val);
 44	return val;
 45}
 46IWL_EXPORT_SYMBOL(iwl_read32);
 47
 48#define IWL_POLL_INTERVAL 10	/* microseconds */
 49
 50int iwl_poll_bits_mask(struct iwl_trans *trans, u32 addr,
 51		       u32 bits, u32 mask, int timeout)
 52{
 53	int t = 0;
 54
 55	do {
 56		if ((iwl_read32(trans, addr) & mask) == (bits & mask))
 57			return 0;
 58		udelay(IWL_POLL_INTERVAL);
 59		t += IWL_POLL_INTERVAL;
 60	} while (t < timeout);
 61
 62	return -ETIMEDOUT;
 63}
 64IWL_EXPORT_SYMBOL(iwl_poll_bits_mask);
 65
 66u32 iwl_read_direct32(struct iwl_trans *trans, u32 reg)
 67{
 68	if (iwl_trans_grab_nic_access(trans)) {
 69		u32 value = iwl_read32(trans, reg);
 70
 71		iwl_trans_release_nic_access(trans);
 72		return value;
 73	}
 74
 75	/* return as if we have a HW timeout/failure */
 76	return 0x5a5a5a5a;
 77}
 78
 79void iwl_write_direct32(struct iwl_trans *trans, u32 reg, u32 value)
 80{
 81	if (iwl_trans_grab_nic_access(trans)) {
 82		iwl_write32(trans, reg, value);
 83		iwl_trans_release_nic_access(trans);
 84	}
 85}
 86IWL_EXPORT_SYMBOL(iwl_write_direct32);
 87
 88void iwl_write_direct64(struct iwl_trans *trans, u64 reg, u64 value)
 89{
 90	if (iwl_trans_grab_nic_access(trans)) {
 91		iwl_write64(trans, reg, value);
 92		iwl_trans_release_nic_access(trans);
 93	}
 94}
 95
 96int iwl_poll_direct_bit(struct iwl_trans *trans, u32 addr, u32 mask,
 97			int timeout)
 98{
 99	int t = 0;
100
101	do {
102		if ((iwl_read_direct32(trans, addr) & mask) == mask)
103			return t;
104		udelay(IWL_POLL_INTERVAL);
105		t += IWL_POLL_INTERVAL;
106	} while (t < timeout);
107
108	return -ETIMEDOUT;
109}
110
111u32 iwl_read_prph_no_grab(struct iwl_trans *trans, u32 ofs)
112{
113	u32 val = iwl_trans_read_prph(trans, ofs);
114	trace_iwlwifi_dev_ioread_prph32(trans->dev, ofs, val);
115	return val;
116}
117
118void iwl_write_prph_no_grab(struct iwl_trans *trans, u32 ofs, u32 val)
119{
120	trace_iwlwifi_dev_iowrite_prph32(trans->dev, ofs, val);
121	iwl_trans_write_prph(trans, ofs, val);
122}
123
124void iwl_write_prph64_no_grab(struct iwl_trans *trans, u64 ofs, u64 val)
125{
126	trace_iwlwifi_dev_iowrite_prph64(trans->dev, ofs, val);
127	iwl_write_prph_no_grab(trans, ofs, val & 0xffffffff);
128	iwl_write_prph_no_grab(trans, ofs + 4, val >> 32);
129}
130
131u32 iwl_read_prph(struct iwl_trans *trans, u32 ofs)
132{
133	if (iwl_trans_grab_nic_access(trans)) {
134		u32 val = iwl_read_prph_no_grab(trans, ofs);
135
136		iwl_trans_release_nic_access(trans);
137
138		return val;
139	}
140
141	/* return as if we have a HW timeout/failure */
142	return 0x5a5a5a5a;
143}
144IWL_EXPORT_SYMBOL(iwl_read_prph);
145
146void iwl_write_prph_delay(struct iwl_trans *trans, u32 ofs, u32 val, u32 delay_ms)
147{
148	if (iwl_trans_grab_nic_access(trans)) {
149		mdelay(delay_ms);
150		iwl_write_prph_no_grab(trans, ofs, val);
151		iwl_trans_release_nic_access(trans);
152	}
153}
154IWL_EXPORT_SYMBOL(iwl_write_prph_delay);
155
156int iwl_poll_prph_bit(struct iwl_trans *trans, u32 addr,
157		      u32 bits, u32 mask, int timeout)
158{
159	int t = 0;
160
161	do {
162		if ((iwl_read_prph(trans, addr) & mask) == (bits & mask))
163			return 0;
164		udelay(IWL_POLL_INTERVAL);
165		t += IWL_POLL_INTERVAL;
166	} while (t < timeout);
167
168	return -ETIMEDOUT;
169}
170
171void iwl_set_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask)
172{
173	if (iwl_trans_grab_nic_access(trans)) {
174		iwl_write_prph_no_grab(trans, ofs,
175				       iwl_read_prph_no_grab(trans, ofs) |
176				       mask);
177		iwl_trans_release_nic_access(trans);
178	}
179}
180IWL_EXPORT_SYMBOL(iwl_set_bits_prph);
181
182void iwl_set_bits_mask_prph(struct iwl_trans *trans, u32 ofs,
183			    u32 bits, u32 mask)
184{
185	if (iwl_trans_grab_nic_access(trans)) {
186		iwl_write_prph_no_grab(trans, ofs,
187				       (iwl_read_prph_no_grab(trans, ofs) &
188					mask) | bits);
189		iwl_trans_release_nic_access(trans);
190	}
191}
192IWL_EXPORT_SYMBOL(iwl_set_bits_mask_prph);
193
194void iwl_clear_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask)
195{
196	u32 val;
197
198	if (iwl_trans_grab_nic_access(trans)) {
199		val = iwl_read_prph_no_grab(trans, ofs);
200		iwl_write_prph_no_grab(trans, ofs, (val & ~mask));
201		iwl_trans_release_nic_access(trans);
202	}
203}
204IWL_EXPORT_SYMBOL(iwl_clear_bits_prph);
205
206void iwl_force_nmi(struct iwl_trans *trans)
207{
208	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_9000)
209		iwl_write_prph_delay(trans, DEVICE_SET_NMI_REG,
210				     DEVICE_SET_NMI_VAL_DRV, 1);
211	else if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
212		iwl_write_umac_prph(trans, UREG_NIC_SET_NMI_DRIVER,
213				UREG_NIC_SET_NMI_DRIVER_NMI_FROM_DRIVER);
214	else if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_BZ)
215		iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,
216				    UREG_DOORBELL_TO_ISR6_NMI_BIT);
217	else
218		iwl_write32(trans, CSR_DOORBELL_VECTOR,
219			    UREG_DOORBELL_TO_ISR6_NMI_BIT);
220}
221IWL_EXPORT_SYMBOL(iwl_force_nmi);
222
223static const char *get_rfh_string(int cmd)
224{
225#define IWL_CMD(x) case x: return #x
226#define IWL_CMD_MQ(arg, reg, q) { if (arg == reg(q)) return #reg; }
227
228	int i;
229
230	for (i = 0; i < IWL_MAX_RX_HW_QUEUES; i++) {
231		IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_BA_LSB, i);
232		IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_WIDX, i);
233		IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_RIDX, i);
234		IWL_CMD_MQ(cmd, RFH_Q_URBD_STTS_WPTR_LSB, i);
235	}
236
237	switch (cmd) {
238	IWL_CMD(RFH_RXF_DMA_CFG);
239	IWL_CMD(RFH_GEN_CFG);
240	IWL_CMD(RFH_GEN_STATUS);
241	IWL_CMD(FH_TSSR_TX_STATUS_REG);
242	IWL_CMD(FH_TSSR_TX_ERROR_REG);
243	default:
244		return "UNKNOWN";
245	}
246#undef IWL_CMD_MQ
247}
248
249struct reg {
250	u32 addr;
251	bool is64;
252};
253
254static int iwl_dump_rfh(struct iwl_trans *trans, char **buf)
255{
256	int i, q;
257	int num_q = trans->info.num_rxqs;
258	static const u32 rfh_tbl[] = {
259		RFH_RXF_DMA_CFG,
260		RFH_GEN_CFG,
261		RFH_GEN_STATUS,
262		FH_TSSR_TX_STATUS_REG,
263		FH_TSSR_TX_ERROR_REG,
264	};
265	static const struct reg rfh_mq_tbl[] = {
266		{ RFH_Q0_FRBDCB_BA_LSB, true },
267		{ RFH_Q0_FRBDCB_WIDX, false },
268		{ RFH_Q0_FRBDCB_RIDX, false },
269		{ RFH_Q0_URBD_STTS_WPTR_LSB, true },
270	};
271
272#ifdef CONFIG_IWLWIFI_DEBUGFS
273	if (buf) {
274		int pos = 0;
275		/*
276		 * Register (up to 34 for name + 8 blank/q for MQ): 40 chars
277		 * Colon + space: 2 characters
278		 * 0X%08x: 10 characters
279		 * New line: 1 character
280		 * Total of 53 characters
281		 */
282		size_t bufsz = ARRAY_SIZE(rfh_tbl) * 53 +
283			       ARRAY_SIZE(rfh_mq_tbl) * 53 * num_q + 40;
284
285		*buf = kmalloc(bufsz, GFP_KERNEL);
286		if (!*buf)
287			return -ENOMEM;
288
289		pos += scnprintf(*buf + pos, bufsz - pos,
290				"RFH register values:\n");
291
292		for (i = 0; i < ARRAY_SIZE(rfh_tbl); i++)
293			pos += scnprintf(*buf + pos, bufsz - pos,
294				"%40s: 0X%08x\n",
295				get_rfh_string(rfh_tbl[i]),
296				iwl_read_prph(trans, rfh_tbl[i]));
297
298		for (i = 0; i < ARRAY_SIZE(rfh_mq_tbl); i++)
299			for (q = 0; q < num_q; q++) {
300				u32 addr = rfh_mq_tbl[i].addr;
301
302				addr += q * (rfh_mq_tbl[i].is64 ? 8 : 4);
303				pos += scnprintf(*buf + pos, bufsz - pos,
304					"%34s(q %2d): 0X%08x\n",
305					get_rfh_string(addr), q,
306					iwl_read_prph(trans, addr));
307			}
308
309		return pos;
310	}
311#endif
312
313	IWL_ERR(trans, "RFH register values:\n");
314	for (i = 0; i < ARRAY_SIZE(rfh_tbl); i++)
315		IWL_ERR(trans, "  %34s: 0X%08x\n",
316			get_rfh_string(rfh_tbl[i]),
317			iwl_read_prph(trans, rfh_tbl[i]));
318
319	for (i = 0; i < ARRAY_SIZE(rfh_mq_tbl); i++)
320		for (q = 0; q < num_q; q++) {
321			u32 addr = rfh_mq_tbl[i].addr;
322
323			addr += q * (rfh_mq_tbl[i].is64 ? 8 : 4);
324			IWL_ERR(trans, "  %34s(q %d): 0X%08x\n",
325				get_rfh_string(addr), q,
326				iwl_read_prph(trans, addr));
327		}
328
329	return 0;
330}
331
332static const char *get_fh_string(int cmd)
333{
334	switch (cmd) {
335	IWL_CMD(FH_RSCSR_CHNL0_STTS_WPTR_REG);
336	IWL_CMD(FH_RSCSR_CHNL0_RBDCB_BASE_REG);
337	IWL_CMD(FH_RSCSR_CHNL0_WPTR);
338	IWL_CMD(FH_MEM_RCSR_CHNL0_CONFIG_REG);
339	IWL_CMD(FH_MEM_RSSR_SHARED_CTRL_REG);
340	IWL_CMD(FH_MEM_RSSR_RX_STATUS_REG);
341	IWL_CMD(FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV);
342	IWL_CMD(FH_TSSR_TX_STATUS_REG);
343	IWL_CMD(FH_TSSR_TX_ERROR_REG);
344	default:
345		return "UNKNOWN";
346	}
347#undef IWL_CMD
348}
349
350int iwl_dump_fh(struct iwl_trans *trans, char **buf)
351{
352	int i;
353	static const u32 fh_tbl[] = {
354		FH_RSCSR_CHNL0_STTS_WPTR_REG,
355		FH_RSCSR_CHNL0_RBDCB_BASE_REG,
356		FH_RSCSR_CHNL0_WPTR,
357		FH_MEM_RCSR_CHNL0_CONFIG_REG,
358		FH_MEM_RSSR_SHARED_CTRL_REG,
359		FH_MEM_RSSR_RX_STATUS_REG,
360		FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV,
361		FH_TSSR_TX_STATUS_REG,
362		FH_TSSR_TX_ERROR_REG
363	};
364
365	if (trans->mac_cfg->mq_rx_supported)
366		return iwl_dump_rfh(trans, buf);
367
368#ifdef CONFIG_IWLWIFI_DEBUGFS
369	if (buf) {
370		int pos = 0;
371		size_t bufsz = ARRAY_SIZE(fh_tbl) * 48 + 40;
372
373		*buf = kmalloc(bufsz, GFP_KERNEL);
374		if (!*buf)
375			return -ENOMEM;
376
377		pos += scnprintf(*buf + pos, bufsz - pos,
378				"FH register values:\n");
379
380		for (i = 0; i < ARRAY_SIZE(fh_tbl); i++)
381			pos += scnprintf(*buf + pos, bufsz - pos,
382				"  %34s: 0X%08x\n",
383				get_fh_string(fh_tbl[i]),
384				iwl_read_direct32(trans, fh_tbl[i]));
385
386		return pos;
387	}
388#endif
389
390	IWL_ERR(trans, "FH register values:\n");
391	for (i = 0; i <  ARRAY_SIZE(fh_tbl); i++)
392		IWL_ERR(trans, "  %34s: 0X%08x\n",
393			get_fh_string(fh_tbl[i]),
394			iwl_read_direct32(trans, fh_tbl[i]));
395
396	return 0;
397}
398
399int iwl_trans_activate_nic(struct iwl_trans *trans)
400{
401	return iwl_pcie_gen1_2_activate_nic(trans);
402}
403IWL_EXPORT_SYMBOL(iwl_trans_activate_nic);
404
405void iwl_trans_sync_nmi_with_addr(struct iwl_trans *trans, u32 inta_addr,
406				  u32 sw_err_bit)
407{
408	unsigned long timeout = jiffies + IWL_TRANS_NMI_TIMEOUT;
409	bool interrupts_enabled = test_bit(STATUS_INT_ENABLED, &trans->status);
410
411	/* if the interrupts were already disabled, there is no point in
412	 * calling iwl_disable_interrupts
413	 */
414	if (interrupts_enabled)
415		iwl_trans_interrupts(trans, false);
416
417	iwl_force_nmi(trans);
418	while (time_after(timeout, jiffies)) {
419		u32 inta_hw = iwl_read32(trans, inta_addr);
420
421		/* Error detected by uCode */
422		if (inta_hw & sw_err_bit) {
423			/* Clear causes register */
424			iwl_write32(trans, inta_addr, inta_hw & sw_err_bit);
425			break;
426		}
427
428		mdelay(1);
429	}
430
431	/* enable interrupts only if there were already enabled before this
432	 * function to avoid a case were the driver enable interrupts before
433	 * proper configurations were made
434	 */
435	if (interrupts_enabled)
436		iwl_trans_interrupts(trans, true);
437
438	iwl_trans_fw_error(trans, IWL_ERR_TYPE_NMI_FORCED);
439}