btintel_pcie.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 *
   4 *  Bluetooth support for Intel PCIe devices
   5 *
   6 *  Copyright (C) 2024  Intel Corporation
   7 */
   8
   9#include <linux/kernel.h>
  10#include <linux/module.h>
  11#include <linux/firmware.h>
  12#include <linux/pci.h>
  13#include <linux/wait.h>
  14#include <linux/delay.h>
  15#include <linux/interrupt.h>
  16
  17#include <linux/unaligned.h>
  18#include <linux/devcoredump.h>
  19
  20#include <net/bluetooth/bluetooth.h>
  21#include <net/bluetooth/hci_core.h>
  22#include <net/bluetooth/hci_drv.h>
  23
  24#include "btintel.h"
  25#include "btintel_pcie.h"
  26
  27#define VERSION "0.1"
  28
  29#define BTINTEL_PCI_DEVICE(dev, subdev)	\
  30	.vendor = PCI_VENDOR_ID_INTEL,	\
  31	.device = (dev),		\
  32	.subvendor = PCI_ANY_ID,	\
  33	.subdevice = (subdev),		\
  34	.driver_data = 0
  35
  36#define POLL_INTERVAL_US	10
  37
  38/* Intel Bluetooth PCIe device id table */
  39static const struct pci_device_id btintel_pcie_table[] = {
  40	/* BlazarI, Wildcat Lake */
  41	{ BTINTEL_PCI_DEVICE(0x4D76, PCI_ANY_ID) },
  42	/* BlazarI, Lunar Lake */
  43	{ BTINTEL_PCI_DEVICE(0xA876, PCI_ANY_ID) },
  44	/* Scorpious, Panther Lake-H484 */
  45	{ BTINTEL_PCI_DEVICE(0xE376, PCI_ANY_ID) },
  46	 /* Scorpious, Panther Lake-H404 */
  47	{ BTINTEL_PCI_DEVICE(0xE476, PCI_ANY_ID) },
  48	{ 0 }
  49};
  50MODULE_DEVICE_TABLE(pci, btintel_pcie_table);
  51
  52struct btintel_pcie_dev_recovery {
  53	struct list_head list;
  54	u8 count;
  55	time64_t last_error;
  56	char name[];
  57};
  58
  59/* Intel PCIe uses 4 bytes of HCI type instead of 1 byte BT SIG HCI type */
  60#define BTINTEL_PCIE_HCI_TYPE_LEN	4
  61#define BTINTEL_PCIE_HCI_CMD_PKT	0x00000001
  62#define BTINTEL_PCIE_HCI_ACL_PKT	0x00000002
  63#define BTINTEL_PCIE_HCI_SCO_PKT	0x00000003
  64#define BTINTEL_PCIE_HCI_EVT_PKT	0x00000004
  65#define BTINTEL_PCIE_HCI_ISO_PKT	0x00000005
  66
  67#define BTINTEL_PCIE_MAGIC_NUM    0xA5A5A5A5
  68
  69#define BTINTEL_PCIE_BLZR_HWEXP_SIZE		1024
  70#define BTINTEL_PCIE_BLZR_HWEXP_DMP_ADDR	0xB00A7C00
  71
  72#define BTINTEL_PCIE_SCP_HWEXP_SIZE		4096
  73#define BTINTEL_PCIE_SCP_HWEXP_DMP_ADDR		0xB030F800
  74
  75#define BTINTEL_PCIE_MAGIC_NUM	0xA5A5A5A5
  76
  77#define BTINTEL_PCIE_TRIGGER_REASON_USER_TRIGGER	0x17A2
  78#define BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT		0x1E61
  79
  80#define BTINTEL_PCIE_RESET_WINDOW_SECS		5
  81#define BTINTEL_PCIE_FLR_MAX_RETRY	1
  82
  83/* Alive interrupt context */
  84enum {
  85	BTINTEL_PCIE_ROM,
  86	BTINTEL_PCIE_FW_DL,
  87	BTINTEL_PCIE_HCI_RESET,
  88	BTINTEL_PCIE_INTEL_HCI_RESET1,
  89	BTINTEL_PCIE_INTEL_HCI_RESET2,
  90	BTINTEL_PCIE_D0,
  91	BTINTEL_PCIE_D3
  92};
  93
  94/* Structure for dbgc fragment buffer
  95 * @buf_addr_lsb: LSB of the buffer's physical address
  96 * @buf_addr_msb: MSB of the buffer's physical address
  97 * @buf_size: Total size of the buffer
  98 */
  99struct btintel_pcie_dbgc_ctxt_buf {
 100	u32	buf_addr_lsb;
 101	u32	buf_addr_msb;
 102	u32	buf_size;
 103};
 104
 105/* Structure for dbgc fragment
 106 * @magic_num: 0XA5A5A5A5
 107 * @ver: For Driver-FW compatibility
 108 * @total_size: Total size of the payload debug info
 109 * @num_buf: Num of allocated debug bufs
 110 * @bufs: All buffer's addresses and sizes
 111 */
 112struct btintel_pcie_dbgc_ctxt {
 113	u32	magic_num;
 114	u32     ver;
 115	u32     total_size;
 116	u32     num_buf;
 117	struct btintel_pcie_dbgc_ctxt_buf bufs[BTINTEL_PCIE_DBGC_BUFFER_COUNT];
 118};
 119
 120struct btintel_pcie_removal {
 121	struct pci_dev *pdev;
 122	struct work_struct work;
 123};
 124
 125static LIST_HEAD(btintel_pcie_recovery_list);
 126static DEFINE_SPINLOCK(btintel_pcie_recovery_lock);
 127
 128static inline char *btintel_pcie_alivectxt_state2str(u32 alive_intr_ctxt)
 129{
 130	switch (alive_intr_ctxt) {
 131	case BTINTEL_PCIE_ROM:
 132		return "rom";
 133	case BTINTEL_PCIE_FW_DL:
 134		return "fw_dl";
 135	case BTINTEL_PCIE_D0:
 136		return "d0";
 137	case BTINTEL_PCIE_D3:
 138		return "d3";
 139	case BTINTEL_PCIE_HCI_RESET:
 140		return "hci_reset";
 141	case BTINTEL_PCIE_INTEL_HCI_RESET1:
 142		return "intel_reset1";
 143	case BTINTEL_PCIE_INTEL_HCI_RESET2:
 144		return "intel_reset2";
 145	default:
 146		return "unknown";
 147	}
 148}
 149
 150/* This function initializes the memory for DBGC buffers and formats the
 151 * DBGC fragment which consists header info and DBGC buffer's LSB, MSB and
 152 * size as the payload
 153 */
 154static int btintel_pcie_setup_dbgc(struct btintel_pcie_data *data)
 155{
 156	struct btintel_pcie_dbgc_ctxt db_frag;
 157	struct data_buf *buf;
 158	int i;
 159
 160	data->dbgc.count = BTINTEL_PCIE_DBGC_BUFFER_COUNT;
 161	data->dbgc.bufs = devm_kcalloc(&data->pdev->dev, data->dbgc.count,
 162				       sizeof(*buf), GFP_KERNEL);
 163	if (!data->dbgc.bufs)
 164		return -ENOMEM;
 165
 166	data->dbgc.buf_v_addr = dmam_alloc_coherent(&data->pdev->dev,
 167						    data->dbgc.count *
 168						    BTINTEL_PCIE_DBGC_BUFFER_SIZE,
 169						    &data->dbgc.buf_p_addr,
 170						    GFP_KERNEL | __GFP_NOWARN);
 171	if (!data->dbgc.buf_v_addr)
 172		return -ENOMEM;
 173
 174	data->dbgc.frag_v_addr = dmam_alloc_coherent(&data->pdev->dev,
 175						     sizeof(struct btintel_pcie_dbgc_ctxt),
 176						     &data->dbgc.frag_p_addr,
 177						     GFP_KERNEL | __GFP_NOWARN);
 178	if (!data->dbgc.frag_v_addr)
 179		return -ENOMEM;
 180
 181	data->dbgc.frag_size = sizeof(struct btintel_pcie_dbgc_ctxt);
 182
 183	db_frag.magic_num = BTINTEL_PCIE_MAGIC_NUM;
 184	db_frag.ver = BTINTEL_PCIE_DBGC_FRAG_VERSION;
 185	db_frag.total_size = BTINTEL_PCIE_DBGC_FRAG_PAYLOAD_SIZE;
 186	db_frag.num_buf = BTINTEL_PCIE_DBGC_FRAG_BUFFER_COUNT;
 187
 188	for (i = 0; i < data->dbgc.count; i++) {
 189		buf = &data->dbgc.bufs[i];
 190		buf->data_p_addr = data->dbgc.buf_p_addr + i * BTINTEL_PCIE_DBGC_BUFFER_SIZE;
 191		buf->data = data->dbgc.buf_v_addr + i * BTINTEL_PCIE_DBGC_BUFFER_SIZE;
 192		db_frag.bufs[i].buf_addr_lsb = lower_32_bits(buf->data_p_addr);
 193		db_frag.bufs[i].buf_addr_msb = upper_32_bits(buf->data_p_addr);
 194		db_frag.bufs[i].buf_size = BTINTEL_PCIE_DBGC_BUFFER_SIZE;
 195	}
 196
 197	memcpy(data->dbgc.frag_v_addr, &db_frag, sizeof(db_frag));
 198	return 0;
 199}
 200
 201static inline void ipc_print_ia_ring(struct hci_dev *hdev, struct ia *ia,
 202				     u16 queue_num)
 203{
 204	bt_dev_dbg(hdev, "IA: %s: tr-h:%02u  tr-t:%02u  cr-h:%02u  cr-t:%02u",
 205		   queue_num == BTINTEL_PCIE_TXQ_NUM ? "TXQ" : "RXQ",
 206		   ia->tr_hia[queue_num], ia->tr_tia[queue_num],
 207		   ia->cr_hia[queue_num], ia->cr_tia[queue_num]);
 208}
 209
 210static inline void ipc_print_urbd1(struct hci_dev *hdev, struct urbd1 *urbd1,
 211				   u16 index)
 212{
 213	bt_dev_dbg(hdev, "RXQ:urbd1(%u) frbd_tag:%u status: 0x%x fixed:0x%x",
 214		   index, urbd1->frbd_tag, urbd1->status, urbd1->fixed);
 215}
 216
 217static struct btintel_pcie_data *btintel_pcie_get_data(struct msix_entry *entry)
 218{
 219	u8 queue = entry->entry;
 220	struct msix_entry *entries = entry - queue;
 221
 222	return container_of(entries, struct btintel_pcie_data, msix_entries[0]);
 223}
 224
 225/* Set the doorbell for TXQ to notify the device that @index (actually index-1)
 226 * of the TFD is updated and ready to transmit.
 227 */
 228static void btintel_pcie_set_tx_db(struct btintel_pcie_data *data, u16 index)
 229{
 230	u32 val;
 231
 232	val = index;
 233	val |= (BTINTEL_PCIE_TX_DB_VEC << 16);
 234
 235	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_HBUS_TARG_WRPTR, val);
 236}
 237
 238/* Copy the data to next(@tfd_index) data buffer and update the TFD(transfer
 239 * descriptor) with the data length and the DMA address of the data buffer.
 240 */
 241static void btintel_pcie_prepare_tx(struct txq *txq, u16 tfd_index,
 242				    struct sk_buff *skb)
 243{
 244	struct data_buf *buf;
 245	struct tfd *tfd;
 246
 247	tfd = &txq->tfds[tfd_index];
 248	memset(tfd, 0, sizeof(*tfd));
 249
 250	buf = &txq->bufs[tfd_index];
 251
 252	tfd->size = skb->len;
 253	tfd->addr = buf->data_p_addr;
 254
 255	/* Copy the outgoing data to DMA buffer */
 256	memcpy(buf->data, skb->data, tfd->size);
 257}
 258
 259static inline void btintel_pcie_dump_debug_registers(struct hci_dev *hdev)
 260{
 261	struct btintel_pcie_data *data = hci_get_drvdata(hdev);
 262	u16 cr_hia, cr_tia;
 263	u32 reg, mbox_reg;
 264	struct sk_buff *skb;
 265	u8 buf[80];
 266
 267	skb = alloc_skb(1024, GFP_ATOMIC);
 268	if (!skb)
 269		return;
 270
 271	snprintf(buf, sizeof(buf), "%s", "---- Dump of debug registers ---");
 272	bt_dev_dbg(hdev, "%s", buf);
 273	skb_put_data(skb, buf, strlen(buf));
 274
 275	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_BOOT_STAGE_REG);
 276	snprintf(buf, sizeof(buf), "boot stage: 0x%8.8x", reg);
 277	bt_dev_dbg(hdev, "%s", buf);
 278	skb_put_data(skb, buf, strlen(buf));
 279	data->boot_stage_cache = reg;
 280
 281	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IPC_STATUS_REG);
 282	snprintf(buf, sizeof(buf), "ipc status: 0x%8.8x", reg);
 283	skb_put_data(skb, buf, strlen(buf));
 284	bt_dev_dbg(hdev, "%s", buf);
 285
 286	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IPC_CONTROL_REG);
 287	snprintf(buf, sizeof(buf), "ipc control: 0x%8.8x", reg);
 288	skb_put_data(skb, buf, strlen(buf));
 289	bt_dev_dbg(hdev, "%s", buf);
 290
 291	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IPC_SLEEP_CTL_REG);
 292	snprintf(buf, sizeof(buf), "ipc sleep control: 0x%8.8x", reg);
 293	skb_put_data(skb, buf, strlen(buf));
 294	bt_dev_dbg(hdev, "%s", buf);
 295
 296	/*Read the Mail box status and registers*/
 297	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MBOX_STATUS_REG);
 298	snprintf(buf, sizeof(buf), "mbox status: 0x%8.8x", reg);
 299	skb_put_data(skb, buf, strlen(buf));
 300	if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX1) {
 301		mbox_reg = btintel_pcie_rd_reg32(data,
 302						 BTINTEL_PCIE_CSR_MBOX_1_REG);
 303		snprintf(buf, sizeof(buf), "mbox_1: 0x%8.8x", mbox_reg);
 304		skb_put_data(skb, buf, strlen(buf));
 305		bt_dev_dbg(hdev, "%s", buf);
 306	}
 307
 308	if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX2) {
 309		mbox_reg = btintel_pcie_rd_reg32(data,
 310						 BTINTEL_PCIE_CSR_MBOX_2_REG);
 311		snprintf(buf, sizeof(buf), "mbox_2: 0x%8.8x", mbox_reg);
 312		skb_put_data(skb, buf, strlen(buf));
 313		bt_dev_dbg(hdev, "%s", buf);
 314	}
 315
 316	if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX3) {
 317		mbox_reg = btintel_pcie_rd_reg32(data,
 318						 BTINTEL_PCIE_CSR_MBOX_3_REG);
 319		snprintf(buf, sizeof(buf), "mbox_3: 0x%8.8x", mbox_reg);
 320		skb_put_data(skb, buf, strlen(buf));
 321		bt_dev_dbg(hdev, "%s", buf);
 322	}
 323
 324	if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX4) {
 325		mbox_reg = btintel_pcie_rd_reg32(data,
 326						 BTINTEL_PCIE_CSR_MBOX_4_REG);
 327		snprintf(buf, sizeof(buf), "mbox_4: 0x%8.8x", mbox_reg);
 328		skb_put_data(skb, buf, strlen(buf));
 329		bt_dev_dbg(hdev, "%s", buf);
 330	}
 331
 332	cr_hia = data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM];
 333	cr_tia = data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM];
 334	snprintf(buf, sizeof(buf), "rxq: cr_tia: %u cr_hia: %u", cr_tia, cr_hia);
 335	skb_put_data(skb, buf, strlen(buf));
 336	bt_dev_dbg(hdev, "%s", buf);
 337
 338	cr_hia = data->ia.cr_hia[BTINTEL_PCIE_TXQ_NUM];
 339	cr_tia = data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM];
 340	snprintf(buf, sizeof(buf), "txq: cr_tia: %u cr_hia: %u", cr_tia, cr_hia);
 341	skb_put_data(skb, buf, strlen(buf));
 342	bt_dev_dbg(hdev, "%s", buf);
 343	snprintf(buf, sizeof(buf), "--------------------------------");
 344	bt_dev_dbg(hdev, "%s", buf);
 345
 346	hci_recv_diag(hdev, skb);
 347}
 348
 349static int btintel_pcie_send_sync(struct btintel_pcie_data *data,
 350				  struct sk_buff *skb, u32 pkt_type, u16 opcode)
 351{
 352	int ret;
 353	u16 tfd_index;
 354	u32 old_ctxt;
 355	bool wait_on_alive = false;
 356	struct hci_dev *hdev = data->hdev;
 357
 358	struct txq *txq = &data->txq;
 359
 360	tfd_index = data->ia.tr_hia[BTINTEL_PCIE_TXQ_NUM];
 361
 362	if (tfd_index > txq->count)
 363		return -ERANGE;
 364
 365	/* Firmware raises alive interrupt on HCI_OP_RESET or
 366	 * BTINTEL_HCI_OP_RESET
 367	 */
 368	wait_on_alive = (pkt_type == BTINTEL_PCIE_HCI_CMD_PKT &&
 369		(opcode == BTINTEL_HCI_OP_RESET || opcode == HCI_OP_RESET));
 370
 371	if (wait_on_alive) {
 372		data->gp0_received = false;
 373		old_ctxt = data->alive_intr_ctxt;
 374		data->alive_intr_ctxt =
 375			(opcode == BTINTEL_HCI_OP_RESET ? BTINTEL_PCIE_INTEL_HCI_RESET1 :
 376				BTINTEL_PCIE_HCI_RESET);
 377		bt_dev_dbg(data->hdev, "sending cmd: 0x%4.4x alive context changed: %s  ->  %s",
 378			   opcode, btintel_pcie_alivectxt_state2str(old_ctxt),
 379			   btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt));
 380	}
 381
 382	memcpy(skb_push(skb, BTINTEL_PCIE_HCI_TYPE_LEN), &pkt_type,
 383	       BTINTEL_PCIE_HCI_TYPE_LEN);
 384
 385	/* Prepare for TX. It updates the TFD with the length of data and
 386	 * address of the DMA buffer, and copy the data to the DMA buffer
 387	 */
 388	btintel_pcie_prepare_tx(txq, tfd_index, skb);
 389
 390	tfd_index = (tfd_index + 1) % txq->count;
 391	data->ia.tr_hia[BTINTEL_PCIE_TXQ_NUM] = tfd_index;
 392
 393	/* Arm wait event condition */
 394	data->tx_wait_done = false;
 395
 396	/* Set the doorbell to notify the device */
 397	btintel_pcie_set_tx_db(data, tfd_index);
 398
 399	/* Wait for the complete interrupt - URBD0 */
 400	ret = wait_event_timeout(data->tx_wait_q, data->tx_wait_done,
 401				 msecs_to_jiffies(BTINTEL_PCIE_TX_WAIT_TIMEOUT_MS));
 402	if (!ret) {
 403		bt_dev_err(data->hdev, "Timeout (%u ms) on tx completion",
 404			   BTINTEL_PCIE_TX_WAIT_TIMEOUT_MS);
 405		btintel_pcie_dump_debug_registers(data->hdev);
 406		return -ETIME;
 407	}
 408
 409	if (wait_on_alive) {
 410		ret = wait_event_timeout(data->gp0_wait_q,
 411					 data->gp0_received,
 412					 msecs_to_jiffies(BTINTEL_DEFAULT_INTR_TIMEOUT_MS));
 413		if (!ret) {
 414			hdev->stat.err_tx++;
 415			bt_dev_err(hdev, "Timeout (%u ms)  on alive interrupt, alive context: %s",
 416				   BTINTEL_DEFAULT_INTR_TIMEOUT_MS,
 417				   btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt));
 418			return  -ETIME;
 419		}
 420	}
 421	return 0;
 422}
 423
 424/* Set the doorbell for RXQ to notify the device that @index (actually index-1)
 425 * is available to receive the data
 426 */
 427static void btintel_pcie_set_rx_db(struct btintel_pcie_data *data, u16 index)
 428{
 429	u32 val;
 430
 431	val = index;
 432	val |= (BTINTEL_PCIE_RX_DB_VEC << 16);
 433
 434	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_HBUS_TARG_WRPTR, val);
 435}
 436
 437/* Update the FRBD (free buffer descriptor) with the @frbd_index and the
 438 * DMA address of the free buffer.
 439 */
 440static void btintel_pcie_prepare_rx(struct rxq *rxq, u16 frbd_index)
 441{
 442	struct data_buf *buf;
 443	struct frbd *frbd;
 444
 445	/* Get the buffer of the FRBD for DMA */
 446	buf = &rxq->bufs[frbd_index];
 447
 448	frbd = &rxq->frbds[frbd_index];
 449	memset(frbd, 0, sizeof(*frbd));
 450
 451	/* Update FRBD */
 452	frbd->tag = frbd_index;
 453	frbd->addr = buf->data_p_addr;
 454}
 455
 456static int btintel_pcie_submit_rx(struct btintel_pcie_data *data)
 457{
 458	u16 frbd_index;
 459	struct rxq *rxq = &data->rxq;
 460
 461	frbd_index = data->ia.tr_hia[BTINTEL_PCIE_RXQ_NUM];
 462
 463	if (frbd_index > rxq->count)
 464		return -ERANGE;
 465
 466	/* Prepare for RX submit. It updates the FRBD with the address of DMA
 467	 * buffer
 468	 */
 469	btintel_pcie_prepare_rx(rxq, frbd_index);
 470
 471	frbd_index = (frbd_index + 1) % rxq->count;
 472	data->ia.tr_hia[BTINTEL_PCIE_RXQ_NUM] = frbd_index;
 473	ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_RXQ_NUM);
 474
 475	/* Set the doorbell to notify the device */
 476	btintel_pcie_set_rx_db(data, frbd_index);
 477
 478	return 0;
 479}
 480
 481static int btintel_pcie_start_rx(struct btintel_pcie_data *data)
 482{
 483	int i, ret;
 484	struct rxq *rxq = &data->rxq;
 485
 486	/* Post (BTINTEL_PCIE_RX_DESCS_COUNT - 3) buffers to overcome the
 487	 * hardware issues leading to race condition at the firmware.
 488	 */
 489
 490	for (i = 0; i < rxq->count - 3; i++) {
 491		ret = btintel_pcie_submit_rx(data);
 492		if (ret)
 493			return ret;
 494	}
 495
 496	return 0;
 497}
 498
 499static void btintel_pcie_reset_ia(struct btintel_pcie_data *data)
 500{
 501	memset(data->ia.tr_hia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES);
 502	memset(data->ia.tr_tia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES);
 503	memset(data->ia.cr_hia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES);
 504	memset(data->ia.cr_tia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES);
 505}
 506
 507static int btintel_pcie_reset_bt(struct btintel_pcie_data *data)
 508{
 509	u32 reg;
 510	int retry = 3;
 511
 512	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
 513
 514	reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA |
 515			BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT |
 516			BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT);
 517	reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON;
 518
 519	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
 520
 521	do {
 522		reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
 523		if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_STS)
 524			break;
 525		usleep_range(10000, 12000);
 526
 527	} while (--retry > 0);
 528	usleep_range(10000, 12000);
 529
 530	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
 531
 532	reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA |
 533			BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT |
 534			BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT);
 535	reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_SW_RESET;
 536	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
 537	usleep_range(10000, 12000);
 538
 539	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
 540	bt_dev_dbg(data->hdev, "csr register after reset: 0x%8.8x", reg);
 541
 542	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_BOOT_STAGE_REG);
 543
 544	/* If shared hardware reset is success then boot stage register shall be
 545	 * set to 0
 546	 */
 547	return reg == 0 ? 0 : -ENODEV;
 548}
 549
 550static void btintel_pcie_mac_init(struct btintel_pcie_data *data)
 551{
 552	u32 reg;
 553
 554	/* Set MAC_INIT bit to start primary bootloader */
 555	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
 556	reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT |
 557			BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON |
 558			BTINTEL_PCIE_CSR_FUNC_CTRL_SW_RESET);
 559	reg |= (BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA |
 560			BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT);
 561	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
 562}
 563
 564static int btintel_pcie_get_mac_access(struct btintel_pcie_data *data)
 565{
 566	u32 reg;
 567	int retry = 15;
 568
 569	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
 570
 571	reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_STOP_MAC_ACCESS_DIS;
 572	reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_XTAL_CLK_REQ;
 573	if ((reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_STS) == 0)
 574		reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ;
 575
 576	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
 577
 578	do {
 579		reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
 580		if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_STS)
 581			return 0;
 582		/* Need delay here for Target Access harwdware to settle down*/
 583		usleep_range(1000, 1200);
 584
 585	} while (--retry > 0);
 586
 587	return -ETIME;
 588}
 589
 590static void btintel_pcie_release_mac_access(struct btintel_pcie_data *data)
 591{
 592	u32 reg;
 593
 594	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
 595
 596	if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ)
 597		reg &= ~BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ;
 598
 599	if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_STOP_MAC_ACCESS_DIS)
 600		reg &= ~BTINTEL_PCIE_CSR_FUNC_CTRL_STOP_MAC_ACCESS_DIS;
 601
 602	if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_XTAL_CLK_REQ)
 603		reg &= ~BTINTEL_PCIE_CSR_FUNC_CTRL_XTAL_CLK_REQ;
 604
 605	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
 606}
 607
 608static void *btintel_pcie_copy_tlv(void *dest, enum btintel_pcie_tlv_type type,
 609				   void *data, size_t size)
 610{
 611	struct intel_tlv *tlv;
 612
 613	tlv = dest;
 614	tlv->type = type;
 615	tlv->len = size;
 616	memcpy(tlv->val, data, tlv->len);
 617	return dest + sizeof(*tlv) + size;
 618}
 619
 620static int btintel_pcie_read_dram_buffers(struct btintel_pcie_data *data)
 621{
 622	u32 offset, prev_size, wr_ptr_status, dump_size, data_len;
 623	struct btintel_pcie_dbgc *dbgc = &data->dbgc;
 624	struct hci_dev *hdev = data->hdev;
 625	u8 *pdata, *p, buf_idx;
 626	struct intel_tlv *tlv;
 627	struct timespec64 now;
 628	struct tm tm_now;
 629	char fw_build[128];
 630	char ts[128];
 631	char vendor[64];
 632	char driver[64];
 633
 634	if (!IS_ENABLED(CONFIG_DEV_COREDUMP))
 635		return -EOPNOTSUPP;
 636
 637
 638	wr_ptr_status = btintel_pcie_rd_dev_mem(data, BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS);
 639	offset = wr_ptr_status & BTINTEL_PCIE_DBG_OFFSET_BIT_MASK;
 640
 641	buf_idx = BTINTEL_PCIE_DBGC_DBG_BUF_IDX(wr_ptr_status);
 642	if (buf_idx > dbgc->count) {
 643		bt_dev_warn(hdev, "Buffer index is invalid");
 644		return -EINVAL;
 645	}
 646
 647	prev_size = buf_idx * BTINTEL_PCIE_DBGC_BUFFER_SIZE;
 648	if (prev_size + offset >= prev_size)
 649		data->dmp_hdr.write_ptr = prev_size + offset;
 650	else
 651		return -EINVAL;
 652
 653	snprintf(vendor, sizeof(vendor), "Vendor: Intel\n");
 654	snprintf(driver, sizeof(driver), "Driver: %s\n",
 655		 data->dmp_hdr.driver_name);
 656
 657	ktime_get_real_ts64(&now);
 658	time64_to_tm(now.tv_sec, 0, &tm_now);
 659	snprintf(ts, sizeof(ts), "Dump Time: %02d-%02d-%04ld %02d:%02d:%02d",
 660				 tm_now.tm_mday, tm_now.tm_mon + 1, tm_now.tm_year + 1900,
 661				 tm_now.tm_hour, tm_now.tm_min, tm_now.tm_sec);
 662
 663	snprintf(fw_build, sizeof(fw_build),
 664			    "Firmware Timestamp: Year %u WW %02u buildtype %u build %u",
 665			    2000 + (data->dmp_hdr.fw_timestamp >> 8),
 666			    data->dmp_hdr.fw_timestamp & 0xff, data->dmp_hdr.fw_build_type,
 667			    data->dmp_hdr.fw_build_num);
 668
 669	data_len = sizeof(*tlv) + sizeof(data->dmp_hdr.cnvi_bt) +
 670		sizeof(*tlv) + sizeof(data->dmp_hdr.write_ptr) +
 671		sizeof(*tlv) + sizeof(data->dmp_hdr.wrap_ctr) +
 672		sizeof(*tlv) + sizeof(data->dmp_hdr.trigger_reason) +
 673		sizeof(*tlv) + sizeof(data->dmp_hdr.fw_git_sha1) +
 674		sizeof(*tlv) + sizeof(data->dmp_hdr.cnvr_top) +
 675		sizeof(*tlv) + sizeof(data->dmp_hdr.cnvi_top) +
 676		sizeof(*tlv) + strlen(ts) +
 677		sizeof(*tlv) + strlen(fw_build) +
 678		sizeof(*tlv) + strlen(vendor) +
 679		sizeof(*tlv) + strlen(driver);
 680
 681	/*
 682	 * sizeof(u32) - signature
 683	 * sizeof(data_len) - to store tlv data size
 684	 * data_len - TLV data
 685	 */
 686	dump_size = sizeof(u32) + sizeof(data_len) + data_len;
 687
 688
 689	/* Add debug buffers data length to dump size */
 690	dump_size += BTINTEL_PCIE_DBGC_BUFFER_SIZE * dbgc->count;
 691
 692	pdata = vmalloc(dump_size);
 693	if (!pdata)
 694		return -ENOMEM;
 695	p = pdata;
 696
 697	*(u32 *)p = BTINTEL_PCIE_MAGIC_NUM;
 698	p += sizeof(u32);
 699
 700	*(u32 *)p = data_len;
 701	p += sizeof(u32);
 702
 703
 704	p = btintel_pcie_copy_tlv(p, BTINTEL_VENDOR, vendor, strlen(vendor));
 705	p = btintel_pcie_copy_tlv(p, BTINTEL_DRIVER, driver, strlen(driver));
 706	p = btintel_pcie_copy_tlv(p, BTINTEL_DUMP_TIME, ts, strlen(ts));
 707	p = btintel_pcie_copy_tlv(p, BTINTEL_FW_BUILD, fw_build,
 708				  strlen(fw_build));
 709	p = btintel_pcie_copy_tlv(p, BTINTEL_CNVI_BT, &data->dmp_hdr.cnvi_bt,
 710				  sizeof(data->dmp_hdr.cnvi_bt));
 711	p = btintel_pcie_copy_tlv(p, BTINTEL_WRITE_PTR, &data->dmp_hdr.write_ptr,
 712				  sizeof(data->dmp_hdr.write_ptr));
 713	p = btintel_pcie_copy_tlv(p, BTINTEL_WRAP_CTR, &data->dmp_hdr.wrap_ctr,
 714				  sizeof(data->dmp_hdr.wrap_ctr));
 715
 716	data->dmp_hdr.wrap_ctr = btintel_pcie_rd_dev_mem(data,
 717							 BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND);
 718
 719	p = btintel_pcie_copy_tlv(p, BTINTEL_TRIGGER_REASON, &data->dmp_hdr.trigger_reason,
 720				  sizeof(data->dmp_hdr.trigger_reason));
 721	p = btintel_pcie_copy_tlv(p, BTINTEL_FW_SHA, &data->dmp_hdr.fw_git_sha1,
 722				  sizeof(data->dmp_hdr.fw_git_sha1));
 723	p = btintel_pcie_copy_tlv(p, BTINTEL_CNVR_TOP, &data->dmp_hdr.cnvr_top,
 724				  sizeof(data->dmp_hdr.cnvr_top));
 725	p = btintel_pcie_copy_tlv(p, BTINTEL_CNVI_TOP, &data->dmp_hdr.cnvi_top,
 726				  sizeof(data->dmp_hdr.cnvi_top));
 727
 728	memcpy(p, dbgc->bufs[0].data, dbgc->count * BTINTEL_PCIE_DBGC_BUFFER_SIZE);
 729	dev_coredumpv(&hdev->dev, pdata, dump_size, GFP_KERNEL);
 730	return 0;
 731}
 732
 733static void btintel_pcie_dump_traces(struct hci_dev *hdev)
 734{
 735	struct btintel_pcie_data *data = hci_get_drvdata(hdev);
 736	int ret = 0;
 737
 738	ret = btintel_pcie_get_mac_access(data);
 739	if (ret) {
 740		bt_dev_err(hdev, "Failed to get mac access: (%d)", ret);
 741		return;
 742	}
 743
 744	ret = btintel_pcie_read_dram_buffers(data);
 745
 746	btintel_pcie_release_mac_access(data);
 747
 748	if (ret)
 749		bt_dev_err(hdev, "Failed to dump traces: (%d)", ret);
 750}
 751
 752/* This function enables BT function by setting BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT bit in
 753 * BTINTEL_PCIE_CSR_FUNC_CTRL_REG register and wait for MSI-X with
 754 * BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0.
 755 * Then the host reads firmware version from BTINTEL_CSR_F2D_MBX and the boot stage
 756 * from BTINTEL_PCIE_CSR_BOOT_STAGE_REG.
 757 */
 758static int btintel_pcie_enable_bt(struct btintel_pcie_data *data)
 759{
 760	int err;
 761	u32 reg;
 762
 763	data->gp0_received = false;
 764
 765	/* Update the DMA address of CI struct to CSR */
 766	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_CI_ADDR_LSB_REG,
 767			      data->ci_p_addr & 0xffffffff);
 768	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_CI_ADDR_MSB_REG,
 769			      (u64)data->ci_p_addr >> 32);
 770
 771	/* Reset the cached value of boot stage. it is updated by the MSI-X
 772	 * gp0 interrupt handler.
 773	 */
 774	data->boot_stage_cache = 0x0;
 775
 776	/* Set MAC_INIT bit to start primary bootloader */
 777	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
 778	reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT |
 779			BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON |
 780			BTINTEL_PCIE_CSR_FUNC_CTRL_SW_RESET);
 781	reg |= (BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA |
 782			BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT);
 783
 784	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
 785
 786	/* MAC is ready. Enable BT FUNC */
 787	btintel_pcie_set_reg_bits(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG,
 788				  BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT);
 789
 790	btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
 791
 792	/* wait for interrupt from the device after booting up to primary
 793	 * bootloader.
 794	 */
 795	data->alive_intr_ctxt = BTINTEL_PCIE_ROM;
 796	err = wait_event_timeout(data->gp0_wait_q, data->gp0_received,
 797				 msecs_to_jiffies(BTINTEL_DEFAULT_INTR_TIMEOUT_MS));
 798	if (!err)
 799		return -ETIME;
 800
 801	/* Check cached boot stage is BTINTEL_PCIE_CSR_BOOT_STAGE_ROM(BIT(0)) */
 802	if (~data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_ROM)
 803		return -ENODEV;
 804
 805	return 0;
 806}
 807
 808static inline bool btintel_pcie_in_op(struct btintel_pcie_data *data)
 809{
 810	return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_OPFW;
 811}
 812
 813static inline bool btintel_pcie_in_iml(struct btintel_pcie_data *data)
 814{
 815	return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_IML &&
 816		!(data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_OPFW);
 817}
 818
 819static inline bool btintel_pcie_in_d3(struct btintel_pcie_data *data)
 820{
 821	return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_D3_STATE_READY;
 822}
 823
 824static inline bool btintel_pcie_in_d0(struct btintel_pcie_data *data)
 825{
 826	return !(data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_D3_STATE_READY);
 827}
 828
 829static inline bool btintel_pcie_in_device_halt(struct btintel_pcie_data *data)
 830{
 831	return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_HALTED;
 832}
 833
 834static void btintel_pcie_wr_sleep_cntrl(struct btintel_pcie_data *data,
 835					u32 dxstate)
 836{
 837	bt_dev_dbg(data->hdev, "writing sleep_ctl_reg: 0x%8.8x", dxstate);
 838	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_IPC_SLEEP_CTL_REG, dxstate);
 839}
 840
 841static int btintel_pcie_read_device_mem(struct btintel_pcie_data *data,
 842					void *buf, u32 dev_addr, int len)
 843{
 844	int err;
 845	u32 *val = buf;
 846
 847	/* Get device mac access */
 848	err = btintel_pcie_get_mac_access(data);
 849	if (err) {
 850		bt_dev_err(data->hdev, "Failed to get mac access %d", err);
 851		return err;
 852	}
 853
 854	for (; len > 0; len -= 4, dev_addr += 4, val++)
 855		*val = btintel_pcie_rd_dev_mem(data, dev_addr);
 856
 857	btintel_pcie_release_mac_access(data);
 858
 859	return 0;
 860}
 861
 862static inline bool btintel_pcie_in_lockdown(struct btintel_pcie_data *data)
 863{
 864	return (data->boot_stage_cache &
 865		BTINTEL_PCIE_CSR_BOOT_STAGE_ROM_LOCKDOWN) ||
 866		(data->boot_stage_cache &
 867		 BTINTEL_PCIE_CSR_BOOT_STAGE_IML_LOCKDOWN);
 868}
 869
 870static inline bool btintel_pcie_in_error(struct btintel_pcie_data *data)
 871{
 872	return (data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_ERR) ||
 873		(data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_ABORT_HANDLER);
 874}
 875
 876static void btintel_pcie_msix_gp1_handler(struct btintel_pcie_data *data)
 877{
 878	bt_dev_err(data->hdev, "Received gp1 mailbox interrupt");
 879	btintel_pcie_dump_debug_registers(data->hdev);
 880}
 881
 882/* This function handles the MSI-X interrupt for gp0 cause (bit 0 in
 883 * BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES) which is sent for boot stage and image response.
 884 */
 885static void btintel_pcie_msix_gp0_handler(struct btintel_pcie_data *data)
 886{
 887	bool submit_rx, signal_waitq;
 888	u32 reg, old_ctxt;
 889
 890	/* This interrupt is for three different causes and it is not easy to
 891	 * know what causes the interrupt. So, it compares each register value
 892	 * with cached value and update it before it wake up the queue.
 893	 */
 894	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_BOOT_STAGE_REG);
 895	if (reg != data->boot_stage_cache)
 896		data->boot_stage_cache = reg;
 897
 898	bt_dev_dbg(data->hdev, "Alive context: %s old_boot_stage: 0x%8.8x new_boot_stage: 0x%8.8x",
 899		   btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt),
 900		   data->boot_stage_cache, reg);
 901	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IMG_RESPONSE_REG);
 902	if (reg != data->img_resp_cache)
 903		data->img_resp_cache = reg;
 904
 905	if (btintel_pcie_in_error(data)) {
 906		bt_dev_err(data->hdev, "Controller in error state");
 907		btintel_pcie_dump_debug_registers(data->hdev);
 908		return;
 909	}
 910
 911	if (btintel_pcie_in_lockdown(data)) {
 912		bt_dev_err(data->hdev, "Controller in lockdown state");
 913		btintel_pcie_dump_debug_registers(data->hdev);
 914		return;
 915	}
 916
 917	data->gp0_received = true;
 918
 919	old_ctxt = data->alive_intr_ctxt;
 920	submit_rx = false;
 921	signal_waitq = false;
 922
 923	switch (data->alive_intr_ctxt) {
 924	case BTINTEL_PCIE_ROM:
 925		data->alive_intr_ctxt = BTINTEL_PCIE_FW_DL;
 926		signal_waitq = true;
 927		break;
 928	case BTINTEL_PCIE_FW_DL:
 929		/* Error case is already handled. Ideally control shall not
 930		 * reach here
 931		 */
 932		break;
 933	case BTINTEL_PCIE_INTEL_HCI_RESET1:
 934		if (btintel_pcie_in_op(data)) {
 935			submit_rx = true;
 936			signal_waitq = true;
 937			break;
 938		}
 939
 940		if (btintel_pcie_in_iml(data)) {
 941			submit_rx = true;
 942			signal_waitq = true;
 943			data->alive_intr_ctxt = BTINTEL_PCIE_FW_DL;
 944			break;
 945		}
 946		break;
 947	case BTINTEL_PCIE_INTEL_HCI_RESET2:
 948		if (btintel_test_and_clear_flag(data->hdev, INTEL_WAIT_FOR_D0)) {
 949			btintel_wake_up_flag(data->hdev, INTEL_WAIT_FOR_D0);
 950			data->alive_intr_ctxt = BTINTEL_PCIE_D0;
 951		}
 952		break;
 953	case BTINTEL_PCIE_D0:
 954		if (btintel_pcie_in_d3(data)) {
 955			data->alive_intr_ctxt = BTINTEL_PCIE_D3;
 956			signal_waitq = true;
 957			break;
 958		}
 959		break;
 960	case BTINTEL_PCIE_D3:
 961		if (btintel_pcie_in_d0(data)) {
 962			data->alive_intr_ctxt = BTINTEL_PCIE_D0;
 963			submit_rx = true;
 964			signal_waitq = true;
 965			break;
 966		}
 967		break;
 968	case BTINTEL_PCIE_HCI_RESET:
 969		data->alive_intr_ctxt = BTINTEL_PCIE_D0;
 970		submit_rx = true;
 971		signal_waitq = true;
 972		break;
 973	default:
 974		bt_dev_err(data->hdev, "Unknown state: 0x%2.2x",
 975			   data->alive_intr_ctxt);
 976		break;
 977	}
 978
 979	if (submit_rx) {
 980		btintel_pcie_reset_ia(data);
 981		btintel_pcie_start_rx(data);
 982	}
 983
 984	if (signal_waitq) {
 985		bt_dev_dbg(data->hdev, "wake up gp0 wait_q");
 986		wake_up(&data->gp0_wait_q);
 987	}
 988
 989	if (old_ctxt != data->alive_intr_ctxt)
 990		bt_dev_dbg(data->hdev, "alive context changed: %s  ->  %s",
 991			   btintel_pcie_alivectxt_state2str(old_ctxt),
 992			   btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt));
 993}
 994
 995/* This function handles the MSX-X interrupt for rx queue 0 which is for TX
 996 */
 997static void btintel_pcie_msix_tx_handle(struct btintel_pcie_data *data)
 998{
 999	u16 cr_tia, cr_hia;
1000	struct txq *txq;
1001	struct urbd0 *urbd0;
1002
1003	cr_tia = data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM];
1004	cr_hia = data->ia.cr_hia[BTINTEL_PCIE_TXQ_NUM];
1005
1006	if (cr_tia == cr_hia)
1007		return;
1008
1009	txq = &data->txq;
1010
1011	while (cr_tia != cr_hia) {
1012		data->tx_wait_done = true;
1013		wake_up(&data->tx_wait_q);
1014
1015		urbd0 = &txq->urbd0s[cr_tia];
1016
1017		if (urbd0->tfd_index > txq->count)
1018			return;
1019
1020		cr_tia = (cr_tia + 1) % txq->count;
1021		data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM] = cr_tia;
1022		ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_TXQ_NUM);
1023	}
1024}
1025
1026static int btintel_pcie_recv_event(struct hci_dev *hdev, struct sk_buff *skb)
1027{
1028	struct hci_event_hdr *hdr = (void *)skb->data;
1029	struct btintel_pcie_data *data = hci_get_drvdata(hdev);
1030
1031	if (skb->len > HCI_EVENT_HDR_SIZE && hdr->evt == 0xff &&
1032	    hdr->plen > 0) {
1033		const void *ptr = skb->data + HCI_EVENT_HDR_SIZE + 1;
1034		unsigned int len = skb->len - HCI_EVENT_HDR_SIZE - 1;
1035
1036		if (btintel_test_flag(hdev, INTEL_BOOTLOADER)) {
1037			switch (skb->data[2]) {
1038			case 0x02:
1039				/* When switching to the operational firmware
1040				 * the device sends a vendor specific event
1041				 * indicating that the bootup completed.
1042				 */
1043				btintel_bootup(hdev, ptr, len);
1044
1045				/* If bootup event is from operational image,
1046				 * driver needs to write sleep control register to
1047				 * move into D0 state
1048				 */
1049				if (btintel_pcie_in_op(data)) {
1050					btintel_pcie_wr_sleep_cntrl(data, BTINTEL_PCIE_STATE_D0);
1051					data->alive_intr_ctxt = BTINTEL_PCIE_INTEL_HCI_RESET2;
1052					kfree_skb(skb);
1053					return 0;
1054				}
1055
1056				if (btintel_pcie_in_iml(data)) {
1057					/* In case of IML, there is no concept
1058					 * of D0 transition. Just mimic as if
1059					 * IML moved to D0 by clearing INTEL_WAIT_FOR_D0
1060					 * bit and waking up the task waiting on
1061					 * INTEL_WAIT_FOR_D0. This is required
1062					 * as intel_boot() is common function for
1063					 * both IML and OP image loading.
1064					 */
1065					if (btintel_test_and_clear_flag(data->hdev,
1066									INTEL_WAIT_FOR_D0))
1067						btintel_wake_up_flag(data->hdev,
1068								     INTEL_WAIT_FOR_D0);
1069				}
1070				kfree_skb(skb);
1071				return 0;
1072			case 0x06:
1073				/* When the firmware loading completes the
1074				 * device sends out a vendor specific event
1075				 * indicating the result of the firmware
1076				 * loading.
1077				 */
1078				btintel_secure_send_result(hdev, ptr, len);
1079				kfree_skb(skb);
1080				return 0;
1081			}
1082		}
1083
1084		/* This is a debug event that comes from IML and OP image when it
1085		 * starts execution. There is no need pass this event to stack.
1086		 */
1087		if (skb->data[2] == 0x97) {
1088			hci_recv_diag(hdev, skb);
1089			return 0;
1090		}
1091	}
1092
1093	return hci_recv_frame(hdev, skb);
1094}
1095/* Process the received rx data
1096 * It check the frame header to identify the data type and create skb
1097 * and calling HCI API
1098 */
1099static int btintel_pcie_recv_frame(struct btintel_pcie_data *data,
1100				       struct sk_buff *skb)
1101{
1102	int ret;
1103	u8 pkt_type;
1104	u16 plen;
1105	u32 pcie_pkt_type;
1106	void *pdata;
1107	struct hci_dev *hdev = data->hdev;
1108
1109	spin_lock(&data->hci_rx_lock);
1110
1111	/* The first 4 bytes indicates the Intel PCIe specific packet type */
1112	pdata = skb_pull_data(skb, BTINTEL_PCIE_HCI_TYPE_LEN);
1113	if (!pdata) {
1114		bt_dev_err(hdev, "Corrupted packet received");
1115		ret = -EILSEQ;
1116		goto exit_error;
1117	}
1118
1119	pcie_pkt_type = get_unaligned_le32(pdata);
1120
1121	switch (pcie_pkt_type) {
1122	case BTINTEL_PCIE_HCI_ACL_PKT:
1123		if (skb->len >= HCI_ACL_HDR_SIZE) {
1124			plen = HCI_ACL_HDR_SIZE + __le16_to_cpu(hci_acl_hdr(skb)->dlen);
1125			pkt_type = HCI_ACLDATA_PKT;
1126		} else {
1127			bt_dev_err(hdev, "ACL packet is too short");
1128			ret = -EILSEQ;
1129			goto exit_error;
1130		}
1131		break;
1132
1133	case BTINTEL_PCIE_HCI_SCO_PKT:
1134		if (skb->len >= HCI_SCO_HDR_SIZE) {
1135			plen = HCI_SCO_HDR_SIZE + hci_sco_hdr(skb)->dlen;
1136			pkt_type = HCI_SCODATA_PKT;
1137		} else {
1138			bt_dev_err(hdev, "SCO packet is too short");
1139			ret = -EILSEQ;
1140			goto exit_error;
1141		}
1142		break;
1143
1144	case BTINTEL_PCIE_HCI_EVT_PKT:
1145		if (skb->len >= HCI_EVENT_HDR_SIZE) {
1146			plen = HCI_EVENT_HDR_SIZE + hci_event_hdr(skb)->plen;
1147			pkt_type = HCI_EVENT_PKT;
1148		} else {
1149			bt_dev_err(hdev, "Event packet is too short");
1150			ret = -EILSEQ;
1151			goto exit_error;
1152		}
1153		break;
1154
1155	case BTINTEL_PCIE_HCI_ISO_PKT:
1156		if (skb->len >= HCI_ISO_HDR_SIZE) {
1157			plen = HCI_ISO_HDR_SIZE + __le16_to_cpu(hci_iso_hdr(skb)->dlen);
1158			pkt_type = HCI_ISODATA_PKT;
1159		} else {
1160			bt_dev_err(hdev, "ISO packet is too short");
1161			ret = -EILSEQ;
1162			goto exit_error;
1163		}
1164		break;
1165
1166	default:
1167		bt_dev_err(hdev, "Invalid packet type received: 0x%4.4x",
1168			   pcie_pkt_type);
1169		ret = -EINVAL;
1170		goto exit_error;
1171	}
1172
1173	if (skb->len < plen) {
1174		bt_dev_err(hdev, "Received corrupted packet. type: 0x%2.2x",
1175			   pkt_type);
1176		ret = -EILSEQ;
1177		goto exit_error;
1178	}
1179
1180	bt_dev_dbg(hdev, "pkt_type: 0x%2.2x len: %u", pkt_type, plen);
1181
1182	hci_skb_pkt_type(skb) = pkt_type;
1183	hdev->stat.byte_rx += plen;
1184	skb_trim(skb, plen);
1185
1186	if (pcie_pkt_type == BTINTEL_PCIE_HCI_EVT_PKT)
1187		ret = btintel_pcie_recv_event(hdev, skb);
1188	else
1189		ret = hci_recv_frame(hdev, skb);
1190	skb = NULL; /* skb is freed in the callee  */
1191
1192exit_error:
1193	if (skb)
1194		kfree_skb(skb);
1195
1196	if (ret)
1197		hdev->stat.err_rx++;
1198
1199	spin_unlock(&data->hci_rx_lock);
1200
1201	return ret;
1202}
1203
1204static void btintel_pcie_read_hwexp(struct btintel_pcie_data *data)
1205{
1206	int len, err, offset, pending;
1207	struct sk_buff *skb;
1208	u8 *buf, prefix[64];
1209	u32 addr, val;
1210	u16 pkt_len;
1211
1212	struct tlv {
1213		u8	type;
1214		__le16	len;
1215		u8	val[];
1216	} __packed;
1217
1218	struct tlv *tlv;
1219
1220	switch (data->dmp_hdr.cnvi_top & 0xfff) {
1221	case BTINTEL_CNVI_BLAZARI:
1222	case BTINTEL_CNVI_BLAZARIW:
1223		/* only from step B0 onwards */
1224		if (INTEL_CNVX_TOP_STEP(data->dmp_hdr.cnvi_top) != 0x01)
1225			return;
1226		len = BTINTEL_PCIE_BLZR_HWEXP_SIZE; /* exception data length */
1227		addr = BTINTEL_PCIE_BLZR_HWEXP_DMP_ADDR;
1228	break;
1229	case BTINTEL_CNVI_SCP:
1230		len = BTINTEL_PCIE_SCP_HWEXP_SIZE;
1231		addr = BTINTEL_PCIE_SCP_HWEXP_DMP_ADDR;
1232	break;
1233	default:
1234		bt_dev_err(data->hdev, "Unsupported cnvi 0x%8.8x", data->dmp_hdr.cnvi_top);
1235		return;
1236	}
1237
1238	buf = kzalloc(len, GFP_KERNEL);
1239	if (!buf)
1240		goto exit_on_error;
1241
1242	btintel_pcie_mac_init(data);
1243
1244	err = btintel_pcie_read_device_mem(data, buf, addr, len);
1245	if (err)
1246		goto exit_on_error;
1247
1248	val = get_unaligned_le32(buf);
1249	if (val != BTINTEL_PCIE_MAGIC_NUM) {
1250		bt_dev_err(data->hdev, "Invalid exception dump signature: 0x%8.8x",
1251			   val);
1252		goto exit_on_error;
1253	}
1254
1255	snprintf(prefix, sizeof(prefix), "Bluetooth: %s: ", bt_dev_name(data->hdev));
1256
1257	offset = 4;
1258	do {
1259		pending = len - offset;
1260		if (pending < sizeof(*tlv))
1261			break;
1262		tlv = (struct tlv *)(buf + offset);
1263
1264		/* If type == 0, then there are no more TLVs to be parsed */
1265		if (!tlv->type) {
1266			bt_dev_dbg(data->hdev, "Invalid TLV type 0");
1267			break;
1268		}
1269		pkt_len = le16_to_cpu(tlv->len);
1270		offset += sizeof(*tlv);
1271		pending = len - offset;
1272		if (pkt_len > pending)
1273			break;
1274
1275		offset += pkt_len;
1276
1277		 /* Only TLVs of type == 1 are HCI events, no need to process other
1278		  * TLVs
1279		  */
1280		if (tlv->type != 1)
1281			continue;
1282
1283		bt_dev_dbg(data->hdev, "TLV packet length: %u", pkt_len);
1284		if (pkt_len > HCI_MAX_EVENT_SIZE)
1285			break;
1286		skb = bt_skb_alloc(pkt_len, GFP_KERNEL);
1287		if (!skb)
1288			goto exit_on_error;
1289		hci_skb_pkt_type(skb) = HCI_EVENT_PKT;
1290		skb_put_data(skb, tlv->val, pkt_len);
1291
1292		/* copy Intel specific pcie packet type */
1293		val = BTINTEL_PCIE_HCI_EVT_PKT;
1294		memcpy(skb_push(skb, BTINTEL_PCIE_HCI_TYPE_LEN), &val,
1295		       BTINTEL_PCIE_HCI_TYPE_LEN);
1296
1297		print_hex_dump(KERN_DEBUG, prefix, DUMP_PREFIX_OFFSET, 16, 1,
1298			       tlv->val, pkt_len, false);
1299
1300		btintel_pcie_recv_frame(data, skb);
1301	} while (offset < len);
1302
1303exit_on_error:
1304	kfree(buf);
1305}
1306
1307static void btintel_pcie_msix_hw_exp_handler(struct btintel_pcie_data *data)
1308{
1309	bt_dev_err(data->hdev, "Received hw exception interrupt");
1310
1311	if (test_and_set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags))
1312		return;
1313
1314	if (test_and_set_bit(BTINTEL_PCIE_HWEXP_INPROGRESS, &data->flags))
1315		return;
1316
1317	/* Trigger device core dump when there is HW  exception */
1318	if (!test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags))
1319		data->dmp_hdr.trigger_reason = BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT;
1320
1321	queue_work(data->workqueue, &data->rx_work);
1322}
1323
1324static void btintel_pcie_rx_work(struct work_struct *work)
1325{
1326	struct btintel_pcie_data *data = container_of(work,
1327					struct btintel_pcie_data, rx_work);
1328	struct sk_buff *skb;
1329
1330	if (test_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags)) {
1331		btintel_pcie_dump_traces(data->hdev);
1332		clear_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags);
1333	}
1334
1335	if (test_bit(BTINTEL_PCIE_HWEXP_INPROGRESS, &data->flags)) {
1336		/* Unlike usb products, controller will not send hardware
1337		 * exception event on exception. Instead controller writes the
1338		 * hardware event to device memory along with optional debug
1339		 * events, raises MSIX and halts. Driver shall read the
1340		 * exception event from device memory and passes it stack for
1341		 * further processing.
1342		 */
1343		btintel_pcie_read_hwexp(data);
1344		clear_bit(BTINTEL_PCIE_HWEXP_INPROGRESS, &data->flags);
1345	}
1346
1347	/* Process the sk_buf in queue and send to the HCI layer */
1348	while ((skb = skb_dequeue(&data->rx_skb_q))) {
1349		btintel_pcie_recv_frame(data, skb);
1350	}
1351}
1352
1353/* create sk_buff with data and save it to queue and start RX work */
1354static int btintel_pcie_submit_rx_work(struct btintel_pcie_data *data, u8 status,
1355				       void *buf)
1356{
1357	int ret, len;
1358	struct rfh_hdr *rfh_hdr;
1359	struct sk_buff *skb;
1360
1361	rfh_hdr = buf;
1362
1363	len = rfh_hdr->packet_len;
1364	if (len <= 0) {
1365		ret = -EINVAL;
1366		goto resubmit;
1367	}
1368
1369	/* Remove RFH header */
1370	buf += sizeof(*rfh_hdr);
1371
1372	skb = alloc_skb(len, GFP_ATOMIC);
1373	if (!skb)
1374		goto resubmit;
1375
1376	skb_put_data(skb, buf, len);
1377	skb_queue_tail(&data->rx_skb_q, skb);
1378	queue_work(data->workqueue, &data->rx_work);
1379
1380resubmit:
1381	ret = btintel_pcie_submit_rx(data);
1382
1383	return ret;
1384}
1385
1386/* Handles the MSI-X interrupt for rx queue 1 which is for RX */
1387static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data)
1388{
1389	u16 cr_hia, cr_tia;
1390	struct rxq *rxq;
1391	struct urbd1 *urbd1;
1392	struct data_buf *buf;
1393	int ret;
1394	struct hci_dev *hdev = data->hdev;
1395
1396	cr_hia = data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM];
1397	cr_tia = data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM];
1398
1399	bt_dev_dbg(hdev, "RXQ: cr_hia: %u  cr_tia: %u", cr_hia, cr_tia);
1400
1401	/* Check CR_TIA and CR_HIA for change */
1402	if (cr_tia == cr_hia)
1403		return;
1404
1405	rxq = &data->rxq;
1406
1407	/* The firmware sends multiple CD in a single MSI-X and it needs to
1408	 * process all received CDs in this interrupt.
1409	 */
1410	while (cr_tia != cr_hia) {
1411		urbd1 = &rxq->urbd1s[cr_tia];
1412		ipc_print_urbd1(data->hdev, urbd1, cr_tia);
1413
1414		buf = &rxq->bufs[urbd1->frbd_tag];
1415		if (!buf) {
1416			bt_dev_err(hdev, "RXQ: failed to get the DMA buffer for %d",
1417				   urbd1->frbd_tag);
1418			return;
1419		}
1420
1421		ret = btintel_pcie_submit_rx_work(data, urbd1->status,
1422						  buf->data);
1423		if (ret) {
1424			bt_dev_err(hdev, "RXQ: failed to submit rx request");
1425			return;
1426		}
1427
1428		cr_tia = (cr_tia + 1) % rxq->count;
1429		data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM] = cr_tia;
1430		ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_RXQ_NUM);
1431	}
1432}
1433
1434static inline bool btintel_pcie_is_rxq_empty(struct btintel_pcie_data *data)
1435{
1436	return data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM] == data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM];
1437}
1438
1439static inline bool btintel_pcie_is_txackq_empty(struct btintel_pcie_data *data)
1440{
1441	return data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM] == data->ia.cr_hia[BTINTEL_PCIE_TXQ_NUM];
1442}
1443
1444static irqreturn_t btintel_pcie_irq_msix_handler(int irq, void *dev_id)
1445{
1446	struct msix_entry *entry = dev_id;
1447	struct btintel_pcie_data *data = btintel_pcie_get_data(entry);
1448	u32 intr_fh, intr_hw;
1449
1450	spin_lock(&data->irq_lock);
1451	intr_fh = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_CAUSES);
1452	intr_hw = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES);
1453
1454	/* Clear causes registers to avoid being handling the same cause */
1455	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_CAUSES, intr_fh);
1456	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES, intr_hw);
1457	spin_unlock(&data->irq_lock);
1458
1459	if (unlikely(!(intr_fh | intr_hw))) {
1460		/* Ignore interrupt, inta == 0 */
1461		return IRQ_NONE;
1462	}
1463
1464	/* This interrupt is raised when there is an hardware exception */
1465	if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_HWEXP)
1466		btintel_pcie_msix_hw_exp_handler(data);
1467
1468	if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP1)
1469		btintel_pcie_msix_gp1_handler(data);
1470
1471
1472	/* For TX */
1473	if (intr_fh & BTINTEL_PCIE_MSIX_FH_INT_CAUSES_0) {
1474		btintel_pcie_msix_tx_handle(data);
1475		if (!btintel_pcie_is_rxq_empty(data))
1476			btintel_pcie_msix_rx_handle(data);
1477	}
1478
1479	/* For RX */
1480	if (intr_fh & BTINTEL_PCIE_MSIX_FH_INT_CAUSES_1) {
1481		btintel_pcie_msix_rx_handle(data);
1482		if (!btintel_pcie_is_txackq_empty(data))
1483			btintel_pcie_msix_tx_handle(data);
1484	}
1485
1486	/* This interrupt is triggered by the firmware after updating
1487	 * boot_stage register and image_response register
1488	 */
1489	if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0)
1490		btintel_pcie_msix_gp0_handler(data);
1491
1492	/*
1493	 * Before sending the interrupt the HW disables it to prevent a nested
1494	 * interrupt. This is done by writing 1 to the corresponding bit in
1495	 * the mask register. After handling the interrupt, it should be
1496	 * re-enabled by clearing this bit. This register is defined as write 1
1497	 * clear (W1C) register, meaning that it's cleared by writing 1
1498	 * to the bit.
1499	 */
1500	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_AUTOMASK_ST,
1501			      BIT(entry->entry));
1502
1503	return IRQ_HANDLED;
1504}
1505
1506/* This function requests the irq for MSI-X and registers the handlers per irq.
1507 * Currently, it requests only 1 irq for all interrupt causes.
1508 */
1509static int btintel_pcie_setup_irq(struct btintel_pcie_data *data)
1510{
1511	int err;
1512	int num_irqs, i;
1513
1514	for (i = 0; i < BTINTEL_PCIE_MSIX_VEC_MAX; i++)
1515		data->msix_entries[i].entry = i;
1516
1517	num_irqs = pci_alloc_irq_vectors(data->pdev, BTINTEL_PCIE_MSIX_VEC_MIN,
1518					 BTINTEL_PCIE_MSIX_VEC_MAX, PCI_IRQ_MSIX);
1519	if (num_irqs < 0)
1520		return num_irqs;
1521
1522	data->alloc_vecs = num_irqs;
1523	data->msix_enabled = 1;
1524	data->def_irq = 0;
1525
1526	/* setup irq handler */
1527	for (i = 0; i < data->alloc_vecs; i++) {
1528		struct msix_entry *msix_entry;
1529
1530		msix_entry = &data->msix_entries[i];
1531		msix_entry->vector = pci_irq_vector(data->pdev, i);
1532
1533		err = devm_request_threaded_irq(&data->pdev->dev,
1534						msix_entry->vector,
1535						NULL,
1536						btintel_pcie_irq_msix_handler,
1537						IRQF_ONESHOT | IRQF_SHARED,
1538						KBUILD_MODNAME,
1539						msix_entry);
1540		if (err) {
1541			pci_free_irq_vectors(data->pdev);
1542			data->alloc_vecs = 0;
1543			return err;
1544		}
1545	}
1546	return 0;
1547}
1548
1549struct btintel_pcie_causes_list {
1550	u32 cause;
1551	u32 mask_reg;
1552	u8 cause_num;
1553};
1554
1555static struct btintel_pcie_causes_list causes_list[] = {
1556	{ BTINTEL_PCIE_MSIX_FH_INT_CAUSES_0,	BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK,	0x00 },
1557	{ BTINTEL_PCIE_MSIX_FH_INT_CAUSES_1,	BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK,	0x01 },
1558	{ BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0,	BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK,	0x20 },
1559	{ BTINTEL_PCIE_MSIX_HW_INT_CAUSES_HWEXP, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK,	0x23 },
1560};
1561
1562/* This function configures the interrupt masks for both HW_INT_CAUSES and
1563 * FH_INT_CAUSES which are meaningful to us.
1564 *
1565 * After resetting BT function via PCIE FLR or FUNC_CTRL reset, the driver
1566 * need to call this function again to configure since the masks
1567 * are reset to 0xFFFFFFFF after reset.
1568 */
1569static void btintel_pcie_config_msix(struct btintel_pcie_data *data)
1570{
1571	int i;
1572	int val = data->def_irq | BTINTEL_PCIE_MSIX_NON_AUTO_CLEAR_CAUSE;
1573
1574	/* Set Non Auto Clear Cause */
1575	for (i = 0; i < ARRAY_SIZE(causes_list); i++) {
1576		btintel_pcie_wr_reg8(data,
1577				     BTINTEL_PCIE_CSR_MSIX_IVAR(causes_list[i].cause_num),
1578				     val);
1579		btintel_pcie_clr_reg_bits(data,
1580					  causes_list[i].mask_reg,
1581					  causes_list[i].cause);
1582	}
1583
1584	/* Save the initial interrupt mask */
1585	data->fh_init_mask = ~btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK);
1586	data->hw_init_mask = ~btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK);
1587}
1588
1589static int btintel_pcie_config_pcie(struct pci_dev *pdev,
1590				    struct btintel_pcie_data *data)
1591{
1592	int err;
1593
1594	err = pcim_enable_device(pdev);
1595	if (err)
1596		return err;
1597
1598	pci_set_master(pdev);
1599
1600	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1601	if (err) {
1602		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1603		if (err)
1604			return err;
1605	}
1606
1607	data->base_addr = pcim_iomap_region(pdev, 0, KBUILD_MODNAME);
1608	if (IS_ERR(data->base_addr))
1609		return PTR_ERR(data->base_addr);
1610
1611	err = btintel_pcie_setup_irq(data);
1612	if (err)
1613		return err;
1614
1615	/* Configure MSI-X with causes list */
1616	btintel_pcie_config_msix(data);
1617
1618	return 0;
1619}
1620
1621static void btintel_pcie_init_ci(struct btintel_pcie_data *data,
1622				 struct ctx_info *ci)
1623{
1624	ci->version = 0x1;
1625	ci->size = sizeof(*ci);
1626	ci->config = 0x0000;
1627	ci->addr_cr_hia = data->ia.cr_hia_p_addr;
1628	ci->addr_tr_tia = data->ia.tr_tia_p_addr;
1629	ci->addr_cr_tia = data->ia.cr_tia_p_addr;
1630	ci->addr_tr_hia = data->ia.tr_hia_p_addr;
1631	ci->num_cr_ia = BTINTEL_PCIE_NUM_QUEUES;
1632	ci->num_tr_ia = BTINTEL_PCIE_NUM_QUEUES;
1633	ci->addr_urbdq0 = data->txq.urbd0s_p_addr;
1634	ci->addr_tfdq = data->txq.tfds_p_addr;
1635	ci->num_tfdq = data->txq.count;
1636	ci->num_urbdq0 = data->txq.count;
1637	ci->tfdq_db_vec = BTINTEL_PCIE_TXQ_NUM;
1638	ci->urbdq0_db_vec = BTINTEL_PCIE_TXQ_NUM;
1639	ci->rbd_size = BTINTEL_PCIE_RBD_SIZE_4K;
1640	ci->addr_frbdq = data->rxq.frbds_p_addr;
1641	ci->num_frbdq = data->rxq.count;
1642	ci->frbdq_db_vec = BTINTEL_PCIE_RXQ_NUM;
1643	ci->addr_urbdq1 = data->rxq.urbd1s_p_addr;
1644	ci->num_urbdq1 = data->rxq.count;
1645	ci->urbdq_db_vec = BTINTEL_PCIE_RXQ_NUM;
1646
1647	ci->dbg_output_mode = 0x01;
1648	ci->dbgc_addr = data->dbgc.frag_p_addr;
1649	ci->dbgc_size = data->dbgc.frag_size;
1650	ci->dbg_preset = 0x00;
1651}
1652
1653static void btintel_pcie_free_txq_bufs(struct btintel_pcie_data *data,
1654				       struct txq *txq)
1655{
1656	/* Free data buffers first */
1657	dma_free_coherent(&data->pdev->dev, txq->count * BTINTEL_PCIE_BUFFER_SIZE,
1658			  txq->buf_v_addr, txq->buf_p_addr);
1659	kfree(txq->bufs);
1660}
1661
1662static int btintel_pcie_setup_txq_bufs(struct btintel_pcie_data *data,
1663				       struct txq *txq)
1664{
1665	int i;
1666	struct data_buf *buf;
1667
1668	/* Allocate the same number of buffers as the descriptor */
1669	txq->bufs = kmalloc_array(txq->count, sizeof(*buf), GFP_KERNEL);
1670	if (!txq->bufs)
1671		return -ENOMEM;
1672
1673	/* Allocate full chunk of data buffer for DMA first and do indexing and
1674	 * initialization next, so it can be freed easily
1675	 */
1676	txq->buf_v_addr = dma_alloc_coherent(&data->pdev->dev,
1677					     txq->count * BTINTEL_PCIE_BUFFER_SIZE,
1678					     &txq->buf_p_addr,
1679					     GFP_KERNEL | __GFP_NOWARN);
1680	if (!txq->buf_v_addr) {
1681		kfree(txq->bufs);
1682		return -ENOMEM;
1683	}
1684
1685	/* Setup the allocated DMA buffer to bufs. Each data_buf should
1686	 * have virtual address and physical address
1687	 */
1688	for (i = 0; i < txq->count; i++) {
1689		buf = &txq->bufs[i];
1690		buf->data_p_addr = txq->buf_p_addr + (i * BTINTEL_PCIE_BUFFER_SIZE);
1691		buf->data = txq->buf_v_addr + (i * BTINTEL_PCIE_BUFFER_SIZE);
1692	}
1693
1694	return 0;
1695}
1696
1697static void btintel_pcie_free_rxq_bufs(struct btintel_pcie_data *data,
1698				       struct rxq *rxq)
1699{
1700	/* Free data buffers first */
1701	dma_free_coherent(&data->pdev->dev, rxq->count * BTINTEL_PCIE_BUFFER_SIZE,
1702			  rxq->buf_v_addr, rxq->buf_p_addr);
1703	kfree(rxq->bufs);
1704}
1705
1706static int btintel_pcie_setup_rxq_bufs(struct btintel_pcie_data *data,
1707				       struct rxq *rxq)
1708{
1709	int i;
1710	struct data_buf *buf;
1711
1712	/* Allocate the same number of buffers as the descriptor */
1713	rxq->bufs = kmalloc_array(rxq->count, sizeof(*buf), GFP_KERNEL);
1714	if (!rxq->bufs)
1715		return -ENOMEM;
1716
1717	/* Allocate full chunk of data buffer for DMA first and do indexing and
1718	 * initialization next, so it can be freed easily
1719	 */
1720	rxq->buf_v_addr = dma_alloc_coherent(&data->pdev->dev,
1721					     rxq->count * BTINTEL_PCIE_BUFFER_SIZE,
1722					     &rxq->buf_p_addr,
1723					     GFP_KERNEL | __GFP_NOWARN);
1724	if (!rxq->buf_v_addr) {
1725		kfree(rxq->bufs);
1726		return -ENOMEM;
1727	}
1728
1729	/* Setup the allocated DMA buffer to bufs. Each data_buf should
1730	 * have virtual address and physical address
1731	 */
1732	for (i = 0; i < rxq->count; i++) {
1733		buf = &rxq->bufs[i];
1734		buf->data_p_addr = rxq->buf_p_addr + (i * BTINTEL_PCIE_BUFFER_SIZE);
1735		buf->data = rxq->buf_v_addr + (i * BTINTEL_PCIE_BUFFER_SIZE);
1736	}
1737
1738	return 0;
1739}
1740
1741static void btintel_pcie_setup_ia(struct btintel_pcie_data *data,
1742				  dma_addr_t p_addr, void *v_addr,
1743				  struct ia *ia)
1744{
1745	/* TR Head Index Array */
1746	ia->tr_hia_p_addr = p_addr;
1747	ia->tr_hia = v_addr;
1748
1749	/* TR Tail Index Array */
1750	ia->tr_tia_p_addr = p_addr + sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES;
1751	ia->tr_tia = v_addr + sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES;
1752
1753	/* CR Head index Array */
1754	ia->cr_hia_p_addr = p_addr + (sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 2);
1755	ia->cr_hia = v_addr + (sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 2);
1756
1757	/* CR Tail Index Array */
1758	ia->cr_tia_p_addr = p_addr + (sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 3);
1759	ia->cr_tia = v_addr + (sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 3);
1760}
1761
1762static void btintel_pcie_free(struct btintel_pcie_data *data)
1763{
1764	btintel_pcie_free_rxq_bufs(data, &data->rxq);
1765	btintel_pcie_free_txq_bufs(data, &data->txq);
1766
1767	dma_pool_free(data->dma_pool, data->dma_v_addr, data->dma_p_addr);
1768	dma_pool_destroy(data->dma_pool);
1769}
1770
1771/* Allocate tx and rx queues, any related data structures and buffers.
1772 */
1773static int btintel_pcie_alloc(struct btintel_pcie_data *data)
1774{
1775	int err = 0;
1776	size_t total;
1777	dma_addr_t p_addr;
1778	void *v_addr;
1779
1780	/* Allocate the chunk of DMA memory for descriptors, index array, and
1781	 * context information, instead of allocating individually.
1782	 * The DMA memory for data buffer is allocated while setting up the
1783	 * each queue.
1784	 *
1785	 * Total size is sum of the following
1786	 *  + size of TFD * Number of descriptors in queue
1787	 *  + size of URBD0 * Number of descriptors in queue
1788	 *  + size of FRBD * Number of descriptors in queue
1789	 *  + size of URBD1 * Number of descriptors in queue
1790	 *  + size of index * Number of queues(2) * type of index array(4)
1791	 *  + size of context information
1792	 */
1793	total = (sizeof(struct tfd) + sizeof(struct urbd0)) * BTINTEL_PCIE_TX_DESCS_COUNT;
1794	total += (sizeof(struct frbd) + sizeof(struct urbd1)) * BTINTEL_PCIE_RX_DESCS_COUNT;
1795
1796	/* Add the sum of size of index array and size of ci struct */
1797	total += (sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 4) + sizeof(struct ctx_info);
1798
1799	/* Allocate DMA Pool */
1800	data->dma_pool = dma_pool_create(KBUILD_MODNAME, &data->pdev->dev,
1801					 total, BTINTEL_PCIE_DMA_POOL_ALIGNMENT, 0);
1802	if (!data->dma_pool) {
1803		err = -ENOMEM;
1804		goto exit_error;
1805	}
1806
1807	v_addr = dma_pool_zalloc(data->dma_pool, GFP_KERNEL | __GFP_NOWARN,
1808				 &p_addr);
1809	if (!v_addr) {
1810		dma_pool_destroy(data->dma_pool);
1811		err = -ENOMEM;
1812		goto exit_error;
1813	}
1814
1815	data->dma_p_addr = p_addr;
1816	data->dma_v_addr = v_addr;
1817
1818	/* Setup descriptor count */
1819	data->txq.count = BTINTEL_PCIE_TX_DESCS_COUNT;
1820	data->rxq.count = BTINTEL_PCIE_RX_DESCS_COUNT;
1821
1822	/* Setup tfds */
1823	data->txq.tfds_p_addr = p_addr;
1824	data->txq.tfds = v_addr;
1825
1826	p_addr += (sizeof(struct tfd) * BTINTEL_PCIE_TX_DESCS_COUNT);
1827	v_addr += (sizeof(struct tfd) * BTINTEL_PCIE_TX_DESCS_COUNT);
1828
1829	/* Setup urbd0 */
1830	data->txq.urbd0s_p_addr = p_addr;
1831	data->txq.urbd0s = v_addr;
1832
1833	p_addr += (sizeof(struct urbd0) * BTINTEL_PCIE_TX_DESCS_COUNT);
1834	v_addr += (sizeof(struct urbd0) * BTINTEL_PCIE_TX_DESCS_COUNT);
1835
1836	/* Setup FRBD*/
1837	data->rxq.frbds_p_addr = p_addr;
1838	data->rxq.frbds = v_addr;
1839
1840	p_addr += (sizeof(struct frbd) * BTINTEL_PCIE_RX_DESCS_COUNT);
1841	v_addr += (sizeof(struct frbd) * BTINTEL_PCIE_RX_DESCS_COUNT);
1842
1843	/* Setup urbd1 */
1844	data->rxq.urbd1s_p_addr = p_addr;
1845	data->rxq.urbd1s = v_addr;
1846
1847	p_addr += (sizeof(struct urbd1) * BTINTEL_PCIE_RX_DESCS_COUNT);
1848	v_addr += (sizeof(struct urbd1) * BTINTEL_PCIE_RX_DESCS_COUNT);
1849
1850	/* Setup data buffers for txq */
1851	err = btintel_pcie_setup_txq_bufs(data, &data->txq);
1852	if (err)
1853		goto exit_error_pool;
1854
1855	/* Setup data buffers for rxq */
1856	err = btintel_pcie_setup_rxq_bufs(data, &data->rxq);
1857	if (err)
1858		goto exit_error_txq;
1859
1860	/* Setup Index Array */
1861	btintel_pcie_setup_ia(data, p_addr, v_addr, &data->ia);
1862
1863	/* Setup data buffers for dbgc */
1864	err = btintel_pcie_setup_dbgc(data);
1865	if (err)
1866		goto exit_error_txq;
1867
1868	/* Setup Context Information */
1869	p_addr += sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 4;
1870	v_addr += sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 4;
1871
1872	data->ci = v_addr;
1873	data->ci_p_addr = p_addr;
1874
1875	/* Initialize the CI */
1876	btintel_pcie_init_ci(data, data->ci);
1877
1878	return 0;
1879
1880exit_error_txq:
1881	btintel_pcie_free_txq_bufs(data, &data->txq);
1882exit_error_pool:
1883	dma_pool_free(data->dma_pool, data->dma_v_addr, data->dma_p_addr);
1884	dma_pool_destroy(data->dma_pool);
1885exit_error:
1886	return err;
1887}
1888
1889static int btintel_pcie_open(struct hci_dev *hdev)
1890{
1891	bt_dev_dbg(hdev, "");
1892
1893	return 0;
1894}
1895
1896static int btintel_pcie_close(struct hci_dev *hdev)
1897{
1898	bt_dev_dbg(hdev, "");
1899
1900	return 0;
1901}
1902
1903static int btintel_pcie_inject_cmd_complete(struct hci_dev *hdev, __u16 opcode)
1904{
1905	struct sk_buff *skb;
1906	struct hci_event_hdr *hdr;
1907	struct hci_ev_cmd_complete *evt;
1908
1909	skb = bt_skb_alloc(sizeof(*hdr) + sizeof(*evt) + 1, GFP_KERNEL);
1910	if (!skb)
1911		return -ENOMEM;
1912
1913	hdr = (struct hci_event_hdr *)skb_put(skb, sizeof(*hdr));
1914	hdr->evt = HCI_EV_CMD_COMPLETE;
1915	hdr->plen = sizeof(*evt) + 1;
1916
1917	evt = (struct hci_ev_cmd_complete *)skb_put(skb, sizeof(*evt));
1918	evt->ncmd = 0x01;
1919	evt->opcode = cpu_to_le16(opcode);
1920
1921	*(u8 *)skb_put(skb, 1) = 0x00;
1922
1923	hci_skb_pkt_type(skb) = HCI_EVENT_PKT;
1924
1925	return hci_recv_frame(hdev, skb);
1926}
1927
1928static int btintel_pcie_send_frame(struct hci_dev *hdev,
1929				       struct sk_buff *skb)
1930{
1931	struct btintel_pcie_data *data = hci_get_drvdata(hdev);
1932	struct hci_command_hdr *cmd;
1933	__u16 opcode = ~0;
1934	int ret;
1935	u32 type;
1936
1937	if (test_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags))
1938		return -ENODEV;
1939
1940	/* Due to the fw limitation, the type header of the packet should be
1941	 * 4 bytes unlike 1 byte for UART. In UART, the firmware can read
1942	 * the first byte to get the packet type and redirect the rest of data
1943	 * packet to the right handler.
1944	 *
1945	 * But for PCIe, THF(Transfer Flow Handler) fetches the 4 bytes of data
1946	 * from DMA memory and by the time it reads the first 4 bytes, it has
1947	 * already consumed some part of packet. Thus the packet type indicator
1948	 * for iBT PCIe is 4 bytes.
1949	 *
1950	 * Luckily, when HCI core creates the skb, it allocates 8 bytes of
1951	 * head room for profile and driver use, and before sending the data
1952	 * to the device, append the iBT PCIe packet type in the front.
1953	 */
1954	switch (hci_skb_pkt_type(skb)) {
1955	case HCI_COMMAND_PKT:
1956		type = BTINTEL_PCIE_HCI_CMD_PKT;
1957		cmd = (void *)skb->data;
1958		opcode = le16_to_cpu(cmd->opcode);
1959		if (btintel_test_flag(hdev, INTEL_BOOTLOADER)) {
1960			struct hci_command_hdr *cmd = (void *)skb->data;
1961			__u16 opcode = le16_to_cpu(cmd->opcode);
1962
1963			/* When the BTINTEL_HCI_OP_RESET command is issued to
1964			 * boot into the operational firmware, it will actually
1965			 * not send a command complete event. To keep the flow
1966			 * control working inject that event here.
1967			 */
1968			if (opcode == BTINTEL_HCI_OP_RESET)
1969				btintel_pcie_inject_cmd_complete(hdev, opcode);
1970		}
1971
1972		hdev->stat.cmd_tx++;
1973		break;
1974	case HCI_ACLDATA_PKT:
1975		type = BTINTEL_PCIE_HCI_ACL_PKT;
1976		hdev->stat.acl_tx++;
1977		break;
1978	case HCI_SCODATA_PKT:
1979		type = BTINTEL_PCIE_HCI_SCO_PKT;
1980		hdev->stat.sco_tx++;
1981		break;
1982	case HCI_ISODATA_PKT:
1983		type = BTINTEL_PCIE_HCI_ISO_PKT;
1984		break;
1985	default:
1986		bt_dev_err(hdev, "Unknown HCI packet type");
1987		return -EILSEQ;
1988	}
1989
1990	ret = btintel_pcie_send_sync(data, skb, type, opcode);
1991	if (ret) {
1992		hdev->stat.err_tx++;
1993		bt_dev_err(hdev, "Failed to send frame (%d)", ret);
1994		goto exit_error;
1995	}
1996
1997	hdev->stat.byte_tx += skb->len;
1998	kfree_skb(skb);
1999
2000exit_error:
2001	return ret;
2002}
2003
2004static void btintel_pcie_release_hdev(struct btintel_pcie_data *data)
2005{
2006	struct hci_dev *hdev;
2007
2008	hdev = data->hdev;
2009	hci_unregister_dev(hdev);
2010	hci_free_dev(hdev);
2011	data->hdev = NULL;
2012}
2013
2014static void btintel_pcie_disable_interrupts(struct btintel_pcie_data *data)
2015{
2016	spin_lock(&data->irq_lock);
2017	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, data->fh_init_mask);
2018	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, data->hw_init_mask);
2019	spin_unlock(&data->irq_lock);
2020}
2021
2022static void btintel_pcie_enable_interrupts(struct btintel_pcie_data *data)
2023{
2024	spin_lock(&data->irq_lock);
2025	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, ~data->fh_init_mask);
2026	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, ~data->hw_init_mask);
2027	spin_unlock(&data->irq_lock);
2028}
2029
2030static void btintel_pcie_synchronize_irqs(struct btintel_pcie_data *data)
2031{
2032	for (int i = 0; i < data->alloc_vecs; i++)
2033		synchronize_irq(data->msix_entries[i].vector);
2034}
2035
2036static int btintel_pcie_setup_internal(struct hci_dev *hdev)
2037{
2038	struct btintel_pcie_data *data = hci_get_drvdata(hdev);
2039	const u8 param[1] = { 0xFF };
2040	struct intel_version_tlv ver_tlv;
2041	struct sk_buff *skb;
2042	int err;
2043
2044	BT_DBG("%s", hdev->name);
2045
2046	skb = __hci_cmd_sync(hdev, 0xfc05, 1, param, HCI_CMD_TIMEOUT);
2047	if (IS_ERR(skb)) {
2048		bt_dev_err(hdev, "Reading Intel version command failed (%ld)",
2049			   PTR_ERR(skb));
2050		return PTR_ERR(skb);
2051	}
2052
2053	/* Check the status */
2054	if (skb->data[0]) {
2055		bt_dev_err(hdev, "Intel Read Version command failed (%02x)",
2056			   skb->data[0]);
2057		err = -EIO;
2058		goto exit_error;
2059	}
2060
2061	/* Apply the common HCI quirks for Intel device */
2062	hci_set_quirk(hdev, HCI_QUIRK_STRICT_DUPLICATE_FILTER);
2063	hci_set_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY);
2064	hci_set_quirk(hdev, HCI_QUIRK_NON_PERSISTENT_DIAG);
2065
2066	/* Set up the quality report callback for Intel devices */
2067	hdev->set_quality_report = btintel_set_quality_report;
2068
2069	memset(&ver_tlv, 0, sizeof(ver_tlv));
2070	/* For TLV type device, parse the tlv data */
2071	err = btintel_parse_version_tlv(hdev, &ver_tlv, skb);
2072	if (err) {
2073		bt_dev_err(hdev, "Failed to parse TLV version information");
2074		goto exit_error;
2075	}
2076
2077	switch (INTEL_HW_PLATFORM(ver_tlv.cnvi_bt)) {
2078	case 0x37:
2079		break;
2080	default:
2081		bt_dev_err(hdev, "Unsupported Intel hardware platform (0x%2x)",
2082			   INTEL_HW_PLATFORM(ver_tlv.cnvi_bt));
2083		err = -EINVAL;
2084		goto exit_error;
2085	}
2086
2087	/* Check for supported iBT hardware variants of this firmware
2088	 * loading method.
2089	 *
2090	 * This check has been put in place to ensure correct forward
2091	 * compatibility options when newer hardware variants come
2092	 * along.
2093	 */
2094	switch (INTEL_HW_VARIANT(ver_tlv.cnvi_bt)) {
2095	case 0x1e:	/* BzrI */
2096	case 0x1f:	/* ScP  */
2097	case 0x22:	/* BzrIW */
2098		/* Display version information of TLV type */
2099		btintel_version_info_tlv(hdev, &ver_tlv);
2100
2101		/* Apply the device specific HCI quirks for TLV based devices
2102		 *
2103		 * All TLV based devices support WBS
2104		 */
2105		hci_set_quirk(hdev, HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED);
2106
2107		/* Setup MSFT Extension support */
2108		btintel_set_msft_opcode(hdev,
2109					INTEL_HW_VARIANT(ver_tlv.cnvi_bt));
2110
2111		err = btintel_bootloader_setup_tlv(hdev, &ver_tlv);
2112		if (err)
2113			goto exit_error;
2114		break;
2115	default:
2116		bt_dev_err(hdev, "Unsupported Intel hw variant (%u)",
2117			   INTEL_HW_VARIANT(ver_tlv.cnvi_bt));
2118		err = -EINVAL;
2119		goto exit_error;
2120		break;
2121	}
2122
2123	data->dmp_hdr.cnvi_top = ver_tlv.cnvi_top;
2124	data->dmp_hdr.cnvr_top = ver_tlv.cnvr_top;
2125	data->dmp_hdr.fw_timestamp = ver_tlv.timestamp;
2126	data->dmp_hdr.fw_build_type = ver_tlv.build_type;
2127	data->dmp_hdr.fw_build_num = ver_tlv.build_num;
2128	data->dmp_hdr.cnvi_bt = ver_tlv.cnvi_bt;
2129
2130	if (ver_tlv.img_type == 0x02 || ver_tlv.img_type == 0x03)
2131		data->dmp_hdr.fw_git_sha1 = ver_tlv.git_sha1;
2132
2133	btintel_print_fseq_info(hdev);
2134exit_error:
2135	kfree_skb(skb);
2136
2137	return err;
2138}
2139
2140static int btintel_pcie_setup(struct hci_dev *hdev)
2141{
2142	int err, fw_dl_retry = 0;
2143	struct btintel_pcie_data *data = hci_get_drvdata(hdev);
2144
2145	while ((err = btintel_pcie_setup_internal(hdev)) && fw_dl_retry++ < 1) {
2146		bt_dev_err(hdev, "Firmware download retry count: %d",
2147			   fw_dl_retry);
2148		btintel_pcie_dump_debug_registers(hdev);
2149		btintel_pcie_disable_interrupts(data);
2150		btintel_pcie_synchronize_irqs(data);
2151		err = btintel_pcie_reset_bt(data);
2152		if (err) {
2153			bt_dev_err(hdev, "Failed to do shr reset: %d", err);
2154			break;
2155		}
2156		usleep_range(10000, 12000);
2157		btintel_pcie_reset_ia(data);
2158		btintel_pcie_enable_interrupts(data);
2159		btintel_pcie_config_msix(data);
2160		err = btintel_pcie_enable_bt(data);
2161		if (err) {
2162			bt_dev_err(hdev, "Failed to enable hardware: %d", err);
2163			break;
2164		}
2165		btintel_pcie_start_rx(data);
2166	}
2167
2168	if (!err)
2169		set_bit(BTINTEL_PCIE_SETUP_DONE, &data->flags);
2170	return err;
2171}
2172
2173static struct btintel_pcie_dev_recovery *
2174btintel_pcie_get_recovery(struct pci_dev *pdev, struct device *dev)
2175{
2176	struct btintel_pcie_dev_recovery *tmp, *data = NULL;
2177	const char *name = pci_name(pdev);
2178	const size_t name_len = strlen(name) + 1;
2179	struct hci_dev *hdev = to_hci_dev(dev);
2180
2181	spin_lock(&btintel_pcie_recovery_lock);
2182	list_for_each_entry(tmp, &btintel_pcie_recovery_list, list) {
2183		if (strcmp(tmp->name, name))
2184			continue;
2185		data = tmp;
2186		break;
2187	}
2188	spin_unlock(&btintel_pcie_recovery_lock);
2189
2190	if (data) {
2191		bt_dev_dbg(hdev, "Found restart data for BDF: %s", data->name);
2192		return data;
2193	}
2194
2195	data = kzalloc(struct_size(data, name, name_len), GFP_ATOMIC);
2196	if (!data)
2197		return NULL;
2198
2199	strscpy(data->name, name, name_len);
2200	spin_lock(&btintel_pcie_recovery_lock);
2201	list_add_tail(&data->list, &btintel_pcie_recovery_list);
2202	spin_unlock(&btintel_pcie_recovery_lock);
2203
2204	return data;
2205}
2206
2207static void btintel_pcie_free_restart_list(void)
2208{
2209	struct btintel_pcie_dev_recovery *tmp;
2210
2211	while ((tmp = list_first_entry_or_null(&btintel_pcie_recovery_list,
2212					       typeof(*tmp), list))) {
2213		list_del(&tmp->list);
2214		kfree(tmp);
2215	}
2216}
2217
2218static void btintel_pcie_inc_recovery_count(struct pci_dev *pdev,
2219					    struct device *dev)
2220{
2221	struct btintel_pcie_dev_recovery *data;
2222	time64_t retry_window;
2223
2224	data = btintel_pcie_get_recovery(pdev, dev);
2225	if (!data)
2226		return;
2227
2228	retry_window = ktime_get_boottime_seconds() - data->last_error;
2229	if (data->count == 0) {
2230		data->last_error = ktime_get_boottime_seconds();
2231		data->count++;
2232	} else if (retry_window < BTINTEL_PCIE_RESET_WINDOW_SECS &&
2233		   data->count <= BTINTEL_PCIE_FLR_MAX_RETRY) {
2234		data->count++;
2235	} else if (retry_window > BTINTEL_PCIE_RESET_WINDOW_SECS) {
2236		data->last_error = 0;
2237		data->count = 0;
2238	}
2239}
2240
2241static int btintel_pcie_setup_hdev(struct btintel_pcie_data *data);
2242
2243static void btintel_pcie_removal_work(struct work_struct *wk)
2244{
2245	struct btintel_pcie_removal *removal =
2246		container_of(wk, struct btintel_pcie_removal, work);
2247	struct pci_dev *pdev = removal->pdev;
2248	struct btintel_pcie_data *data;
2249	int err;
2250
2251	pci_lock_rescan_remove();
2252
2253	if (!pdev->bus)
2254		goto error;
2255
2256	data = pci_get_drvdata(pdev);
2257
2258	btintel_pcie_disable_interrupts(data);
2259	btintel_pcie_synchronize_irqs(data);
2260
2261	flush_work(&data->rx_work);
2262
2263	bt_dev_dbg(data->hdev, "Release bluetooth interface");
2264	btintel_pcie_release_hdev(data);
2265
2266	err = pci_reset_function(pdev);
2267	if (err) {
2268		BT_ERR("Failed resetting the pcie device (%d)", err);
2269		goto error;
2270	}
2271
2272	btintel_pcie_enable_interrupts(data);
2273	btintel_pcie_config_msix(data);
2274
2275	err = btintel_pcie_enable_bt(data);
2276	if (err) {
2277		BT_ERR("Failed to enable bluetooth hardware after reset (%d)",
2278		       err);
2279		goto error;
2280	}
2281
2282	btintel_pcie_reset_ia(data);
2283	btintel_pcie_start_rx(data);
2284	data->flags = 0;
2285
2286	err = btintel_pcie_setup_hdev(data);
2287	if (err) {
2288		BT_ERR("Failed registering hdev (%d)", err);
2289		goto error;
2290	}
2291error:
2292	pci_dev_put(pdev);
2293	pci_unlock_rescan_remove();
2294	kfree(removal);
2295}
2296
2297static void btintel_pcie_reset(struct hci_dev *hdev)
2298{
2299	struct btintel_pcie_removal *removal;
2300	struct btintel_pcie_data *data;
2301
2302	data = hci_get_drvdata(hdev);
2303
2304	if (!test_bit(BTINTEL_PCIE_SETUP_DONE, &data->flags))
2305		return;
2306
2307	if (test_and_set_bit(BTINTEL_PCIE_RECOVERY_IN_PROGRESS, &data->flags))
2308		return;
2309
2310	removal = kzalloc(sizeof(*removal), GFP_ATOMIC);
2311	if (!removal)
2312		return;
2313
2314	removal->pdev = data->pdev;
2315	INIT_WORK(&removal->work, btintel_pcie_removal_work);
2316	pci_dev_get(removal->pdev);
2317	schedule_work(&removal->work);
2318}
2319
2320static void btintel_pcie_hw_error(struct hci_dev *hdev, u8 code)
2321{
2322	struct btintel_pcie_dev_recovery *data;
2323	struct btintel_pcie_data *dev_data = hci_get_drvdata(hdev);
2324	struct pci_dev *pdev = dev_data->pdev;
2325	time64_t retry_window;
2326
2327	if (code == 0x13) {
2328		bt_dev_err(hdev, "Encountered top exception");
2329		return;
2330	}
2331
2332	data = btintel_pcie_get_recovery(pdev, &hdev->dev);
2333	if (!data)
2334		return;
2335
2336	retry_window = ktime_get_boottime_seconds() - data->last_error;
2337
2338	if (retry_window < BTINTEL_PCIE_RESET_WINDOW_SECS &&
2339	    data->count >= BTINTEL_PCIE_FLR_MAX_RETRY) {
2340		bt_dev_err(hdev, "Exhausted maximum: %d recovery attempts: %d",
2341			   BTINTEL_PCIE_FLR_MAX_RETRY, data->count);
2342		bt_dev_dbg(hdev, "Boot time: %lld seconds",
2343			   ktime_get_boottime_seconds());
2344		bt_dev_dbg(hdev, "last error at: %lld seconds",
2345			   data->last_error);
2346		return;
2347	}
2348	btintel_pcie_inc_recovery_count(pdev, &hdev->dev);
2349	btintel_pcie_reset(hdev);
2350}
2351
2352static bool btintel_pcie_wakeup(struct hci_dev *hdev)
2353{
2354	struct btintel_pcie_data *data = hci_get_drvdata(hdev);
2355
2356	return device_may_wakeup(&data->pdev->dev);
2357}
2358
2359static const struct {
2360	u16 opcode;
2361	const char *desc;
2362} btintel_pcie_hci_drv_supported_commands[] = {
2363	/* Common commands */
2364	{ HCI_DRV_OP_READ_INFO, "Read Info" },
2365};
2366
2367static int btintel_pcie_hci_drv_read_info(struct hci_dev *hdev, void *data,
2368					  u16 data_len)
2369{
2370	struct hci_drv_rp_read_info *rp;
2371	size_t rp_size;
2372	int err, i;
2373	u16 opcode, num_supported_commands =
2374		ARRAY_SIZE(btintel_pcie_hci_drv_supported_commands);
2375
2376	rp_size = sizeof(*rp) + num_supported_commands * 2;
2377
2378	rp = kmalloc(rp_size, GFP_KERNEL);
2379	if (!rp)
2380		return -ENOMEM;
2381
2382	strscpy_pad(rp->driver_name, KBUILD_MODNAME);
2383
2384	rp->num_supported_commands = cpu_to_le16(num_supported_commands);
2385	for (i = 0; i < num_supported_commands; i++) {
2386		opcode = btintel_pcie_hci_drv_supported_commands[i].opcode;
2387		bt_dev_dbg(hdev,
2388			    "Supported HCI Drv command (0x%02x|0x%04x): %s",
2389			    hci_opcode_ogf(opcode),
2390			    hci_opcode_ocf(opcode),
2391			    btintel_pcie_hci_drv_supported_commands[i].desc);
2392		rp->supported_commands[i] = cpu_to_le16(opcode);
2393	}
2394
2395	err = hci_drv_cmd_complete(hdev, HCI_DRV_OP_READ_INFO,
2396				   HCI_DRV_STATUS_SUCCESS,
2397				   rp, rp_size);
2398
2399	kfree(rp);
2400	return err;
2401}
2402
2403static const struct hci_drv_handler btintel_pcie_hci_drv_common_handlers[] = {
2404	{ btintel_pcie_hci_drv_read_info,       HCI_DRV_READ_INFO_SIZE },
2405};
2406
2407static const struct hci_drv_handler btintel_pcie_hci_drv_specific_handlers[] = {};
2408
2409static struct hci_drv btintel_pcie_hci_drv = {
2410	.common_handler_count   = ARRAY_SIZE(btintel_pcie_hci_drv_common_handlers),
2411	.common_handlers        = btintel_pcie_hci_drv_common_handlers,
2412	.specific_handler_count = ARRAY_SIZE(btintel_pcie_hci_drv_specific_handlers),
2413	.specific_handlers      = btintel_pcie_hci_drv_specific_handlers,
2414};
2415
2416static int btintel_pcie_setup_hdev(struct btintel_pcie_data *data)
2417{
2418	int err;
2419	struct hci_dev *hdev;
2420
2421	hdev = hci_alloc_dev_priv(sizeof(struct btintel_data));
2422	if (!hdev)
2423		return -ENOMEM;
2424
2425	hdev->bus = HCI_PCI;
2426	hci_set_drvdata(hdev, data);
2427
2428	data->hdev = hdev;
2429	SET_HCIDEV_DEV(hdev, &data->pdev->dev);
2430
2431	hdev->manufacturer = 2;
2432	hdev->open = btintel_pcie_open;
2433	hdev->close = btintel_pcie_close;
2434	hdev->send = btintel_pcie_send_frame;
2435	hdev->setup = btintel_pcie_setup;
2436	hdev->shutdown = btintel_shutdown_combined;
2437	hdev->hw_error = btintel_pcie_hw_error;
2438	hdev->set_diag = btintel_set_diag;
2439	hdev->set_bdaddr = btintel_set_bdaddr;
2440	hdev->reset = btintel_pcie_reset;
2441	hdev->wakeup = btintel_pcie_wakeup;
2442	hdev->hci_drv = &btintel_pcie_hci_drv;
2443
2444	err = hci_register_dev(hdev);
2445	if (err < 0) {
2446		BT_ERR("Failed to register to hdev (%d)", err);
2447		goto exit_error;
2448	}
2449
2450	data->dmp_hdr.driver_name = KBUILD_MODNAME;
2451	return 0;
2452
2453exit_error:
2454	hci_free_dev(hdev);
2455	return err;
2456}
2457
2458static int btintel_pcie_probe(struct pci_dev *pdev,
2459			      const struct pci_device_id *ent)
2460{
2461	int err;
2462	struct btintel_pcie_data *data;
2463
2464	if (!pdev)
2465		return -ENODEV;
2466
2467	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
2468	if (!data)
2469		return -ENOMEM;
2470
2471	data->pdev = pdev;
2472
2473	spin_lock_init(&data->irq_lock);
2474	spin_lock_init(&data->hci_rx_lock);
2475
2476	init_waitqueue_head(&data->gp0_wait_q);
2477	data->gp0_received = false;
2478
2479	init_waitqueue_head(&data->tx_wait_q);
2480	data->tx_wait_done = false;
2481
2482	data->workqueue = alloc_ordered_workqueue(KBUILD_MODNAME, WQ_HIGHPRI);
2483	if (!data->workqueue)
2484		return -ENOMEM;
2485
2486	skb_queue_head_init(&data->rx_skb_q);
2487	INIT_WORK(&data->rx_work, btintel_pcie_rx_work);
2488
2489	data->boot_stage_cache = 0x00;
2490	data->img_resp_cache = 0x00;
2491
2492	err = btintel_pcie_config_pcie(pdev, data);
2493	if (err)
2494		goto exit_error;
2495
2496	pci_set_drvdata(pdev, data);
2497
2498	err = btintel_pcie_alloc(data);
2499	if (err)
2500		goto exit_error;
2501
2502	err = btintel_pcie_enable_bt(data);
2503	if (err)
2504		goto exit_error;
2505
2506	/* CNV information (CNVi and CNVr) is in CSR */
2507	data->cnvi = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_HW_REV_REG);
2508
2509	data->cnvr = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_RF_ID_REG);
2510
2511	err = btintel_pcie_start_rx(data);
2512	if (err)
2513		goto exit_error;
2514
2515	err = btintel_pcie_setup_hdev(data);
2516	if (err)
2517		goto exit_error;
2518
2519	bt_dev_dbg(data->hdev, "cnvi: 0x%8.8x cnvr: 0x%8.8x", data->cnvi,
2520		   data->cnvr);
2521	return 0;
2522
2523exit_error:
2524	/* reset device before exit */
2525	btintel_pcie_reset_bt(data);
2526
2527	pci_clear_master(pdev);
2528
2529	pci_set_drvdata(pdev, NULL);
2530
2531	return err;
2532}
2533
2534static void btintel_pcie_remove(struct pci_dev *pdev)
2535{
2536	struct btintel_pcie_data *data;
2537
2538	data = pci_get_drvdata(pdev);
2539
2540	btintel_pcie_disable_interrupts(data);
2541
2542	btintel_pcie_synchronize_irqs(data);
2543
2544	flush_work(&data->rx_work);
2545
2546	btintel_pcie_reset_bt(data);
2547	for (int i = 0; i < data->alloc_vecs; i++) {
2548		struct msix_entry *msix_entry;
2549
2550		msix_entry = &data->msix_entries[i];
2551		free_irq(msix_entry->vector, msix_entry);
2552	}
2553
2554	pci_free_irq_vectors(pdev);
2555
2556	btintel_pcie_release_hdev(data);
2557
2558	destroy_workqueue(data->workqueue);
2559
2560	btintel_pcie_free(data);
2561
2562	pci_clear_master(pdev);
2563
2564	pci_set_drvdata(pdev, NULL);
2565}
2566
2567#ifdef CONFIG_DEV_COREDUMP
2568static void btintel_pcie_coredump(struct device *dev)
2569{
2570	struct  pci_dev *pdev = to_pci_dev(dev);
2571	struct btintel_pcie_data *data = pci_get_drvdata(pdev);
2572
2573	if (test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags))
2574		return;
2575
2576	data->dmp_hdr.trigger_reason  = BTINTEL_PCIE_TRIGGER_REASON_USER_TRIGGER;
2577	queue_work(data->workqueue, &data->rx_work);
2578}
2579#endif
2580
2581static int btintel_pcie_set_dxstate(struct btintel_pcie_data *data, u32 dxstate)
2582{
2583	int retry = 0, status;
2584	u32 dx_intr_timeout_ms = 200;
2585
2586	do {
2587		data->gp0_received = false;
2588
2589		btintel_pcie_wr_sleep_cntrl(data, dxstate);
2590
2591		status = wait_event_timeout(data->gp0_wait_q, data->gp0_received,
2592			msecs_to_jiffies(dx_intr_timeout_ms));
2593
2594		if (status)
2595			return 0;
2596
2597		bt_dev_warn(data->hdev,
2598			   "Timeout (%u ms) on alive interrupt for D%d entry, retry count %d",
2599			   dx_intr_timeout_ms, dxstate, retry);
2600
2601		/* clear gp0 cause */
2602		btintel_pcie_clr_reg_bits(data,
2603					  BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES,
2604					  BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0);
2605
2606		/* A hardware bug may cause the alive interrupt to be missed.
2607		 * Check if the controller reached the expected state and retry
2608		 * the operation only if it hasn't.
2609		 */
2610		if (dxstate == BTINTEL_PCIE_STATE_D0) {
2611			if (btintel_pcie_in_d0(data))
2612				return 0;
2613		} else {
2614			if (btintel_pcie_in_d3(data))
2615				return 0;
2616		}
2617
2618	} while (++retry < BTINTEL_PCIE_DX_TRANSITION_MAX_RETRIES);
2619
2620	return -EBUSY;
2621}
2622
2623static int btintel_pcie_suspend_late(struct device *dev, pm_message_t mesg)
2624{
2625	struct pci_dev *pdev = to_pci_dev(dev);
2626	struct btintel_pcie_data *data;
2627	ktime_t start;
2628	u32 dxstate;
2629	int err;
2630
2631	data = pci_get_drvdata(pdev);
2632
2633	dxstate = (mesg.event == PM_EVENT_SUSPEND ?
2634		   BTINTEL_PCIE_STATE_D3_HOT : BTINTEL_PCIE_STATE_D3_COLD);
2635
2636	data->pm_sx_event = mesg.event;
2637
2638	start = ktime_get();
2639
2640	/* Refer: 6.4.11.7 -> Platform power management */
2641	err = btintel_pcie_set_dxstate(data, dxstate);
2642
2643	if (err)
2644		return err;
2645
2646	bt_dev_dbg(data->hdev,
2647		   "device entered into d3 state from d0 in %lld us",
2648		   ktime_to_us(ktime_get() - start));
2649	return err;
2650}
2651
2652static int btintel_pcie_suspend(struct device *dev)
2653{
2654	return btintel_pcie_suspend_late(dev, PMSG_SUSPEND);
2655}
2656
2657static int btintel_pcie_hibernate(struct device *dev)
2658{
2659	return btintel_pcie_suspend_late(dev, PMSG_HIBERNATE);
2660}
2661
2662static int btintel_pcie_freeze(struct device *dev)
2663{
2664	return btintel_pcie_suspend_late(dev, PMSG_FREEZE);
2665}
2666
2667static int btintel_pcie_resume(struct device *dev)
2668{
2669	struct pci_dev *pdev = to_pci_dev(dev);
2670	struct btintel_pcie_data *data;
2671	ktime_t start;
2672	int err;
2673
2674	data = pci_get_drvdata(pdev);
2675	data->gp0_received = false;
2676
2677	start = ktime_get();
2678
2679	/* When the system enters S4 (hibernate) mode, bluetooth device loses
2680	 * power, which results in the erasure of its loaded firmware.
2681	 * Consequently, function level reset (flr) is required on system
2682	 * resume to bring the controller back into an operational state by
2683	 * initiating a new firmware download.
2684	 */
2685
2686	if (data->pm_sx_event == PM_EVENT_FREEZE ||
2687	    data->pm_sx_event == PM_EVENT_HIBERNATE) {
2688		set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags);
2689		btintel_pcie_reset(data->hdev);
2690		return 0;
2691	}
2692
2693	/* Refer: 6.4.11.7 -> Platform power management */
2694	err = btintel_pcie_set_dxstate(data, BTINTEL_PCIE_STATE_D0);
2695
2696	if (err == 0) {
2697		bt_dev_dbg(data->hdev,
2698			   "device entered into d0 state from d3 in %lld us",
2699			   ktime_to_us(ktime_get() - start));
2700		return err;
2701	}
2702
2703	/* Trigger function level reset if the controller is in error
2704	 * state during resume() to bring back the controller to
2705	 * operational mode
2706	 */
2707
2708	data->boot_stage_cache = btintel_pcie_rd_reg32(data,
2709			BTINTEL_PCIE_CSR_BOOT_STAGE_REG);
2710	if (btintel_pcie_in_error(data) ||
2711			btintel_pcie_in_device_halt(data)) {
2712		bt_dev_err(data->hdev, "Controller in error state for D0 entry");
2713		if (!test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS,
2714				      &data->flags)) {
2715			data->dmp_hdr.trigger_reason =
2716				BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT;
2717			queue_work(data->workqueue, &data->rx_work);
2718		}
2719		set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags);
2720		btintel_pcie_reset(data->hdev);
2721	}
2722	return err;
2723}
2724
2725static const struct dev_pm_ops btintel_pcie_pm_ops = {
2726	.suspend = btintel_pcie_suspend,
2727	.resume = btintel_pcie_resume,
2728	.freeze = btintel_pcie_freeze,
2729	.thaw = btintel_pcie_resume,
2730	.poweroff = btintel_pcie_hibernate,
2731	.restore = btintel_pcie_resume,
2732};
2733
2734static struct pci_driver btintel_pcie_driver = {
2735	.name = KBUILD_MODNAME,
2736	.id_table = btintel_pcie_table,
2737	.probe = btintel_pcie_probe,
2738	.remove = btintel_pcie_remove,
2739	.driver.pm = pm_sleep_ptr(&btintel_pcie_pm_ops),
2740#ifdef CONFIG_DEV_COREDUMP
2741	.driver.coredump = btintel_pcie_coredump
2742#endif
2743};
2744
2745static int __init btintel_pcie_init(void)
2746{
2747	return pci_register_driver(&btintel_pcie_driver);
2748}
2749
2750static void __exit btintel_pcie_exit(void)
2751{
2752	pci_unregister_driver(&btintel_pcie_driver);
2753	btintel_pcie_free_restart_list();
2754}
2755
2756module_init(btintel_pcie_init);
2757module_exit(btintel_pcie_exit);
2758
2759MODULE_AUTHOR("Tedd Ho-Jeong An <tedd.an@intel.com>");
2760MODULE_DESCRIPTION("Intel Bluetooth PCIe transport driver ver " VERSION);
2761MODULE_VERSION(VERSION);
2762MODULE_LICENSE("GPL");