FILE
pci.c
File preview with syntax highlighting.
linux-6.19.14/drivers/net/wireless/realtek/rtw89/pci.c
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2/* Copyright(c) 2020 Realtek Corporation 3 */ 4 5#include <linux/pci.h> 6 7#include "mac.h" 8#include "pci.h" 9#include "reg.h" 10#include "ser.h" 11 12static bool rtw89_pci_disable_clkreq; 13static bool rtw89_pci_disable_aspm_l1; 14static bool rtw89_pci_disable_l1ss; 15module_param_named(disable_clkreq, rtw89_pci_disable_clkreq, bool, 0644); 16module_param_named(disable_aspm_l1, rtw89_pci_disable_aspm_l1, bool, 0644); 17module_param_named(disable_aspm_l1ss, rtw89_pci_disable_l1ss, bool, 0644); 18MODULE_PARM_DESC(disable_clkreq, "Set Y to disable PCI clkreq support"); 19MODULE_PARM_DESC(disable_aspm_l1, "Set Y to disable PCI ASPM L1 support"); 20MODULE_PARM_DESC(disable_aspm_l1ss, "Set Y to disable PCI L1SS support"); 21 22static int rtw89_pci_get_phy_offset_by_link_speed(struct rtw89_dev *rtwdev, 23 u32 *phy_offset) 24{ 25 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 26 struct pci_dev *pdev = rtwpci->pdev; 27 u32 val; 28 int ret; 29 30 ret = pci_read_config_dword(pdev, RTW89_PCIE_L1_STS_V1, &val); 31 if (ret) 32 return ret; 33 34 val = u32_get_bits(val, RTW89_BCFG_LINK_SPEED_MASK); 35 if (val == RTW89_PCIE_GEN1_SPEED) { 36 *phy_offset = R_RAC_DIRECT_OFFSET_G1; 37 } else if (val == RTW89_PCIE_GEN2_SPEED) { 38 *phy_offset = R_RAC_DIRECT_OFFSET_G2; 39 } else { 40 rtw89_warn(rtwdev, "Unknown PCI link speed %d\n", val); 41 return -EFAULT; 42 } 43 44 return 0; 45} 46 47static int rtw89_pci_rst_bdram_ax(struct rtw89_dev *rtwdev) 48{ 49 u32 val; 50 int ret; 51 52 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RST_BDRAM); 53 54 ret = read_poll_timeout_atomic(rtw89_read32, val, !(val & B_AX_RST_BDRAM), 55 1, RTW89_PCI_POLL_BDRAM_RST_CNT, false, 56 rtwdev, R_AX_PCIE_INIT_CFG1); 57 58 return ret; 59} 60 61static u32 rtw89_pci_dma_recalc(struct rtw89_dev *rtwdev, 62 struct rtw89_pci_dma_ring *bd_ring, 63 u32 cur_idx, bool tx) 64{ 65 const struct rtw89_pci_info *info = rtwdev->pci_info; 66 u32 cnt, cur_rp, wp, rp, len; 67 68 rp = bd_ring->rp; 69 wp = bd_ring->wp; 70 len = bd_ring->len; 71 72 cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx); 73 if (tx) { 74 cnt = cur_rp >= rp ? cur_rp - rp : len - (rp - cur_rp); 75 } else { 76 if (info->rx_ring_eq_is_full) 77 wp += 1; 78 79 cnt = cur_rp >= wp ? cur_rp - wp : len - (wp - cur_rp); 80 } 81 82 bd_ring->rp = cur_rp; 83 84 return cnt; 85} 86 87static u32 rtw89_pci_txbd_recalc(struct rtw89_dev *rtwdev, 88 struct rtw89_pci_tx_ring *tx_ring) 89{ 90 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring; 91 u32 addr_idx = bd_ring->addr.idx; 92 u32 cnt, idx; 93 94 idx = rtw89_read32(rtwdev, addr_idx); 95 cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, true); 96 97 return cnt; 98} 99 100static void rtw89_pci_release_fwcmd(struct rtw89_dev *rtwdev, 101 struct rtw89_pci *rtwpci, 102 u32 cnt, bool release_all) 103{ 104 struct rtw89_pci_tx_data *tx_data; 105 struct sk_buff *skb; 106 u32 qlen; 107 108 while (cnt--) { 109 skb = skb_dequeue(&rtwpci->h2c_queue); 110 if (!skb) { 111 rtw89_err(rtwdev, "failed to pre-release fwcmd\n"); 112 return; 113 } 114 skb_queue_tail(&rtwpci->h2c_release_queue, skb); 115 } 116 117 qlen = skb_queue_len(&rtwpci->h2c_release_queue); 118 if (!release_all) 119 qlen = qlen > RTW89_PCI_MULTITAG ? qlen - RTW89_PCI_MULTITAG : 0; 120 121 while (qlen--) { 122 skb = skb_dequeue(&rtwpci->h2c_release_queue); 123 if (!skb) { 124 rtw89_err(rtwdev, "failed to release fwcmd\n"); 125 return; 126 } 127 tx_data = RTW89_PCI_TX_SKB_CB(skb); 128 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len, 129 DMA_TO_DEVICE); 130 dev_kfree_skb_any(skb); 131 } 132} 133 134static void rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev *rtwdev, 135 struct rtw89_pci *rtwpci) 136{ 137 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[RTW89_TXCH_CH12]; 138 u32 cnt; 139 140 cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring); 141 if (!cnt) 142 return; 143 rtw89_pci_release_fwcmd(rtwdev, rtwpci, cnt, false); 144} 145 146static u32 rtw89_pci_rxbd_recalc(struct rtw89_dev *rtwdev, 147 struct rtw89_pci_rx_ring *rx_ring) 148{ 149 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring; 150 u32 addr_idx = bd_ring->addr.idx; 151 u32 cnt, idx; 152 153 idx = rtw89_read32(rtwdev, addr_idx); 154 cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, false); 155 156 return cnt; 157} 158 159static void rtw89_pci_sync_skb_for_cpu(struct rtw89_dev *rtwdev, 160 struct sk_buff *skb) 161{ 162 struct rtw89_pci_rx_info *rx_info; 163 dma_addr_t dma; 164 165 rx_info = RTW89_PCI_RX_SKB_CB(skb); 166 dma = rx_info->dma; 167 dma_sync_single_for_cpu(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE, 168 DMA_FROM_DEVICE); 169} 170 171static void rtw89_pci_sync_skb_for_device(struct rtw89_dev *rtwdev, 172 struct sk_buff *skb) 173{ 174 struct rtw89_pci_rx_info *rx_info; 175 dma_addr_t dma; 176 177 rx_info = RTW89_PCI_RX_SKB_CB(skb); 178 dma = rx_info->dma; 179 dma_sync_single_for_device(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE, 180 DMA_FROM_DEVICE); 181} 182 183static void rtw89_pci_rxbd_info_update(struct rtw89_dev *rtwdev, 184 struct sk_buff *skb) 185{ 186 struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb); 187 struct rtw89_pci_rxbd_info *rxbd_info; 188 __le32 info; 189 190 rxbd_info = (struct rtw89_pci_rxbd_info *)skb->data; 191 info = rxbd_info->dword; 192 193 rx_info->fs = le32_get_bits(info, RTW89_PCI_RXBD_FS); 194 rx_info->ls = le32_get_bits(info, RTW89_PCI_RXBD_LS); 195 rx_info->len = le32_get_bits(info, RTW89_PCI_RXBD_WRITE_SIZE); 196 rx_info->tag = le32_get_bits(info, RTW89_PCI_RXBD_TAG); 197} 198 199static int rtw89_pci_validate_rx_tag(struct rtw89_dev *rtwdev, 200 struct rtw89_pci_rx_ring *rx_ring, 201 struct sk_buff *skb) 202{ 203 struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb); 204 const struct rtw89_pci_info *info = rtwdev->pci_info; 205 u32 target_rx_tag; 206 207 if (!info->check_rx_tag) 208 return 0; 209 210 /* valid range is 1 ~ 0x1FFF */ 211 if (rx_ring->target_rx_tag == 0) 212 target_rx_tag = 1; 213 else 214 target_rx_tag = rx_ring->target_rx_tag; 215 216 if (rx_info->tag != target_rx_tag) { 217 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "mismatch RX tag 0x%x 0x%x\n", 218 rx_info->tag, target_rx_tag); 219 return -EAGAIN; 220 } 221 222 return 0; 223} 224 225static 226int rtw89_pci_sync_skb_for_device_and_validate_rx_info(struct rtw89_dev *rtwdev, 227 struct rtw89_pci_rx_ring *rx_ring, 228 struct sk_buff *skb) 229{ 230 struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb); 231 int rx_tag_retry = 1000; 232 int ret; 233 234 do { 235 rtw89_pci_sync_skb_for_cpu(rtwdev, skb); 236 rtw89_pci_rxbd_info_update(rtwdev, skb); 237 238 ret = rtw89_pci_validate_rx_tag(rtwdev, rx_ring, skb); 239 if (ret != -EAGAIN) 240 break; 241 } while (rx_tag_retry--); 242 243 /* update target rx_tag for next RX */ 244 rx_ring->target_rx_tag = rx_info->tag + 1; 245 246 return ret; 247} 248 249static void rtw89_pci_ctrl_txdma_ch_ax(struct rtw89_dev *rtwdev, bool enable) 250{ 251 const struct rtw89_pci_info *info = rtwdev->pci_info; 252 const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1; 253 const struct rtw89_reg_def *dma_stop2 = &info->dma_stop2; 254 255 if (enable) { 256 rtw89_write32_clr(rtwdev, dma_stop1->addr, dma_stop1->mask); 257 if (dma_stop2->addr) 258 rtw89_write32_clr(rtwdev, dma_stop2->addr, dma_stop2->mask); 259 } else { 260 rtw89_write32_set(rtwdev, dma_stop1->addr, dma_stop1->mask); 261 if (dma_stop2->addr) 262 rtw89_write32_set(rtwdev, dma_stop2->addr, dma_stop2->mask); 263 } 264} 265 266static void rtw89_pci_ctrl_txdma_fw_ch_ax(struct rtw89_dev *rtwdev, bool enable) 267{ 268 const struct rtw89_pci_info *info = rtwdev->pci_info; 269 const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1; 270 271 if (enable) 272 rtw89_write32_clr(rtwdev, dma_stop1->addr, B_AX_STOP_CH12); 273 else 274 rtw89_write32_set(rtwdev, dma_stop1->addr, B_AX_STOP_CH12); 275} 276 277static bool 278rtw89_skb_put_rx_data(struct rtw89_dev *rtwdev, bool fs, bool ls, 279 struct sk_buff *new, 280 const struct sk_buff *skb, u32 offset, 281 const struct rtw89_pci_rx_info *rx_info, 282 const struct rtw89_rx_desc_info *desc_info) 283{ 284 u32 copy_len = rx_info->len - offset; 285 286 if (unlikely(skb_tailroom(new) < copy_len)) { 287 rtw89_debug(rtwdev, RTW89_DBG_TXRX, 288 "invalid rx data length bd_len=%d desc_len=%d offset=%d (fs=%d ls=%d)\n", 289 rx_info->len, desc_info->pkt_size, offset, fs, ls); 290 rtw89_hex_dump(rtwdev, RTW89_DBG_TXRX, "rx_data: ", 291 skb->data, rx_info->len); 292 /* length of a single segment skb is desc_info->pkt_size */ 293 if (fs && ls) { 294 copy_len = desc_info->pkt_size; 295 } else { 296 rtw89_info(rtwdev, "drop rx data due to invalid length\n"); 297 return false; 298 } 299 } 300 301 skb_put_data(new, skb->data + offset, copy_len); 302 303 return true; 304} 305 306static u32 rtw89_pci_get_rx_skb_idx(struct rtw89_dev *rtwdev, 307 struct rtw89_pci_dma_ring *bd_ring) 308{ 309 const struct rtw89_pci_info *info = rtwdev->pci_info; 310 u32 wp = bd_ring->wp; 311 312 if (!info->rx_ring_eq_is_full) 313 return wp; 314 315 if (++wp >= bd_ring->len) 316 wp = 0; 317 318 return wp; 319} 320 321static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev, 322 struct rtw89_pci_rx_ring *rx_ring) 323{ 324 struct rtw89_rx_desc_info *desc_info = &rx_ring->diliver_desc; 325 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring; 326 const struct rtw89_pci_info *info = rtwdev->pci_info; 327 struct sk_buff *new = rx_ring->diliver_skb; 328 struct rtw89_pci_rx_info *rx_info; 329 struct sk_buff *skb; 330 u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info); 331 u32 skb_idx; 332 u32 offset; 333 u32 cnt = 1; 334 bool fs, ls; 335 int ret; 336 337 skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring); 338 skb = rx_ring->buf[skb_idx]; 339 340 ret = rtw89_pci_sync_skb_for_device_and_validate_rx_info(rtwdev, rx_ring, skb); 341 if (ret) { 342 rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n", 343 bd_ring->wp, ret); 344 goto err_sync_device; 345 } 346 347 rx_info = RTW89_PCI_RX_SKB_CB(skb); 348 fs = info->no_rxbd_fs ? !new : rx_info->fs; 349 ls = rx_info->ls; 350 351 if (unlikely(!fs || !ls)) 352 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, 353 "unexpected fs/ls=%d/%d tag=%u len=%u new->len=%u\n", 354 fs, ls, rx_info->tag, rx_info->len, new ? new->len : 0); 355 356 if (fs) { 357 if (new) { 358 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, 359 "skb should not be ready before first segment start\n"); 360 goto err_sync_device; 361 } 362 if (desc_info->ready) { 363 rtw89_warn(rtwdev, "desc info should not be ready before first segment start\n"); 364 goto err_sync_device; 365 } 366 367 rtw89_chip_query_rxdesc(rtwdev, desc_info, skb->data, rxinfo_size); 368 369 new = rtw89_alloc_skb_for_rx(rtwdev, desc_info->pkt_size); 370 if (!new) 371 goto err_sync_device; 372 373 rx_ring->diliver_skb = new; 374 375 /* first segment has RX desc */ 376 offset = desc_info->offset + desc_info->rxd_len; 377 } else { 378 offset = sizeof(struct rtw89_pci_rxbd_info); 379 if (!new) { 380 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "no last skb\n"); 381 goto err_sync_device; 382 } 383 } 384 if (!rtw89_skb_put_rx_data(rtwdev, fs, ls, new, skb, offset, rx_info, desc_info)) 385 goto err_sync_device; 386 rtw89_pci_sync_skb_for_device(rtwdev, skb); 387 rtw89_pci_rxbd_increase(rx_ring, 1); 388 389 if (!desc_info->ready) { 390 rtw89_warn(rtwdev, "no rx desc information\n"); 391 goto err_free_resource; 392 } 393 if (ls) { 394 rtw89_core_rx(rtwdev, desc_info, new); 395 rx_ring->diliver_skb = NULL; 396 desc_info->ready = false; 397 } 398 399 return cnt; 400 401err_sync_device: 402 rtw89_pci_sync_skb_for_device(rtwdev, skb); 403 rtw89_pci_rxbd_increase(rx_ring, 1); 404err_free_resource: 405 if (new) 406 dev_kfree_skb_any(new); 407 rx_ring->diliver_skb = NULL; 408 desc_info->ready = false; 409 410 return cnt; 411} 412 413static void rtw89_pci_rxbd_deliver(struct rtw89_dev *rtwdev, 414 struct rtw89_pci_rx_ring *rx_ring, 415 u32 cnt) 416{ 417 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring; 418 u32 rx_cnt; 419 420 while (cnt && rtwdev->napi_budget_countdown > 0) { 421 rx_cnt = rtw89_pci_rxbd_deliver_skbs(rtwdev, rx_ring); 422 if (!rx_cnt) { 423 rtw89_err(rtwdev, "failed to deliver RXBD skb\n"); 424 425 /* skip the rest RXBD bufs */ 426 rtw89_pci_rxbd_increase(rx_ring, cnt); 427 break; 428 } 429 430 cnt -= rx_cnt; 431 } 432 433 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp); 434} 435 436static int rtw89_pci_poll_rxq_dma(struct rtw89_dev *rtwdev, 437 struct rtw89_pci *rtwpci, int budget) 438{ 439 struct rtw89_pci_rx_ring *rx_ring; 440 int countdown = rtwdev->napi_budget_countdown; 441 u32 cnt; 442 443 rx_ring = &rtwpci->rx.rings[RTW89_RXCH_RXQ]; 444 445 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring); 446 if (!cnt) 447 return 0; 448 449 cnt = min_t(u32, budget, cnt); 450 451 rtw89_pci_rxbd_deliver(rtwdev, rx_ring, cnt); 452 453 /* In case of flushing pending SKBs, the countdown may exceed. */ 454 if (rtwdev->napi_budget_countdown <= 0) 455 return budget; 456 457 return budget - countdown; 458} 459 460static void rtw89_pci_tx_status(struct rtw89_dev *rtwdev, 461 struct rtw89_pci_tx_ring *tx_ring, 462 struct sk_buff *skb, u8 tx_status) 463{ 464 struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb); 465 struct ieee80211_tx_info *info; 466 467 if (rtw89_core_tx_wait_complete(rtwdev, skb_data, tx_status)) 468 return; 469 470 info = IEEE80211_SKB_CB(skb); 471 ieee80211_tx_info_clear_status(info); 472 473 if (info->flags & IEEE80211_TX_CTL_NO_ACK) 474 info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED; 475 if (tx_status == RTW89_TX_DONE) { 476 info->flags |= IEEE80211_TX_STAT_ACK; 477 tx_ring->tx_acked++; 478 } else { 479 if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) 480 rtw89_debug(rtwdev, RTW89_DBG_FW, 481 "failed to TX of status %x\n", tx_status); 482 switch (tx_status) { 483 case RTW89_TX_RETRY_LIMIT: 484 tx_ring->tx_retry_lmt++; 485 break; 486 case RTW89_TX_LIFE_TIME: 487 tx_ring->tx_life_time++; 488 break; 489 case RTW89_TX_MACID_DROP: 490 tx_ring->tx_mac_id_drop++; 491 break; 492 default: 493 rtw89_warn(rtwdev, "invalid TX status %x\n", tx_status); 494 break; 495 } 496 } 497 498 ieee80211_tx_status_ni(rtwdev->hw, skb); 499} 500 501static void rtw89_pci_reclaim_txbd(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring) 502{ 503 struct rtw89_pci_tx_wd *txwd; 504 u32 cnt; 505 506 cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring); 507 while (cnt--) { 508 txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list); 509 if (!txwd) { 510 rtw89_warn(rtwdev, "No busy txwd pages available\n"); 511 break; 512 } 513 514 list_del_init(&txwd->list); 515 516 /* this skb has been freed by RPP */ 517 if (skb_queue_len(&txwd->queue) == 0) 518 rtw89_pci_enqueue_txwd(tx_ring, txwd); 519 } 520} 521 522static void rtw89_pci_release_busy_txwd(struct rtw89_dev *rtwdev, 523 struct rtw89_pci_tx_ring *tx_ring) 524{ 525 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring; 526 struct rtw89_pci_tx_wd *txwd; 527 int i; 528 529 for (i = 0; i < wd_ring->page_num; i++) { 530 txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list); 531 if (!txwd) 532 break; 533 534 list_del_init(&txwd->list); 535 } 536} 537 538static void rtw89_pci_release_txwd_skb(struct rtw89_dev *rtwdev, 539 struct rtw89_pci_tx_ring *tx_ring, 540 struct rtw89_pci_tx_wd *txwd, u16 seq, 541 u8 tx_status) 542{ 543 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 544 struct rtw89_pci_tx_data *tx_data; 545 struct sk_buff *skb, *tmp; 546 u8 txch = tx_ring->txch; 547 548 if (!list_empty(&txwd->list)) { 549 rtw89_pci_reclaim_txbd(rtwdev, tx_ring); 550 /* In low power mode, RPP can receive before updating of TX BD. 551 * In normal mode, it should not happen so give it a warning. 552 */ 553 if (!rtwpci->low_power && !list_empty(&txwd->list)) 554 rtw89_warn(rtwdev, "queue %d txwd %d is not idle\n", 555 txch, seq); 556 } 557 558 skb_queue_walk_safe(&txwd->queue, skb, tmp) { 559 skb_unlink(skb, &txwd->queue); 560 561 tx_data = RTW89_PCI_TX_SKB_CB(skb); 562 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len, 563 DMA_TO_DEVICE); 564 565 rtw89_pci_tx_status(rtwdev, tx_ring, skb, tx_status); 566 } 567 568 if (list_empty(&txwd->list)) 569 rtw89_pci_enqueue_txwd(tx_ring, txwd); 570} 571 572void rtw89_pci_parse_rpp(struct rtw89_dev *rtwdev, void *_rpp, 573 struct rtw89_pci_rpp_info *rpp_info) 574{ 575 const struct rtw89_pci_rpp_fmt *rpp = _rpp; 576 577 rpp_info->seq = le32_get_bits(rpp->dword, RTW89_PCI_RPP_SEQ); 578 rpp_info->qsel = le32_get_bits(rpp->dword, RTW89_PCI_RPP_QSEL); 579 rpp_info->tx_status = le32_get_bits(rpp->dword, RTW89_PCI_RPP_TX_STATUS); 580 rpp_info->txch = rtw89_chip_get_ch_dma(rtwdev, rpp_info->qsel); 581} 582EXPORT_SYMBOL(rtw89_pci_parse_rpp); 583 584void rtw89_pci_parse_rpp_v1(struct rtw89_dev *rtwdev, void *_rpp, 585 struct rtw89_pci_rpp_info *rpp_info) 586{ 587 const struct rtw89_pci_rpp_fmt_v1 *rpp = _rpp; 588 589 rpp_info->seq = le32_get_bits(rpp->w0, RTW89_PCI_RPP_W0_PCIE_SEQ_V1_MASK); 590 rpp_info->qsel = le32_get_bits(rpp->w1, RTW89_PCI_RPP_W1_QSEL_V1_MASK); 591 rpp_info->tx_status = le32_get_bits(rpp->w0, RTW89_PCI_RPP_W0_TX_STATUS_V1_MASK); 592 rpp_info->txch = le32_get_bits(rpp->w0, RTW89_PCI_RPP_W0_DMA_CH_MASK); 593} 594EXPORT_SYMBOL(rtw89_pci_parse_rpp_v1); 595 596static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev, void *rpp) 597{ 598 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 599 const struct rtw89_pci_info *info = rtwdev->pci_info; 600 struct rtw89_pci_rpp_info rpp_info = {}; 601 struct rtw89_pci_tx_wd_ring *wd_ring; 602 struct rtw89_pci_tx_ring *tx_ring; 603 struct rtw89_pci_tx_wd *txwd; 604 605 info->parse_rpp(rtwdev, rpp, &rpp_info); 606 607 if (unlikely(rpp_info.txch >= RTW89_TXCH_NUM || 608 info->tx_dma_ch_mask & BIT(rpp_info.txch))) { 609 rtw89_warn(rtwdev, "should no release report on txch %d\n", 610 rpp_info.txch); 611 return; 612 } 613 614 if (unlikely(rpp_info.seq >= RTW89_PCI_TXWD_NUM_MAX)) { 615 rtw89_warn(rtwdev, "invalid seq %d\n", rpp_info.seq); 616 return; 617 } 618 619 tx_ring = &rtwpci->tx.rings[rpp_info.txch]; 620 wd_ring = &tx_ring->wd_ring; 621 txwd = &wd_ring->pages[rpp_info.seq]; 622 623 rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, rpp_info.seq, 624 rpp_info.tx_status); 625} 626 627static void rtw89_pci_release_pending_txwd_skb(struct rtw89_dev *rtwdev, 628 struct rtw89_pci_tx_ring *tx_ring) 629{ 630 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring; 631 struct rtw89_pci_tx_wd *txwd; 632 int i; 633 634 for (i = 0; i < wd_ring->page_num; i++) { 635 txwd = &wd_ring->pages[i]; 636 637 if (!list_empty(&txwd->list)) 638 continue; 639 640 rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, i, RTW89_TX_MACID_DROP); 641 } 642} 643 644static u32 rtw89_pci_release_tx_skbs(struct rtw89_dev *rtwdev, 645 struct rtw89_pci_rx_ring *rx_ring, 646 u32 max_cnt) 647{ 648 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring; 649 const struct rtw89_pci_info *info = rtwdev->pci_info; 650 struct rtw89_rx_desc_info desc_info = {}; 651 struct rtw89_pci_rx_info *rx_info; 652 struct sk_buff *skb; 653 void *rpp; 654 u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info); 655 u32 rpp_size = info->rpp_fmt_size; 656 u32 cnt = 0; 657 u32 skb_idx; 658 u32 offset; 659 int ret; 660 661 skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring); 662 skb = rx_ring->buf[skb_idx]; 663 664 ret = rtw89_pci_sync_skb_for_device_and_validate_rx_info(rtwdev, rx_ring, skb); 665 if (ret) { 666 rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n", 667 bd_ring->wp, ret); 668 goto err_sync_device; 669 } 670 671 rx_info = RTW89_PCI_RX_SKB_CB(skb); 672 if (!rx_info->fs || !rx_info->ls) { 673 rtw89_err(rtwdev, "cannot process RP frame not set FS/LS\n"); 674 return cnt; 675 } 676 677 rtw89_chip_query_rxdesc(rtwdev, &desc_info, skb->data, rxinfo_size); 678 679 /* first segment has RX desc */ 680 offset = desc_info.offset + desc_info.rxd_len; 681 for (; offset + rpp_size <= rx_info->len; offset += rpp_size) { 682 rpp = skb->data + offset; 683 rtw89_pci_release_rpp(rtwdev, rpp); 684 } 685 686 rtw89_pci_sync_skb_for_device(rtwdev, skb); 687 rtw89_pci_rxbd_increase(rx_ring, 1); 688 cnt++; 689 690 return cnt; 691 692err_sync_device: 693 rtw89_pci_sync_skb_for_device(rtwdev, skb); 694 return 0; 695} 696 697static void rtw89_pci_release_tx(struct rtw89_dev *rtwdev, 698 struct rtw89_pci_rx_ring *rx_ring, 699 u32 cnt) 700{ 701 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring; 702 u32 release_cnt; 703 704 while (cnt) { 705 release_cnt = rtw89_pci_release_tx_skbs(rtwdev, rx_ring, cnt); 706 if (!release_cnt) { 707 rtw89_err(rtwdev, "failed to release TX skbs\n"); 708 709 /* skip the rest RXBD bufs */ 710 rtw89_pci_rxbd_increase(rx_ring, cnt); 711 break; 712 } 713 714 cnt -= release_cnt; 715 } 716 717 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp); 718} 719 720static int rtw89_pci_poll_rpq_dma(struct rtw89_dev *rtwdev, 721 struct rtw89_pci *rtwpci, int budget) 722{ 723 struct rtw89_pci_rx_ring *rx_ring; 724 u32 cnt; 725 int work_done; 726 727 rx_ring = &rtwpci->rx.rings[RTW89_RXCH_RPQ]; 728 729 spin_lock_bh(&rtwpci->trx_lock); 730 731 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring); 732 if (cnt == 0) 733 goto out_unlock; 734 735 rtw89_pci_release_tx(rtwdev, rx_ring, cnt); 736 737out_unlock: 738 spin_unlock_bh(&rtwpci->trx_lock); 739 740 /* always release all RPQ */ 741 work_done = min_t(int, cnt, budget); 742 rtwdev->napi_budget_countdown -= work_done; 743 744 return work_done; 745} 746 747static void rtw89_pci_isr_rxd_unavail(struct rtw89_dev *rtwdev, 748 struct rtw89_pci *rtwpci) 749{ 750 struct rtw89_pci_rx_ring *rx_ring; 751 struct rtw89_pci_dma_ring *bd_ring; 752 u32 reg_idx; 753 u16 hw_idx, hw_idx_next, host_idx; 754 int i; 755 756 for (i = 0; i < RTW89_RXCH_NUM; i++) { 757 rx_ring = &rtwpci->rx.rings[i]; 758 bd_ring = &rx_ring->bd_ring; 759 760 reg_idx = rtw89_read32(rtwdev, bd_ring->addr.idx); 761 hw_idx = FIELD_GET(TXBD_HW_IDX_MASK, reg_idx); 762 host_idx = FIELD_GET(TXBD_HOST_IDX_MASK, reg_idx); 763 hw_idx_next = (hw_idx + 1) % bd_ring->len; 764 765 if (hw_idx_next == host_idx) 766 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "%d RXD unavailable\n", i); 767 768 rtw89_debug(rtwdev, RTW89_DBG_TXRX, 769 "%d RXD unavailable, idx=0x%08x, len=%d\n", 770 i, reg_idx, bd_ring->len); 771 } 772} 773 774void rtw89_pci_recognize_intrs(struct rtw89_dev *rtwdev, 775 struct rtw89_pci *rtwpci, 776 struct rtw89_pci_isrs *isrs) 777{ 778 isrs->halt_c2h_isrs = rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs; 779 isrs->isrs[0] = rtw89_read32(rtwdev, R_AX_PCIE_HISR00) & rtwpci->intrs[0]; 780 isrs->isrs[1] = rtw89_read32(rtwdev, R_AX_PCIE_HISR10) & rtwpci->intrs[1]; 781 782 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs); 783 rtw89_write32(rtwdev, R_AX_PCIE_HISR00, isrs->isrs[0]); 784 rtw89_write32(rtwdev, R_AX_PCIE_HISR10, isrs->isrs[1]); 785} 786EXPORT_SYMBOL(rtw89_pci_recognize_intrs); 787 788void rtw89_pci_recognize_intrs_v1(struct rtw89_dev *rtwdev, 789 struct rtw89_pci *rtwpci, 790 struct rtw89_pci_isrs *isrs) 791{ 792 isrs->ind_isrs = rtw89_read32(rtwdev, R_AX_PCIE_HISR00_V1) & rtwpci->ind_intrs; 793 isrs->halt_c2h_isrs = isrs->ind_isrs & B_AX_HS0ISR_IND_INT_EN ? 794 rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs : 0; 795 isrs->isrs[0] = isrs->ind_isrs & B_AX_HCI_AXIDMA_INT_EN ? 796 rtw89_read32(rtwdev, R_AX_HAXI_HISR00) & rtwpci->intrs[0] : 0; 797 isrs->isrs[1] = isrs->ind_isrs & B_AX_HS1ISR_IND_INT_EN ? 798 rtw89_read32(rtwdev, R_AX_HISR1) & rtwpci->intrs[1] : 0; 799 800 if (isrs->halt_c2h_isrs) 801 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs); 802 if (isrs->isrs[0]) 803 rtw89_write32(rtwdev, R_AX_HAXI_HISR00, isrs->isrs[0]); 804 if (isrs->isrs[1]) 805 rtw89_write32(rtwdev, R_AX_HISR1, isrs->isrs[1]); 806} 807EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v1); 808 809void rtw89_pci_recognize_intrs_v2(struct rtw89_dev *rtwdev, 810 struct rtw89_pci *rtwpci, 811 struct rtw89_pci_isrs *isrs) 812{ 813 isrs->ind_isrs = rtw89_read32(rtwdev, R_BE_PCIE_HISR) & rtwpci->ind_intrs; 814 isrs->halt_c2h_isrs = isrs->ind_isrs & B_BE_HS0ISR_IND_INT ? 815 rtw89_read32(rtwdev, R_BE_HISR0) & rtwpci->halt_c2h_intrs : 0; 816 isrs->isrs[0] = isrs->ind_isrs & B_BE_HCI_AXIDMA_INT ? 817 rtw89_read32(rtwdev, R_BE_HAXI_HISR00) & rtwpci->intrs[0] : 0; 818 isrs->isrs[1] = rtw89_read32(rtwdev, R_BE_PCIE_DMA_ISR) & rtwpci->intrs[1]; 819 820 if (isrs->halt_c2h_isrs) 821 rtw89_write32(rtwdev, R_BE_HISR0, isrs->halt_c2h_isrs); 822 if (isrs->isrs[0]) 823 rtw89_write32(rtwdev, R_BE_HAXI_HISR00, isrs->isrs[0]); 824 if (isrs->isrs[1]) 825 rtw89_write32(rtwdev, R_BE_PCIE_DMA_ISR, isrs->isrs[1]); 826 rtw89_write32(rtwdev, R_BE_PCIE_HISR, isrs->ind_isrs); 827} 828EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v2); 829 830void rtw89_pci_recognize_intrs_v3(struct rtw89_dev *rtwdev, 831 struct rtw89_pci *rtwpci, 832 struct rtw89_pci_isrs *isrs) 833{ 834 isrs->ind_isrs = rtw89_read32(rtwdev, R_BE_PCIE_HISR) & rtwpci->ind_intrs; 835 isrs->halt_c2h_isrs = isrs->ind_isrs & B_BE_HS0ISR_IND_INT ? 836 rtw89_read32(rtwdev, R_BE_HISR0) & rtwpci->halt_c2h_intrs : 0; 837 isrs->isrs[1] = rtw89_read32(rtwdev, R_BE_PCIE_DMA_ISR) & rtwpci->intrs[1]; 838 839 /* isrs[0] is not used, so borrow to store RDU status to share common 840 * flow in rtw89_pci_interrupt_threadfn(). 841 */ 842 isrs->isrs[0] = isrs->isrs[1] & (B_BE_PCIE_RDU_CH1_INT | 843 B_BE_PCIE_RDU_CH0_INT); 844 845 if (isrs->halt_c2h_isrs) 846 rtw89_write32(rtwdev, R_BE_HISR0, isrs->halt_c2h_isrs); 847 if (isrs->isrs[1]) 848 rtw89_write32(rtwdev, R_BE_PCIE_DMA_ISR, isrs->isrs[1]); 849 rtw89_write32(rtwdev, R_BE_PCIE_HISR, isrs->ind_isrs); 850} 851EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v3); 852 853void rtw89_pci_enable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci) 854{ 855 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs); 856 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, rtwpci->intrs[0]); 857 rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, rtwpci->intrs[1]); 858} 859EXPORT_SYMBOL(rtw89_pci_enable_intr); 860 861void rtw89_pci_disable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci) 862{ 863 rtw89_write32(rtwdev, R_AX_HIMR0, 0); 864 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, 0); 865 rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, 0); 866} 867EXPORT_SYMBOL(rtw89_pci_disable_intr); 868 869void rtw89_pci_enable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci) 870{ 871 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, rtwpci->ind_intrs); 872 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs); 873 rtw89_write32(rtwdev, R_AX_HAXI_HIMR00, rtwpci->intrs[0]); 874 rtw89_write32(rtwdev, R_AX_HIMR1, rtwpci->intrs[1]); 875} 876EXPORT_SYMBOL(rtw89_pci_enable_intr_v1); 877 878void rtw89_pci_disable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci) 879{ 880 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, 0); 881} 882EXPORT_SYMBOL(rtw89_pci_disable_intr_v1); 883 884void rtw89_pci_enable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci) 885{ 886 rtw89_write32(rtwdev, R_BE_HIMR0, rtwpci->halt_c2h_intrs); 887 rtw89_write32(rtwdev, R_BE_HAXI_HIMR00, rtwpci->intrs[0]); 888 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, rtwpci->intrs[1]); 889 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, rtwpci->ind_intrs); 890} 891EXPORT_SYMBOL(rtw89_pci_enable_intr_v2); 892 893void rtw89_pci_disable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci) 894{ 895 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, 0); 896 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, 0); 897} 898EXPORT_SYMBOL(rtw89_pci_disable_intr_v2); 899 900void rtw89_pci_enable_intr_v3(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci) 901{ 902 rtw89_write32(rtwdev, R_BE_HIMR0, rtwpci->halt_c2h_intrs); 903 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, rtwpci->intrs[1]); 904 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, rtwpci->ind_intrs); 905} 906EXPORT_SYMBOL(rtw89_pci_enable_intr_v3); 907 908void rtw89_pci_disable_intr_v3(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci) 909{ 910 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, 0); 911 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, 0); 912} 913EXPORT_SYMBOL(rtw89_pci_disable_intr_v3); 914 915static void rtw89_pci_ops_recovery_start(struct rtw89_dev *rtwdev) 916{ 917 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 918 unsigned long flags; 919 920 spin_lock_irqsave(&rtwpci->irq_lock, flags); 921 rtw89_chip_disable_intr(rtwdev, rtwpci); 922 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_START); 923 rtw89_chip_enable_intr(rtwdev, rtwpci); 924 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); 925} 926 927static void rtw89_pci_ops_recovery_complete(struct rtw89_dev *rtwdev) 928{ 929 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 930 unsigned long flags; 931 932 spin_lock_irqsave(&rtwpci->irq_lock, flags); 933 rtw89_chip_disable_intr(rtwdev, rtwpci); 934 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_COMPLETE); 935 rtw89_chip_enable_intr(rtwdev, rtwpci); 936 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); 937} 938 939static void rtw89_pci_low_power_interrupt_handler(struct rtw89_dev *rtwdev) 940{ 941 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 942 int budget = NAPI_POLL_WEIGHT; 943 944 /* To prevent RXQ get stuck due to run out of budget. */ 945 rtwdev->napi_budget_countdown = budget; 946 947 rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, budget); 948 rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, budget); 949} 950 951static irqreturn_t rtw89_pci_interrupt_threadfn(int irq, void *dev) 952{ 953 struct rtw89_dev *rtwdev = dev; 954 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 955 const struct rtw89_pci_info *info = rtwdev->pci_info; 956 const struct rtw89_pci_isr_def *isr_def = info->isr_def; 957 struct rtw89_pci_isrs isrs; 958 unsigned long flags; 959 960 spin_lock_irqsave(&rtwpci->irq_lock, flags); 961 rtw89_chip_recognize_intrs(rtwdev, rtwpci, &isrs); 962 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); 963 964 if (unlikely(isrs.isrs[0] & isr_def->isr_rdu)) 965 rtw89_pci_isr_rxd_unavail(rtwdev, rtwpci); 966 967 if (unlikely(isrs.halt_c2h_isrs & isr_def->isr_halt_c2h)) 968 rtw89_ser_notify(rtwdev, rtw89_mac_get_err_status(rtwdev)); 969 970 if (unlikely(isrs.halt_c2h_isrs & isr_def->isr_wdt_timeout)) 971 rtw89_ser_notify(rtwdev, MAC_AX_ERR_L2_ERR_WDT_TIMEOUT_INT); 972 973 if (unlikely(rtwpci->under_recovery)) 974 goto enable_intr; 975 976 if (unlikely(rtwpci->low_power)) { 977 rtw89_pci_low_power_interrupt_handler(rtwdev); 978 goto enable_intr; 979 } 980 981 if (likely(rtwpci->running)) { 982 local_bh_disable(); 983 napi_schedule(&rtwdev->napi); 984 local_bh_enable(); 985 } 986 987 return IRQ_HANDLED; 988 989enable_intr: 990 spin_lock_irqsave(&rtwpci->irq_lock, flags); 991 if (likely(rtwpci->running)) 992 rtw89_chip_enable_intr(rtwdev, rtwpci); 993 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); 994 return IRQ_HANDLED; 995} 996 997static irqreturn_t rtw89_pci_interrupt_handler(int irq, void *dev) 998{ 999 struct rtw89_dev *rtwdev = dev; 1000 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1001 unsigned long flags; 1002 irqreturn_t irqret = IRQ_WAKE_THREAD; 1003 1004 spin_lock_irqsave(&rtwpci->irq_lock, flags); 1005 1006 /* If interrupt event is on the road, it is still trigger interrupt 1007 * even we have done pci_stop() to turn off IMR. 1008 */ 1009 if (unlikely(!rtwpci->running)) { 1010 irqret = IRQ_HANDLED; 1011 goto exit; 1012 } 1013 1014 rtw89_chip_disable_intr(rtwdev, rtwpci); 1015exit: 1016 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); 1017 1018 return irqret; 1019} 1020 1021#define DEF_TXCHADDRS_TYPE3(gen, ch_idx, txch, v...) \ 1022 [RTW89_TXCH_##ch_idx] = { \ 1023 .num = R_##gen##_##txch##_TXBD_CFG, \ 1024 .idx = R_##gen##_##txch##_TXBD_IDX ##v, \ 1025 .bdram = 0, \ 1026 .desa_l = 0, \ 1027 .desa_h = 0, \ 1028 } 1029 1030#define DEF_TXCHADDRS_TYPE3_GRP_BASE(gen, ch_idx, txch, grp, v...) \ 1031 [RTW89_TXCH_##ch_idx] = { \ 1032 .num = R_##gen##_##txch##_TXBD_CFG, \ 1033 .idx = R_##gen##_##txch##_TXBD_IDX ##v, \ 1034 .bdram = 0, \ 1035 .desa_l = R_##gen##_##grp##_TXBD_DESA_L, \ 1036 .desa_h = R_##gen##_##grp##_TXBD_DESA_H, \ 1037 } 1038 1039#define DEF_TXCHADDRS_TYPE2(gen, ch_idx, txch, v...) \ 1040 [RTW89_TXCH_##ch_idx] = { \ 1041 .num = R_##gen##_##txch##_TXBD_NUM ##v, \ 1042 .idx = R_##gen##_##txch##_TXBD_IDX ##v, \ 1043 .bdram = 0, \ 1044 .desa_l = R_##gen##_##txch##_TXBD_DESA_L ##v, \ 1045 .desa_h = R_##gen##_##txch##_TXBD_DESA_H ##v, \ 1046 } 1047 1048#define DEF_TXCHADDRS_TYPE1(info, txch, v...) \ 1049 [RTW89_TXCH_##txch] = { \ 1050 .num = R_AX_##txch##_TXBD_NUM ##v, \ 1051 .idx = R_AX_##txch##_TXBD_IDX ##v, \ 1052 .bdram = R_AX_##txch##_BDRAM_CTRL ##v, \ 1053 .desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \ 1054 .desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \ 1055 } 1056 1057#define DEF_TXCHADDRS(info, txch, v...) \ 1058 [RTW89_TXCH_##txch] = { \ 1059 .num = R_AX_##txch##_TXBD_NUM, \ 1060 .idx = R_AX_##txch##_TXBD_IDX, \ 1061 .bdram = R_AX_##txch##_BDRAM_CTRL ##v, \ 1062 .desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \ 1063 .desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \ 1064 } 1065 1066#define DEF_RXCHADDRS_TYPE3(gen, ch_idx, rxch, v...) \ 1067 [RTW89_RXCH_##ch_idx] = { \ 1068 .num = R_##gen##_RX_##rxch##_RXBD_CONFIG, \ 1069 .idx = R_##gen##_##ch_idx##0_RXBD_IDX ##v, \ 1070 .desa_l = 0, \ 1071 .desa_h = 0, \ 1072 } 1073 1074#define DEF_RXCHADDRS_TYPE3_GRP_BASE(gen, ch_idx, rxch, grp, v...) \ 1075 [RTW89_RXCH_##ch_idx] = { \ 1076 .num = R_##gen##_RX_##rxch##_RXBD_CONFIG, \ 1077 .idx = R_##gen##_##ch_idx##0_RXBD_IDX ##v, \ 1078 .desa_l = R_##gen##_##grp##_RXBD_DESA_L, \ 1079 .desa_h = R_##gen##_##grp##_RXBD_DESA_H, \ 1080 } 1081 1082#define DEF_RXCHADDRS(gen, ch_idx, rxch, v...) \ 1083 [RTW89_RXCH_##ch_idx] = { \ 1084 .num = R_##gen##_##rxch##_RXBD_NUM ##v, \ 1085 .idx = R_##gen##_##rxch##_RXBD_IDX ##v, \ 1086 .desa_l = R_##gen##_##rxch##_RXBD_DESA_L ##v, \ 1087 .desa_h = R_##gen##_##rxch##_RXBD_DESA_H ##v, \ 1088 } 1089 1090const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set = { 1091 .tx = { 1092 DEF_TXCHADDRS(info, ACH0), 1093 DEF_TXCHADDRS(info, ACH1), 1094 DEF_TXCHADDRS(info, ACH2), 1095 DEF_TXCHADDRS(info, ACH3), 1096 DEF_TXCHADDRS(info, ACH4), 1097 DEF_TXCHADDRS(info, ACH5), 1098 DEF_TXCHADDRS(info, ACH6), 1099 DEF_TXCHADDRS(info, ACH7), 1100 DEF_TXCHADDRS(info, CH8), 1101 DEF_TXCHADDRS(info, CH9), 1102 DEF_TXCHADDRS_TYPE1(info, CH10), 1103 DEF_TXCHADDRS_TYPE1(info, CH11), 1104 DEF_TXCHADDRS(info, CH12), 1105 }, 1106 .rx = { 1107 DEF_RXCHADDRS(AX, RXQ, RXQ), 1108 DEF_RXCHADDRS(AX, RPQ, RPQ), 1109 }, 1110}; 1111EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set); 1112 1113const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_v1 = { 1114 .tx = { 1115 DEF_TXCHADDRS(info, ACH0, _V1), 1116 DEF_TXCHADDRS(info, ACH1, _V1), 1117 DEF_TXCHADDRS(info, ACH2, _V1), 1118 DEF_TXCHADDRS(info, ACH3, _V1), 1119 DEF_TXCHADDRS(info, ACH4, _V1), 1120 DEF_TXCHADDRS(info, ACH5, _V1), 1121 DEF_TXCHADDRS(info, ACH6, _V1), 1122 DEF_TXCHADDRS(info, ACH7, _V1), 1123 DEF_TXCHADDRS(info, CH8, _V1), 1124 DEF_TXCHADDRS(info, CH9, _V1), 1125 DEF_TXCHADDRS_TYPE1(info, CH10, _V1), 1126 DEF_TXCHADDRS_TYPE1(info, CH11, _V1), 1127 DEF_TXCHADDRS(info, CH12, _V1), 1128 }, 1129 .rx = { 1130 DEF_RXCHADDRS(AX, RXQ, RXQ, _V1), 1131 DEF_RXCHADDRS(AX, RPQ, RPQ, _V1), 1132 }, 1133}; 1134EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_v1); 1135 1136const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_be = { 1137 .tx = { 1138 DEF_TXCHADDRS_TYPE2(BE, ACH0, CH0, _V1), 1139 DEF_TXCHADDRS_TYPE2(BE, ACH1, CH1, _V1), 1140 DEF_TXCHADDRS_TYPE2(BE, ACH2, CH2, _V1), 1141 DEF_TXCHADDRS_TYPE2(BE, ACH3, CH3, _V1), 1142 DEF_TXCHADDRS_TYPE2(BE, ACH4, CH4, _V1), 1143 DEF_TXCHADDRS_TYPE2(BE, ACH5, CH5, _V1), 1144 DEF_TXCHADDRS_TYPE2(BE, ACH6, CH6, _V1), 1145 DEF_TXCHADDRS_TYPE2(BE, ACH7, CH7, _V1), 1146 DEF_TXCHADDRS_TYPE2(BE, CH8, CH8, _V1), 1147 DEF_TXCHADDRS_TYPE2(BE, CH9, CH9, _V1), 1148 DEF_TXCHADDRS_TYPE2(BE, CH10, CH10, _V1), 1149 DEF_TXCHADDRS_TYPE2(BE, CH11, CH11, _V1), 1150 DEF_TXCHADDRS_TYPE2(BE, CH12, CH12, _V1), 1151 }, 1152 .rx = { 1153 DEF_RXCHADDRS(BE, RXQ, RXQ0, _V1), 1154 DEF_RXCHADDRS(BE, RPQ, RPQ0, _V1), 1155 }, 1156}; 1157EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_be); 1158 1159const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_be_v1 = { 1160 .tx = { 1161 DEF_TXCHADDRS_TYPE3_GRP_BASE(BE, ACH0, CH0, ACQ, _V1), 1162 /* no CH1 */ 1163 DEF_TXCHADDRS_TYPE3(BE, ACH2, CH2, _V1), 1164 /* no CH3 */ 1165 DEF_TXCHADDRS_TYPE3(BE, ACH4, CH4, _V1), 1166 /* no CH5 */ 1167 DEF_TXCHADDRS_TYPE3(BE, ACH6, CH6, _V1), 1168 /* no CH7 */ 1169 DEF_TXCHADDRS_TYPE3_GRP_BASE(BE, CH8, CH8, NACQ, _V1), 1170 /* no CH9 */ 1171 DEF_TXCHADDRS_TYPE3(BE, CH10, CH10, _V1), 1172 /* no CH11 */ 1173 DEF_TXCHADDRS_TYPE3(BE, CH12, CH12, _V1), 1174 }, 1175 .rx = { 1176 DEF_RXCHADDRS_TYPE3_GRP_BASE(BE, RXQ, CH0, HOST0, _V1), 1177 DEF_RXCHADDRS_TYPE3(BE, RPQ, CH1, _V1), 1178 }, 1179}; 1180EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_be_v1); 1181 1182#undef DEF_TXCHADDRS_TYPE3 1183#undef DEF_TXCHADDRS_TYPE3_GRP_BASE 1184#undef DEF_TXCHADDRS_TYPE2 1185#undef DEF_TXCHADDRS_TYPE1 1186#undef DEF_TXCHADDRS 1187#undef DEF_RXCHADDRS_TYPE3 1188#undef DEF_RXCHADDRS_TYPE3_GRP_BASE 1189#undef DEF_RXCHADDRS 1190 1191static int rtw89_pci_get_txch_addrs(struct rtw89_dev *rtwdev, 1192 enum rtw89_tx_channel txch, 1193 const struct rtw89_pci_ch_dma_addr **addr) 1194{ 1195 const struct rtw89_pci_info *info = rtwdev->pci_info; 1196 1197 if (txch >= RTW89_TXCH_NUM) 1198 return -EINVAL; 1199 1200 *addr = &info->dma_addr_set->tx[txch]; 1201 1202 return 0; 1203} 1204 1205static int rtw89_pci_get_rxch_addrs(struct rtw89_dev *rtwdev, 1206 enum rtw89_rx_channel rxch, 1207 const struct rtw89_pci_ch_dma_addr **addr) 1208{ 1209 const struct rtw89_pci_info *info = rtwdev->pci_info; 1210 1211 if (rxch >= RTW89_RXCH_NUM) 1212 return -EINVAL; 1213 1214 *addr = &info->dma_addr_set->rx[rxch]; 1215 1216 return 0; 1217} 1218 1219static u32 rtw89_pci_get_avail_txbd_num(struct rtw89_pci_tx_ring *ring) 1220{ 1221 struct rtw89_pci_dma_ring *bd_ring = &ring->bd_ring; 1222 1223 /* reserved 1 desc check ring is full or not */ 1224 if (bd_ring->rp > bd_ring->wp) 1225 return bd_ring->rp - bd_ring->wp - 1; 1226 1227 return bd_ring->len - (bd_ring->wp - bd_ring->rp) - 1; 1228} 1229 1230static 1231u32 __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev *rtwdev) 1232{ 1233 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1234 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[RTW89_TXCH_CH12]; 1235 u32 cnt; 1236 1237 spin_lock_bh(&rtwpci->trx_lock); 1238 rtw89_pci_reclaim_tx_fwcmd(rtwdev, rtwpci); 1239 cnt = rtw89_pci_get_avail_txbd_num(tx_ring); 1240 spin_unlock_bh(&rtwpci->trx_lock); 1241 1242 return cnt; 1243} 1244 1245static 1246u32 __rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev *rtwdev, 1247 u8 txch) 1248{ 1249 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1250 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[txch]; 1251 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring; 1252 u32 cnt; 1253 1254 spin_lock_bh(&rtwpci->trx_lock); 1255 cnt = rtw89_pci_get_avail_txbd_num(tx_ring); 1256 if (txch != RTW89_TXCH_CH12) 1257 cnt = min(cnt, wd_ring->curr_num); 1258 spin_unlock_bh(&rtwpci->trx_lock); 1259 1260 return cnt; 1261} 1262 1263static u32 __rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev, 1264 u8 txch) 1265{ 1266 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1267 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[txch]; 1268 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring; 1269 const struct rtw89_chip_info *chip = rtwdev->chip; 1270 u32 bd_cnt, wd_cnt, min_cnt = 0; 1271 struct rtw89_pci_rx_ring *rx_ring; 1272 enum rtw89_debug_mask debug_mask; 1273 u32 cnt; 1274 1275 rx_ring = &rtwpci->rx.rings[RTW89_RXCH_RPQ]; 1276 1277 spin_lock_bh(&rtwpci->trx_lock); 1278 bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring); 1279 wd_cnt = wd_ring->curr_num; 1280 1281 if (wd_cnt == 0 || bd_cnt == 0) { 1282 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring); 1283 if (cnt) 1284 rtw89_pci_release_tx(rtwdev, rx_ring, cnt); 1285 else if (wd_cnt == 0) 1286 goto out_unlock; 1287 1288 bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring); 1289 if (bd_cnt == 0) 1290 rtw89_pci_reclaim_txbd(rtwdev, tx_ring); 1291 } 1292 1293 bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring); 1294 wd_cnt = wd_ring->curr_num; 1295 min_cnt = min(bd_cnt, wd_cnt); 1296 if (min_cnt == 0) { 1297 /* This message can be frequently shown in low power mode or 1298 * high traffic with small FIFO chips, and we have recognized it as normal 1299 * behavior, so print with mask RTW89_DBG_TXRX in these situations. 1300 */ 1301 if (rtwpci->low_power || chip->small_fifo_size) 1302 debug_mask = RTW89_DBG_TXRX; 1303 else 1304 debug_mask = RTW89_DBG_UNEXP; 1305 1306 rtw89_debug(rtwdev, debug_mask, 1307 "still no tx resource after reclaim: wd_cnt=%d bd_cnt=%d\n", 1308 wd_cnt, bd_cnt); 1309 } 1310 1311out_unlock: 1312 spin_unlock_bh(&rtwpci->trx_lock); 1313 1314 return min_cnt; 1315} 1316 1317static u32 rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev, 1318 u8 txch) 1319{ 1320 if (rtwdev->hci.paused) 1321 return __rtw89_pci_check_and_reclaim_tx_resource_noio(rtwdev, txch); 1322 1323 if (txch == RTW89_TXCH_CH12) 1324 return __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(rtwdev); 1325 1326 return __rtw89_pci_check_and_reclaim_tx_resource(rtwdev, txch); 1327} 1328 1329static void __rtw89_pci_tx_kick_off(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring) 1330{ 1331 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1332 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring; 1333 u32 host_idx, addr; 1334 1335 spin_lock_bh(&rtwpci->trx_lock); 1336 1337 addr = bd_ring->addr.idx; 1338 host_idx = bd_ring->wp; 1339 rtw89_write16(rtwdev, addr, host_idx); 1340 1341 spin_unlock_bh(&rtwpci->trx_lock); 1342} 1343 1344static void rtw89_pci_tx_bd_ring_update(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring, 1345 int n_txbd) 1346{ 1347 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring; 1348 u32 host_idx, len; 1349 1350 len = bd_ring->len; 1351 host_idx = bd_ring->wp + n_txbd; 1352 host_idx = host_idx < len ? host_idx : host_idx - len; 1353 1354 bd_ring->wp = host_idx; 1355} 1356 1357static void rtw89_pci_ops_tx_kick_off(struct rtw89_dev *rtwdev, u8 txch) 1358{ 1359 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1360 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[txch]; 1361 1362 if (rtwdev->hci.paused) { 1363 set_bit(txch, rtwpci->kick_map); 1364 return; 1365 } 1366 1367 __rtw89_pci_tx_kick_off(rtwdev, tx_ring); 1368} 1369 1370static void rtw89_pci_tx_kick_off_pending(struct rtw89_dev *rtwdev) 1371{ 1372 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1373 struct rtw89_pci_tx_ring *tx_ring; 1374 int txch; 1375 1376 for (txch = 0; txch < RTW89_TXCH_NUM; txch++) { 1377 if (!test_and_clear_bit(txch, rtwpci->kick_map)) 1378 continue; 1379 1380 tx_ring = &rtwpci->tx.rings[txch]; 1381 __rtw89_pci_tx_kick_off(rtwdev, tx_ring); 1382 } 1383} 1384 1385static void __pci_flush_txch(struct rtw89_dev *rtwdev, u8 txch, bool drop) 1386{ 1387 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1388 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[txch]; 1389 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring; 1390 u32 cur_idx, cur_rp; 1391 u8 i; 1392 1393 /* Because the time taked by the I/O is a bit dynamic, it's hard to 1394 * define a reasonable fixed total timeout to use read_poll_timeout* 1395 * helper. Instead, we can ensure a reasonable polling times, so we 1396 * just use for loop with udelay here. 1397 */ 1398 for (i = 0; i < 60; i++) { 1399 cur_idx = rtw89_read32(rtwdev, bd_ring->addr.idx); 1400 cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx); 1401 if (cur_rp == bd_ring->wp) 1402 return; 1403 1404 udelay(1); 1405 } 1406 1407 if (!drop) 1408 rtw89_info(rtwdev, "timed out to flush pci txch: %d\n", txch); 1409} 1410 1411static void __rtw89_pci_ops_flush_txchs(struct rtw89_dev *rtwdev, u32 txchs, 1412 bool drop) 1413{ 1414 const struct rtw89_pci_info *info = rtwdev->pci_info; 1415 u8 i; 1416 1417 for (i = 0; i < RTW89_TXCH_NUM; i++) { 1418 /* It may be unnecessary to flush FWCMD queue. */ 1419 if (i == RTW89_TXCH_CH12) 1420 continue; 1421 if (info->tx_dma_ch_mask & BIT(i)) 1422 continue; 1423 1424 if (txchs & BIT(i)) 1425 __pci_flush_txch(rtwdev, i, drop); 1426 } 1427} 1428 1429static void rtw89_pci_ops_flush_queues(struct rtw89_dev *rtwdev, u32 queues, 1430 bool drop) 1431{ 1432 __rtw89_pci_ops_flush_txchs(rtwdev, BIT(RTW89_TXCH_NUM) - 1, drop); 1433} 1434 1435u32 rtw89_pci_fill_txaddr_info(struct rtw89_dev *rtwdev, 1436 void *txaddr_info_addr, u32 total_len, 1437 dma_addr_t dma, u8 *add_info_nr) 1438{ 1439 struct rtw89_pci_tx_addr_info_32 *txaddr_info = txaddr_info_addr; 1440 __le16 option; 1441 1442 txaddr_info->length = cpu_to_le16(total_len); 1443 option = cpu_to_le16(RTW89_PCI_ADDR_MSDU_LS | RTW89_PCI_ADDR_NUM(1)); 1444 option |= le16_encode_bits(upper_32_bits(dma), RTW89_PCI_ADDR_HIGH_MASK); 1445 txaddr_info->option = option; 1446 txaddr_info->dma = cpu_to_le32(dma); 1447 1448 *add_info_nr = 1; 1449 1450 return sizeof(*txaddr_info); 1451} 1452EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info); 1453 1454u32 rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev *rtwdev, 1455 void *txaddr_info_addr, u32 total_len, 1456 dma_addr_t dma, u8 *add_info_nr) 1457{ 1458 struct rtw89_pci_tx_addr_info_32_v1 *txaddr_info = txaddr_info_addr; 1459 u32 remain = total_len; 1460 u32 len; 1461 u16 length_option; 1462 int n; 1463 1464 for (n = 0; n < RTW89_TXADDR_INFO_NR_V1 && remain; n++) { 1465 len = remain >= TXADDR_INFO_LENTHG_V1_MAX ? 1466 TXADDR_INFO_LENTHG_V1_MAX : remain; 1467 remain -= len; 1468 1469 length_option = FIELD_PREP(B_PCIADDR_LEN_V1_MASK, len) | 1470 FIELD_PREP(B_PCIADDR_HIGH_SEL_V1_MASK, 0) | 1471 FIELD_PREP(B_PCIADDR_LS_V1_MASK, remain == 0); 1472 length_option |= u16_encode_bits(upper_32_bits(dma), 1473 B_PCIADDR_HIGH_SEL_V1_MASK); 1474 txaddr_info->length_opt = cpu_to_le16(length_option); 1475 txaddr_info->dma_low_lsb = cpu_to_le16(FIELD_GET(GENMASK(15, 0), dma)); 1476 txaddr_info->dma_low_msb = cpu_to_le16(FIELD_GET(GENMASK(31, 16), dma)); 1477 1478 dma += len; 1479 txaddr_info++; 1480 } 1481 1482 WARN_ONCE(remain, "length overflow remain=%u total_len=%u", 1483 remain, total_len); 1484 1485 *add_info_nr = n; 1486 1487 return n * sizeof(*txaddr_info); 1488} 1489EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info_v1); 1490 1491static int rtw89_pci_txwd_submit(struct rtw89_dev *rtwdev, 1492 struct rtw89_pci_tx_ring *tx_ring, 1493 struct rtw89_pci_tx_wd *txwd, 1494 struct rtw89_core_tx_request *tx_req) 1495{ 1496 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1497 const struct rtw89_chip_info *chip = rtwdev->chip; 1498 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; 1499 struct rtw89_pci_tx_wp_info *txwp_info; 1500 void *txaddr_info_addr; 1501 struct pci_dev *pdev = rtwpci->pdev; 1502 struct sk_buff *skb = tx_req->skb; 1503 struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb); 1504 bool en_wd_info = desc_info->en_wd_info; 1505 u32 txwd_len; 1506 u32 txwp_len; 1507 u32 txaddr_info_len; 1508 dma_addr_t dma; 1509 int ret; 1510 1511 dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE); 1512 if (dma_mapping_error(&pdev->dev, dma)) { 1513 rtw89_err(rtwdev, "failed to map skb dma data\n"); 1514 ret = -EBUSY; 1515 goto err; 1516 } 1517 1518 tx_data->dma = dma; 1519 1520 txwp_len = sizeof(*txwp_info); 1521 txwd_len = chip->txwd_body_size; 1522 txwd_len += en_wd_info ? chip->txwd_info_size : 0; 1523 1524 txwp_info = txwd->vaddr + txwd_len; 1525 txwp_info->seq0 = cpu_to_le16(txwd->seq | RTW89_PCI_TXWP_VALID); 1526 txwp_info->seq1 = 0; 1527 txwp_info->seq2 = 0; 1528 txwp_info->seq3 = 0; 1529 1530 tx_ring->tx_cnt++; 1531 txaddr_info_addr = txwd->vaddr + txwd_len + txwp_len; 1532 txaddr_info_len = 1533 rtw89_chip_fill_txaddr_info(rtwdev, txaddr_info_addr, skb->len, 1534 dma, &desc_info->addr_info_nr); 1535 1536 txwd->len = txwd_len + txwp_len + txaddr_info_len; 1537 1538 rtw89_chip_fill_txdesc(rtwdev, desc_info, txwd->vaddr); 1539 1540 skb_queue_tail(&txwd->queue, skb); 1541 1542 return 0; 1543 1544err: 1545 return ret; 1546} 1547 1548static int rtw89_pci_fwcmd_submit(struct rtw89_dev *rtwdev, 1549 struct rtw89_pci_tx_ring *tx_ring, 1550 struct rtw89_pci_tx_bd_32 *txbd, 1551 struct rtw89_core_tx_request *tx_req) 1552{ 1553 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1554 const struct rtw89_chip_info *chip = rtwdev->chip; 1555 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; 1556 void *txdesc; 1557 int txdesc_size = chip->h2c_desc_size; 1558 struct pci_dev *pdev = rtwpci->pdev; 1559 struct sk_buff *skb = tx_req->skb; 1560 struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb); 1561 dma_addr_t dma; 1562 __le16 opt; 1563 1564 txdesc = skb_push(skb, txdesc_size); 1565 memset(txdesc, 0, txdesc_size); 1566 rtw89_chip_fill_txdesc_fwcmd(rtwdev, desc_info, txdesc); 1567 1568 dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE); 1569 if (dma_mapping_error(&pdev->dev, dma)) { 1570 rtw89_err(rtwdev, "failed to map fwcmd dma data\n"); 1571 return -EBUSY; 1572 } 1573 1574 tx_data->dma = dma; 1575 opt = cpu_to_le16(RTW89_PCI_TXBD_OPT_LS); 1576 opt |= le16_encode_bits(upper_32_bits(dma), RTW89_PCI_TXBD_OPT_DMA_HI); 1577 txbd->opt = opt; 1578 txbd->length = cpu_to_le16(skb->len); 1579 txbd->dma = cpu_to_le32(tx_data->dma); 1580 skb_queue_tail(&rtwpci->h2c_queue, skb); 1581 1582 rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1); 1583 1584 return 0; 1585} 1586 1587static int rtw89_pci_txbd_submit(struct rtw89_dev *rtwdev, 1588 struct rtw89_pci_tx_ring *tx_ring, 1589 struct rtw89_pci_tx_bd_32 *txbd, 1590 struct rtw89_core_tx_request *tx_req) 1591{ 1592 struct rtw89_pci_tx_wd *txwd; 1593 __le16 opt; 1594 int ret; 1595 1596 /* FWCMD queue doesn't have wd pages. Instead, it submits the CMD 1597 * buffer with WD BODY only. So here we don't need to check the free 1598 * pages of the wd ring. 1599 */ 1600 if (tx_ring->txch == RTW89_TXCH_CH12) 1601 return rtw89_pci_fwcmd_submit(rtwdev, tx_ring, txbd, tx_req); 1602 1603 txwd = rtw89_pci_dequeue_txwd(tx_ring); 1604 if (!txwd) { 1605 rtw89_err(rtwdev, "no available TXWD\n"); 1606 ret = -ENOSPC; 1607 goto err; 1608 } 1609 1610 ret = rtw89_pci_txwd_submit(rtwdev, tx_ring, txwd, tx_req); 1611 if (ret) { 1612 rtw89_err(rtwdev, "failed to submit TXWD %d\n", txwd->seq); 1613 goto err_enqueue_wd; 1614 } 1615 1616 list_add_tail(&txwd->list, &tx_ring->busy_pages); 1617 1618 opt = cpu_to_le16(RTW89_PCI_TXBD_OPT_LS); 1619 opt |= le16_encode_bits(upper_32_bits(txwd->paddr), RTW89_PCI_TXBD_OPT_DMA_HI); 1620 txbd->opt = opt; 1621 txbd->length = cpu_to_le16(txwd->len); 1622 txbd->dma = cpu_to_le32(txwd->paddr); 1623 1624 rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1); 1625 1626 return 0; 1627 1628err_enqueue_wd: 1629 rtw89_pci_enqueue_txwd(tx_ring, txwd); 1630err: 1631 return ret; 1632} 1633 1634static int rtw89_pci_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req, 1635 u8 txch) 1636{ 1637 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1638 struct rtw89_pci_tx_ring *tx_ring; 1639 struct rtw89_pci_tx_bd_32 *txbd; 1640 u32 n_avail_txbd; 1641 int ret = 0; 1642 1643 /* check the tx type and dma channel for fw cmd queue */ 1644 if ((txch == RTW89_TXCH_CH12 || 1645 tx_req->tx_type == RTW89_CORE_TX_TYPE_FWCMD) && 1646 (txch != RTW89_TXCH_CH12 || 1647 tx_req->tx_type != RTW89_CORE_TX_TYPE_FWCMD)) { 1648 rtw89_err(rtwdev, "only fw cmd uses dma channel 12\n"); 1649 return -EINVAL; 1650 } 1651 1652 tx_ring = &rtwpci->tx.rings[txch]; 1653 spin_lock_bh(&rtwpci->trx_lock); 1654 1655 n_avail_txbd = rtw89_pci_get_avail_txbd_num(tx_ring); 1656 if (n_avail_txbd == 0) { 1657 rtw89_err(rtwdev, "no available TXBD\n"); 1658 ret = -ENOSPC; 1659 goto err_unlock; 1660 } 1661 1662 txbd = rtw89_pci_get_next_txbd(tx_ring); 1663 ret = rtw89_pci_txbd_submit(rtwdev, tx_ring, txbd, tx_req); 1664 if (ret) { 1665 rtw89_err(rtwdev, "failed to submit TXBD\n"); 1666 goto err_unlock; 1667 } 1668 1669 spin_unlock_bh(&rtwpci->trx_lock); 1670 return 0; 1671 1672err_unlock: 1673 spin_unlock_bh(&rtwpci->trx_lock); 1674 return ret; 1675} 1676 1677static int rtw89_pci_ops_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req) 1678{ 1679 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; 1680 int ret; 1681 1682 ret = rtw89_pci_tx_write(rtwdev, tx_req, desc_info->ch_dma); 1683 if (ret) { 1684 rtw89_err(rtwdev, "failed to TX Queue %d\n", desc_info->ch_dma); 1685 return ret; 1686 } 1687 1688 return 0; 1689} 1690 1691const struct rtw89_pci_bd_ram rtw89_bd_ram_table_dual[RTW89_TXCH_NUM] = { 1692 [RTW89_TXCH_ACH0] = {.start_idx = 0, .max_num = 5, .min_num = 2}, 1693 [RTW89_TXCH_ACH1] = {.start_idx = 5, .max_num = 5, .min_num = 2}, 1694 [RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2}, 1695 [RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2}, 1696 [RTW89_TXCH_ACH4] = {.start_idx = 20, .max_num = 5, .min_num = 2}, 1697 [RTW89_TXCH_ACH5] = {.start_idx = 25, .max_num = 5, .min_num = 2}, 1698 [RTW89_TXCH_ACH6] = {.start_idx = 30, .max_num = 5, .min_num = 2}, 1699 [RTW89_TXCH_ACH7] = {.start_idx = 35, .max_num = 5, .min_num = 2}, 1700 [RTW89_TXCH_CH8] = {.start_idx = 40, .max_num = 5, .min_num = 1}, 1701 [RTW89_TXCH_CH9] = {.start_idx = 45, .max_num = 5, .min_num = 1}, 1702 [RTW89_TXCH_CH10] = {.start_idx = 50, .max_num = 5, .min_num = 1}, 1703 [RTW89_TXCH_CH11] = {.start_idx = 55, .max_num = 5, .min_num = 1}, 1704 [RTW89_TXCH_CH12] = {.start_idx = 60, .max_num = 4, .min_num = 1}, 1705}; 1706EXPORT_SYMBOL(rtw89_bd_ram_table_dual); 1707 1708const struct rtw89_pci_bd_ram rtw89_bd_ram_table_single[RTW89_TXCH_NUM] = { 1709 [RTW89_TXCH_ACH0] = {.start_idx = 0, .max_num = 5, .min_num = 2}, 1710 [RTW89_TXCH_ACH1] = {.start_idx = 5, .max_num = 5, .min_num = 2}, 1711 [RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2}, 1712 [RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2}, 1713 [RTW89_TXCH_CH8] = {.start_idx = 20, .max_num = 4, .min_num = 1}, 1714 [RTW89_TXCH_CH9] = {.start_idx = 24, .max_num = 4, .min_num = 1}, 1715 [RTW89_TXCH_CH12] = {.start_idx = 28, .max_num = 4, .min_num = 1}, 1716}; 1717EXPORT_SYMBOL(rtw89_bd_ram_table_single); 1718 1719static void rtw89_pci_init_wp_16sel(struct rtw89_dev *rtwdev) 1720{ 1721 const struct rtw89_pci_info *info = rtwdev->pci_info; 1722 u32 addr = info->wp_sel_addr; 1723 u32 val; 1724 int i; 1725 1726 if (!info->wp_sel_addr) 1727 return; 1728 1729 for (i = 0; i < 16; i += 4) { 1730 val = u32_encode_bits(i + 0, MASKBYTE0) | 1731 u32_encode_bits(i + 1, MASKBYTE1) | 1732 u32_encode_bits(i + 2, MASKBYTE2) | 1733 u32_encode_bits(i + 3, MASKBYTE3); 1734 rtw89_write32(rtwdev, addr + i, val); 1735 } 1736} 1737 1738static u16 rtw89_pci_enc_bd_cfg(struct rtw89_dev *rtwdev, u16 bd_num, 1739 u32 dma_offset) 1740{ 1741 u16 dma_offset_sel; 1742 u16 num_sel; 1743 1744 /* B_BE_TX_NUM_SEL_MASK, B_BE_RX_NUM_SEL_MASK: 1745 * 0 -> 0 1746 * 1 -> 64 = 2^6 1747 * 2 -> 128 = 2^7 1748 * ... 1749 * 7 -> 4096 = 2^12 1750 */ 1751 num_sel = ilog2(bd_num) - 5; 1752 1753 if (hweight16(bd_num) != 1) 1754 rtw89_warn(rtwdev, "bd_num %u is not power of 2\n", bd_num); 1755 1756 /* B_BE_TX_START_OFFSET_MASK, B_BE_RX_START_OFFSET_MASK: 1757 * 0 -> 0 = 0 * 2^9 1758 * 1 -> 512 = 1 * 2^9 1759 * 2 -> 1024 = 2 * 2^9 1760 * 3 -> 1536 = 3 * 2^9 1761 * ... 1762 * 255 -> 130560 = 255 * 2^9 1763 */ 1764 dma_offset_sel = dma_offset >> 9; 1765 1766 if (dma_offset % 512) 1767 rtw89_warn(rtwdev, "offset %u is not multiple of 512\n", dma_offset); 1768 1769 return u16_encode_bits(num_sel, B_BE_TX_NUM_SEL_MASK) | 1770 u16_encode_bits(dma_offset_sel, B_BE_TX_START_OFFSET_MASK); 1771} 1772 1773static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev) 1774{ 1775 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1776 const struct rtw89_pci_info *info = rtwdev->pci_info; 1777 const struct rtw89_pci_bd_ram *bd_ram_table = *info->bd_ram_table; 1778 struct rtw89_pci_tx_ring *tx_ring; 1779 struct rtw89_pci_rx_ring *rx_ring; 1780 struct rtw89_pci_dma_ring *bd_ring; 1781 const struct rtw89_pci_bd_ram *bd_ram; 1782 dma_addr_t group_dma_base = 0; 1783 u16 num_or_offset; 1784 u32 addr_desa_l; 1785 u32 addr_bdram; 1786 u32 addr_num; 1787 u32 addr_idx; 1788 u32 val32; 1789 int i; 1790 1791 for (i = 0; i < RTW89_TXCH_NUM; i++) { 1792 if (info->tx_dma_ch_mask & BIT(i)) 1793 continue; 1794 1795 tx_ring = &rtwpci->tx.rings[i]; 1796 bd_ring = &tx_ring->bd_ring; 1797 bd_ram = bd_ram_table ? &bd_ram_table[i] : NULL; 1798 addr_num = bd_ring->addr.num; 1799 addr_bdram = bd_ring->addr.bdram; 1800 addr_desa_l = bd_ring->addr.desa_l; 1801 bd_ring->wp = 0; 1802 bd_ring->rp = 0; 1803 1804 if (info->group_bd_addr) { 1805 if (addr_desa_l) 1806 group_dma_base = bd_ring->dma; 1807 1808 num_or_offset = 1809 rtw89_pci_enc_bd_cfg(rtwdev, bd_ring->len, 1810 bd_ring->dma - group_dma_base); 1811 } else { 1812 num_or_offset = bd_ring->len; 1813 } 1814 rtw89_write16(rtwdev, addr_num, num_or_offset); 1815 1816 if (addr_bdram && bd_ram) { 1817 val32 = FIELD_PREP(BDRAM_SIDX_MASK, bd_ram->start_idx) | 1818 FIELD_PREP(BDRAM_MAX_MASK, bd_ram->max_num) | 1819 FIELD_PREP(BDRAM_MIN_MASK, bd_ram->min_num); 1820 1821 rtw89_write32(rtwdev, addr_bdram, val32); 1822 } 1823 if (addr_desa_l) { 1824 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma); 1825 rtw89_write32(rtwdev, addr_desa_l + 4, upper_32_bits(bd_ring->dma)); 1826 } 1827 } 1828 1829 for (i = 0; i < RTW89_RXCH_NUM; i++) { 1830 rx_ring = &rtwpci->rx.rings[i]; 1831 bd_ring = &rx_ring->bd_ring; 1832 addr_num = bd_ring->addr.num; 1833 addr_idx = bd_ring->addr.idx; 1834 addr_desa_l = bd_ring->addr.desa_l; 1835 if (info->rx_ring_eq_is_full) 1836 bd_ring->wp = bd_ring->len - 1; 1837 else 1838 bd_ring->wp = 0; 1839 bd_ring->rp = 0; 1840 rx_ring->diliver_skb = NULL; 1841 rx_ring->diliver_desc.ready = false; 1842 rx_ring->target_rx_tag = 0; 1843 1844 if (info->group_bd_addr) { 1845 if (addr_desa_l) 1846 group_dma_base = bd_ring->dma; 1847 1848 num_or_offset = 1849 rtw89_pci_enc_bd_cfg(rtwdev, bd_ring->len, 1850 bd_ring->dma - group_dma_base); 1851 } else { 1852 num_or_offset = bd_ring->len; 1853 } 1854 rtw89_write16(rtwdev, addr_num, num_or_offset); 1855 1856 if (addr_desa_l) { 1857 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma); 1858 rtw89_write32(rtwdev, addr_desa_l + 4, upper_32_bits(bd_ring->dma)); 1859 } 1860 1861 if (info->rx_ring_eq_is_full) 1862 rtw89_write16(rtwdev, addr_idx, bd_ring->wp); 1863 } 1864 1865 rtw89_pci_init_wp_16sel(rtwdev); 1866} 1867 1868static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev, 1869 struct rtw89_pci_tx_ring *tx_ring) 1870{ 1871 rtw89_pci_release_busy_txwd(rtwdev, tx_ring); 1872 rtw89_pci_release_pending_txwd_skb(rtwdev, tx_ring); 1873} 1874 1875void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev) 1876{ 1877 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1878 const struct rtw89_pci_info *info = rtwdev->pci_info; 1879 int txch; 1880 1881 rtw89_pci_reset_trx_rings(rtwdev); 1882 1883 spin_lock_bh(&rtwpci->trx_lock); 1884 for (txch = 0; txch < RTW89_TXCH_NUM; txch++) { 1885 if (info->tx_dma_ch_mask & BIT(txch)) 1886 continue; 1887 if (txch == RTW89_TXCH_CH12) { 1888 rtw89_pci_release_fwcmd(rtwdev, rtwpci, 1889 skb_queue_len(&rtwpci->h2c_queue), true); 1890 continue; 1891 } 1892 rtw89_pci_release_tx_ring(rtwdev, &rtwpci->tx.rings[txch]); 1893 } 1894 spin_unlock_bh(&rtwpci->trx_lock); 1895} 1896 1897static void rtw89_pci_enable_intr_lock(struct rtw89_dev *rtwdev) 1898{ 1899 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1900 unsigned long flags; 1901 1902 spin_lock_irqsave(&rtwpci->irq_lock, flags); 1903 rtwpci->running = true; 1904 rtw89_chip_enable_intr(rtwdev, rtwpci); 1905 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); 1906} 1907 1908static void rtw89_pci_disable_intr_lock(struct rtw89_dev *rtwdev) 1909{ 1910 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1911 unsigned long flags; 1912 1913 spin_lock_irqsave(&rtwpci->irq_lock, flags); 1914 rtwpci->running = false; 1915 rtw89_chip_disable_intr(rtwdev, rtwpci); 1916 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); 1917} 1918 1919static int rtw89_pci_ops_start(struct rtw89_dev *rtwdev) 1920{ 1921 rtw89_core_napi_start(rtwdev); 1922 rtw89_pci_enable_intr_lock(rtwdev); 1923 1924 return 0; 1925} 1926 1927static void rtw89_pci_ops_stop(struct rtw89_dev *rtwdev) 1928{ 1929 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1930 struct pci_dev *pdev = rtwpci->pdev; 1931 1932 rtw89_pci_disable_intr_lock(rtwdev); 1933 synchronize_irq(pdev->irq); 1934 rtw89_core_napi_stop(rtwdev); 1935} 1936 1937static void rtw89_pci_ops_pause(struct rtw89_dev *rtwdev, bool pause) 1938{ 1939 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1940 struct pci_dev *pdev = rtwpci->pdev; 1941 1942 if (pause) { 1943 rtw89_pci_disable_intr_lock(rtwdev); 1944 synchronize_irq(pdev->irq); 1945 if (test_bit(RTW89_FLAG_NAPI_RUNNING, rtwdev->flags)) 1946 napi_synchronize(&rtwdev->napi); 1947 } else { 1948 rtw89_pci_enable_intr_lock(rtwdev); 1949 rtw89_pci_tx_kick_off_pending(rtwdev); 1950 } 1951} 1952 1953static 1954void rtw89_pci_switch_bd_idx_addr(struct rtw89_dev *rtwdev, bool low_power) 1955{ 1956 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1957 const struct rtw89_pci_info *info = rtwdev->pci_info; 1958 const struct rtw89_pci_bd_idx_addr *bd_idx_addr = info->bd_idx_addr_low_power; 1959 const struct rtw89_pci_ch_dma_addr_set *dma_addr_set = info->dma_addr_set; 1960 struct rtw89_pci_tx_ring *tx_ring; 1961 struct rtw89_pci_rx_ring *rx_ring; 1962 int i; 1963 1964 if (WARN(!bd_idx_addr, "only HCI with low power mode needs this\n")) 1965 return; 1966 1967 for (i = 0; i < RTW89_TXCH_NUM; i++) { 1968 tx_ring = &rtwpci->tx.rings[i]; 1969 tx_ring->bd_ring.addr.idx = low_power ? 1970 bd_idx_addr->tx_bd_addrs[i] : 1971 dma_addr_set->tx[i].idx; 1972 } 1973 1974 for (i = 0; i < RTW89_RXCH_NUM; i++) { 1975 rx_ring = &rtwpci->rx.rings[i]; 1976 rx_ring->bd_ring.addr.idx = low_power ? 1977 bd_idx_addr->rx_bd_addrs[i] : 1978 dma_addr_set->rx[i].idx; 1979 } 1980} 1981 1982static void rtw89_pci_ops_switch_mode(struct rtw89_dev *rtwdev, bool low_power) 1983{ 1984 enum rtw89_pci_intr_mask_cfg cfg; 1985 1986 WARN(!rtwdev->hci.paused, "HCI isn't paused\n"); 1987 1988 cfg = low_power ? RTW89_PCI_INTR_MASK_LOW_POWER : RTW89_PCI_INTR_MASK_NORMAL; 1989 rtw89_chip_config_intr_mask(rtwdev, cfg); 1990 rtw89_pci_switch_bd_idx_addr(rtwdev, low_power); 1991} 1992 1993static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data); 1994 1995static u32 rtw89_pci_ops_read32_cmac(struct rtw89_dev *rtwdev, u32 addr) 1996{ 1997 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 1998 u32 val = readl(rtwpci->mmap + addr); 1999 int count; 2000 2001 for (count = 0; ; count++) { 2002 if (val != RTW89_R32_DEAD) 2003 return val; 2004 if (count >= MAC_REG_POOL_COUNT) { 2005 rtw89_warn(rtwdev, "addr %#x = %#x\n", addr, val); 2006 return RTW89_R32_DEAD; 2007 } 2008 rtw89_pci_ops_write32(rtwdev, R_AX_CK_EN, B_AX_CMAC_ALLCKEN); 2009 val = readl(rtwpci->mmap + addr); 2010 } 2011 2012 return val; 2013} 2014 2015static u8 rtw89_pci_ops_read8(struct rtw89_dev *rtwdev, u32 addr) 2016{ 2017 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 2018 u32 addr32, val32, shift; 2019 2020 if (!ACCESS_CMAC(addr)) 2021 return readb(rtwpci->mmap + addr); 2022 2023 addr32 = addr & ~0x3; 2024 shift = (addr & 0x3) * 8; 2025 val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32); 2026 return val32 >> shift; 2027} 2028 2029static u16 rtw89_pci_ops_read16(struct rtw89_dev *rtwdev, u32 addr) 2030{ 2031 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 2032 u32 addr32, val32, shift; 2033 2034 if (!ACCESS_CMAC(addr)) 2035 return readw(rtwpci->mmap + addr); 2036 2037 addr32 = addr & ~0x3; 2038 shift = (addr & 0x3) * 8; 2039 val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32); 2040 return val32 >> shift; 2041} 2042 2043static u32 rtw89_pci_ops_read32(struct rtw89_dev *rtwdev, u32 addr) 2044{ 2045 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 2046 2047 if (!ACCESS_CMAC(addr)) 2048 return readl(rtwpci->mmap + addr); 2049 2050 return rtw89_pci_ops_read32_cmac(rtwdev, addr); 2051} 2052 2053static void rtw89_pci_ops_write8(struct rtw89_dev *rtwdev, u32 addr, u8 data) 2054{ 2055 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 2056 2057 writeb(data, rtwpci->mmap + addr); 2058} 2059 2060static void rtw89_pci_ops_write16(struct rtw89_dev *rtwdev, u32 addr, u16 data) 2061{ 2062 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 2063 2064 writew(data, rtwpci->mmap + addr); 2065} 2066 2067static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data) 2068{ 2069 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 2070 2071 writel(data, rtwpci->mmap + addr); 2072} 2073 2074static u32 rtw89_pci_ops_read32_pci_cfg(struct rtw89_dev *rtwdev, u32 addr) 2075{ 2076 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 2077 struct pci_dev *pdev = rtwpci->pdev; 2078 u32 value; 2079 int ret; 2080 2081 ret = pci_read_config_dword(pdev, addr, &value); 2082 if (ret) 2083 return RTW89_R32_EA; 2084 2085 return value; 2086} 2087 2088static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable) 2089{ 2090 const struct rtw89_pci_info *info = rtwdev->pci_info; 2091 2092 if (enable) 2093 rtw89_write32_set(rtwdev, info->init_cfg_reg, 2094 info->rxhci_en_bit | info->txhci_en_bit); 2095 else 2096 rtw89_write32_clr(rtwdev, info->init_cfg_reg, 2097 info->rxhci_en_bit | info->txhci_en_bit); 2098} 2099 2100static void rtw89_pci_ctrl_dma_io(struct rtw89_dev *rtwdev, bool enable) 2101{ 2102 const struct rtw89_pci_info *info = rtwdev->pci_info; 2103 const struct rtw89_reg_def *reg = &info->dma_io_stop; 2104 2105 if (enable) 2106 rtw89_write32_clr(rtwdev, reg->addr, reg->mask); 2107 else 2108 rtw89_write32_set(rtwdev, reg->addr, reg->mask); 2109} 2110 2111void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable) 2112{ 2113 rtw89_pci_ctrl_dma_io(rtwdev, enable); 2114 rtw89_pci_ctrl_dma_trx(rtwdev, enable); 2115} 2116 2117static int rtw89_pci_check_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 rw_bit) 2118{ 2119 u16 val; 2120 2121 rtw89_write8(rtwdev, R_AX_MDIO_CFG, addr & 0x1F); 2122 2123 val = rtw89_read16(rtwdev, R_AX_MDIO_CFG); 2124 switch (speed) { 2125 case PCIE_PHY_GEN1: 2126 if (addr < 0x20) 2127 val = u16_replace_bits(val, MDIO_PG0_G1, B_AX_MDIO_PHY_ADDR_MASK); 2128 else 2129 val = u16_replace_bits(val, MDIO_PG1_G1, B_AX_MDIO_PHY_ADDR_MASK); 2130 break; 2131 case PCIE_PHY_GEN2: 2132 if (addr < 0x20) 2133 val = u16_replace_bits(val, MDIO_PG0_G2, B_AX_MDIO_PHY_ADDR_MASK); 2134 else 2135 val = u16_replace_bits(val, MDIO_PG1_G2, B_AX_MDIO_PHY_ADDR_MASK); 2136 break; 2137 default: 2138 rtw89_err(rtwdev, "[ERR]Error Speed %d!\n", speed); 2139 return -EINVAL; 2140 } 2141 rtw89_write16(rtwdev, R_AX_MDIO_CFG, val); 2142 rtw89_write16_set(rtwdev, R_AX_MDIO_CFG, rw_bit); 2143 2144 return read_poll_timeout(rtw89_read16, val, !(val & rw_bit), 10, 2000, 2145 false, rtwdev, R_AX_MDIO_CFG); 2146} 2147 2148static int 2149rtw89_read16_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 *val) 2150{ 2151 int ret; 2152 2153 ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_RFLAG); 2154 if (ret) { 2155 rtw89_err(rtwdev, "[ERR]MDIO R16 0x%X fail ret=%d!\n", addr, ret); 2156 return ret; 2157 } 2158 *val = rtw89_read16(rtwdev, R_AX_MDIO_RDATA); 2159 2160 return 0; 2161} 2162 2163static int 2164rtw89_write16_mdio(struct rtw89_dev *rtwdev, u8 addr, u16 data, u8 speed) 2165{ 2166 int ret; 2167 2168 rtw89_write16(rtwdev, R_AX_MDIO_WDATA, data); 2169 ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_WFLAG); 2170 if (ret) { 2171 rtw89_err(rtwdev, "[ERR]MDIO W16 0x%X = %x fail ret=%d!\n", addr, data, ret); 2172 return ret; 2173 } 2174 2175 return 0; 2176} 2177 2178static int 2179rtw89_write16_mdio_mask(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u16 data, u8 speed) 2180{ 2181 u32 shift; 2182 int ret; 2183 u16 val; 2184 2185 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val); 2186 if (ret) 2187 return ret; 2188 2189 shift = __ffs(mask); 2190 val &= ~mask; 2191 val |= ((data << shift) & mask); 2192 2193 ret = rtw89_write16_mdio(rtwdev, addr, val, speed); 2194 if (ret) 2195 return ret; 2196 2197 return 0; 2198} 2199 2200static int rtw89_write16_mdio_set(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed) 2201{ 2202 int ret; 2203 u16 val; 2204 2205 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val); 2206 if (ret) 2207 return ret; 2208 ret = rtw89_write16_mdio(rtwdev, addr, val | mask, speed); 2209 if (ret) 2210 return ret; 2211 2212 return 0; 2213} 2214 2215static int rtw89_write16_mdio_clr(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed) 2216{ 2217 int ret; 2218 u16 val; 2219 2220 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val); 2221 if (ret) 2222 return ret; 2223 ret = rtw89_write16_mdio(rtwdev, addr, val & ~mask, speed); 2224 if (ret) 2225 return ret; 2226 2227 return 0; 2228} 2229 2230static int rtw89_dbi_write8(struct rtw89_dev *rtwdev, u16 addr, u8 data) 2231{ 2232 u16 addr_2lsb = addr & B_AX_DBI_2LSB; 2233 u16 write_addr; 2234 u8 flag; 2235 int ret; 2236 2237 write_addr = addr & B_AX_DBI_ADDR_MSK; 2238 write_addr |= u16_encode_bits(BIT(addr_2lsb), B_AX_DBI_WREN_MSK); 2239 rtw89_write8(rtwdev, R_AX_DBI_WDATA + addr_2lsb, data); 2240 rtw89_write16(rtwdev, R_AX_DBI_FLAG, write_addr); 2241 rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_WFLAG >> 16); 2242 2243 ret = read_poll_timeout_atomic(rtw89_read8, flag, !flag, 10, 2244 10 * RTW89_PCI_WR_RETRY_CNT, false, 2245 rtwdev, R_AX_DBI_FLAG + 2); 2246 if (ret) 2247 rtw89_err(rtwdev, "failed to write DBI register, addr=0x%X\n", 2248 addr); 2249 2250 return ret; 2251} 2252 2253static int rtw89_dbi_read8(struct rtw89_dev *rtwdev, u16 addr, u8 *value) 2254{ 2255 u16 read_addr = addr & B_AX_DBI_ADDR_MSK; 2256 u8 flag; 2257 int ret; 2258 2259 rtw89_write16(rtwdev, R_AX_DBI_FLAG, read_addr); 2260 rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_RFLAG >> 16); 2261 2262 ret = read_poll_timeout_atomic(rtw89_read8, flag, !flag, 10, 2263 10 * RTW89_PCI_WR_RETRY_CNT, false, 2264 rtwdev, R_AX_DBI_FLAG + 2); 2265 if (ret) { 2266 rtw89_err(rtwdev, "failed to read DBI register, addr=0x%X\n", 2267 addr); 2268 return ret; 2269 } 2270 2271 read_addr = R_AX_DBI_RDATA + (addr & 3); 2272 *value = rtw89_read8(rtwdev, read_addr); 2273 2274 return 0; 2275} 2276 2277static int rtw89_pci_write_config_byte(struct rtw89_dev *rtwdev, u16 addr, 2278 u8 data) 2279{ 2280 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 2281 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; 2282 struct pci_dev *pdev = rtwpci->pdev; 2283 int ret; 2284 2285 ret = pci_write_config_byte(pdev, addr, data); 2286 if (!ret) 2287 return 0; 2288 2289 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) 2290 ret = rtw89_dbi_write8(rtwdev, addr, data); 2291 2292 return ret; 2293} 2294 2295static int rtw89_pci_read_config_byte(struct rtw89_dev *rtwdev, u16 addr, 2296 u8 *value) 2297{ 2298 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 2299 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; 2300 struct pci_dev *pdev = rtwpci->pdev; 2301 int ret; 2302 2303 ret = pci_read_config_byte(pdev, addr, value); 2304 if (!ret) 2305 return 0; 2306 2307 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) 2308 ret = rtw89_dbi_read8(rtwdev, addr, value); 2309 2310 return ret; 2311} 2312 2313static int rtw89_pci_config_byte_set(struct rtw89_dev *rtwdev, u16 addr, 2314 u8 bit) 2315{ 2316 u8 value; 2317 int ret; 2318 2319 ret = rtw89_pci_read_config_byte(rtwdev, addr, &value); 2320 if (ret) 2321 return ret; 2322 2323 value |= bit; 2324 ret = rtw89_pci_write_config_byte(rtwdev, addr, value); 2325 2326 return ret; 2327} 2328 2329static int rtw89_pci_config_byte_clr(struct rtw89_dev *rtwdev, u16 addr, 2330 u8 bit) 2331{ 2332 u8 value; 2333 int ret; 2334 2335 ret = rtw89_pci_read_config_byte(rtwdev, addr, &value); 2336 if (ret) 2337 return ret; 2338 2339 value &= ~bit; 2340 ret = rtw89_pci_write_config_byte(rtwdev, addr, value); 2341 2342 return ret; 2343} 2344 2345static int 2346__get_target(struct rtw89_dev *rtwdev, u16 *target, enum rtw89_pcie_phy phy_rate) 2347{ 2348 u16 val, tar; 2349 int ret; 2350 2351 /* Enable counter */ 2352 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val); 2353 if (ret) 2354 return ret; 2355 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN, 2356 phy_rate); 2357 if (ret) 2358 return ret; 2359 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val | B_AX_CLK_CALIB_EN, 2360 phy_rate); 2361 if (ret) 2362 return ret; 2363 2364 fsleep(300); 2365 2366 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &tar); 2367 if (ret) 2368 return ret; 2369 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN, 2370 phy_rate); 2371 if (ret) 2372 return ret; 2373 2374 tar = tar & 0x0FFF; 2375 if (tar == 0 || tar == 0x0FFF) { 2376 rtw89_err(rtwdev, "[ERR]Get target failed.\n"); 2377 return -EINVAL; 2378 } 2379 2380 *target = tar; 2381 2382 return 0; 2383} 2384 2385static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev) 2386{ 2387 int ret; 2388 2389 if (!rtw89_is_rtl885xb(rtwdev)) 2390 return 0; 2391 2392 ret = rtw89_write16_mdio_mask(rtwdev, RAC_REG_FLD_0, BAC_AUTOK_N_MASK, 2393 PCIE_AUTOK_4, PCIE_PHY_GEN1); 2394 return ret; 2395} 2396 2397static int rtw89_pci_auto_refclk_cal(struct rtw89_dev *rtwdev, bool autook_en) 2398{ 2399 enum rtw89_pcie_phy phy_rate; 2400 u16 val16, mgn_set, div_set, tar; 2401 u8 val8, bdr_ori; 2402 bool l1_flag = false; 2403 int ret = 0; 2404 2405 if (!rtw89_is_rtl885xb(rtwdev)) 2406 return 0; 2407 2408 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_PHY_RATE, &val8); 2409 if (ret) { 2410 rtw89_err(rtwdev, "[ERR]pci config read %X\n", 2411 RTW89_PCIE_PHY_RATE); 2412 return ret; 2413 } 2414 2415 if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x1) { 2416 phy_rate = PCIE_PHY_GEN1; 2417 } else if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x2) { 2418 phy_rate = PCIE_PHY_GEN2; 2419 } else { 2420 rtw89_err(rtwdev, "[ERR]PCIe PHY rate %#x not support\n", val8); 2421 return -EOPNOTSUPP; 2422 } 2423 /* Disable L1BD */ 2424 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_L1_CTRL, &bdr_ori); 2425 if (ret) { 2426 rtw89_err(rtwdev, "[ERR]pci config read %X\n", RTW89_PCIE_L1_CTRL); 2427 return ret; 2428 } 2429 2430 if (bdr_ori & RTW89_PCIE_BIT_L1) { 2431 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL, 2432 bdr_ori & ~RTW89_PCIE_BIT_L1); 2433 if (ret) { 2434 rtw89_err(rtwdev, "[ERR]pci config write %X\n", 2435 RTW89_PCIE_L1_CTRL); 2436 return ret; 2437 } 2438 l1_flag = true; 2439 } 2440 2441 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16); 2442 if (ret) { 2443 rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1); 2444 goto end; 2445 } 2446 2447 if (val16 & B_AX_CALIB_EN) { 2448 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, 2449 val16 & ~B_AX_CALIB_EN, phy_rate); 2450 if (ret) { 2451 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1); 2452 goto end; 2453 } 2454 } 2455 2456 if (!autook_en) 2457 goto end; 2458 /* Set div */ 2459 ret = rtw89_write16_mdio_clr(rtwdev, RAC_CTRL_PPR_V1, B_AX_DIV, phy_rate); 2460 if (ret) { 2461 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1); 2462 goto end; 2463 } 2464 2465 /* Obtain div and margin */ 2466 ret = __get_target(rtwdev, &tar, phy_rate); 2467 if (ret) { 2468 rtw89_err(rtwdev, "[ERR]1st get target fail %d\n", ret); 2469 goto end; 2470 } 2471 2472 mgn_set = tar * INTF_INTGRA_HOSTREF_V1 / INTF_INTGRA_MINREF_V1 - tar; 2473 2474 if (mgn_set >= 128) { 2475 div_set = 0x0003; 2476 mgn_set = 0x000F; 2477 } else if (mgn_set >= 64) { 2478 div_set = 0x0003; 2479 mgn_set >>= 3; 2480 } else if (mgn_set >= 32) { 2481 div_set = 0x0002; 2482 mgn_set >>= 2; 2483 } else if (mgn_set >= 16) { 2484 div_set = 0x0001; 2485 mgn_set >>= 1; 2486 } else if (mgn_set == 0) { 2487 rtw89_err(rtwdev, "[ERR]cal mgn is 0,tar = %d\n", tar); 2488 goto end; 2489 } else { 2490 div_set = 0x0000; 2491 } 2492 2493 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16); 2494 if (ret) { 2495 rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1); 2496 goto end; 2497 } 2498 2499 val16 |= u16_encode_bits(div_set, B_AX_DIV); 2500 2501 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val16, phy_rate); 2502 if (ret) { 2503 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1); 2504 goto end; 2505 } 2506 2507 ret = __get_target(rtwdev, &tar, phy_rate); 2508 if (ret) { 2509 rtw89_err(rtwdev, "[ERR]2nd get target fail %d\n", ret); 2510 goto end; 2511 } 2512 2513 rtw89_debug(rtwdev, RTW89_DBG_HCI, "[TRACE]target = 0x%X, div = 0x%X, margin = 0x%X\n", 2514 tar, div_set, mgn_set); 2515 ret = rtw89_write16_mdio(rtwdev, RAC_SET_PPR_V1, 2516 (tar & 0x0FFF) | (mgn_set << 12), phy_rate); 2517 if (ret) { 2518 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_SET_PPR_V1); 2519 goto end; 2520 } 2521 2522 /* Enable function */ 2523 ret = rtw89_write16_mdio_set(rtwdev, RAC_CTRL_PPR_V1, B_AX_CALIB_EN, phy_rate); 2524 if (ret) { 2525 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1); 2526 goto end; 2527 } 2528 2529 /* CLK delay = 0 */ 2530 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL, 2531 PCIE_CLKDLY_HW_0); 2532 2533end: 2534 /* Set L1BD to ori */ 2535 if (l1_flag) { 2536 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL, 2537 bdr_ori); 2538 if (ret) { 2539 rtw89_err(rtwdev, "[ERR]pci config write %X\n", 2540 RTW89_PCIE_L1_CTRL); 2541 return ret; 2542 } 2543 } 2544 2545 return ret; 2546} 2547 2548static int rtw89_pci_deglitch_setting(struct rtw89_dev *rtwdev) 2549{ 2550 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; 2551 int ret; 2552 2553 if (chip_id == RTL8852A) { 2554 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH, 2555 PCIE_PHY_GEN1); 2556 if (ret) 2557 return ret; 2558 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH, 2559 PCIE_PHY_GEN2); 2560 if (ret) 2561 return ret; 2562 } else if (chip_id == RTL8852C) { 2563 rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA24 * 2, 2564 B_AX_DEGLITCH); 2565 rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA24 * 2, 2566 B_AX_DEGLITCH); 2567 } 2568 2569 return 0; 2570} 2571 2572static void rtw89_pci_disable_eq_ax(struct rtw89_dev *rtwdev) 2573{ 2574 u16 g1_oobs, g2_oobs; 2575 u32 backup_aspm; 2576 u32 phy_offset; 2577 u16 offset_cal; 2578 u16 oobs_val; 2579 int ret; 2580 u8 gen; 2581 2582 if (rtwdev->chip->chip_id != RTL8852C) 2583 return; 2584 2585 g1_oobs = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 + 2586 RAC_ANA09 * RAC_MULT, BAC_OOBS_SEL); 2587 g2_oobs = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G2 + 2588 RAC_ANA09 * RAC_MULT, BAC_OOBS_SEL); 2589 if (g1_oobs && g2_oobs) 2590 return; 2591 2592 backup_aspm = rtw89_read32(rtwdev, R_AX_PCIE_MIX_CFG_V1); 2593 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK); 2594 2595 ret = rtw89_pci_get_phy_offset_by_link_speed(rtwdev, &phy_offset); 2596 if (ret) 2597 goto out; 2598 2599 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0D * RAC_MULT, BAC_RX_TEST_EN); 2600 rtw89_write16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT, ADDR_SEL_PINOUT_DIS_VAL); 2601 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT, B_PCIE_BIT_RD_SEL); 2602 2603 oobs_val = rtw89_read16_mask(rtwdev, phy_offset + RAC_ANA1F * RAC_MULT, 2604 OOBS_LEVEL_MASK); 2605 2606 rtw89_write16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA03 * RAC_MULT, 2607 OOBS_SEN_MASK, oobs_val); 2608 rtw89_write16_set(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA09 * RAC_MULT, 2609 BAC_OOBS_SEL); 2610 2611 rtw89_write16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA03 * RAC_MULT, 2612 OOBS_SEN_MASK, oobs_val); 2613 rtw89_write16_set(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA09 * RAC_MULT, 2614 BAC_OOBS_SEL); 2615 2616 /* offset K */ 2617 for (gen = 1; gen <= 2; gen++) { 2618 phy_offset = gen == 1 ? R_RAC_DIRECT_OFFSET_G1 : 2619 R_RAC_DIRECT_OFFSET_G2; 2620 2621 rtw89_write16_clr(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT, 2622 B_PCIE_BIT_RD_SEL); 2623 } 2624 2625 offset_cal = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 + 2626 RAC_ANA1F * RAC_MULT, OFFSET_CAL_MASK); 2627 2628 for (gen = 1; gen <= 2; gen++) { 2629 phy_offset = gen == 1 ? R_RAC_DIRECT_OFFSET_G1 : 2630 R_RAC_DIRECT_OFFSET_G2; 2631 2632 rtw89_write16_mask(rtwdev, phy_offset + RAC_ANA0B * RAC_MULT, 2633 MANUAL_LVL_MASK, offset_cal); 2634 rtw89_write16_clr(rtwdev, phy_offset + RAC_ANA0D * RAC_MULT, 2635 OFFSET_CAL_MODE); 2636 } 2637 2638out: 2639 rtw89_write32(rtwdev, R_AX_PCIE_MIX_CFG_V1, backup_aspm); 2640} 2641 2642static void rtw89_pci_ber(struct rtw89_dev *rtwdev) 2643{ 2644 u32 phy_offset; 2645 2646 if (!test_bit(RTW89_QUIRK_PCI_BER, rtwdev->quirks)) 2647 return; 2648 2649 phy_offset = R_RAC_DIRECT_OFFSET_G1; 2650 rtw89_write16(rtwdev, phy_offset + RAC_ANA1E * RAC_MULT, RAC_ANA1E_G1_VAL); 2651 rtw89_write16(rtwdev, phy_offset + RAC_ANA2E * RAC_MULT, RAC_ANA2E_VAL); 2652 2653 phy_offset = R_RAC_DIRECT_OFFSET_G2; 2654 rtw89_write16(rtwdev, phy_offset + RAC_ANA1E * RAC_MULT, RAC_ANA1E_G2_VAL); 2655 rtw89_write16(rtwdev, phy_offset + RAC_ANA2E * RAC_MULT, RAC_ANA2E_VAL); 2656} 2657 2658static void rtw89_pci_rxdma_prefth(struct rtw89_dev *rtwdev) 2659{ 2660 if (rtwdev->chip->chip_id != RTL8852A) 2661 return; 2662 2663 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_DIS_RXDMA_PRE); 2664} 2665 2666static void rtw89_pci_l1off_pwroff(struct rtw89_dev *rtwdev) 2667{ 2668 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; 2669 2670 if (chip_id != RTL8852A && !rtw89_is_rtl885xb(rtwdev)) 2671 return; 2672 2673 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL, B_AX_L1OFF_PWR_OFF_EN); 2674} 2675 2676static u32 rtw89_pci_l2_rxen_lat(struct rtw89_dev *rtwdev) 2677{ 2678 int ret; 2679 2680 if (rtwdev->chip->chip_id != RTL8852A) 2681 return 0; 2682 2683 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN, 2684 PCIE_PHY_GEN1); 2685 if (ret) 2686 return ret; 2687 2688 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN, 2689 PCIE_PHY_GEN2); 2690 if (ret) 2691 return ret; 2692 2693 return 0; 2694} 2695 2696static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev) 2697{ 2698 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; 2699 2700 if (chip_id != RTL8852A && !rtw89_is_rtl885xb(rtwdev)) 2701 return; 2702 2703 rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_PSUS_OFF_CAPC_EN); 2704} 2705 2706static void rtw89_pci_hci_ldo(struct rtw89_dev *rtwdev) 2707{ 2708 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; 2709 2710 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { 2711 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, 2712 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI); 2713 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL, 2714 B_AX_PCIE_DIS_WLSUS_AFT_PDN); 2715 } else if (rtwdev->chip->chip_id == RTL8852C) { 2716 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL, 2717 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI); 2718 } 2719} 2720 2721static int rtw89_pci_dphy_delay(struct rtw89_dev *rtwdev) 2722{ 2723 if (!rtw89_is_rtl885xb(rtwdev)) 2724 return 0; 2725 2726 return rtw89_write16_mdio_mask(rtwdev, RAC_REG_REV2, BAC_CMU_EN_DLY_MASK, 2727 PCIE_DPHY_DLY_25US, PCIE_PHY_GEN1); 2728} 2729 2730static void rtw89_pci_power_wake_ax(struct rtw89_dev *rtwdev, bool pwr_up) 2731{ 2732 if (pwr_up) 2733 rtw89_write32_set(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL); 2734 else 2735 rtw89_write32_clr(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL); 2736} 2737 2738static void rtw89_pci_autoload_hang(struct rtw89_dev *rtwdev) 2739{ 2740 if (rtwdev->chip->chip_id != RTL8852C) 2741 return; 2742 2743 rtw89_write32_set(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3); 2744 rtw89_write32_clr(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3); 2745} 2746 2747static void rtw89_pci_l12_vmain(struct rtw89_dev *rtwdev) 2748{ 2749 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV)) 2750 return; 2751 2752 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_FORCE_PWR_NGAT); 2753} 2754 2755static void rtw89_pci_gen2_force_ib(struct rtw89_dev *rtwdev) 2756{ 2757 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV)) 2758 return; 2759 2760 rtw89_write32_set(rtwdev, R_AX_PMC_DBG_CTRL2, 2761 B_AX_SYSON_DIS_PMCR_AX_WRMSK); 2762 rtw89_write32_set(rtwdev, R_AX_HCI_BG_CTRL, B_AX_BG_CLR_ASYNC_M3); 2763 rtw89_write32_clr(rtwdev, R_AX_PMC_DBG_CTRL2, 2764 B_AX_SYSON_DIS_PMCR_AX_WRMSK); 2765} 2766 2767static void rtw89_pci_l1_ent_lat(struct rtw89_dev *rtwdev) 2768{ 2769 if (rtwdev->chip->chip_id != RTL8852C) 2770 return; 2771 2772 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_SEL_REQ_ENTR_L1); 2773} 2774 2775static void rtw89_pci_wd_exit_l1(struct rtw89_dev *rtwdev) 2776{ 2777 if (rtwdev->chip->chip_id != RTL8852C) 2778 return; 2779 2780 rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_DMAC0_EXIT_L1_EN); 2781} 2782 2783static void rtw89_pci_set_sic(struct rtw89_dev *rtwdev) 2784{ 2785 if (rtwdev->chip->chip_id == RTL8852C) 2786 return; 2787 2788 rtw89_write32_clr(rtwdev, R_AX_PCIE_EXP_CTRL, 2789 B_AX_SIC_EN_FORCE_CLKREQ); 2790} 2791 2792static void rtw89_pci_set_lbc(struct rtw89_dev *rtwdev) 2793{ 2794 const struct rtw89_pci_info *info = rtwdev->pci_info; 2795 u32 lbc; 2796 2797 if (rtwdev->chip->chip_id == RTL8852C) 2798 return; 2799 2800 lbc = rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG); 2801 if (info->lbc_en == MAC_AX_PCIE_ENABLE) { 2802 lbc = u32_replace_bits(lbc, info->lbc_tmr, B_AX_LBC_TIMER); 2803 lbc |= B_AX_LBC_FLAG | B_AX_LBC_EN; 2804 rtw89_write32(rtwdev, R_AX_LBC_WATCHDOG, lbc); 2805 } else { 2806 lbc &= ~B_AX_LBC_EN; 2807 } 2808 rtw89_write32_set(rtwdev, R_AX_LBC_WATCHDOG, lbc); 2809} 2810 2811static void rtw89_pci_set_io_rcy(struct rtw89_dev *rtwdev) 2812{ 2813 const struct rtw89_pci_info *info = rtwdev->pci_info; 2814 u32 val32; 2815 2816 if (rtwdev->chip->chip_id != RTL8852C) 2817 return; 2818 2819 if (info->io_rcy_en == MAC_AX_PCIE_ENABLE) { 2820 val32 = FIELD_PREP(B_AX_PCIE_WDT_TIMER_M1_MASK, 2821 info->io_rcy_tmr); 2822 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M1, val32); 2823 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M2, val32); 2824 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_E0, val32); 2825 2826 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1); 2827 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2); 2828 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0); 2829 } else { 2830 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1); 2831 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2); 2832 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0); 2833 } 2834 2835 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_S1, B_AX_PCIE_IO_RCY_WDT_MODE_S1); 2836} 2837 2838static void rtw89_pci_set_dbg(struct rtw89_dev *rtwdev) 2839{ 2840 if (rtwdev->chip->chip_id == RTL8852C) 2841 return; 2842 2843 rtw89_write32_set(rtwdev, R_AX_PCIE_DBG_CTRL, 2844 B_AX_ASFF_FULL_NO_STK | B_AX_EN_STUCK_DBG); 2845 2846 rtw89_write32_mask(rtwdev, R_AX_PCIE_EXP_CTRL, 2847 B_AX_EN_STUCK_DBG | B_AX_ASFF_FULL_NO_STK, 2848 B_AX_EN_STUCK_DBG); 2849 2850 if (rtwdev->chip->chip_id == RTL8852A) 2851 rtw89_write32_set(rtwdev, R_AX_PCIE_EXP_CTRL, 2852 B_AX_EN_CHKDSC_NO_RX_STUCK); 2853} 2854 2855static void rtw89_pci_set_keep_reg(struct rtw89_dev *rtwdev) 2856{ 2857 if (rtwdev->chip->chip_id == RTL8852C) 2858 return; 2859 2860 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, 2861 B_AX_PCIE_TXRST_KEEP_REG | B_AX_PCIE_RXRST_KEEP_REG); 2862} 2863 2864static void rtw89_pci_clr_idx_all_ax(struct rtw89_dev *rtwdev) 2865{ 2866 const struct rtw89_pci_info *info = rtwdev->pci_info; 2867 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; 2868 u32 val = B_AX_CLR_ACH0_IDX | B_AX_CLR_ACH1_IDX | B_AX_CLR_ACH2_IDX | 2869 B_AX_CLR_ACH3_IDX | B_AX_CLR_CH8_IDX | B_AX_CLR_CH9_IDX | 2870 B_AX_CLR_CH12_IDX; 2871 u32 rxbd_rwptr_clr = info->rxbd_rwptr_clr_reg; 2872 u32 txbd_rwptr_clr2 = info->txbd_rwptr_clr2_reg; 2873 2874 if (chip_id == RTL8852A || chip_id == RTL8852C) 2875 val |= B_AX_CLR_ACH4_IDX | B_AX_CLR_ACH5_IDX | 2876 B_AX_CLR_ACH6_IDX | B_AX_CLR_ACH7_IDX; 2877 /* clear DMA indexes */ 2878 rtw89_write32_set(rtwdev, R_AX_TXBD_RWPTR_CLR1, val); 2879 if (chip_id == RTL8852A || chip_id == RTL8852C) 2880 rtw89_write32_set(rtwdev, txbd_rwptr_clr2, 2881 B_AX_CLR_CH10_IDX | B_AX_CLR_CH11_IDX); 2882 rtw89_write32_set(rtwdev, rxbd_rwptr_clr, 2883 B_AX_CLR_RXQ_IDX | B_AX_CLR_RPQ_IDX); 2884} 2885 2886static int rtw89_pci_poll_txdma_ch_idle_ax(struct rtw89_dev *rtwdev) 2887{ 2888 const struct rtw89_pci_info *info = rtwdev->pci_info; 2889 u32 dma_busy1 = info->dma_busy1.addr; 2890 u32 dma_busy2 = info->dma_busy2_reg; 2891 u32 check, dma_busy; 2892 int ret; 2893 2894 check = info->dma_busy1.mask; 2895 2896 ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0, 2897 10, 100, false, rtwdev, dma_busy1); 2898 if (ret) 2899 return ret; 2900 2901 if (!dma_busy2) 2902 return 0; 2903 2904 check = B_AX_CH10_BUSY | B_AX_CH11_BUSY; 2905 2906 ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0, 2907 10, 100, false, rtwdev, dma_busy2); 2908 if (ret) 2909 return ret; 2910 2911 return 0; 2912} 2913 2914static int rtw89_pci_poll_rxdma_ch_idle_ax(struct rtw89_dev *rtwdev) 2915{ 2916 const struct rtw89_pci_info *info = rtwdev->pci_info; 2917 u32 dma_busy3 = info->dma_busy3_reg; 2918 u32 check, dma_busy; 2919 int ret; 2920 2921 check = B_AX_RXQ_BUSY | B_AX_RPQ_BUSY; 2922 2923 ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0, 2924 10, 100, false, rtwdev, dma_busy3); 2925 if (ret) 2926 return ret; 2927 2928 return 0; 2929} 2930 2931static int rtw89_pci_poll_dma_all_idle(struct rtw89_dev *rtwdev) 2932{ 2933 int ret; 2934 2935 ret = rtw89_pci_poll_txdma_ch_idle_ax(rtwdev); 2936 if (ret) { 2937 rtw89_err(rtwdev, "txdma ch busy\n"); 2938 return ret; 2939 } 2940 2941 ret = rtw89_pci_poll_rxdma_ch_idle_ax(rtwdev); 2942 if (ret) { 2943 rtw89_err(rtwdev, "rxdma ch busy\n"); 2944 return ret; 2945 } 2946 2947 return 0; 2948} 2949 2950static int rtw89_pci_mode_op(struct rtw89_dev *rtwdev) 2951{ 2952 const struct rtw89_pci_info *info = rtwdev->pci_info; 2953 enum mac_ax_bd_trunc_mode txbd_trunc_mode = info->txbd_trunc_mode; 2954 enum mac_ax_bd_trunc_mode rxbd_trunc_mode = info->rxbd_trunc_mode; 2955 enum mac_ax_rxbd_mode rxbd_mode = info->rxbd_mode; 2956 enum mac_ax_tag_mode tag_mode = info->tag_mode; 2957 enum mac_ax_wd_dma_intvl wd_dma_idle_intvl = info->wd_dma_idle_intvl; 2958 enum mac_ax_wd_dma_intvl wd_dma_act_intvl = info->wd_dma_act_intvl; 2959 enum mac_ax_tx_burst tx_burst = info->tx_burst; 2960 enum mac_ax_rx_burst rx_burst = info->rx_burst; 2961 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; 2962 u8 cv = rtwdev->hal.cv; 2963 u32 val32; 2964 2965 if (txbd_trunc_mode == MAC_AX_BD_TRUNC) { 2966 if (chip_id == RTL8852A && cv == CHIP_CBV) 2967 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE); 2968 } else if (txbd_trunc_mode == MAC_AX_BD_NORM) { 2969 if (chip_id == RTL8852A || chip_id == RTL8852B) 2970 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE); 2971 } 2972 2973 if (rxbd_trunc_mode == MAC_AX_BD_TRUNC) { 2974 if (chip_id == RTL8852A && cv == CHIP_CBV) 2975 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE); 2976 } else if (rxbd_trunc_mode == MAC_AX_BD_NORM) { 2977 if (chip_id == RTL8852A || chip_id == RTL8852B) 2978 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE); 2979 } 2980 2981 if (rxbd_mode == MAC_AX_RXBD_PKT) { 2982 rtw89_write32_clr(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit); 2983 } else if (rxbd_mode == MAC_AX_RXBD_SEP) { 2984 rtw89_write32_set(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit); 2985 2986 if (chip_id == RTL8852A || chip_id == RTL8852B) 2987 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, 2988 B_AX_PCIE_RX_APPLEN_MASK, 0); 2989 } 2990 2991 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { 2992 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_TXDMA_MASK, tx_burst); 2993 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_RXDMA_MASK, rx_burst); 2994 } else if (chip_id == RTL8852C) { 2995 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_TXDMA_MASK, tx_burst); 2996 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_RXDMA_MASK, rx_burst); 2997 } 2998 2999 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { 3000 if (tag_mode == MAC_AX_TAG_SGL) { 3001 val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) & 3002 ~B_AX_LATENCY_CONTROL; 3003 rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32); 3004 } else if (tag_mode == MAC_AX_TAG_MULTI) { 3005 val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) | 3006 B_AX_LATENCY_CONTROL; 3007 rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32); 3008 } 3009 } 3010 3011 rtw89_write32_mask(rtwdev, info->exp_ctrl_reg, info->max_tag_num_mask, 3012 info->multi_tag_num); 3013 3014 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { 3015 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_IDLE, 3016 wd_dma_idle_intvl); 3017 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_ACT, 3018 wd_dma_act_intvl); 3019 } else if (chip_id == RTL8852C) { 3020 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_IDLE_V1_MASK, 3021 wd_dma_idle_intvl); 3022 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_ACT_V1_MASK, 3023 wd_dma_act_intvl); 3024 } 3025 3026 if (txbd_trunc_mode == MAC_AX_BD_TRUNC) { 3027 rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING, 3028 B_AX_HOST_ADDR_INFO_8B_SEL); 3029 rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH); 3030 } else if (txbd_trunc_mode == MAC_AX_BD_NORM) { 3031 rtw89_write32_clr(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING, 3032 B_AX_HOST_ADDR_INFO_8B_SEL); 3033 rtw89_write32_set(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH); 3034 } 3035 3036 return 0; 3037} 3038 3039static int rtw89_pci_ops_deinit(struct rtw89_dev *rtwdev) 3040{ 3041 const struct rtw89_pci_info *info = rtwdev->pci_info; 3042 3043 rtw89_pci_power_wake(rtwdev, false); 3044 3045 if (rtwdev->chip->chip_id == RTL8852A) { 3046 /* ltr sw trigger */ 3047 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_IDLE); 3048 } 3049 info->ltr_set(rtwdev, false); 3050 rtw89_pci_ctrl_dma_all(rtwdev, false); 3051 rtw89_pci_clr_idx_all(rtwdev); 3052 3053 return 0; 3054} 3055 3056static int rtw89_pci_ops_mac_pre_init_ax(struct rtw89_dev *rtwdev) 3057{ 3058 const struct rtw89_pci_info *info = rtwdev->pci_info; 3059 int ret; 3060 3061 rtw89_pci_ber(rtwdev); 3062 rtw89_pci_rxdma_prefth(rtwdev); 3063 rtw89_pci_l1off_pwroff(rtwdev); 3064 rtw89_pci_deglitch_setting(rtwdev); 3065 ret = rtw89_pci_l2_rxen_lat(rtwdev); 3066 if (ret) { 3067 rtw89_err(rtwdev, "[ERR] pcie l2 rxen lat %d\n", ret); 3068 return ret; 3069 } 3070 3071 rtw89_pci_aphy_pwrcut(rtwdev); 3072 rtw89_pci_hci_ldo(rtwdev); 3073 rtw89_pci_dphy_delay(rtwdev); 3074 3075 ret = rtw89_pci_autok_x(rtwdev); 3076 if (ret) { 3077 rtw89_err(rtwdev, "[ERR] pcie autok_x fail %d\n", ret); 3078 return ret; 3079 } 3080 3081 ret = rtw89_pci_auto_refclk_cal(rtwdev, false); 3082 if (ret) { 3083 rtw89_err(rtwdev, "[ERR] pcie autok fail %d\n", ret); 3084 return ret; 3085 } 3086 3087 rtw89_pci_power_wake_ax(rtwdev, true); 3088 rtw89_pci_autoload_hang(rtwdev); 3089 rtw89_pci_l12_vmain(rtwdev); 3090 rtw89_pci_gen2_force_ib(rtwdev); 3091 rtw89_pci_l1_ent_lat(rtwdev); 3092 rtw89_pci_wd_exit_l1(rtwdev); 3093 rtw89_pci_set_sic(rtwdev); 3094 rtw89_pci_set_lbc(rtwdev); 3095 rtw89_pci_set_io_rcy(rtwdev); 3096 rtw89_pci_set_dbg(rtwdev); 3097 rtw89_pci_set_keep_reg(rtwdev); 3098 3099 rtw89_write32_set(rtwdev, info->dma_stop1.addr, B_AX_STOP_WPDMA); 3100 3101 /* stop DMA activities */ 3102 rtw89_pci_ctrl_dma_all(rtwdev, false); 3103 3104 ret = rtw89_pci_poll_dma_all_idle(rtwdev); 3105 if (ret) { 3106 rtw89_err(rtwdev, "[ERR] poll pcie dma all idle\n"); 3107 return ret; 3108 } 3109 3110 rtw89_pci_clr_idx_all(rtwdev); 3111 rtw89_pci_mode_op(rtwdev); 3112 3113 /* fill TRX BD indexes */ 3114 rtw89_pci_ops_reset(rtwdev); 3115 3116 ret = rtw89_pci_rst_bdram_ax(rtwdev); 3117 if (ret) { 3118 rtw89_warn(rtwdev, "reset bdram busy\n"); 3119 return ret; 3120 } 3121 3122 /* disable all channels except to FW CMD channel to download firmware */ 3123 rtw89_pci_ctrl_txdma_ch_ax(rtwdev, false); 3124 rtw89_pci_ctrl_txdma_fw_ch_ax(rtwdev, true); 3125 3126 /* start DMA activities */ 3127 rtw89_pci_ctrl_dma_all(rtwdev, true); 3128 3129 return 0; 3130} 3131 3132static int rtw89_pci_ops_mac_pre_deinit_ax(struct rtw89_dev *rtwdev) 3133{ 3134 rtw89_pci_power_wake_ax(rtwdev, false); 3135 3136 return 0; 3137} 3138 3139int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en) 3140{ 3141 u32 val; 3142 3143 if (!en) 3144 return 0; 3145 3146 val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0); 3147 if (rtw89_pci_ltr_is_err_reg_val(val)) 3148 return -EINVAL; 3149 val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1); 3150 if (rtw89_pci_ltr_is_err_reg_val(val)) 3151 return -EINVAL; 3152 val = rtw89_read32(rtwdev, R_AX_LTR_IDLE_LATENCY); 3153 if (rtw89_pci_ltr_is_err_reg_val(val)) 3154 return -EINVAL; 3155 val = rtw89_read32(rtwdev, R_AX_LTR_ACTIVE_LATENCY); 3156 if (rtw89_pci_ltr_is_err_reg_val(val)) 3157 return -EINVAL; 3158 3159 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_HW_EN | B_AX_LTR_EN | 3160 B_AX_LTR_WD_NOEMP_CHK); 3161 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_SPACE_IDX_MASK, 3162 PCI_LTR_SPC_500US); 3163 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK, 3164 PCI_LTR_IDLE_TIMER_3_2MS); 3165 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28); 3166 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28); 3167 rtw89_write32(rtwdev, R_AX_LTR_IDLE_LATENCY, 0x90039003); 3168 rtw89_write32(rtwdev, R_AX_LTR_ACTIVE_LATENCY, 0x880b880b); 3169 3170 return 0; 3171} 3172EXPORT_SYMBOL(rtw89_pci_ltr_set); 3173 3174int rtw89_pci_ltr_set_v1(struct rtw89_dev *rtwdev, bool en) 3175{ 3176 u32 dec_ctrl; 3177 u32 val32; 3178 3179 val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0); 3180 if (rtw89_pci_ltr_is_err_reg_val(val32)) 3181 return -EINVAL; 3182 val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1); 3183 if (rtw89_pci_ltr_is_err_reg_val(val32)) 3184 return -EINVAL; 3185 dec_ctrl = rtw89_read32(rtwdev, R_AX_LTR_DEC_CTRL); 3186 if (rtw89_pci_ltr_is_err_reg_val(dec_ctrl)) 3187 return -EINVAL; 3188 val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX3); 3189 if (rtw89_pci_ltr_is_err_reg_val(val32)) 3190 return -EINVAL; 3191 val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX0); 3192 if (rtw89_pci_ltr_is_err_reg_val(val32)) 3193 return -EINVAL; 3194 3195 if (!en) { 3196 dec_ctrl &= ~(LTR_EN_BITS | B_AX_LTR_IDX_DRV_MASK | B_AX_LTR_HW_DEC_EN); 3197 dec_ctrl |= FIELD_PREP(B_AX_LTR_IDX_DRV_MASK, PCIE_LTR_IDX_IDLE) | 3198 B_AX_LTR_REQ_DRV; 3199 } else { 3200 dec_ctrl |= B_AX_LTR_HW_DEC_EN; 3201 } 3202 3203 dec_ctrl &= ~B_AX_LTR_SPACE_IDX_V1_MASK; 3204 dec_ctrl |= FIELD_PREP(B_AX_LTR_SPACE_IDX_V1_MASK, PCI_LTR_SPC_500US); 3205 3206 if (en) 3207 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, 3208 B_AX_LTR_WD_NOEMP_CHK_V1 | B_AX_LTR_HW_EN); 3209 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK, 3210 PCI_LTR_IDLE_TIMER_3_2MS); 3211 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28); 3212 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28); 3213 rtw89_write32(rtwdev, R_AX_LTR_DEC_CTRL, dec_ctrl); 3214 rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX3, 0x90039003); 3215 rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX0, 0x880b880b); 3216 3217 return 0; 3218} 3219EXPORT_SYMBOL(rtw89_pci_ltr_set_v1); 3220 3221static int rtw89_pci_ops_mac_post_init_ax(struct rtw89_dev *rtwdev) 3222{ 3223 const struct rtw89_pci_info *info = rtwdev->pci_info; 3224 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; 3225 int ret; 3226 3227 ret = info->ltr_set(rtwdev, true); 3228 if (ret) { 3229 rtw89_err(rtwdev, "pci ltr set fail\n"); 3230 return ret; 3231 } 3232 if (chip_id == RTL8852A) { 3233 /* ltr sw trigger */ 3234 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_ACT); 3235 } 3236 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { 3237 /* ADDR info 8-byte mode */ 3238 rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING, 3239 B_AX_HOST_ADDR_INFO_8B_SEL); 3240 rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH); 3241 } 3242 3243 /* enable DMA for all queues */ 3244 rtw89_pci_ctrl_txdma_ch_ax(rtwdev, true); 3245 3246 /* Release PCI IO */ 3247 rtw89_write32_clr(rtwdev, info->dma_stop1.addr, 3248 B_AX_STOP_WPDMA | B_AX_STOP_PCIEIO); 3249 3250 return 0; 3251} 3252 3253static int rtw89_pci_claim_device(struct rtw89_dev *rtwdev, 3254 struct pci_dev *pdev) 3255{ 3256 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3257 int ret; 3258 3259 ret = pci_enable_device(pdev); 3260 if (ret) { 3261 rtw89_err(rtwdev, "failed to enable pci device\n"); 3262 return ret; 3263 } 3264 3265 pci_set_master(pdev); 3266 pci_set_drvdata(pdev, rtwdev->hw); 3267 3268 rtwpci->pdev = pdev; 3269 3270 return 0; 3271} 3272 3273static void rtw89_pci_declaim_device(struct rtw89_dev *rtwdev, 3274 struct pci_dev *pdev) 3275{ 3276 pci_disable_device(pdev); 3277} 3278 3279static bool rtw89_pci_chip_is_manual_dac(struct rtw89_dev *rtwdev) 3280{ 3281 const struct rtw89_chip_info *chip = rtwdev->chip; 3282 3283 switch (chip->chip_id) { 3284 case RTL8852A: 3285 case RTL8852B: 3286 case RTL8851B: 3287 case RTL8852BT: 3288 return true; 3289 default: 3290 return false; 3291 } 3292} 3293 3294static bool rtw89_pci_is_dac_compatible_bridge(struct rtw89_dev *rtwdev) 3295{ 3296 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3297 struct pci_dev *bridge = pci_upstream_bridge(rtwpci->pdev); 3298 3299 if (!rtw89_pci_chip_is_manual_dac(rtwdev)) 3300 return true; 3301 3302 if (!bridge) 3303 return false; 3304 3305 switch (bridge->vendor) { 3306 case PCI_VENDOR_ID_INTEL: 3307 return true; 3308 case PCI_VENDOR_ID_ASMEDIA: 3309 if (bridge->device == 0x2806) 3310 return true; 3311 break; 3312 } 3313 3314 return false; 3315} 3316 3317static int rtw89_pci_cfg_dac(struct rtw89_dev *rtwdev, bool force) 3318{ 3319 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3320 struct pci_dev *pdev = rtwpci->pdev; 3321 int ret; 3322 u8 val; 3323 3324 if (!rtwpci->enable_dac && !force) 3325 return 0; 3326 3327 if (!rtw89_pci_chip_is_manual_dac(rtwdev)) 3328 return 0; 3329 3330 /* Configure DAC only via PCI config API, not DBI interfaces */ 3331 ret = pci_read_config_byte(pdev, RTW89_PCIE_L1_CTRL, &val); 3332 if (ret) 3333 return ret; 3334 3335 val |= RTW89_PCIE_BIT_EN_64BITS; 3336 return pci_write_config_byte(pdev, RTW89_PCIE_L1_CTRL, val); 3337} 3338 3339static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev, 3340 struct pci_dev *pdev) 3341{ 3342 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3343 unsigned long resource_len; 3344 u8 bar_id = 2; 3345 int ret; 3346 3347 ret = pci_request_regions(pdev, KBUILD_MODNAME); 3348 if (ret) { 3349 rtw89_err(rtwdev, "failed to request pci regions\n"); 3350 goto err; 3351 } 3352 3353 if (!rtw89_pci_is_dac_compatible_bridge(rtwdev)) 3354 goto try_dac_done; 3355 3356 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(36)); 3357 if (!ret) { 3358 ret = rtw89_pci_cfg_dac(rtwdev, true); 3359 if (!ret) { 3360 rtwpci->enable_dac = true; 3361 goto try_dac_done; 3362 } 3363 3364 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 3365 if (ret) { 3366 rtw89_err(rtwdev, 3367 "failed to set dma and consistent mask to 32/36-bit\n"); 3368 goto err_release_regions; 3369 } 3370 } 3371try_dac_done: 3372 3373 resource_len = pci_resource_len(pdev, bar_id); 3374 rtwpci->mmap = pci_iomap(pdev, bar_id, resource_len); 3375 if (!rtwpci->mmap) { 3376 rtw89_err(rtwdev, "failed to map pci io\n"); 3377 ret = -EIO; 3378 goto err_release_regions; 3379 } 3380 3381 return 0; 3382 3383err_release_regions: 3384 pci_release_regions(pdev); 3385err: 3386 return ret; 3387} 3388 3389static void rtw89_pci_clear_mapping(struct rtw89_dev *rtwdev, 3390 struct pci_dev *pdev) 3391{ 3392 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3393 3394 if (rtwpci->mmap) { 3395 pci_iounmap(pdev, rtwpci->mmap); 3396 pci_release_regions(pdev); 3397 } 3398} 3399 3400static void rtw89_pci_free_tx_wd_ring(struct rtw89_dev *rtwdev, 3401 struct pci_dev *pdev, 3402 struct rtw89_pci_tx_ring *tx_ring) 3403{ 3404 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring; 3405 u8 *head = wd_ring->head; 3406 dma_addr_t dma = wd_ring->dma; 3407 u32 page_size = wd_ring->page_size; 3408 u32 page_num = wd_ring->page_num; 3409 u32 ring_sz = page_size * page_num; 3410 3411 dma_free_coherent(&pdev->dev, ring_sz, head, dma); 3412 wd_ring->head = NULL; 3413} 3414 3415static void rtw89_pci_free_tx_ring(struct rtw89_dev *rtwdev, 3416 struct pci_dev *pdev, 3417 struct rtw89_pci_tx_ring *tx_ring) 3418{ 3419 tx_ring->bd_ring.head = NULL; 3420} 3421 3422static void rtw89_pci_free_tx_rings(struct rtw89_dev *rtwdev, 3423 struct pci_dev *pdev) 3424{ 3425 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3426 struct rtw89_pci_dma_pool *bd_pool = &rtwpci->tx.bd_pool; 3427 const struct rtw89_pci_info *info = rtwdev->pci_info; 3428 struct rtw89_pci_tx_ring *tx_ring; 3429 int i; 3430 3431 for (i = 0; i < RTW89_TXCH_NUM; i++) { 3432 if (info->tx_dma_ch_mask & BIT(i)) 3433 continue; 3434 tx_ring = &rtwpci->tx.rings[i]; 3435 rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring); 3436 rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring); 3437 } 3438 3439 dma_free_coherent(&pdev->dev, bd_pool->size, bd_pool->head, bd_pool->dma); 3440} 3441 3442static void rtw89_pci_free_rx_ring(struct rtw89_dev *rtwdev, 3443 struct pci_dev *pdev, 3444 struct rtw89_pci_rx_ring *rx_ring) 3445{ 3446 struct rtw89_pci_rx_info *rx_info; 3447 struct sk_buff *skb; 3448 dma_addr_t dma; 3449 u32 buf_sz; 3450 int i; 3451 3452 buf_sz = rx_ring->buf_sz; 3453 for (i = 0; i < rx_ring->bd_ring.len; i++) { 3454 skb = rx_ring->buf[i]; 3455 if (!skb) 3456 continue; 3457 3458 rx_info = RTW89_PCI_RX_SKB_CB(skb); 3459 dma = rx_info->dma; 3460 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE); 3461 dev_kfree_skb(skb); 3462 rx_ring->buf[i] = NULL; 3463 } 3464 3465 rx_ring->bd_ring.head = NULL; 3466} 3467 3468static void rtw89_pci_free_rx_rings(struct rtw89_dev *rtwdev, 3469 struct pci_dev *pdev) 3470{ 3471 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3472 struct rtw89_pci_dma_pool *bd_pool = &rtwpci->rx.bd_pool; 3473 struct rtw89_pci_rx_ring *rx_ring; 3474 int i; 3475 3476 for (i = 0; i < RTW89_RXCH_NUM; i++) { 3477 rx_ring = &rtwpci->rx.rings[i]; 3478 rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring); 3479 } 3480 3481 dma_free_coherent(&pdev->dev, bd_pool->size, bd_pool->head, bd_pool->dma); 3482} 3483 3484static void rtw89_pci_free_trx_rings(struct rtw89_dev *rtwdev, 3485 struct pci_dev *pdev) 3486{ 3487 rtw89_pci_free_rx_rings(rtwdev, pdev); 3488 rtw89_pci_free_tx_rings(rtwdev, pdev); 3489} 3490 3491static int rtw89_pci_init_rx_bd(struct rtw89_dev *rtwdev, struct pci_dev *pdev, 3492 struct rtw89_pci_rx_ring *rx_ring, 3493 struct sk_buff *skb, int buf_sz, u32 idx) 3494{ 3495 struct rtw89_pci_rx_info *rx_info; 3496 struct rtw89_pci_rx_bd_32 *rx_bd; 3497 dma_addr_t dma; 3498 3499 if (!skb) 3500 return -EINVAL; 3501 3502 dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE); 3503 if (dma_mapping_error(&pdev->dev, dma)) 3504 return -EBUSY; 3505 3506 rx_info = RTW89_PCI_RX_SKB_CB(skb); 3507 rx_bd = RTW89_PCI_RX_BD(rx_ring, idx); 3508 3509 memset(rx_bd, 0, sizeof(*rx_bd)); 3510 rx_bd->buf_size = cpu_to_le16(buf_sz); 3511 rx_bd->dma = cpu_to_le32(dma); 3512 rx_bd->opt = le16_encode_bits(upper_32_bits(dma), RTW89_PCI_RXBD_OPT_DMA_HI); 3513 rx_info->dma = dma; 3514 3515 return 0; 3516} 3517 3518static int rtw89_pci_alloc_tx_wd_ring(struct rtw89_dev *rtwdev, 3519 struct pci_dev *pdev, 3520 struct rtw89_pci_tx_ring *tx_ring, 3521 enum rtw89_tx_channel txch) 3522{ 3523 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring; 3524 struct rtw89_pci_tx_wd *txwd; 3525 dma_addr_t dma; 3526 dma_addr_t cur_paddr; 3527 u8 *head; 3528 u8 *cur_vaddr; 3529 u32 page_size = RTW89_PCI_TXWD_PAGE_SIZE; 3530 u32 page_num = RTW89_PCI_TXWD_NUM_MAX; 3531 u32 ring_sz = page_size * page_num; 3532 u32 page_offset; 3533 int i; 3534 3535 /* FWCMD queue doesn't use txwd as pages */ 3536 if (txch == RTW89_TXCH_CH12) 3537 return 0; 3538 3539 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); 3540 if (!head) 3541 return -ENOMEM; 3542 3543 INIT_LIST_HEAD(&wd_ring->free_pages); 3544 wd_ring->head = head; 3545 wd_ring->dma = dma; 3546 wd_ring->page_size = page_size; 3547 wd_ring->page_num = page_num; 3548 3549 page_offset = 0; 3550 for (i = 0; i < page_num; i++) { 3551 txwd = &wd_ring->pages[i]; 3552 cur_paddr = dma + page_offset; 3553 cur_vaddr = head + page_offset; 3554 3555 skb_queue_head_init(&txwd->queue); 3556 INIT_LIST_HEAD(&txwd->list); 3557 txwd->paddr = cur_paddr; 3558 txwd->vaddr = cur_vaddr; 3559 txwd->len = page_size; 3560 txwd->seq = i; 3561 rtw89_pci_enqueue_txwd(tx_ring, txwd); 3562 3563 page_offset += page_size; 3564 } 3565 3566 return 0; 3567} 3568 3569static int rtw89_pci_alloc_tx_ring(struct rtw89_dev *rtwdev, 3570 struct pci_dev *pdev, 3571 struct rtw89_pci_tx_ring *tx_ring, 3572 u32 desc_size, u32 len, 3573 enum rtw89_tx_channel txch, 3574 void *head, dma_addr_t dma) 3575{ 3576 const struct rtw89_pci_ch_dma_addr *txch_addr; 3577 int ret; 3578 3579 ret = rtw89_pci_alloc_tx_wd_ring(rtwdev, pdev, tx_ring, txch); 3580 if (ret) { 3581 rtw89_err(rtwdev, "failed to alloc txwd ring of txch %d\n", txch); 3582 goto err; 3583 } 3584 3585 ret = rtw89_pci_get_txch_addrs(rtwdev, txch, &txch_addr); 3586 if (ret) { 3587 rtw89_err(rtwdev, "failed to get address of txch %d", txch); 3588 goto err_free_wd_ring; 3589 } 3590 3591 INIT_LIST_HEAD(&tx_ring->busy_pages); 3592 tx_ring->bd_ring.head = head; 3593 tx_ring->bd_ring.dma = dma; 3594 tx_ring->bd_ring.len = len; 3595 tx_ring->bd_ring.desc_size = desc_size; 3596 tx_ring->bd_ring.addr = *txch_addr; 3597 tx_ring->bd_ring.wp = 0; 3598 tx_ring->bd_ring.rp = 0; 3599 tx_ring->txch = txch; 3600 3601 return 0; 3602 3603err_free_wd_ring: 3604 rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring); 3605err: 3606 return ret; 3607} 3608 3609static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev, 3610 struct pci_dev *pdev) 3611{ 3612 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3613 struct rtw89_pci_dma_pool *bd_pool = &rtwpci->tx.bd_pool; 3614 const struct rtw89_pci_info *info = rtwdev->pci_info; 3615 struct rtw89_pci_tx_ring *tx_ring; 3616 u32 i, tx_allocated; 3617 dma_addr_t dma; 3618 u32 desc_size; 3619 u32 ring_sz; 3620 u32 pool_sz; 3621 u32 ch_num; 3622 void *head; 3623 u32 len; 3624 int ret; 3625 3626 BUILD_BUG_ON(RTW89_PCI_TXBD_NUM_MAX % 16); 3627 3628 desc_size = sizeof(struct rtw89_pci_tx_bd_32); 3629 len = RTW89_PCI_TXBD_NUM_MAX; 3630 ch_num = RTW89_TXCH_NUM - hweight32(info->tx_dma_ch_mask); 3631 ring_sz = desc_size * len; 3632 pool_sz = ring_sz * ch_num; 3633 3634 head = dma_alloc_coherent(&pdev->dev, pool_sz, &dma, GFP_KERNEL); 3635 if (!head) 3636 return -ENOMEM; 3637 3638 bd_pool->head = head; 3639 bd_pool->dma = dma; 3640 bd_pool->size = pool_sz; 3641 3642 for (i = 0; i < RTW89_TXCH_NUM; i++) { 3643 if (info->tx_dma_ch_mask & BIT(i)) 3644 continue; 3645 tx_ring = &rtwpci->tx.rings[i]; 3646 ret = rtw89_pci_alloc_tx_ring(rtwdev, pdev, tx_ring, 3647 desc_size, len, i, head, dma); 3648 if (ret) { 3649 rtw89_err(rtwdev, "failed to alloc tx ring %d\n", i); 3650 goto err_free; 3651 } 3652 3653 head += ring_sz; 3654 dma += ring_sz; 3655 } 3656 3657 return 0; 3658 3659err_free: 3660 tx_allocated = i; 3661 for (i = 0; i < tx_allocated; i++) { 3662 tx_ring = &rtwpci->tx.rings[i]; 3663 rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring); 3664 } 3665 3666 dma_free_coherent(&pdev->dev, bd_pool->size, bd_pool->head, bd_pool->dma); 3667 3668 return ret; 3669} 3670 3671static int rtw89_pci_alloc_rx_ring(struct rtw89_dev *rtwdev, 3672 struct pci_dev *pdev, 3673 struct rtw89_pci_rx_ring *rx_ring, 3674 u32 desc_size, u32 len, u32 rxch, 3675 void *head, dma_addr_t dma) 3676{ 3677 const struct rtw89_pci_info *info = rtwdev->pci_info; 3678 const struct rtw89_pci_ch_dma_addr *rxch_addr; 3679 struct sk_buff *skb; 3680 int buf_sz = RTW89_PCI_RX_BUF_SIZE; 3681 int i, allocated; 3682 int ret; 3683 3684 ret = rtw89_pci_get_rxch_addrs(rtwdev, rxch, &rxch_addr); 3685 if (ret) { 3686 rtw89_err(rtwdev, "failed to get address of rxch %d", rxch); 3687 return ret; 3688 } 3689 3690 rx_ring->bd_ring.head = head; 3691 rx_ring->bd_ring.dma = dma; 3692 rx_ring->bd_ring.len = len; 3693 rx_ring->bd_ring.desc_size = desc_size; 3694 rx_ring->bd_ring.addr = *rxch_addr; 3695 if (info->rx_ring_eq_is_full) 3696 rx_ring->bd_ring.wp = len - 1; 3697 else 3698 rx_ring->bd_ring.wp = 0; 3699 rx_ring->bd_ring.rp = 0; 3700 rx_ring->buf_sz = buf_sz; 3701 rx_ring->diliver_skb = NULL; 3702 rx_ring->diliver_desc.ready = false; 3703 rx_ring->target_rx_tag = 0; 3704 3705 for (i = 0; i < len; i++) { 3706 skb = dev_alloc_skb(buf_sz); 3707 if (!skb) { 3708 ret = -ENOMEM; 3709 goto err_free; 3710 } 3711 3712 memset(skb->data, 0, buf_sz); 3713 rx_ring->buf[i] = skb; 3714 ret = rtw89_pci_init_rx_bd(rtwdev, pdev, rx_ring, skb, 3715 buf_sz, i); 3716 if (ret) { 3717 rtw89_err(rtwdev, "failed to init rx buf %d\n", i); 3718 dev_kfree_skb_any(skb); 3719 rx_ring->buf[i] = NULL; 3720 goto err_free; 3721 } 3722 } 3723 3724 return 0; 3725 3726err_free: 3727 allocated = i; 3728 for (i = 0; i < allocated; i++) { 3729 skb = rx_ring->buf[i]; 3730 if (!skb) 3731 continue; 3732 dma = *((dma_addr_t *)skb->cb); 3733 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE); 3734 dev_kfree_skb(skb); 3735 rx_ring->buf[i] = NULL; 3736 } 3737 3738 rx_ring->bd_ring.head = NULL; 3739 3740 return ret; 3741} 3742 3743static int rtw89_pci_alloc_rx_rings(struct rtw89_dev *rtwdev, 3744 struct pci_dev *pdev) 3745{ 3746 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3747 struct rtw89_pci_dma_pool *bd_pool = &rtwpci->rx.bd_pool; 3748 struct rtw89_pci_rx_ring *rx_ring; 3749 int i, rx_allocated; 3750 dma_addr_t dma; 3751 u32 desc_size; 3752 u32 ring_sz; 3753 u32 pool_sz; 3754 void *head; 3755 u32 len; 3756 int ret; 3757 3758 desc_size = sizeof(struct rtw89_pci_rx_bd_32); 3759 len = RTW89_PCI_RXBD_NUM_MAX; 3760 ring_sz = desc_size * len; 3761 pool_sz = ring_sz * RTW89_RXCH_NUM; 3762 3763 head = dma_alloc_coherent(&pdev->dev, pool_sz, &dma, GFP_KERNEL); 3764 if (!head) 3765 return -ENOMEM; 3766 3767 bd_pool->head = head; 3768 bd_pool->dma = dma; 3769 bd_pool->size = pool_sz; 3770 3771 for (i = 0; i < RTW89_RXCH_NUM; i++) { 3772 rx_ring = &rtwpci->rx.rings[i]; 3773 3774 ret = rtw89_pci_alloc_rx_ring(rtwdev, pdev, rx_ring, 3775 desc_size, len, i, 3776 head, dma); 3777 if (ret) { 3778 rtw89_err(rtwdev, "failed to alloc rx ring %d\n", i); 3779 goto err_free; 3780 } 3781 3782 head += ring_sz; 3783 dma += ring_sz; 3784 } 3785 3786 return 0; 3787 3788err_free: 3789 rx_allocated = i; 3790 for (i = 0; i < rx_allocated; i++) { 3791 rx_ring = &rtwpci->rx.rings[i]; 3792 rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring); 3793 } 3794 3795 dma_free_coherent(&pdev->dev, bd_pool->size, bd_pool->head, bd_pool->dma); 3796 3797 return ret; 3798} 3799 3800static int rtw89_pci_alloc_trx_rings(struct rtw89_dev *rtwdev, 3801 struct pci_dev *pdev) 3802{ 3803 int ret; 3804 3805 ret = rtw89_pci_alloc_tx_rings(rtwdev, pdev); 3806 if (ret) { 3807 rtw89_err(rtwdev, "failed to alloc dma tx rings\n"); 3808 goto err; 3809 } 3810 3811 ret = rtw89_pci_alloc_rx_rings(rtwdev, pdev); 3812 if (ret) { 3813 rtw89_err(rtwdev, "failed to alloc dma rx rings\n"); 3814 goto err_free_tx_rings; 3815 } 3816 3817 return 0; 3818 3819err_free_tx_rings: 3820 rtw89_pci_free_tx_rings(rtwdev, pdev); 3821err: 3822 return ret; 3823} 3824 3825static void rtw89_pci_h2c_init(struct rtw89_dev *rtwdev, 3826 struct rtw89_pci *rtwpci) 3827{ 3828 skb_queue_head_init(&rtwpci->h2c_queue); 3829 skb_queue_head_init(&rtwpci->h2c_release_queue); 3830} 3831 3832static int rtw89_pci_setup_resource(struct rtw89_dev *rtwdev, 3833 struct pci_dev *pdev) 3834{ 3835 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3836 int ret; 3837 3838 ret = rtw89_pci_setup_mapping(rtwdev, pdev); 3839 if (ret) { 3840 rtw89_err(rtwdev, "failed to setup pci mapping\n"); 3841 goto err; 3842 } 3843 3844 ret = rtw89_pci_alloc_trx_rings(rtwdev, pdev); 3845 if (ret) { 3846 rtw89_err(rtwdev, "failed to alloc pci trx rings\n"); 3847 goto err_pci_unmap; 3848 } 3849 3850 rtw89_pci_h2c_init(rtwdev, rtwpci); 3851 3852 spin_lock_init(&rtwpci->irq_lock); 3853 spin_lock_init(&rtwpci->trx_lock); 3854 3855 return 0; 3856 3857err_pci_unmap: 3858 rtw89_pci_clear_mapping(rtwdev, pdev); 3859err: 3860 return ret; 3861} 3862 3863static void rtw89_pci_clear_resource(struct rtw89_dev *rtwdev, 3864 struct pci_dev *pdev) 3865{ 3866 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3867 3868 rtw89_pci_free_trx_rings(rtwdev, pdev); 3869 rtw89_pci_clear_mapping(rtwdev, pdev); 3870 rtw89_pci_release_fwcmd(rtwdev, rtwpci, 3871 skb_queue_len(&rtwpci->h2c_queue), true); 3872} 3873 3874void rtw89_pci_config_intr_mask(struct rtw89_dev *rtwdev) 3875{ 3876 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3877 const struct rtw89_chip_info *chip = rtwdev->chip; 3878 u32 hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN; 3879 3880 if (chip->chip_id == RTL8851B) 3881 hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN_WKARND; 3882 3883 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | 0; 3884 3885 if (rtwpci->under_recovery) { 3886 rtwpci->intrs[0] = hs0isr_ind_int_en; 3887 rtwpci->intrs[1] = 0; 3888 } else { 3889 rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN | 3890 B_AX_RXDMA_INT_EN | 3891 B_AX_RXP1DMA_INT_EN | 3892 B_AX_RPQDMA_INT_EN | 3893 B_AX_RXDMA_STUCK_INT_EN | 3894 B_AX_RDU_INT_EN | 3895 B_AX_RPQBD_FULL_INT_EN | 3896 hs0isr_ind_int_en; 3897 3898 rtwpci->intrs[1] = B_AX_HC10ISR_IND_INT_EN; 3899 } 3900} 3901EXPORT_SYMBOL(rtw89_pci_config_intr_mask); 3902 3903static void rtw89_pci_recovery_intr_mask_v1(struct rtw89_dev *rtwdev) 3904{ 3905 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3906 3907 rtwpci->ind_intrs = B_AX_HS0ISR_IND_INT_EN; 3908 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN; 3909 rtwpci->intrs[0] = 0; 3910 rtwpci->intrs[1] = 0; 3911} 3912 3913static void rtw89_pci_default_intr_mask_v1(struct rtw89_dev *rtwdev) 3914{ 3915 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3916 3917 rtwpci->ind_intrs = B_AX_HCI_AXIDMA_INT_EN | 3918 B_AX_HS1ISR_IND_INT_EN | 3919 B_AX_HS0ISR_IND_INT_EN; 3920 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN; 3921 rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN | 3922 B_AX_RXDMA_INT_EN | 3923 B_AX_RXP1DMA_INT_EN | 3924 B_AX_RPQDMA_INT_EN | 3925 B_AX_RXDMA_STUCK_INT_EN | 3926 B_AX_RDU_INT_EN | 3927 B_AX_RPQBD_FULL_INT_EN; 3928 rtwpci->intrs[1] = B_AX_GPIO18_INT_EN; 3929} 3930 3931static void rtw89_pci_low_power_intr_mask_v1(struct rtw89_dev *rtwdev) 3932{ 3933 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3934 3935 rtwpci->ind_intrs = B_AX_HS1ISR_IND_INT_EN | 3936 B_AX_HS0ISR_IND_INT_EN; 3937 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN; 3938 rtwpci->intrs[0] = 0; 3939 rtwpci->intrs[1] = B_AX_GPIO18_INT_EN; 3940} 3941 3942void rtw89_pci_config_intr_mask_v1(struct rtw89_dev *rtwdev) 3943{ 3944 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3945 3946 if (rtwpci->under_recovery) 3947 rtw89_pci_recovery_intr_mask_v1(rtwdev); 3948 else if (rtwpci->low_power) 3949 rtw89_pci_low_power_intr_mask_v1(rtwdev); 3950 else 3951 rtw89_pci_default_intr_mask_v1(rtwdev); 3952} 3953EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v1); 3954 3955static void rtw89_pci_recovery_intr_mask_v2(struct rtw89_dev *rtwdev) 3956{ 3957 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3958 3959 rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0; 3960 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN; 3961 rtwpci->intrs[0] = 0; 3962 rtwpci->intrs[1] = 0; 3963} 3964 3965static void rtw89_pci_default_intr_mask_v2(struct rtw89_dev *rtwdev) 3966{ 3967 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3968 3969 rtwpci->ind_intrs = B_BE_HCI_AXIDMA_INT_EN0 | 3970 B_BE_HS0_IND_INT_EN0; 3971 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN; 3972 rtwpci->intrs[0] = B_BE_RDU_CH1_INT_IMR_V1 | 3973 B_BE_RDU_CH0_INT_IMR_V1; 3974 rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 | 3975 B_BE_PCIE_RX_RPQ0_IMR0_V1; 3976} 3977 3978static void rtw89_pci_low_power_intr_mask_v2(struct rtw89_dev *rtwdev) 3979{ 3980 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3981 3982 rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0 | 3983 B_BE_HS1_IND_INT_EN0; 3984 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN; 3985 rtwpci->intrs[0] = 0; 3986 rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 | 3987 B_BE_PCIE_RX_RPQ0_IMR0_V1; 3988} 3989 3990void rtw89_pci_config_intr_mask_v2(struct rtw89_dev *rtwdev) 3991{ 3992 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 3993 3994 if (rtwpci->under_recovery) 3995 rtw89_pci_recovery_intr_mask_v2(rtwdev); 3996 else if (rtwpci->low_power) 3997 rtw89_pci_low_power_intr_mask_v2(rtwdev); 3998 else 3999 rtw89_pci_default_intr_mask_v2(rtwdev); 4000} 4001EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v2); 4002 4003static void rtw89_pci_recovery_intr_mask_v3(struct rtw89_dev *rtwdev) 4004{ 4005 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 4006 4007 rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0; 4008 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN; 4009 rtwpci->intrs[0] = 0; 4010 rtwpci->intrs[1] = 0; 4011} 4012 4013static void rtw89_pci_default_intr_mask_v3(struct rtw89_dev *rtwdev) 4014{ 4015 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 4016 4017 rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0; 4018 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN; 4019 rtwpci->intrs[0] = 0; 4020 rtwpci->intrs[1] = B_BE_PCIE_RDU_CH1_IMR | 4021 B_BE_PCIE_RDU_CH0_IMR | 4022 B_BE_PCIE_RX_RX0P2_IMR0_V1 | 4023 B_BE_PCIE_RX_RPQ0_IMR0_V1; 4024} 4025 4026void rtw89_pci_config_intr_mask_v3(struct rtw89_dev *rtwdev) 4027{ 4028 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 4029 4030 if (rtwpci->under_recovery) 4031 rtw89_pci_recovery_intr_mask_v3(rtwdev); 4032 else 4033 rtw89_pci_default_intr_mask_v3(rtwdev); 4034} 4035EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v3); 4036 4037static int rtw89_pci_request_irq(struct rtw89_dev *rtwdev, 4038 struct pci_dev *pdev) 4039{ 4040 unsigned long flags = 0; 4041 int ret; 4042 4043 flags |= PCI_IRQ_INTX | PCI_IRQ_MSI; 4044 ret = pci_alloc_irq_vectors(pdev, 1, 1, flags); 4045 if (ret < 0) { 4046 rtw89_err(rtwdev, "failed to alloc irq vectors, ret %d\n", ret); 4047 goto err; 4048 } 4049 4050 ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq, 4051 rtw89_pci_interrupt_handler, 4052 rtw89_pci_interrupt_threadfn, 4053 IRQF_SHARED, KBUILD_MODNAME, rtwdev); 4054 if (ret) { 4055 rtw89_err(rtwdev, "failed to request threaded irq\n"); 4056 goto err_free_vector; 4057 } 4058 4059 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RESET); 4060 4061 return 0; 4062 4063err_free_vector: 4064 pci_free_irq_vectors(pdev); 4065err: 4066 return ret; 4067} 4068 4069static void rtw89_pci_free_irq(struct rtw89_dev *rtwdev, 4070 struct pci_dev *pdev) 4071{ 4072 devm_free_irq(rtwdev->dev, pdev->irq, rtwdev); 4073 pci_free_irq_vectors(pdev); 4074} 4075 4076static u16 gray_code_to_bin(u16 gray_code) 4077{ 4078 u16 binary = gray_code; 4079 4080 while (gray_code) { 4081 gray_code >>= 1; 4082 binary ^= gray_code; 4083 } 4084 4085 return binary; 4086} 4087 4088static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev) 4089{ 4090 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 4091 struct pci_dev *pdev = rtwpci->pdev; 4092 u16 val16, filter_out_val; 4093 u32 val, phy_offset; 4094 int ret; 4095 4096 if (rtwdev->chip->chip_id != RTL8852C) 4097 return 0; 4098 4099 val = rtw89_read32_mask(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK); 4100 if (val == B_AX_ASPM_CTRL_L1) 4101 return 0; 4102 4103 ret = pci_read_config_dword(pdev, RTW89_PCIE_L1_STS_V1, &val); 4104 if (ret) 4105 return ret; 4106 4107 val = FIELD_GET(RTW89_BCFG_LINK_SPEED_MASK, val); 4108 if (val == RTW89_PCIE_GEN1_SPEED) { 4109 phy_offset = R_RAC_DIRECT_OFFSET_G1; 4110 } else if (val == RTW89_PCIE_GEN2_SPEED) { 4111 phy_offset = R_RAC_DIRECT_OFFSET_G2; 4112 val16 = rtw89_read16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT); 4113 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT, 4114 val16 | B_PCIE_BIT_PINOUT_DIS); 4115 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT, 4116 val16 & ~B_PCIE_BIT_RD_SEL); 4117 4118 val16 = rtw89_read16_mask(rtwdev, 4119 phy_offset + RAC_ANA1F * RAC_MULT, 4120 FILTER_OUT_EQ_MASK); 4121 val16 = gray_code_to_bin(val16); 4122 filter_out_val = rtw89_read16(rtwdev, phy_offset + RAC_ANA24 * 4123 RAC_MULT); 4124 filter_out_val &= ~REG_FILTER_OUT_MASK; 4125 filter_out_val |= FIELD_PREP(REG_FILTER_OUT_MASK, val16); 4126 4127 rtw89_write16(rtwdev, phy_offset + RAC_ANA24 * RAC_MULT, 4128 filter_out_val); 4129 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0A * RAC_MULT, 4130 B_BAC_EQ_SEL); 4131 rtw89_write16_set(rtwdev, 4132 R_RAC_DIRECT_OFFSET_G1 + RAC_ANA0C * RAC_MULT, 4133 B_PCIE_BIT_PSAVE); 4134 } else { 4135 return -EOPNOTSUPP; 4136 } 4137 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0C * RAC_MULT, 4138 B_PCIE_BIT_PSAVE); 4139 4140 return 0; 4141} 4142 4143static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable) 4144{ 4145 const struct rtw89_pci_info *info = rtwdev->pci_info; 4146 const struct rtw89_pci_gen_def *gen_def = info->gen_def; 4147 4148 if (rtw89_pci_disable_clkreq) 4149 return; 4150 4151 gen_def->clkreq_set(rtwdev, enable); 4152} 4153 4154static void rtw89_pci_clkreq_set_ax(struct rtw89_dev *rtwdev, bool enable) 4155{ 4156 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; 4157 int ret; 4158 4159 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL, 4160 PCIE_CLKDLY_HW_30US); 4161 if (ret) 4162 rtw89_err(rtwdev, "failed to set CLKREQ Delay\n"); 4163 4164 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { 4165 if (enable) 4166 ret = rtw89_pci_config_byte_set(rtwdev, 4167 RTW89_PCIE_L1_CTRL, 4168 RTW89_PCIE_BIT_CLK); 4169 else 4170 ret = rtw89_pci_config_byte_clr(rtwdev, 4171 RTW89_PCIE_L1_CTRL, 4172 RTW89_PCIE_BIT_CLK); 4173 if (ret) 4174 rtw89_err(rtwdev, "failed to %s CLKREQ_L1, ret=%d", 4175 enable ? "set" : "unset", ret); 4176 } else if (chip_id == RTL8852C) { 4177 rtw89_write32_set(rtwdev, R_AX_PCIE_LAT_CTRL, 4178 B_AX_CLK_REQ_SEL_OPT | B_AX_CLK_REQ_SEL); 4179 if (enable) 4180 rtw89_write32_set(rtwdev, R_AX_L1_CLK_CTRL, 4181 B_AX_CLK_REQ_N); 4182 else 4183 rtw89_write32_clr(rtwdev, R_AX_L1_CLK_CTRL, 4184 B_AX_CLK_REQ_N); 4185 } 4186} 4187 4188static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable) 4189{ 4190 const struct rtw89_pci_info *info = rtwdev->pci_info; 4191 const struct rtw89_pci_gen_def *gen_def = info->gen_def; 4192 4193 if (rtw89_pci_disable_aspm_l1) 4194 return; 4195 4196 gen_def->aspm_set(rtwdev, enable); 4197} 4198 4199static void rtw89_pci_aspm_set_ax(struct rtw89_dev *rtwdev, bool enable) 4200{ 4201 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; 4202 u8 value = 0; 4203 int ret; 4204 4205 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, &value); 4206 if (ret) 4207 rtw89_warn(rtwdev, "failed to read ASPM Delay\n"); 4208 4209 u8p_replace_bits(&value, PCIE_L1DLY_16US, RTW89_L1DLY_MASK); 4210 u8p_replace_bits(&value, PCIE_L0SDLY_4US, RTW89_L0DLY_MASK); 4211 4212 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, value); 4213 if (ret) 4214 rtw89_warn(rtwdev, "failed to read ASPM Delay\n"); 4215 4216 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { 4217 if (enable) 4218 ret = rtw89_pci_config_byte_set(rtwdev, 4219 RTW89_PCIE_L1_CTRL, 4220 RTW89_PCIE_BIT_L1); 4221 else 4222 ret = rtw89_pci_config_byte_clr(rtwdev, 4223 RTW89_PCIE_L1_CTRL, 4224 RTW89_PCIE_BIT_L1); 4225 } else if (chip_id == RTL8852C) { 4226 if (enable) 4227 rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1, 4228 B_AX_ASPM_CTRL_L1); 4229 else 4230 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1, 4231 B_AX_ASPM_CTRL_L1); 4232 } 4233 if (ret) 4234 rtw89_err(rtwdev, "failed to %s ASPM L1, ret=%d", 4235 enable ? "set" : "unset", ret); 4236} 4237 4238static void rtw89_pci_recalc_int_mit(struct rtw89_dev *rtwdev) 4239{ 4240 enum rtw89_chip_gen chip_gen = rtwdev->chip->chip_gen; 4241 const struct rtw89_pci_info *info = rtwdev->pci_info; 4242 struct rtw89_traffic_stats *stats = &rtwdev->stats; 4243 enum rtw89_tfc_lv tx_tfc_lv = stats->tx_tfc_lv; 4244 enum rtw89_tfc_lv rx_tfc_lv = stats->rx_tfc_lv; 4245 u32 val = 0; 4246 4247 if (rtwdev->scanning || 4248 (tx_tfc_lv < RTW89_TFC_HIGH && rx_tfc_lv < RTW89_TFC_HIGH)) 4249 goto out; 4250 4251 if (chip_gen == RTW89_CHIP_BE) 4252 val = B_BE_PCIE_MIT_RX0P2_EN | B_BE_PCIE_MIT_RX0P1_EN; 4253 else 4254 val = B_AX_RXMIT_RXP2_SEL | B_AX_RXMIT_RXP1_SEL | 4255 FIELD_PREP(B_AX_RXCOUNTER_MATCH_MASK, RTW89_PCI_RXBD_NUM_MAX / 2) | 4256 FIELD_PREP(B_AX_RXTIMER_UNIT_MASK, AX_RXTIMER_UNIT_64US) | 4257 FIELD_PREP(B_AX_RXTIMER_MATCH_MASK, 2048 / 64); 4258 4259out: 4260 rtw89_write32(rtwdev, info->mit_addr, val); 4261} 4262 4263static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev) 4264{ 4265 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 4266 struct pci_dev *pdev = rtwpci->pdev; 4267 u16 link_ctrl; 4268 int ret; 4269 4270 /* Though there is standard PCIE configuration space to set the 4271 * link control register, but by Realtek's design, driver should 4272 * check if host supports CLKREQ/ASPM to enable the HW module. 4273 * 4274 * These functions are implemented by two HW modules associated, 4275 * one is responsible to access PCIE configuration space to 4276 * follow the host settings, and another is in charge of doing 4277 * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes 4278 * the host does not support it, and due to some reasons or wrong 4279 * settings (ex. CLKREQ# not Bi-Direction), it could lead to device 4280 * loss if HW misbehaves on the link. 4281 * 4282 * Hence it's designed that driver should first check the PCIE 4283 * configuration space is sync'ed and enabled, then driver can turn 4284 * on the other module that is actually working on the mechanism. 4285 */ 4286 ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl); 4287 if (ret) { 4288 rtw89_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret); 4289 return; 4290 } 4291 4292 if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN) 4293 rtw89_pci_clkreq_set(rtwdev, true); 4294 4295 if (link_ctrl & PCI_EXP_LNKCTL_ASPM_L1) 4296 rtw89_pci_aspm_set(rtwdev, true); 4297} 4298 4299static void rtw89_pci_l1ss_set(struct rtw89_dev *rtwdev, bool enable) 4300{ 4301 const struct rtw89_pci_info *info = rtwdev->pci_info; 4302 const struct rtw89_pci_gen_def *gen_def = info->gen_def; 4303 4304 if (rtw89_pci_disable_l1ss) 4305 return; 4306 4307 gen_def->l1ss_set(rtwdev, enable); 4308} 4309 4310static void rtw89_pci_l1ss_set_ax(struct rtw89_dev *rtwdev, bool enable) 4311{ 4312 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; 4313 int ret; 4314 4315 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { 4316 if (enable) 4317 ret = rtw89_pci_config_byte_set(rtwdev, 4318 RTW89_PCIE_TIMER_CTRL, 4319 RTW89_PCIE_BIT_L1SUB); 4320 else 4321 ret = rtw89_pci_config_byte_clr(rtwdev, 4322 RTW89_PCIE_TIMER_CTRL, 4323 RTW89_PCIE_BIT_L1SUB); 4324 if (ret) 4325 rtw89_err(rtwdev, "failed to %s L1SS, ret=%d", 4326 enable ? "set" : "unset", ret); 4327 } else if (chip_id == RTL8852C) { 4328 ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1SS_STS_V1, 4329 RTW89_PCIE_BIT_ASPM_L11 | 4330 RTW89_PCIE_BIT_PCI_L11); 4331 if (ret) 4332 rtw89_warn(rtwdev, "failed to unset ASPM L1.1, ret=%d", ret); 4333 if (enable) 4334 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1, 4335 B_AX_L1SUB_DISABLE); 4336 else 4337 rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1, 4338 B_AX_L1SUB_DISABLE); 4339 } 4340} 4341 4342static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev) 4343{ 4344 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 4345 struct pci_dev *pdev = rtwpci->pdev; 4346 u32 l1ss_cap_ptr, l1ss_ctrl; 4347 4348 if (rtw89_pci_disable_l1ss) 4349 return; 4350 4351 l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS); 4352 if (!l1ss_cap_ptr) 4353 return; 4354 4355 pci_read_config_dword(pdev, l1ss_cap_ptr + PCI_L1SS_CTL1, &l1ss_ctrl); 4356 4357 if (l1ss_ctrl & PCI_L1SS_CTL1_L1SS_MASK) 4358 rtw89_pci_l1ss_set(rtwdev, true); 4359} 4360 4361static void rtw89_pci_cpl_timeout_cfg(struct rtw89_dev *rtwdev) 4362{ 4363 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 4364 struct pci_dev *pdev = rtwpci->pdev; 4365 4366 pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2, 4367 PCI_EXP_DEVCTL2_COMP_TMOUT_DIS); 4368} 4369 4370static int rtw89_pci_poll_io_idle_ax(struct rtw89_dev *rtwdev) 4371{ 4372 int ret = 0; 4373 u32 sts; 4374 u32 busy = B_AX_PCIEIO_BUSY | B_AX_PCIEIO_TX_BUSY | B_AX_PCIEIO_RX_BUSY; 4375 4376 ret = read_poll_timeout_atomic(rtw89_read32, sts, (sts & busy) == 0x0, 4377 10, 1000, false, rtwdev, 4378 R_AX_PCIE_DMA_BUSY1); 4379 if (ret) { 4380 rtw89_err(rtwdev, "pci dmach busy1 0x%X\n", 4381 rtw89_read32(rtwdev, R_AX_PCIE_DMA_BUSY1)); 4382 return -EINVAL; 4383 } 4384 return ret; 4385} 4386 4387static int rtw89_pci_lv1rst_stop_dma_ax(struct rtw89_dev *rtwdev) 4388{ 4389 u32 val; 4390 int ret; 4391 4392 if (rtwdev->chip->chip_id == RTL8852C) 4393 return 0; 4394 4395 rtw89_pci_ctrl_dma_all(rtwdev, false); 4396 ret = rtw89_pci_poll_io_idle_ax(rtwdev); 4397 if (ret) { 4398 val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG); 4399 rtw89_debug(rtwdev, RTW89_DBG_HCI, 4400 "[PCIe] poll_io_idle fail, before 0x%08x: 0x%08x\n", 4401 R_AX_DBG_ERR_FLAG, val); 4402 if (val & B_AX_TX_STUCK || val & B_AX_PCIE_TXBD_LEN0) 4403 rtw89_mac_ctrl_hci_dma_tx(rtwdev, false); 4404 if (val & B_AX_RX_STUCK) 4405 rtw89_mac_ctrl_hci_dma_rx(rtwdev, false); 4406 rtw89_mac_ctrl_hci_dma_trx(rtwdev, true); 4407 ret = rtw89_pci_poll_io_idle_ax(rtwdev); 4408 val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG); 4409 rtw89_debug(rtwdev, RTW89_DBG_HCI, 4410 "[PCIe] poll_io_idle fail, after 0x%08x: 0x%08x\n", 4411 R_AX_DBG_ERR_FLAG, val); 4412 } 4413 4414 return ret; 4415} 4416 4417static int rtw89_pci_lv1rst_start_dma_ax(struct rtw89_dev *rtwdev) 4418{ 4419 int ret; 4420 4421 if (rtwdev->chip->chip_id == RTL8852C) 4422 return 0; 4423 4424 rtw89_mac_ctrl_hci_dma_trx(rtwdev, false); 4425 rtw89_mac_ctrl_hci_dma_trx(rtwdev, true); 4426 rtw89_pci_clr_idx_all(rtwdev); 4427 4428 ret = rtw89_pci_rst_bdram_ax(rtwdev); 4429 if (ret) 4430 return ret; 4431 4432 rtw89_pci_ctrl_dma_all(rtwdev, true); 4433 return 0; 4434} 4435 4436static int rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev *rtwdev, 4437 enum rtw89_lv1_rcvy_step step) 4438{ 4439 const struct rtw89_pci_info *info = rtwdev->pci_info; 4440 const struct rtw89_pci_gen_def *gen_def = info->gen_def; 4441 int ret; 4442 4443 switch (step) { 4444 case RTW89_LV1_RCVY_STEP_1: 4445 ret = gen_def->lv1rst_stop_dma(rtwdev); 4446 if (ret) 4447 rtw89_err(rtwdev, "lv1 rcvy pci stop dma fail\n"); 4448 4449 break; 4450 4451 case RTW89_LV1_RCVY_STEP_2: 4452 ret = gen_def->lv1rst_start_dma(rtwdev); 4453 if (ret) 4454 rtw89_err(rtwdev, "lv1 rcvy pci start dma fail\n"); 4455 break; 4456 4457 default: 4458 return -EINVAL; 4459 } 4460 4461 return ret; 4462} 4463 4464static void rtw89_pci_ops_dump_err_status(struct rtw89_dev *rtwdev) 4465{ 4466 if (rtwdev->chip->chip_gen == RTW89_CHIP_BE) 4467 return; 4468 4469 if (rtwdev->chip->chip_id == RTL8852C) { 4470 rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n", 4471 rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG_V1)); 4472 rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n", 4473 rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG_V1)); 4474 } else { 4475 rtw89_info(rtwdev, "R_AX_RPQ_RXBD_IDX =0x%08x\n", 4476 rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX)); 4477 rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n", 4478 rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG)); 4479 rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n", 4480 rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG)); 4481 } 4482} 4483 4484static int rtw89_pci_napi_poll(struct napi_struct *napi, int budget) 4485{ 4486 struct rtw89_dev *rtwdev = container_of(napi, struct rtw89_dev, napi); 4487 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; 4488 const struct rtw89_pci_info *info = rtwdev->pci_info; 4489 const struct rtw89_pci_isr_def *isr_def = info->isr_def; 4490 unsigned long flags; 4491 int work_done; 4492 4493 rtwdev->napi_budget_countdown = budget; 4494 4495 rtw89_write32(rtwdev, isr_def->isr_clear_rpq.addr, isr_def->isr_clear_rpq.data); 4496 work_done = rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown); 4497 if (work_done == budget) 4498 return budget; 4499 4500 rtw89_write32(rtwdev, isr_def->isr_clear_rxq.addr, isr_def->isr_clear_rxq.data); 4501 work_done += rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown); 4502 if (work_done < budget && napi_complete_done(napi, work_done)) { 4503 spin_lock_irqsave(&rtwpci->irq_lock, flags); 4504 if (likely(rtwpci->running)) 4505 rtw89_chip_enable_intr(rtwdev, rtwpci); 4506 spin_unlock_irqrestore(&rtwpci->irq_lock, flags); 4507 } 4508 4509 return work_done; 4510} 4511 4512static 4513void rtw89_check_pci_ssid_quirks(struct rtw89_dev *rtwdev, 4514 struct pci_dev *pdev, 4515 const struct rtw89_pci_ssid_quirk *ssid_quirks) 4516{ 4517 int i; 4518 4519 if (!ssid_quirks) 4520 return; 4521 4522 for (i = 0; i < 200; i++, ssid_quirks++) { 4523 if (ssid_quirks->vendor == 0 && ssid_quirks->device == 0) 4524 break; 4525 4526 if (ssid_quirks->vendor != pdev->vendor || 4527 ssid_quirks->device != pdev->device || 4528 ssid_quirks->subsystem_vendor != pdev->subsystem_vendor || 4529 ssid_quirks->subsystem_device != pdev->subsystem_device) 4530 continue; 4531 4532 bitmap_or(rtwdev->quirks, rtwdev->quirks, &ssid_quirks->bitmap, 4533 NUM_OF_RTW89_QUIRKS); 4534 rtwdev->custid = ssid_quirks->custid; 4535 break; 4536 } 4537 4538 rtw89_debug(rtwdev, RTW89_DBG_HCI, "quirks=%*ph custid=%d\n", 4539 (int)sizeof(rtwdev->quirks), rtwdev->quirks, rtwdev->custid); 4540} 4541 4542static int __maybe_unused rtw89_pci_suspend(struct device *dev) 4543{ 4544 struct ieee80211_hw *hw = dev_get_drvdata(dev); 4545 struct rtw89_dev *rtwdev = hw->priv; 4546 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; 4547 4548 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6); 4549 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST); 4550 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6); 4551 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { 4552 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL, 4553 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI); 4554 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, 4555 B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG); 4556 } else { 4557 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, 4558 B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN); 4559 } 4560 4561 return 0; 4562} 4563 4564static void rtw89_pci_l2_hci_ldo(struct rtw89_dev *rtwdev) 4565{ 4566 if (rtwdev->chip->chip_id == RTL8852C) 4567 return; 4568 4569 /* Hardware need write the reg twice to ensure the setting work */ 4570 rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE, 4571 RTW89_PCIE_BIT_CFG_RST_MSTATE); 4572 rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE, 4573 RTW89_PCIE_BIT_CFG_RST_MSTATE); 4574} 4575 4576void rtw89_pci_basic_cfg(struct rtw89_dev *rtwdev, bool resume) 4577{ 4578 if (resume) 4579 rtw89_pci_cfg_dac(rtwdev, false); 4580 4581 rtw89_pci_disable_eq(rtwdev); 4582 rtw89_pci_filter_out(rtwdev); 4583 rtw89_pci_cpl_timeout_cfg(rtwdev); 4584 rtw89_pci_link_cfg(rtwdev); 4585 rtw89_pci_l1ss_cfg(rtwdev); 4586} 4587 4588static int __maybe_unused rtw89_pci_resume(struct device *dev) 4589{ 4590 struct ieee80211_hw *hw = dev_get_drvdata(dev); 4591 struct rtw89_dev *rtwdev = hw->priv; 4592 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; 4593 4594 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6); 4595 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST); 4596 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6); 4597 if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { 4598 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, 4599 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI); 4600 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, 4601 B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG); 4602 } else { 4603 rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1, 4604 B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN); 4605 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, 4606 B_AX_SEL_REQ_ENTR_L1); 4607 } 4608 rtw89_pci_hci_ldo(rtwdev); 4609 rtw89_pci_l2_hci_ldo(rtwdev); 4610 4611 rtw89_pci_basic_cfg(rtwdev, true); 4612 4613 return 0; 4614} 4615 4616SIMPLE_DEV_PM_OPS(rtw89_pm_ops, rtw89_pci_suspend, rtw89_pci_resume); 4617EXPORT_SYMBOL(rtw89_pm_ops); 4618 4619static pci_ers_result_t rtw89_pci_io_error_detected(struct pci_dev *pdev, 4620 pci_channel_state_t state) 4621{ 4622 struct net_device *netdev = pci_get_drvdata(pdev); 4623 4624 netif_device_detach(netdev); 4625 4626 return PCI_ERS_RESULT_NEED_RESET; 4627} 4628 4629static pci_ers_result_t rtw89_pci_io_slot_reset(struct pci_dev *pdev) 4630{ 4631 struct ieee80211_hw *hw = pci_get_drvdata(pdev); 4632 struct rtw89_dev *rtwdev = hw->priv; 4633 4634 rtw89_ser_notify(rtwdev, MAC_AX_ERR_ASSERTION); 4635 4636 return PCI_ERS_RESULT_RECOVERED; 4637} 4638 4639static void rtw89_pci_io_resume(struct pci_dev *pdev) 4640{ 4641 struct net_device *netdev = pci_get_drvdata(pdev); 4642 4643 /* ack any pending wake events, disable PME */ 4644 pci_enable_wake(pdev, PCI_D0, 0); 4645 4646 netif_device_attach(netdev); 4647} 4648 4649const struct pci_error_handlers rtw89_pci_err_handler = { 4650 .error_detected = rtw89_pci_io_error_detected, 4651 .slot_reset = rtw89_pci_io_slot_reset, 4652 .resume = rtw89_pci_io_resume, 4653}; 4654EXPORT_SYMBOL(rtw89_pci_err_handler); 4655 4656const struct rtw89_pci_isr_def rtw89_pci_isr_ax = { 4657 .isr_rdu = B_AX_RDU_INT, 4658 .isr_halt_c2h = B_AX_HALT_C2H_INT_EN, 4659 .isr_wdt_timeout = B_AX_WDT_TIMEOUT_INT_EN, 4660 .isr_clear_rpq = {R_AX_PCIE_HISR00, B_AX_RPQDMA_INT | B_AX_RPQBD_FULL_INT}, 4661 .isr_clear_rxq = {R_AX_PCIE_HISR00, B_AX_RXP1DMA_INT | B_AX_RXDMA_INT | 4662 B_AX_RDU_INT}, 4663}; 4664EXPORT_SYMBOL(rtw89_pci_isr_ax); 4665 4666const struct rtw89_pci_gen_def rtw89_pci_gen_ax = { 4667 .mac_pre_init = rtw89_pci_ops_mac_pre_init_ax, 4668 .mac_pre_deinit = rtw89_pci_ops_mac_pre_deinit_ax, 4669 .mac_post_init = rtw89_pci_ops_mac_post_init_ax, 4670 4671 .clr_idx_all = rtw89_pci_clr_idx_all_ax, 4672 .rst_bdram = rtw89_pci_rst_bdram_ax, 4673 4674 .lv1rst_stop_dma = rtw89_pci_lv1rst_stop_dma_ax, 4675 .lv1rst_start_dma = rtw89_pci_lv1rst_start_dma_ax, 4676 4677 .ctrl_txdma_ch = rtw89_pci_ctrl_txdma_ch_ax, 4678 .ctrl_txdma_fw_ch = rtw89_pci_ctrl_txdma_fw_ch_ax, 4679 .poll_txdma_ch_idle = rtw89_pci_poll_txdma_ch_idle_ax, 4680 4681 .aspm_set = rtw89_pci_aspm_set_ax, 4682 .clkreq_set = rtw89_pci_clkreq_set_ax, 4683 .l1ss_set = rtw89_pci_l1ss_set_ax, 4684 4685 .disable_eq = rtw89_pci_disable_eq_ax, 4686 .power_wake = rtw89_pci_power_wake_ax, 4687}; 4688EXPORT_SYMBOL(rtw89_pci_gen_ax); 4689 4690static const struct rtw89_hci_ops rtw89_pci_ops = { 4691 .tx_write = rtw89_pci_ops_tx_write, 4692 .tx_kick_off = rtw89_pci_ops_tx_kick_off, 4693 .flush_queues = rtw89_pci_ops_flush_queues, 4694 .reset = rtw89_pci_ops_reset, 4695 .start = rtw89_pci_ops_start, 4696 .stop = rtw89_pci_ops_stop, 4697 .pause = rtw89_pci_ops_pause, 4698 .switch_mode = rtw89_pci_ops_switch_mode, 4699 .recalc_int_mit = rtw89_pci_recalc_int_mit, 4700 4701 .read8 = rtw89_pci_ops_read8, 4702 .read16 = rtw89_pci_ops_read16, 4703 .read32 = rtw89_pci_ops_read32, 4704 .write8 = rtw89_pci_ops_write8, 4705 .write16 = rtw89_pci_ops_write16, 4706 .write32 = rtw89_pci_ops_write32, 4707 4708 .read32_pci_cfg = rtw89_pci_ops_read32_pci_cfg, 4709 4710 .mac_pre_init = rtw89_pci_ops_mac_pre_init, 4711 .mac_pre_deinit = rtw89_pci_ops_mac_pre_deinit, 4712 .mac_post_init = rtw89_pci_ops_mac_post_init, 4713 .deinit = rtw89_pci_ops_deinit, 4714 4715 .check_and_reclaim_tx_resource = rtw89_pci_check_and_reclaim_tx_resource, 4716 .mac_lv1_rcvy = rtw89_pci_ops_mac_lv1_recovery, 4717 .dump_err_status = rtw89_pci_ops_dump_err_status, 4718 .napi_poll = rtw89_pci_napi_poll, 4719 4720 .recovery_start = rtw89_pci_ops_recovery_start, 4721 .recovery_complete = rtw89_pci_ops_recovery_complete, 4722 4723 .ctrl_txdma_ch = rtw89_pci_ctrl_txdma_ch, 4724 .ctrl_txdma_fw_ch = rtw89_pci_ctrl_txdma_fw_ch, 4725 .ctrl_trxhci = rtw89_pci_ctrl_dma_trx, 4726 .poll_txdma_ch_idle = rtw89_pci_poll_txdma_ch_idle, 4727 4728 .clr_idx_all = rtw89_pci_clr_idx_all, 4729 .clear = rtw89_pci_clear_resource, 4730 .disable_intr = rtw89_pci_disable_intr_lock, 4731 .enable_intr = rtw89_pci_enable_intr_lock, 4732 .rst_bdram = rtw89_pci_reset_bdram, 4733}; 4734 4735int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 4736{ 4737 struct rtw89_dev *rtwdev; 4738 const struct rtw89_driver_info *info; 4739 const struct rtw89_pci_info *pci_info; 4740 int ret; 4741 4742 info = (const struct rtw89_driver_info *)id->driver_data; 4743 4744 rtwdev = rtw89_alloc_ieee80211_hw(&pdev->dev, 4745 sizeof(struct rtw89_pci), 4746 info->chip, info->variant); 4747 if (!rtwdev) { 4748 dev_err(&pdev->dev, "failed to allocate hw\n"); 4749 return -ENOMEM; 4750 } 4751 4752 pci_info = info->bus.pci; 4753 4754 rtwdev->pci_info = info->bus.pci; 4755 rtwdev->hci.ops = &rtw89_pci_ops; 4756 rtwdev->hci.type = RTW89_HCI_TYPE_PCIE; 4757 rtwdev->hci.dle_type = RTW89_HCI_DLE_TYPE_PCIE; 4758 rtwdev->hci.rpwm_addr = pci_info->rpwm_addr; 4759 rtwdev->hci.cpwm_addr = pci_info->cpwm_addr; 4760 4761 rtw89_check_quirks(rtwdev, info->quirks); 4762 rtw89_check_pci_ssid_quirks(rtwdev, pdev, pci_info->ssid_quirks); 4763 4764 SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev); 4765 4766 ret = rtw89_core_init(rtwdev); 4767 if (ret) { 4768 rtw89_err(rtwdev, "failed to initialise core\n"); 4769 goto err_release_hw; 4770 } 4771 4772 ret = rtw89_pci_claim_device(rtwdev, pdev); 4773 if (ret) { 4774 rtw89_err(rtwdev, "failed to claim pci device\n"); 4775 goto err_core_deinit; 4776 } 4777 4778 ret = rtw89_pci_setup_resource(rtwdev, pdev); 4779 if (ret) { 4780 rtw89_err(rtwdev, "failed to setup pci resource\n"); 4781 goto err_declaim_pci; 4782 } 4783 4784 ret = rtw89_chip_info_setup(rtwdev); 4785 if (ret) { 4786 rtw89_err(rtwdev, "failed to setup chip information\n"); 4787 goto err_clear_resource; 4788 } 4789 4790 rtw89_pci_basic_cfg(rtwdev, false); 4791 4792 ret = rtw89_core_napi_init(rtwdev); 4793 if (ret) { 4794 rtw89_err(rtwdev, "failed to init napi\n"); 4795 goto err_clear_resource; 4796 } 4797 4798 ret = rtw89_pci_request_irq(rtwdev, pdev); 4799 if (ret) { 4800 rtw89_err(rtwdev, "failed to request pci irq\n"); 4801 goto err_deinit_napi; 4802 } 4803 4804 ret = rtw89_core_register(rtwdev); 4805 if (ret) { 4806 rtw89_err(rtwdev, "failed to register core\n"); 4807 goto err_free_irq; 4808 } 4809 4810 set_bit(RTW89_FLAG_PROBE_DONE, rtwdev->flags); 4811 4812 return 0; 4813 4814err_free_irq: 4815 rtw89_pci_free_irq(rtwdev, pdev); 4816err_deinit_napi: 4817 rtw89_core_napi_deinit(rtwdev); 4818err_clear_resource: 4819 rtw89_pci_clear_resource(rtwdev, pdev); 4820err_declaim_pci: 4821 rtw89_pci_declaim_device(rtwdev, pdev); 4822err_core_deinit: 4823 rtw89_core_deinit(rtwdev); 4824err_release_hw: 4825 rtw89_free_ieee80211_hw(rtwdev); 4826 4827 return ret; 4828} 4829EXPORT_SYMBOL(rtw89_pci_probe); 4830 4831void rtw89_pci_remove(struct pci_dev *pdev) 4832{ 4833 struct ieee80211_hw *hw = pci_get_drvdata(pdev); 4834 struct rtw89_dev *rtwdev; 4835 4836 rtwdev = hw->priv; 4837 4838 rtw89_pci_free_irq(rtwdev, pdev); 4839 rtw89_core_napi_deinit(rtwdev); 4840 rtw89_core_unregister(rtwdev); 4841 rtw89_pci_clear_resource(rtwdev, pdev); 4842 rtw89_pci_declaim_device(rtwdev, pdev); 4843 rtw89_core_deinit(rtwdev); 4844 rtw89_free_ieee80211_hw(rtwdev); 4845} 4846EXPORT_SYMBOL(rtw89_pci_remove); 4847 4848MODULE_AUTHOR("Realtek Corporation"); 4849MODULE_DESCRIPTION("Realtek PCI 802.11ax wireless driver"); 4850MODULE_LICENSE("Dual BSD/GPL");
~/
Read-only file preview.