FILE
pcie-designware-ep.c
File preview with syntax highlighting.
linux-6.19.14/drivers/pci/controller/dwc/pcie-designware-ep.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Synopsys DesignWare PCIe Endpoint controller driver 4 * 5 * Copyright (C) 2017 Texas Instruments 6 * Author: Kishon Vijay Abraham I <kishon@ti.com> 7 */ 8 9#include <linux/align.h> 10#include <linux/bitfield.h> 11#include <linux/of.h> 12#include <linux/platform_device.h> 13 14#include "pcie-designware.h" 15#include <linux/pci-epc.h> 16#include <linux/pci-epf.h> 17 18/** 19 * dw_pcie_ep_get_func_from_ep - Get the struct dw_pcie_ep_func corresponding to 20 * the endpoint function 21 * @ep: DWC EP device 22 * @func_no: Function number of the endpoint device 23 * 24 * Return: struct dw_pcie_ep_func if success, NULL otherwise. 25 */ 26struct dw_pcie_ep_func * 27dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no) 28{ 29 struct dw_pcie_ep_func *ep_func; 30 31 list_for_each_entry(ep_func, &ep->func_list, list) { 32 if (ep_func->func_no == func_no) 33 return ep_func; 34 } 35 36 return NULL; 37} 38 39static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no, 40 enum pci_barno bar, int flags) 41{ 42 struct dw_pcie_ep *ep = &pci->ep; 43 u32 reg; 44 45 reg = PCI_BASE_ADDRESS_0 + (4 * bar); 46 dw_pcie_dbi_ro_wr_en(pci); 47 dw_pcie_ep_writel_dbi2(ep, func_no, reg, 0x0); 48 dw_pcie_ep_writel_dbi(ep, func_no, reg, 0x0); 49 if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) { 50 dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, 0x0); 51 dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0x0); 52 } 53 dw_pcie_dbi_ro_wr_dis(pci); 54} 55 56/** 57 * dw_pcie_ep_reset_bar - Reset endpoint BAR 58 * @pci: DWC PCI device 59 * @bar: BAR number of the endpoint 60 */ 61void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) 62{ 63 u8 func_no, funcs; 64 65 funcs = pci->ep.epc->max_functions; 66 67 for (func_no = 0; func_no < funcs; func_no++) 68 __dw_pcie_ep_reset_bar(pci, func_no, bar, 0); 69} 70EXPORT_SYMBOL_GPL(dw_pcie_ep_reset_bar); 71 72static u8 dw_pcie_ep_find_capability(struct dw_pcie_ep *ep, u8 func_no, u8 cap) 73{ 74 return PCI_FIND_NEXT_CAP(dw_pcie_ep_read_cfg, PCI_CAPABILITY_LIST, 75 cap, NULL, ep, func_no); 76} 77 78static u16 dw_pcie_ep_find_ext_capability(struct dw_pcie_ep *ep, 79 u8 func_no, u8 cap) 80{ 81 return PCI_FIND_NEXT_EXT_CAP(dw_pcie_ep_read_cfg, 0, 82 cap, NULL, ep, func_no); 83} 84 85static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 86 struct pci_epf_header *hdr) 87{ 88 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 89 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 90 91 dw_pcie_dbi_ro_wr_en(pci); 92 dw_pcie_ep_writew_dbi(ep, func_no, PCI_VENDOR_ID, hdr->vendorid); 93 dw_pcie_ep_writew_dbi(ep, func_no, PCI_DEVICE_ID, hdr->deviceid); 94 dw_pcie_ep_writeb_dbi(ep, func_no, PCI_REVISION_ID, hdr->revid); 95 dw_pcie_ep_writeb_dbi(ep, func_no, PCI_CLASS_PROG, hdr->progif_code); 96 dw_pcie_ep_writew_dbi(ep, func_no, PCI_CLASS_DEVICE, 97 hdr->subclass_code | hdr->baseclass_code << 8); 98 dw_pcie_ep_writeb_dbi(ep, func_no, PCI_CACHE_LINE_SIZE, 99 hdr->cache_line_size); 100 dw_pcie_ep_writew_dbi(ep, func_no, PCI_SUBSYSTEM_VENDOR_ID, 101 hdr->subsys_vendor_id); 102 dw_pcie_ep_writew_dbi(ep, func_no, PCI_SUBSYSTEM_ID, hdr->subsys_id); 103 dw_pcie_ep_writeb_dbi(ep, func_no, PCI_INTERRUPT_PIN, 104 hdr->interrupt_pin); 105 dw_pcie_dbi_ro_wr_dis(pci); 106 107 return 0; 108} 109 110/* BAR Match Mode inbound iATU mapping */ 111static int dw_pcie_ep_ib_atu_bar(struct dw_pcie_ep *ep, u8 func_no, int type, 112 dma_addr_t parent_bus_addr, enum pci_barno bar, 113 size_t size) 114{ 115 int ret; 116 u32 free_win; 117 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 118 struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 119 120 if (!ep_func) 121 return -EINVAL; 122 123 if (!ep_func->bar_to_atu[bar]) 124 free_win = find_first_zero_bit(ep->ib_window_map, pci->num_ib_windows); 125 else 126 free_win = ep_func->bar_to_atu[bar] - 1; 127 128 if (free_win >= pci->num_ib_windows) { 129 dev_err(pci->dev, "No free inbound window\n"); 130 return -EINVAL; 131 } 132 133 ret = dw_pcie_prog_ep_inbound_atu(pci, func_no, free_win, type, 134 parent_bus_addr, bar, size); 135 if (ret < 0) { 136 dev_err(pci->dev, "Failed to program IB window\n"); 137 return ret; 138 } 139 140 /* 141 * Always increment free_win before assignment, since value 0 is used to identify 142 * unallocated mapping. 143 */ 144 ep_func->bar_to_atu[bar] = free_win + 1; 145 set_bit(free_win, ep->ib_window_map); 146 147 return 0; 148} 149 150static void dw_pcie_ep_clear_ib_maps(struct dw_pcie_ep *ep, u8 func_no, enum pci_barno bar) 151{ 152 struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 153 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 154 struct device *dev = pci->dev; 155 unsigned int i, num; 156 u32 atu_index; 157 u32 *indexes; 158 159 if (!ep_func) 160 return; 161 162 /* Tear down the BAR Match Mode mapping, if any. */ 163 if (ep_func->bar_to_atu[bar]) { 164 atu_index = ep_func->bar_to_atu[bar] - 1; 165 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, atu_index); 166 clear_bit(atu_index, ep->ib_window_map); 167 ep_func->bar_to_atu[bar] = 0; 168 } 169 170 /* Tear down all Address Match Mode mappings, if any. */ 171 indexes = ep_func->ib_atu_indexes[bar]; 172 num = ep_func->num_ib_atu_indexes[bar]; 173 ep_func->ib_atu_indexes[bar] = NULL; 174 ep_func->num_ib_atu_indexes[bar] = 0; 175 if (!indexes) 176 return; 177 for (i = 0; i < num; i++) { 178 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, indexes[i]); 179 clear_bit(indexes[i], ep->ib_window_map); 180 } 181 devm_kfree(dev, indexes); 182} 183 184static u64 dw_pcie_ep_read_bar_assigned(struct dw_pcie_ep *ep, u8 func_no, 185 enum pci_barno bar, int flags) 186{ 187 u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar); 188 u32 lo, hi; 189 u64 addr; 190 191 lo = dw_pcie_ep_readl_dbi(ep, func_no, reg); 192 193 if (flags & PCI_BASE_ADDRESS_SPACE) 194 return lo & PCI_BASE_ADDRESS_IO_MASK; 195 196 addr = lo & PCI_BASE_ADDRESS_MEM_MASK; 197 if (!(flags & PCI_BASE_ADDRESS_MEM_TYPE_64)) 198 return addr; 199 200 hi = dw_pcie_ep_readl_dbi(ep, func_no, reg + 4); 201 return addr | ((u64)hi << 32); 202} 203 204static int dw_pcie_ep_validate_submap(struct dw_pcie_ep *ep, 205 const struct pci_epf_bar_submap *submap, 206 unsigned int num_submap, size_t bar_size) 207{ 208 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 209 u32 align = pci->region_align; 210 size_t off = 0; 211 unsigned int i; 212 size_t size; 213 214 if (!align || !IS_ALIGNED(bar_size, align)) 215 return -EINVAL; 216 217 /* 218 * The submap array order defines the BAR layout (submap[0] starts 219 * at offset 0 and each entry immediately follows the previous 220 * one). Here, validate that it forms a strict, gapless 221 * decomposition of the BAR: 222 * - each entry has a non-zero size 223 * - sizes, implicit offsets and phys_addr are aligned to 224 * pci->region_align 225 * - each entry lies within the BAR range 226 * - the entries exactly cover the whole BAR 227 * 228 * Note: dw_pcie_prog_inbound_atu() also checks alignment for the 229 * PCI address and the target phys_addr, but validating up-front 230 * avoids partially programming iATU windows in vain. 231 */ 232 for (i = 0; i < num_submap; i++) { 233 size = submap[i].size; 234 235 if (!size) 236 return -EINVAL; 237 238 if (!IS_ALIGNED(size, align) || !IS_ALIGNED(off, align)) 239 return -EINVAL; 240 241 if (!IS_ALIGNED(submap[i].phys_addr, align)) 242 return -EINVAL; 243 244 if (off > bar_size || size > bar_size - off) 245 return -EINVAL; 246 247 off += size; 248 } 249 if (off != bar_size) 250 return -EINVAL; 251 252 return 0; 253} 254 255/* Address Match Mode inbound iATU mapping */ 256static int dw_pcie_ep_ib_atu_addr(struct dw_pcie_ep *ep, u8 func_no, int type, 257 const struct pci_epf_bar *epf_bar) 258{ 259 struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 260 const struct pci_epf_bar_submap *submap = epf_bar->submap; 261 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 262 enum pci_barno bar = epf_bar->barno; 263 struct device *dev = pci->dev; 264 u64 pci_addr, parent_bus_addr; 265 u64 size, base, off = 0; 266 int free_win, ret; 267 unsigned int i; 268 u32 *indexes; 269 270 if (!ep_func || !epf_bar->num_submap || !submap || !epf_bar->size) 271 return -EINVAL; 272 273 ret = dw_pcie_ep_validate_submap(ep, submap, epf_bar->num_submap, 274 epf_bar->size); 275 if (ret) 276 return ret; 277 278 base = dw_pcie_ep_read_bar_assigned(ep, func_no, bar, epf_bar->flags); 279 if (!base) { 280 dev_err(dev, 281 "BAR%u not assigned, cannot set up sub-range mappings\n", 282 bar); 283 return -EINVAL; 284 } 285 286 indexes = devm_kcalloc(dev, epf_bar->num_submap, sizeof(*indexes), 287 GFP_KERNEL); 288 if (!indexes) 289 return -ENOMEM; 290 291 ep_func->ib_atu_indexes[bar] = indexes; 292 ep_func->num_ib_atu_indexes[bar] = 0; 293 294 for (i = 0; i < epf_bar->num_submap; i++) { 295 size = submap[i].size; 296 parent_bus_addr = submap[i].phys_addr; 297 298 if (off > (~0ULL) - base) { 299 ret = -EINVAL; 300 goto err; 301 } 302 303 pci_addr = base + off; 304 off += size; 305 306 free_win = find_first_zero_bit(ep->ib_window_map, 307 pci->num_ib_windows); 308 if (free_win >= pci->num_ib_windows) { 309 ret = -ENOSPC; 310 goto err; 311 } 312 313 ret = dw_pcie_prog_inbound_atu(pci, free_win, type, 314 parent_bus_addr, pci_addr, size); 315 if (ret) 316 goto err; 317 318 set_bit(free_win, ep->ib_window_map); 319 indexes[i] = free_win; 320 ep_func->num_ib_atu_indexes[bar] = i + 1; 321 } 322 return 0; 323err: 324 dw_pcie_ep_clear_ib_maps(ep, func_no, bar); 325 return ret; 326} 327 328static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, 329 struct dw_pcie_ob_atu_cfg *atu) 330{ 331 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 332 u32 free_win; 333 int ret; 334 335 free_win = find_first_zero_bit(ep->ob_window_map, pci->num_ob_windows); 336 if (free_win >= pci->num_ob_windows) { 337 dev_err(pci->dev, "No free outbound window\n"); 338 return -EINVAL; 339 } 340 341 atu->index = free_win; 342 ret = dw_pcie_prog_outbound_atu(pci, atu); 343 if (ret) 344 return ret; 345 346 set_bit(free_win, ep->ob_window_map); 347 ep->outbound_addr[free_win] = atu->parent_bus_addr; 348 349 return 0; 350} 351 352static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 353 struct pci_epf_bar *epf_bar) 354{ 355 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 356 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 357 enum pci_barno bar = epf_bar->barno; 358 struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 359 360 if (!ep_func || !ep_func->epf_bar[bar]) 361 return; 362 363 __dw_pcie_ep_reset_bar(pci, func_no, bar, epf_bar->flags); 364 365 dw_pcie_ep_clear_ib_maps(ep, func_no, bar); 366 367 ep_func->epf_bar[bar] = NULL; 368} 369 370static unsigned int dw_pcie_ep_get_rebar_offset(struct dw_pcie_ep *ep, u8 func_no, 371 enum pci_barno bar) 372{ 373 u32 reg, bar_index; 374 unsigned int offset, nbars; 375 int i; 376 377 offset = dw_pcie_ep_find_ext_capability(ep, func_no, PCI_EXT_CAP_ID_REBAR); 378 if (!offset) 379 return offset; 380 381 reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); 382 nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, reg); 383 384 for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) { 385 reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); 386 bar_index = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, reg); 387 if (bar_index == bar) 388 return offset; 389 } 390 391 return 0; 392} 393 394static int dw_pcie_ep_set_bar_resizable(struct dw_pcie_ep *ep, u8 func_no, 395 struct pci_epf_bar *epf_bar) 396{ 397 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 398 enum pci_barno bar = epf_bar->barno; 399 size_t size = epf_bar->size; 400 int flags = epf_bar->flags; 401 u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar); 402 unsigned int rebar_offset; 403 u32 rebar_cap, rebar_ctrl; 404 int ret; 405 406 rebar_offset = dw_pcie_ep_get_rebar_offset(ep, func_no, bar); 407 if (!rebar_offset) 408 return -EINVAL; 409 410 ret = pci_epc_bar_size_to_rebar_cap(size, &rebar_cap); 411 if (ret) 412 return ret; 413 414 dw_pcie_dbi_ro_wr_en(pci); 415 416 /* 417 * A BAR mask should not be written for a resizable BAR. The BAR mask 418 * is automatically derived by the controller every time the "selected 419 * size" bits are updated, see "Figure 3-26 Resizable BAR Example for 420 * 32-bit Memory BAR0" in DWC EP databook 5.96a. We simply need to write 421 * BIT(0) to set the BAR enable bit. 422 */ 423 dw_pcie_ep_writel_dbi2(ep, func_no, reg, BIT(0)); 424 dw_pcie_ep_writel_dbi(ep, func_no, reg, flags); 425 426 if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) { 427 dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, 0); 428 dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0); 429 } 430 431 /* 432 * Bits 31:0 in PCI_REBAR_CAP define "supported sizes" bits for sizes 433 * 1 MB to 128 TB. Bits 31:16 in PCI_REBAR_CTRL define "supported sizes" 434 * bits for sizes 256 TB to 8 EB. Disallow sizes 256 TB to 8 EB. 435 */ 436 rebar_ctrl = dw_pcie_ep_readl_dbi(ep, func_no, rebar_offset + PCI_REBAR_CTRL); 437 rebar_ctrl &= ~GENMASK(31, 16); 438 dw_pcie_ep_writel_dbi(ep, func_no, rebar_offset + PCI_REBAR_CTRL, rebar_ctrl); 439 440 /* 441 * The "selected size" (bits 13:8) in PCI_REBAR_CTRL are automatically 442 * updated when writing PCI_REBAR_CAP, see "Figure 3-26 Resizable BAR 443 * Example for 32-bit Memory BAR0" in DWC EP databook 5.96a. 444 */ 445 dw_pcie_ep_writel_dbi(ep, func_no, rebar_offset + PCI_REBAR_CAP, rebar_cap); 446 447 dw_pcie_dbi_ro_wr_dis(pci); 448 449 return 0; 450} 451 452static int dw_pcie_ep_set_bar_programmable(struct dw_pcie_ep *ep, u8 func_no, 453 struct pci_epf_bar *epf_bar) 454{ 455 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 456 enum pci_barno bar = epf_bar->barno; 457 size_t size = epf_bar->size; 458 int flags = epf_bar->flags; 459 u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar); 460 461 dw_pcie_dbi_ro_wr_en(pci); 462 463 dw_pcie_ep_writel_dbi2(ep, func_no, reg, lower_32_bits(size - 1)); 464 dw_pcie_ep_writel_dbi(ep, func_no, reg, flags); 465 466 if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) { 467 dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, upper_32_bits(size - 1)); 468 dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0); 469 } 470 471 dw_pcie_dbi_ro_wr_dis(pci); 472 473 return 0; 474} 475 476static enum pci_epc_bar_type dw_pcie_ep_get_bar_type(struct dw_pcie_ep *ep, 477 enum pci_barno bar) 478{ 479 const struct pci_epc_features *epc_features; 480 481 if (!ep->ops->get_features) 482 return BAR_PROGRAMMABLE; 483 484 epc_features = ep->ops->get_features(ep); 485 486 return epc_features->bar[bar].type; 487} 488 489static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 490 struct pci_epf_bar *epf_bar) 491{ 492 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 493 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 494 struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 495 enum pci_barno bar = epf_bar->barno; 496 size_t size = epf_bar->size; 497 enum pci_epc_bar_type bar_type; 498 int flags = epf_bar->flags; 499 int ret, type; 500 501 if (!ep_func) 502 return -EINVAL; 503 504 /* 505 * DWC does not allow BAR pairs to overlap, e.g. you cannot combine BARs 506 * 1 and 2 to form a 64-bit BAR. 507 */ 508 if ((flags & PCI_BASE_ADDRESS_MEM_TYPE_64) && (bar & 1)) 509 return -EINVAL; 510 511 /* 512 * Certain EPF drivers dynamically change the physical address of a BAR 513 * (i.e. they call set_bar() twice, without ever calling clear_bar(), as 514 * calling clear_bar() would clear the BAR's PCI address assigned by the 515 * host). 516 */ 517 if (ep_func->epf_bar[bar]) { 518 /* 519 * We can only dynamically change a BAR if the new BAR size and 520 * BAR flags do not differ from the existing configuration. 521 * 522 * Note: this safety check only works when the caller uses 523 * a new struct pci_epf_bar in the second set_bar() call. 524 * If the same instance is updated in place and passed in, 525 * we cannot reliably detect invalid barno/size/flags 526 * changes here. 527 */ 528 if (ep_func->epf_bar[bar]->barno != bar || 529 ep_func->epf_bar[bar]->size != size || 530 ep_func->epf_bar[bar]->flags != flags) 531 return -EINVAL; 532 533 /* 534 * When dynamically changing a BAR, tear down any existing 535 * mappings before re-programming. This is redundant when 536 * both the old and new mappings are BAR Match Mode, but 537 * required to handle in-place updates and match-mode 538 * changes reliably. 539 */ 540 dw_pcie_ep_clear_ib_maps(ep, func_no, bar); 541 542 /* 543 * When dynamically changing a BAR, skip writing the BAR reg, as 544 * that would clear the BAR's PCI address assigned by the host. 545 */ 546 goto config_atu; 547 } else { 548 /* 549 * Subrange mapping is an update-only operation. The BAR 550 * must have been configured once without submaps so that 551 * subsequent set_bar() calls can update inbound mappings 552 * without touching the BAR register (and clobbering the 553 * host-assigned address). 554 */ 555 if (epf_bar->num_submap) 556 return -EINVAL; 557 } 558 559 bar_type = dw_pcie_ep_get_bar_type(ep, bar); 560 switch (bar_type) { 561 case BAR_FIXED: 562 /* 563 * There is no need to write a BAR mask for a fixed BAR (except 564 * to write 1 to the LSB of the BAR mask register, to enable the 565 * BAR). Write the BAR mask regardless. (The fixed bits in the 566 * BAR mask register will be read-only anyway.) 567 */ 568 fallthrough; 569 case BAR_PROGRAMMABLE: 570 ret = dw_pcie_ep_set_bar_programmable(ep, func_no, epf_bar); 571 break; 572 case BAR_RESIZABLE: 573 ret = dw_pcie_ep_set_bar_resizable(ep, func_no, epf_bar); 574 break; 575 default: 576 ret = -EINVAL; 577 dev_err(pci->dev, "Invalid BAR type\n"); 578 break; 579 } 580 581 if (ret) 582 return ret; 583 584config_atu: 585 if (!(flags & PCI_BASE_ADDRESS_SPACE)) 586 type = PCIE_ATU_TYPE_MEM; 587 else 588 type = PCIE_ATU_TYPE_IO; 589 590 if (epf_bar->num_submap) 591 ret = dw_pcie_ep_ib_atu_addr(ep, func_no, type, epf_bar); 592 else 593 ret = dw_pcie_ep_ib_atu_bar(ep, func_no, type, 594 epf_bar->phys_addr, bar, size); 595 596 if (ret) 597 return ret; 598 599 ep_func->epf_bar[bar] = epf_bar; 600 601 return 0; 602} 603 604static int dw_pcie_find_index(struct dw_pcie_ep *ep, phys_addr_t addr, 605 u32 *atu_index) 606{ 607 u32 index; 608 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 609 610 for_each_set_bit(index, ep->ob_window_map, pci->num_ob_windows) { 611 if (ep->outbound_addr[index] != addr) 612 continue; 613 *atu_index = index; 614 return 0; 615 } 616 617 return -EINVAL; 618} 619 620static u64 dw_pcie_ep_align_addr(struct pci_epc *epc, u64 pci_addr, 621 size_t *pci_size, size_t *offset) 622{ 623 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 624 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 625 u64 mask = pci->region_align - 1; 626 size_t ofst = pci_addr & mask; 627 628 *pci_size = ALIGN(ofst + *pci_size, epc->mem->window.page_size); 629 *offset = ofst; 630 631 return pci_addr & ~mask; 632} 633 634static void dw_pcie_ep_unmap_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 635 phys_addr_t addr) 636{ 637 int ret; 638 u32 atu_index; 639 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 640 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 641 642 ret = dw_pcie_find_index(ep, addr - pci->parent_bus_offset, 643 &atu_index); 644 if (ret < 0) 645 return; 646 647 ep->outbound_addr[atu_index] = 0; 648 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_OB, atu_index); 649 clear_bit(atu_index, ep->ob_window_map); 650} 651 652static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 653 phys_addr_t addr, u64 pci_addr, size_t size) 654{ 655 int ret; 656 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 657 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 658 struct dw_pcie_ob_atu_cfg atu = { 0 }; 659 660 atu.func_no = func_no; 661 atu.type = PCIE_ATU_TYPE_MEM; 662 atu.parent_bus_addr = addr - pci->parent_bus_offset; 663 atu.pci_addr = pci_addr; 664 atu.size = size; 665 ret = dw_pcie_ep_outbound_atu(ep, &atu); 666 if (ret) { 667 dev_err(pci->dev, "Failed to enable address\n"); 668 return ret; 669 } 670 671 return 0; 672} 673 674static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no) 675{ 676 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 677 struct dw_pcie_ep_func *ep_func; 678 u32 val, reg; 679 680 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 681 if (!ep_func || !ep_func->msi_cap) 682 return -EINVAL; 683 684 reg = ep_func->msi_cap + PCI_MSI_FLAGS; 685 val = dw_pcie_ep_readw_dbi(ep, func_no, reg); 686 if (!(val & PCI_MSI_FLAGS_ENABLE)) 687 return -EINVAL; 688 689 val = FIELD_GET(PCI_MSI_FLAGS_QSIZE, val); 690 691 return 1 << val; 692} 693 694static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 695 u8 nr_irqs) 696{ 697 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 698 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 699 struct dw_pcie_ep_func *ep_func; 700 u8 mmc = order_base_2(nr_irqs); 701 u32 val, reg; 702 703 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 704 if (!ep_func || !ep_func->msi_cap) 705 return -EINVAL; 706 707 reg = ep_func->msi_cap + PCI_MSI_FLAGS; 708 val = dw_pcie_ep_readw_dbi(ep, func_no, reg); 709 val &= ~PCI_MSI_FLAGS_QMASK; 710 val |= FIELD_PREP(PCI_MSI_FLAGS_QMASK, mmc); 711 dw_pcie_dbi_ro_wr_en(pci); 712 dw_pcie_ep_writew_dbi(ep, func_no, reg, val); 713 dw_pcie_dbi_ro_wr_dis(pci); 714 715 return 0; 716} 717 718static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no) 719{ 720 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 721 struct dw_pcie_ep_func *ep_func; 722 u32 val, reg; 723 724 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 725 if (!ep_func || !ep_func->msix_cap) 726 return -EINVAL; 727 728 reg = ep_func->msix_cap + PCI_MSIX_FLAGS; 729 val = dw_pcie_ep_readw_dbi(ep, func_no, reg); 730 if (!(val & PCI_MSIX_FLAGS_ENABLE)) 731 return -EINVAL; 732 733 val &= PCI_MSIX_FLAGS_QSIZE; 734 735 return val + 1; 736} 737 738static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 739 u16 nr_irqs, enum pci_barno bir, u32 offset) 740{ 741 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 742 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 743 struct dw_pcie_ep_func *ep_func; 744 u32 val, reg; 745 746 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 747 if (!ep_func || !ep_func->msix_cap) 748 return -EINVAL; 749 750 dw_pcie_dbi_ro_wr_en(pci); 751 752 reg = ep_func->msix_cap + PCI_MSIX_FLAGS; 753 val = dw_pcie_ep_readw_dbi(ep, func_no, reg); 754 val &= ~PCI_MSIX_FLAGS_QSIZE; 755 val |= nr_irqs - 1; /* encoded as N-1 */ 756 dw_pcie_writew_dbi(pci, reg, val); 757 758 reg = ep_func->msix_cap + PCI_MSIX_TABLE; 759 val = offset | bir; 760 dw_pcie_ep_writel_dbi(ep, func_no, reg, val); 761 762 reg = ep_func->msix_cap + PCI_MSIX_PBA; 763 val = (offset + (nr_irqs * PCI_MSIX_ENTRY_SIZE)) | bir; 764 dw_pcie_ep_writel_dbi(ep, func_no, reg, val); 765 766 dw_pcie_dbi_ro_wr_dis(pci); 767 768 return 0; 769} 770 771static int dw_pcie_ep_raise_irq(struct pci_epc *epc, u8 func_no, u8 vfunc_no, 772 unsigned int type, u16 interrupt_num) 773{ 774 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 775 776 if (!ep->ops->raise_irq) 777 return -EINVAL; 778 779 return ep->ops->raise_irq(ep, func_no, type, interrupt_num); 780} 781 782static void dw_pcie_ep_stop(struct pci_epc *epc) 783{ 784 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 785 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 786 787 /* 788 * Tear down the dedicated outbound window used for MSI 789 * generation. This avoids leaking an iATU window across 790 * endpoint stop/start cycles. 791 */ 792 if (ep->msi_iatu_mapped) { 793 dw_pcie_ep_unmap_addr(epc, 0, 0, ep->msi_mem_phys); 794 ep->msi_iatu_mapped = false; 795 } 796 797 dw_pcie_stop_link(pci); 798} 799 800static int dw_pcie_ep_start(struct pci_epc *epc) 801{ 802 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 803 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 804 805 return dw_pcie_start_link(pci); 806} 807 808static const struct pci_epc_features* 809dw_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no) 810{ 811 struct dw_pcie_ep *ep = epc_get_drvdata(epc); 812 813 if (!ep->ops->get_features) 814 return NULL; 815 816 return ep->ops->get_features(ep); 817} 818 819static const struct pci_epc_ops epc_ops = { 820 .write_header = dw_pcie_ep_write_header, 821 .set_bar = dw_pcie_ep_set_bar, 822 .clear_bar = dw_pcie_ep_clear_bar, 823 .align_addr = dw_pcie_ep_align_addr, 824 .map_addr = dw_pcie_ep_map_addr, 825 .unmap_addr = dw_pcie_ep_unmap_addr, 826 .set_msi = dw_pcie_ep_set_msi, 827 .get_msi = dw_pcie_ep_get_msi, 828 .set_msix = dw_pcie_ep_set_msix, 829 .get_msix = dw_pcie_ep_get_msix, 830 .raise_irq = dw_pcie_ep_raise_irq, 831 .start = dw_pcie_ep_start, 832 .stop = dw_pcie_ep_stop, 833 .get_features = dw_pcie_ep_get_features, 834}; 835 836/** 837 * dw_pcie_ep_raise_intx_irq - Raise INTx IRQ to the host 838 * @ep: DWC EP device 839 * @func_no: Function number of the endpoint 840 * 841 * Return: 0 if success, errno otherwise. 842 */ 843int dw_pcie_ep_raise_intx_irq(struct dw_pcie_ep *ep, u8 func_no) 844{ 845 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 846 struct device *dev = pci->dev; 847 848 dev_err(dev, "EP cannot raise INTX IRQs\n"); 849 850 return -EINVAL; 851} 852EXPORT_SYMBOL_GPL(dw_pcie_ep_raise_intx_irq); 853 854/** 855 * dw_pcie_ep_raise_msi_irq - Raise MSI IRQ to the host 856 * @ep: DWC EP device 857 * @func_no: Function number of the endpoint 858 * @interrupt_num: Interrupt number to be raised 859 * 860 * Return: 0 if success, errno otherwise. 861 */ 862int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, 863 u8 interrupt_num) 864{ 865 u32 msg_addr_lower, msg_addr_upper, reg; 866 struct dw_pcie_ep_func *ep_func; 867 struct pci_epc *epc = ep->epc; 868 size_t map_size = sizeof(u32); 869 size_t offset; 870 u16 msg_ctrl, msg_data; 871 bool has_upper; 872 u64 msg_addr; 873 int ret; 874 875 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 876 if (!ep_func || !ep_func->msi_cap) 877 return -EINVAL; 878 879 /* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. */ 880 reg = ep_func->msi_cap + PCI_MSI_FLAGS; 881 msg_ctrl = dw_pcie_ep_readw_dbi(ep, func_no, reg); 882 has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT); 883 reg = ep_func->msi_cap + PCI_MSI_ADDRESS_LO; 884 msg_addr_lower = dw_pcie_ep_readl_dbi(ep, func_no, reg); 885 if (has_upper) { 886 reg = ep_func->msi_cap + PCI_MSI_ADDRESS_HI; 887 msg_addr_upper = dw_pcie_ep_readl_dbi(ep, func_no, reg); 888 reg = ep_func->msi_cap + PCI_MSI_DATA_64; 889 msg_data = dw_pcie_ep_readw_dbi(ep, func_no, reg); 890 } else { 891 msg_addr_upper = 0; 892 reg = ep_func->msi_cap + PCI_MSI_DATA_32; 893 msg_data = dw_pcie_ep_readw_dbi(ep, func_no, reg); 894 } 895 msg_addr = ((u64)msg_addr_upper) << 32 | msg_addr_lower; 896 897 msg_addr = dw_pcie_ep_align_addr(epc, msg_addr, &map_size, &offset); 898 899 /* 900 * Program the outbound iATU once and keep it enabled. 901 * 902 * The spec warns that updating iATU registers while there are 903 * operations in flight on the AXI bridge interface is not 904 * supported, so we avoid reprogramming the region on every MSI, 905 * specifically unmapping immediately after writel(). 906 */ 907 if (ep->msi_iatu_mapped && (ep->msi_msg_addr != msg_addr || 908 ep->msi_map_size != map_size)) { 909 /* 910 * The host changed the MSI target address or the required 911 * mapping size changed. Reprogramming the iATU when there are 912 * operations in flight is unsafe on this controller. However, 913 * there is no unified way to check if we have operations in 914 * flight, thus we don't know if we should WARN() or not. 915 */ 916 dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); 917 ep->msi_iatu_mapped = false; 918 } 919 920 if (!ep->msi_iatu_mapped) { 921 ret = dw_pcie_ep_map_addr(epc, func_no, 0, 922 ep->msi_mem_phys, msg_addr, 923 map_size); 924 if (ret) 925 return ret; 926 927 ep->msi_iatu_mapped = true; 928 ep->msi_msg_addr = msg_addr; 929 ep->msi_map_size = map_size; 930 } 931 932 writel(msg_data | (interrupt_num - 1), ep->msi_mem + offset); 933 934 return 0; 935} 936EXPORT_SYMBOL_GPL(dw_pcie_ep_raise_msi_irq); 937 938/** 939 * dw_pcie_ep_raise_msix_irq_doorbell - Raise MSI-X to the host using Doorbell 940 * method 941 * @ep: DWC EP device 942 * @func_no: Function number of the endpoint device 943 * @interrupt_num: Interrupt number to be raised 944 * 945 * Return: 0 if success, errno otherwise. 946 */ 947int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, 948 u16 interrupt_num) 949{ 950 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 951 struct dw_pcie_ep_func *ep_func; 952 u32 msg_data; 953 954 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 955 if (!ep_func || !ep_func->msix_cap) 956 return -EINVAL; 957 958 msg_data = (func_no << PCIE_MSIX_DOORBELL_PF_SHIFT) | 959 (interrupt_num - 1); 960 961 dw_pcie_writel_dbi(pci, PCIE_MSIX_DOORBELL, msg_data); 962 963 return 0; 964} 965 966/** 967 * dw_pcie_ep_raise_msix_irq - Raise MSI-X to the host 968 * @ep: DWC EP device 969 * @func_no: Function number of the endpoint device 970 * @interrupt_num: Interrupt number to be raised 971 * 972 * Return: 0 if success, errno otherwise. 973 */ 974int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, 975 u16 interrupt_num) 976{ 977 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 978 struct pci_epf_msix_tbl *msix_tbl; 979 struct dw_pcie_ep_func *ep_func; 980 struct pci_epc *epc = ep->epc; 981 size_t map_size = sizeof(u32); 982 size_t offset; 983 u32 reg, msg_data, vec_ctrl; 984 u32 tbl_offset; 985 u64 msg_addr; 986 int ret; 987 u8 bir; 988 989 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 990 if (!ep_func || !ep_func->msix_cap) 991 return -EINVAL; 992 993 reg = ep_func->msix_cap + PCI_MSIX_TABLE; 994 tbl_offset = dw_pcie_ep_readl_dbi(ep, func_no, reg); 995 bir = FIELD_GET(PCI_MSIX_TABLE_BIR, tbl_offset); 996 tbl_offset &= PCI_MSIX_TABLE_OFFSET; 997 998 msix_tbl = ep_func->epf_bar[bir]->addr + tbl_offset; 999 msg_addr = msix_tbl[(interrupt_num - 1)].msg_addr; 1000 msg_data = msix_tbl[(interrupt_num - 1)].msg_data; 1001 vec_ctrl = msix_tbl[(interrupt_num - 1)].vector_ctrl; 1002 1003 if (vec_ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT) { 1004 dev_dbg(pci->dev, "MSI-X entry ctrl set\n"); 1005 return -EPERM; 1006 } 1007 1008 msg_addr = dw_pcie_ep_align_addr(epc, msg_addr, &map_size, &offset); 1009 ret = dw_pcie_ep_map_addr(epc, func_no, 0, ep->msi_mem_phys, msg_addr, 1010 map_size); 1011 if (ret) 1012 return ret; 1013 1014 writel(msg_data, ep->msi_mem + offset); 1015 1016 /* flush posted write before unmap */ 1017 readl(ep->msi_mem + offset); 1018 1019 dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); 1020 1021 return 0; 1022} 1023EXPORT_SYMBOL_GPL(dw_pcie_ep_raise_msix_irq); 1024 1025/** 1026 * dw_pcie_ep_cleanup - Cleanup DWC EP resources after fundamental reset 1027 * @ep: DWC EP device 1028 * 1029 * Cleans up the DWC EP specific resources like eDMA etc... after fundamental 1030 * reset like PERST#. Note that this API is only applicable for drivers 1031 * supporting PERST# or any other methods of fundamental reset. 1032 */ 1033void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep) 1034{ 1035 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1036 1037 dwc_pcie_debugfs_deinit(pci); 1038 dw_pcie_edma_remove(pci); 1039} 1040EXPORT_SYMBOL_GPL(dw_pcie_ep_cleanup); 1041 1042/** 1043 * dw_pcie_ep_deinit - Deinitialize the endpoint device 1044 * @ep: DWC EP device 1045 * 1046 * Deinitialize the endpoint device. EPC device is not destroyed since that will 1047 * be taken care by Devres. 1048 */ 1049void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) 1050{ 1051 struct pci_epc *epc = ep->epc; 1052 1053 dw_pcie_ep_cleanup(ep); 1054 1055 pci_epc_mem_free_addr(epc, ep->msi_mem_phys, ep->msi_mem, 1056 epc->mem->window.page_size); 1057 1058 pci_epc_mem_exit(epc); 1059} 1060EXPORT_SYMBOL_GPL(dw_pcie_ep_deinit); 1061 1062static void dw_pcie_ep_init_rebar_registers(struct dw_pcie_ep *ep, u8 func_no) 1063{ 1064 struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 1065 unsigned int offset, nbars; 1066 enum pci_barno bar; 1067 u32 reg, i, val; 1068 1069 if (!ep_func) 1070 return; 1071 1072 offset = dw_pcie_ep_find_ext_capability(ep, func_no, PCI_EXT_CAP_ID_REBAR); 1073 1074 if (offset) { 1075 reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); 1076 nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, reg); 1077 1078 /* 1079 * PCIe r6.0, sec 7.8.6.2 require us to support at least one 1080 * size in the range from 1 MB to 512 GB. Advertise support 1081 * for 1 MB BAR size only. 1082 * 1083 * For a BAR that has been configured via dw_pcie_ep_set_bar(), 1084 * advertise support for only that size instead. 1085 */ 1086 for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) { 1087 /* 1088 * While the RESBAR_CAP_REG_* fields are sticky, the 1089 * RESBAR_CTRL_REG_BAR_SIZE field is non-sticky (it is 1090 * sticky in certain versions of DWC PCIe, but not all). 1091 * 1092 * RESBAR_CTRL_REG_BAR_SIZE is updated automatically by 1093 * the controller when RESBAR_CAP_REG is written, which 1094 * is why RESBAR_CAP_REG is written here. 1095 */ 1096 val = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); 1097 bar = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, val); 1098 if (ep_func->epf_bar[bar]) 1099 pci_epc_bar_size_to_rebar_cap(ep_func->epf_bar[bar]->size, &val); 1100 else 1101 val = BIT(4); 1102 1103 dw_pcie_ep_writel_dbi(ep, func_no, offset + PCI_REBAR_CAP, val); 1104 } 1105 } 1106} 1107 1108static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) 1109{ 1110 struct dw_pcie_ep *ep = &pci->ep; 1111 u8 funcs = ep->epc->max_functions; 1112 u8 func_no; 1113 1114 dw_pcie_dbi_ro_wr_en(pci); 1115 1116 for (func_no = 0; func_no < funcs; func_no++) 1117 dw_pcie_ep_init_rebar_registers(ep, func_no); 1118 1119 dw_pcie_setup(pci); 1120 dw_pcie_dbi_ro_wr_dis(pci); 1121} 1122 1123/** 1124 * dw_pcie_ep_init_registers - Initialize DWC EP specific registers 1125 * @ep: DWC EP device 1126 * 1127 * Initialize the registers (CSRs) specific to DWC EP. This API should be called 1128 * only when the endpoint receives an active refclk (either from host or 1129 * generated locally). 1130 */ 1131int dw_pcie_ep_init_registers(struct dw_pcie_ep *ep) 1132{ 1133 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1134 struct dw_pcie_ep_func *ep_func; 1135 struct device *dev = pci->dev; 1136 struct pci_epc *epc = ep->epc; 1137 u32 ptm_cap_base, reg; 1138 u8 hdr_type; 1139 u8 func_no; 1140 void *addr; 1141 int ret; 1142 1143 hdr_type = dw_pcie_readb_dbi(pci, PCI_HEADER_TYPE) & 1144 PCI_HEADER_TYPE_MASK; 1145 if (hdr_type != PCI_HEADER_TYPE_NORMAL) { 1146 dev_err(pci->dev, 1147 "PCIe controller is not set to EP mode (hdr_type:0x%x)!\n", 1148 hdr_type); 1149 return -EIO; 1150 } 1151 1152 dw_pcie_version_detect(pci); 1153 1154 dw_pcie_iatu_detect(pci); 1155 1156 ret = dw_pcie_edma_detect(pci); 1157 if (ret) 1158 return ret; 1159 1160 ret = -ENOMEM; 1161 if (!ep->ib_window_map) { 1162 ep->ib_window_map = devm_bitmap_zalloc(dev, pci->num_ib_windows, 1163 GFP_KERNEL); 1164 if (!ep->ib_window_map) 1165 goto err_remove_edma; 1166 } 1167 1168 if (!ep->ob_window_map) { 1169 ep->ob_window_map = devm_bitmap_zalloc(dev, pci->num_ob_windows, 1170 GFP_KERNEL); 1171 if (!ep->ob_window_map) 1172 goto err_remove_edma; 1173 } 1174 1175 if (!ep->outbound_addr) { 1176 addr = devm_kcalloc(dev, pci->num_ob_windows, sizeof(phys_addr_t), 1177 GFP_KERNEL); 1178 if (!addr) 1179 goto err_remove_edma; 1180 ep->outbound_addr = addr; 1181 } 1182 1183 for (func_no = 0; func_no < epc->max_functions; func_no++) { 1184 1185 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); 1186 if (ep_func) 1187 continue; 1188 1189 ep_func = devm_kzalloc(dev, sizeof(*ep_func), GFP_KERNEL); 1190 if (!ep_func) 1191 goto err_remove_edma; 1192 1193 ep_func->func_no = func_no; 1194 ep_func->msi_cap = dw_pcie_ep_find_capability(ep, func_no, 1195 PCI_CAP_ID_MSI); 1196 ep_func->msix_cap = dw_pcie_ep_find_capability(ep, func_no, 1197 PCI_CAP_ID_MSIX); 1198 1199 list_add_tail(&ep_func->list, &ep->func_list); 1200 } 1201 1202 if (ep->ops->init) 1203 ep->ops->init(ep); 1204 1205 ptm_cap_base = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_PTM); 1206 1207 /* 1208 * PTM responder capability can be disabled only after disabling 1209 * PTM root capability. 1210 */ 1211 if (ptm_cap_base) { 1212 dw_pcie_dbi_ro_wr_en(pci); 1213 reg = dw_pcie_readl_dbi(pci, ptm_cap_base + PCI_PTM_CAP); 1214 reg &= ~PCI_PTM_CAP_ROOT; 1215 dw_pcie_writel_dbi(pci, ptm_cap_base + PCI_PTM_CAP, reg); 1216 1217 reg = dw_pcie_readl_dbi(pci, ptm_cap_base + PCI_PTM_CAP); 1218 reg &= ~(PCI_PTM_CAP_RES | PCI_PTM_GRANULARITY_MASK); 1219 dw_pcie_writel_dbi(pci, ptm_cap_base + PCI_PTM_CAP, reg); 1220 dw_pcie_dbi_ro_wr_dis(pci); 1221 } 1222 1223 dw_pcie_ep_init_non_sticky_registers(pci); 1224 1225 dwc_pcie_debugfs_init(pci, DW_PCIE_EP_TYPE); 1226 1227 return 0; 1228 1229err_remove_edma: 1230 dw_pcie_edma_remove(pci); 1231 1232 return ret; 1233} 1234EXPORT_SYMBOL_GPL(dw_pcie_ep_init_registers); 1235 1236/** 1237 * dw_pcie_ep_linkup - Notify EPF drivers about Link Up event 1238 * @ep: DWC EP device 1239 */ 1240void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) 1241{ 1242 struct pci_epc *epc = ep->epc; 1243 1244 pci_epc_linkup(epc); 1245} 1246EXPORT_SYMBOL_GPL(dw_pcie_ep_linkup); 1247 1248/** 1249 * dw_pcie_ep_linkdown - Notify EPF drivers about Link Down event 1250 * @ep: DWC EP device 1251 * 1252 * Non-sticky registers are also initialized before sending the notification to 1253 * the EPF drivers. This is needed since the registers need to be initialized 1254 * before the link comes back again. 1255 */ 1256void dw_pcie_ep_linkdown(struct dw_pcie_ep *ep) 1257{ 1258 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1259 struct pci_epc *epc = ep->epc; 1260 1261 /* 1262 * Initialize the non-sticky DWC registers as they would've reset post 1263 * Link Down. This is specifically needed for drivers not supporting 1264 * PERST# as they have no way to reinitialize the registers before the 1265 * link comes back again. 1266 */ 1267 dw_pcie_ep_init_non_sticky_registers(pci); 1268 1269 pci_epc_linkdown(epc); 1270} 1271EXPORT_SYMBOL_GPL(dw_pcie_ep_linkdown); 1272 1273static int dw_pcie_ep_get_resources(struct dw_pcie_ep *ep) 1274{ 1275 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1276 struct device *dev = pci->dev; 1277 struct platform_device *pdev = to_platform_device(dev); 1278 struct device_node *np = dev->of_node; 1279 struct pci_epc *epc = ep->epc; 1280 struct resource *res; 1281 int ret; 1282 1283 ret = dw_pcie_get_resources(pci); 1284 if (ret) 1285 return ret; 1286 1287 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space"); 1288 if (!res) 1289 return -EINVAL; 1290 1291 ep->phys_base = res->start; 1292 ep->addr_size = resource_size(res); 1293 1294 /* 1295 * artpec6_pcie_cpu_addr_fixup() uses ep->phys_base, so call 1296 * dw_pcie_parent_bus_offset() after setting ep->phys_base. 1297 */ 1298 pci->parent_bus_offset = dw_pcie_parent_bus_offset(pci, "addr_space", 1299 ep->phys_base); 1300 1301 ret = of_property_read_u8(np, "max-functions", &epc->max_functions); 1302 if (ret < 0) 1303 epc->max_functions = 1; 1304 1305 return 0; 1306} 1307 1308/** 1309 * dw_pcie_ep_init - Initialize the endpoint device 1310 * @ep: DWC EP device 1311 * 1312 * Initialize the endpoint device. Allocate resources and create the EPC 1313 * device with the endpoint framework. 1314 * 1315 * Return: 0 if success, errno otherwise. 1316 */ 1317int dw_pcie_ep_init(struct dw_pcie_ep *ep) 1318{ 1319 int ret; 1320 struct pci_epc *epc; 1321 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1322 struct device *dev = pci->dev; 1323 1324 INIT_LIST_HEAD(&ep->func_list); 1325 ep->msi_iatu_mapped = false; 1326 ep->msi_msg_addr = 0; 1327 ep->msi_map_size = 0; 1328 1329 epc = devm_pci_epc_create(dev, &epc_ops); 1330 if (IS_ERR(epc)) { 1331 dev_err(dev, "Failed to create epc device\n"); 1332 return PTR_ERR(epc); 1333 } 1334 1335 ep->epc = epc; 1336 epc_set_drvdata(epc, ep); 1337 1338 ret = dw_pcie_ep_get_resources(ep); 1339 if (ret) 1340 return ret; 1341 1342 if (ep->ops->pre_init) 1343 ep->ops->pre_init(ep); 1344 1345 ret = pci_epc_mem_init(epc, ep->phys_base, ep->addr_size, 1346 ep->page_size); 1347 if (ret < 0) { 1348 dev_err(dev, "Failed to initialize address space\n"); 1349 return ret; 1350 } 1351 1352 ep->msi_mem = pci_epc_mem_alloc_addr(epc, &ep->msi_mem_phys, 1353 epc->mem->window.page_size); 1354 if (!ep->msi_mem) { 1355 ret = -ENOMEM; 1356 dev_err(dev, "Failed to reserve memory for MSI/MSI-X\n"); 1357 goto err_exit_epc_mem; 1358 } 1359 1360 return 0; 1361 1362err_exit_epc_mem: 1363 pci_epc_mem_exit(epc); 1364 1365 return ret; 1366} 1367EXPORT_SYMBOL_GPL(dw_pcie_ep_init);
~/
Read-only file preview.