port.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
   3#include <linux/platform_device.h>
   4#include <linux/memregion.h>
   5#include <linux/workqueue.h>
   6#include <linux/debugfs.h>
   7#include <linux/device.h>
   8#include <linux/module.h>
   9#include <linux/pci.h>
  10#include <linux/slab.h>
  11#include <linux/idr.h>
  12#include <linux/node.h>
  13#include <cxl/einj.h>
  14#include <cxlmem.h>
  15#include <cxlpci.h>
  16#include <cxl.h>
  17#include "core.h"
  18
  19/**
  20 * DOC: cxl core
  21 *
  22 * The CXL core provides a set of interfaces that can be consumed by CXL aware
  23 * drivers. The interfaces allow for creation, modification, and destruction of
  24 * regions, memory devices, ports, and decoders. CXL aware drivers must register
  25 * with the CXL core via these interfaces in order to be able to participate in
  26 * cross-device interleave coordination. The CXL core also establishes and
  27 * maintains the bridge to the nvdimm subsystem.
  28 *
  29 * CXL core introduces sysfs hierarchy to control the devices that are
  30 * instantiated by the core.
  31 */
  32
  33static DEFINE_IDA(cxl_port_ida);
  34static DEFINE_XARRAY(cxl_root_buses);
  35
  36/*
  37 * The terminal device in PCI is NULL and @platform_bus
  38 * for platform devices (for cxl_test)
  39 */
  40static bool is_cxl_host_bridge(struct device *dev)
  41{
  42	return (!dev || dev == &platform_bus);
  43}
  44
  45int cxl_num_decoders_committed(struct cxl_port *port)
  46{
  47	lockdep_assert_held(&cxl_rwsem.region);
  48
  49	return port->commit_end + 1;
  50}
  51
  52static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
  53			    char *buf)
  54{
  55	return sysfs_emit(buf, "%s\n", dev->type->name);
  56}
  57static DEVICE_ATTR_RO(devtype);
  58
  59static int cxl_device_id(const struct device *dev)
  60{
  61	if (dev->type == &cxl_nvdimm_bridge_type)
  62		return CXL_DEVICE_NVDIMM_BRIDGE;
  63	if (dev->type == &cxl_nvdimm_type)
  64		return CXL_DEVICE_NVDIMM;
  65	if (dev->type == CXL_PMEM_REGION_TYPE())
  66		return CXL_DEVICE_PMEM_REGION;
  67	if (dev->type == CXL_DAX_REGION_TYPE())
  68		return CXL_DEVICE_DAX_REGION;
  69	if (is_cxl_port(dev)) {
  70		if (is_cxl_root(to_cxl_port(dev)))
  71			return CXL_DEVICE_ROOT;
  72		return CXL_DEVICE_PORT;
  73	}
  74	if (is_cxl_memdev(dev))
  75		return CXL_DEVICE_MEMORY_EXPANDER;
  76	if (dev->type == CXL_REGION_TYPE())
  77		return CXL_DEVICE_REGION;
  78	if (dev->type == &cxl_pmu_type)
  79		return CXL_DEVICE_PMU;
  80	return 0;
  81}
  82
  83static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
  84			     char *buf)
  85{
  86	return sysfs_emit(buf, CXL_MODALIAS_FMT "\n", cxl_device_id(dev));
  87}
  88static DEVICE_ATTR_RO(modalias);
  89
  90static struct attribute *cxl_base_attributes[] = {
  91	&dev_attr_devtype.attr,
  92	&dev_attr_modalias.attr,
  93	NULL,
  94};
  95
  96struct attribute_group cxl_base_attribute_group = {
  97	.attrs = cxl_base_attributes,
  98};
  99
 100static ssize_t start_show(struct device *dev, struct device_attribute *attr,
 101			  char *buf)
 102{
 103	struct cxl_decoder *cxld = to_cxl_decoder(dev);
 104
 105	return sysfs_emit(buf, "%#llx\n", cxld->hpa_range.start);
 106}
 107static DEVICE_ATTR_ADMIN_RO(start);
 108
 109static ssize_t size_show(struct device *dev, struct device_attribute *attr,
 110			char *buf)
 111{
 112	struct cxl_decoder *cxld = to_cxl_decoder(dev);
 113
 114	return sysfs_emit(buf, "%#llx\n", range_len(&cxld->hpa_range));
 115}
 116static DEVICE_ATTR_RO(size);
 117
 118#define CXL_DECODER_FLAG_ATTR(name, flag)                            \
 119static ssize_t name##_show(struct device *dev,                       \
 120			   struct device_attribute *attr, char *buf) \
 121{                                                                    \
 122	struct cxl_decoder *cxld = to_cxl_decoder(dev);              \
 123                                                                     \
 124	return sysfs_emit(buf, "%s\n",                               \
 125			  (cxld->flags & (flag)) ? "1" : "0");       \
 126}                                                                    \
 127static DEVICE_ATTR_RO(name)
 128
 129CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM);
 130CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM);
 131CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2);
 132CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3);
 133CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK);
 134
 135static ssize_t target_type_show(struct device *dev,
 136				struct device_attribute *attr, char *buf)
 137{
 138	struct cxl_decoder *cxld = to_cxl_decoder(dev);
 139
 140	switch (cxld->target_type) {
 141	case CXL_DECODER_DEVMEM:
 142		return sysfs_emit(buf, "accelerator\n");
 143	case CXL_DECODER_HOSTONLYMEM:
 144		return sysfs_emit(buf, "expander\n");
 145	}
 146	return -ENXIO;
 147}
 148static DEVICE_ATTR_RO(target_type);
 149
 150static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf)
 151{
 152	struct cxl_decoder *cxld = &cxlsd->cxld;
 153	ssize_t offset = 0;
 154	int i, rc = 0;
 155
 156	for (i = 0; i < cxld->interleave_ways; i++) {
 157		struct cxl_dport *dport = cxlsd->target[i];
 158		struct cxl_dport *next = NULL;
 159
 160		if (!dport)
 161			break;
 162
 163		if (i + 1 < cxld->interleave_ways)
 164			next = cxlsd->target[i + 1];
 165		rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
 166				   next ? "," : "");
 167		if (rc < 0)
 168			return rc;
 169		offset += rc;
 170	}
 171
 172	return offset;
 173}
 174
 175static ssize_t target_list_show(struct device *dev,
 176				struct device_attribute *attr, char *buf)
 177{
 178	struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
 179	ssize_t offset;
 180	int rc;
 181
 182	guard(rwsem_read)(&cxl_rwsem.region);
 183	rc = emit_target_list(cxlsd, buf);
 184	if (rc < 0)
 185		return rc;
 186	offset = rc;
 187
 188	rc = sysfs_emit_at(buf, offset, "\n");
 189	if (rc < 0)
 190		return rc;
 191
 192	return offset + rc;
 193}
 194static DEVICE_ATTR_RO(target_list);
 195
 196static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
 197			 char *buf)
 198{
 199	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 200	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 201	struct cxl_dev_state *cxlds = cxlmd->cxlds;
 202	/* without @cxl_rwsem.dpa, make sure @part is not reloaded */
 203	int part = READ_ONCE(cxled->part);
 204	const char *desc;
 205
 206	if (part < 0)
 207		desc = "none";
 208	else
 209		desc = cxlds->part[part].res.name;
 210
 211	return sysfs_emit(buf, "%s\n", desc);
 212}
 213
 214static ssize_t mode_store(struct device *dev, struct device_attribute *attr,
 215			  const char *buf, size_t len)
 216{
 217	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 218	enum cxl_partition_mode mode;
 219	ssize_t rc;
 220
 221	if (sysfs_streq(buf, "pmem"))
 222		mode = CXL_PARTMODE_PMEM;
 223	else if (sysfs_streq(buf, "ram"))
 224		mode = CXL_PARTMODE_RAM;
 225	else
 226		return -EINVAL;
 227
 228	rc = cxl_dpa_set_part(cxled, mode);
 229	if (rc)
 230		return rc;
 231
 232	return len;
 233}
 234static DEVICE_ATTR_RW(mode);
 235
 236static ssize_t dpa_resource_show(struct device *dev, struct device_attribute *attr,
 237			    char *buf)
 238{
 239	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 240
 241	guard(rwsem_read)(&cxl_rwsem.dpa);
 242	return sysfs_emit(buf, "%#llx\n", (u64)cxl_dpa_resource_start(cxled));
 243}
 244static DEVICE_ATTR_RO(dpa_resource);
 245
 246static ssize_t dpa_size_show(struct device *dev, struct device_attribute *attr,
 247			     char *buf)
 248{
 249	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 250	resource_size_t size = cxl_dpa_size(cxled);
 251
 252	return sysfs_emit(buf, "%pa\n", &size);
 253}
 254
 255static ssize_t dpa_size_store(struct device *dev, struct device_attribute *attr,
 256			      const char *buf, size_t len)
 257{
 258	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 259	unsigned long long size;
 260	ssize_t rc;
 261
 262	rc = kstrtoull(buf, 0, &size);
 263	if (rc)
 264		return rc;
 265
 266	if (!IS_ALIGNED(size, SZ_256M))
 267		return -EINVAL;
 268
 269	rc = cxl_dpa_free(cxled);
 270	if (rc)
 271		return rc;
 272
 273	if (size == 0)
 274		return len;
 275
 276	rc = cxl_dpa_alloc(cxled, size);
 277	if (rc)
 278		return rc;
 279
 280	return len;
 281}
 282static DEVICE_ATTR_RW(dpa_size);
 283
 284static ssize_t interleave_granularity_show(struct device *dev,
 285					   struct device_attribute *attr,
 286					   char *buf)
 287{
 288	struct cxl_decoder *cxld = to_cxl_decoder(dev);
 289
 290	return sysfs_emit(buf, "%d\n", cxld->interleave_granularity);
 291}
 292
 293static DEVICE_ATTR_RO(interleave_granularity);
 294
 295static ssize_t interleave_ways_show(struct device *dev,
 296				    struct device_attribute *attr, char *buf)
 297{
 298	struct cxl_decoder *cxld = to_cxl_decoder(dev);
 299
 300	return sysfs_emit(buf, "%d\n", cxld->interleave_ways);
 301}
 302
 303static DEVICE_ATTR_RO(interleave_ways);
 304
 305static ssize_t qos_class_show(struct device *dev,
 306			      struct device_attribute *attr, char *buf)
 307{
 308	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
 309
 310	return sysfs_emit(buf, "%d\n", cxlrd->qos_class);
 311}
 312static DEVICE_ATTR_RO(qos_class);
 313
 314static struct attribute *cxl_decoder_base_attrs[] = {
 315	&dev_attr_start.attr,
 316	&dev_attr_size.attr,
 317	&dev_attr_locked.attr,
 318	&dev_attr_interleave_granularity.attr,
 319	&dev_attr_interleave_ways.attr,
 320	NULL,
 321};
 322
 323static struct attribute_group cxl_decoder_base_attribute_group = {
 324	.attrs = cxl_decoder_base_attrs,
 325};
 326
 327static struct attribute *cxl_decoder_root_attrs[] = {
 328	&dev_attr_cap_pmem.attr,
 329	&dev_attr_cap_ram.attr,
 330	&dev_attr_cap_type2.attr,
 331	&dev_attr_cap_type3.attr,
 332	&dev_attr_target_list.attr,
 333	&dev_attr_qos_class.attr,
 334	SET_CXL_REGION_ATTR(create_pmem_region)
 335	SET_CXL_REGION_ATTR(create_ram_region)
 336	SET_CXL_REGION_ATTR(delete_region)
 337	NULL,
 338};
 339
 340static bool can_create_pmem(struct cxl_root_decoder *cxlrd)
 341{
 342	unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_PMEM;
 343
 344	return (cxlrd->cxlsd.cxld.flags & flags) == flags;
 345}
 346
 347static bool can_create_ram(struct cxl_root_decoder *cxlrd)
 348{
 349	unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_RAM;
 350
 351	return (cxlrd->cxlsd.cxld.flags & flags) == flags;
 352}
 353
 354static umode_t cxl_root_decoder_visible(struct kobject *kobj, struct attribute *a, int n)
 355{
 356	struct device *dev = kobj_to_dev(kobj);
 357	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
 358
 359	if (a == CXL_REGION_ATTR(create_pmem_region) && !can_create_pmem(cxlrd))
 360		return 0;
 361
 362	if (a == CXL_REGION_ATTR(create_ram_region) && !can_create_ram(cxlrd))
 363		return 0;
 364
 365	if (a == CXL_REGION_ATTR(delete_region) &&
 366	    !(can_create_pmem(cxlrd) || can_create_ram(cxlrd)))
 367		return 0;
 368
 369	return a->mode;
 370}
 371
 372static struct attribute_group cxl_decoder_root_attribute_group = {
 373	.attrs = cxl_decoder_root_attrs,
 374	.is_visible = cxl_root_decoder_visible,
 375};
 376
 377static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
 378	&cxl_decoder_root_attribute_group,
 379	&cxl_decoder_base_attribute_group,
 380	&cxl_base_attribute_group,
 381	NULL,
 382};
 383
 384static struct attribute *cxl_decoder_switch_attrs[] = {
 385	&dev_attr_target_type.attr,
 386	&dev_attr_target_list.attr,
 387	SET_CXL_REGION_ATTR(region)
 388	NULL,
 389};
 390
 391static struct attribute_group cxl_decoder_switch_attribute_group = {
 392	.attrs = cxl_decoder_switch_attrs,
 393};
 394
 395static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = {
 396	&cxl_decoder_switch_attribute_group,
 397	&cxl_decoder_base_attribute_group,
 398	&cxl_base_attribute_group,
 399	NULL,
 400};
 401
 402static struct attribute *cxl_decoder_endpoint_attrs[] = {
 403	&dev_attr_target_type.attr,
 404	&dev_attr_mode.attr,
 405	&dev_attr_dpa_size.attr,
 406	&dev_attr_dpa_resource.attr,
 407	SET_CXL_REGION_ATTR(region)
 408	NULL,
 409};
 410
 411static struct attribute_group cxl_decoder_endpoint_attribute_group = {
 412	.attrs = cxl_decoder_endpoint_attrs,
 413};
 414
 415static const struct attribute_group *cxl_decoder_endpoint_attribute_groups[] = {
 416	&cxl_decoder_base_attribute_group,
 417	&cxl_decoder_endpoint_attribute_group,
 418	&cxl_base_attribute_group,
 419	NULL,
 420};
 421
 422static void __cxl_decoder_release(struct cxl_decoder *cxld)
 423{
 424	struct cxl_port *port = to_cxl_port(cxld->dev.parent);
 425
 426	ida_free(&port->decoder_ida, cxld->id);
 427	put_device(&port->dev);
 428}
 429
 430static void cxl_endpoint_decoder_release(struct device *dev)
 431{
 432	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
 433
 434	__cxl_decoder_release(&cxled->cxld);
 435	kfree(cxled);
 436}
 437
 438static void cxl_switch_decoder_release(struct device *dev)
 439{
 440	struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
 441
 442	__cxl_decoder_release(&cxlsd->cxld);
 443	kfree(cxlsd);
 444}
 445
 446struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev)
 447{
 448	if (dev_WARN_ONCE(dev, !is_root_decoder(dev),
 449			  "not a cxl_root_decoder device\n"))
 450		return NULL;
 451	return container_of(dev, struct cxl_root_decoder, cxlsd.cxld.dev);
 452}
 453EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, "CXL");
 454
 455static void cxl_root_decoder_release(struct device *dev)
 456{
 457	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
 458
 459	if (atomic_read(&cxlrd->region_id) >= 0)
 460		memregion_free(atomic_read(&cxlrd->region_id));
 461	__cxl_decoder_release(&cxlrd->cxlsd.cxld);
 462	kfree(cxlrd);
 463}
 464
 465static const struct device_type cxl_decoder_endpoint_type = {
 466	.name = "cxl_decoder_endpoint",
 467	.release = cxl_endpoint_decoder_release,
 468	.groups = cxl_decoder_endpoint_attribute_groups,
 469};
 470
 471static const struct device_type cxl_decoder_switch_type = {
 472	.name = "cxl_decoder_switch",
 473	.release = cxl_switch_decoder_release,
 474	.groups = cxl_decoder_switch_attribute_groups,
 475};
 476
 477static const struct device_type cxl_decoder_root_type = {
 478	.name = "cxl_decoder_root",
 479	.release = cxl_root_decoder_release,
 480	.groups = cxl_decoder_root_attribute_groups,
 481};
 482
 483bool is_endpoint_decoder(struct device *dev)
 484{
 485	return dev->type == &cxl_decoder_endpoint_type;
 486}
 487EXPORT_SYMBOL_NS_GPL(is_endpoint_decoder, "CXL");
 488
 489bool is_root_decoder(struct device *dev)
 490{
 491	return dev->type == &cxl_decoder_root_type;
 492}
 493EXPORT_SYMBOL_NS_GPL(is_root_decoder, "CXL");
 494
 495bool is_switch_decoder(struct device *dev)
 496{
 497	return is_root_decoder(dev) || dev->type == &cxl_decoder_switch_type;
 498}
 499EXPORT_SYMBOL_NS_GPL(is_switch_decoder, "CXL");
 500
 501struct cxl_decoder *to_cxl_decoder(struct device *dev)
 502{
 503	if (dev_WARN_ONCE(dev,
 504			  !is_switch_decoder(dev) && !is_endpoint_decoder(dev),
 505			  "not a cxl_decoder device\n"))
 506		return NULL;
 507	return container_of(dev, struct cxl_decoder, dev);
 508}
 509EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, "CXL");
 510
 511struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev)
 512{
 513	if (dev_WARN_ONCE(dev, !is_endpoint_decoder(dev),
 514			  "not a cxl_endpoint_decoder device\n"))
 515		return NULL;
 516	return container_of(dev, struct cxl_endpoint_decoder, cxld.dev);
 517}
 518EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, "CXL");
 519
 520struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev)
 521{
 522	if (dev_WARN_ONCE(dev, !is_switch_decoder(dev),
 523			  "not a cxl_switch_decoder device\n"))
 524		return NULL;
 525	return container_of(dev, struct cxl_switch_decoder, cxld.dev);
 526}
 527EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, "CXL");
 528
 529static void cxl_ep_release(struct cxl_ep *ep)
 530{
 531	put_device(ep->ep);
 532	kfree(ep);
 533}
 534
 535static void cxl_ep_remove(struct cxl_port *port, struct cxl_ep *ep)
 536{
 537	if (!ep)
 538		return;
 539	xa_erase(&port->endpoints, (unsigned long) ep->ep);
 540	cxl_ep_release(ep);
 541}
 542
 543static void cxl_port_release(struct device *dev)
 544{
 545	struct cxl_port *port = to_cxl_port(dev);
 546	unsigned long index;
 547	struct cxl_ep *ep;
 548
 549	xa_for_each(&port->endpoints, index, ep)
 550		cxl_ep_remove(port, ep);
 551	xa_destroy(&port->endpoints);
 552	xa_destroy(&port->dports);
 553	xa_destroy(&port->regions);
 554	ida_free(&cxl_port_ida, port->id);
 555
 556	if (is_cxl_root(port)) {
 557		kfree(to_cxl_root(port));
 558	} else {
 559		put_device(dev->parent);
 560		kfree(port);
 561	}
 562}
 563
 564static ssize_t decoders_committed_show(struct device *dev,
 565				       struct device_attribute *attr, char *buf)
 566{
 567	struct cxl_port *port = to_cxl_port(dev);
 568
 569	guard(rwsem_read)(&cxl_rwsem.region);
 570	return sysfs_emit(buf, "%d\n", cxl_num_decoders_committed(port));
 571}
 572
 573static DEVICE_ATTR_RO(decoders_committed);
 574
 575static struct attribute *cxl_port_attrs[] = {
 576	&dev_attr_decoders_committed.attr,
 577	NULL,
 578};
 579
 580static struct attribute_group cxl_port_attribute_group = {
 581	.attrs = cxl_port_attrs,
 582};
 583
 584static const struct attribute_group *cxl_port_attribute_groups[] = {
 585	&cxl_base_attribute_group,
 586	&cxl_port_attribute_group,
 587	NULL,
 588};
 589
 590static const struct device_type cxl_port_type = {
 591	.name = "cxl_port",
 592	.release = cxl_port_release,
 593	.groups = cxl_port_attribute_groups,
 594};
 595
 596bool is_cxl_port(const struct device *dev)
 597{
 598	return dev->type == &cxl_port_type;
 599}
 600EXPORT_SYMBOL_NS_GPL(is_cxl_port, "CXL");
 601
 602struct cxl_port *to_cxl_port(const struct device *dev)
 603{
 604	if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
 605			  "not a cxl_port device\n"))
 606		return NULL;
 607	return container_of(dev, struct cxl_port, dev);
 608}
 609EXPORT_SYMBOL_NS_GPL(to_cxl_port, "CXL");
 610
 611struct cxl_port *parent_port_of(struct cxl_port *port)
 612{
 613	if (!port || !port->parent_dport)
 614		return NULL;
 615	return port->parent_dport->port;
 616}
 617
 618static void unregister_port(void *_port)
 619{
 620	struct cxl_port *port = _port;
 621	struct cxl_port *parent = parent_port_of(port);
 622	struct device *lock_dev;
 623
 624	/*
 625	 * CXL root port's and the first level of ports are unregistered
 626	 * under the platform firmware device lock, all other ports are
 627	 * unregistered while holding their parent port lock.
 628	 */
 629	if (!parent)
 630		lock_dev = port->uport_dev;
 631	else if (is_cxl_root(parent))
 632		lock_dev = parent->uport_dev;
 633	else
 634		lock_dev = &parent->dev;
 635
 636	device_lock_assert(lock_dev);
 637	port->dead = true;
 638	device_unregister(&port->dev);
 639}
 640
 641static void cxl_unlink_uport(void *_port)
 642{
 643	struct cxl_port *port = _port;
 644
 645	sysfs_remove_link(&port->dev.kobj, "uport");
 646}
 647
 648static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
 649{
 650	int rc;
 651
 652	rc = sysfs_create_link(&port->dev.kobj, &port->uport_dev->kobj,
 653			       "uport");
 654	if (rc)
 655		return rc;
 656	return devm_add_action_or_reset(host, cxl_unlink_uport, port);
 657}
 658
 659static void cxl_unlink_parent_dport(void *_port)
 660{
 661	struct cxl_port *port = _port;
 662
 663	sysfs_remove_link(&port->dev.kobj, "parent_dport");
 664}
 665
 666static int devm_cxl_link_parent_dport(struct device *host,
 667				      struct cxl_port *port,
 668				      struct cxl_dport *parent_dport)
 669{
 670	int rc;
 671
 672	if (!parent_dport)
 673		return 0;
 674
 675	rc = sysfs_create_link(&port->dev.kobj, &parent_dport->dport_dev->kobj,
 676			       "parent_dport");
 677	if (rc)
 678		return rc;
 679	return devm_add_action_or_reset(host, cxl_unlink_parent_dport, port);
 680}
 681
 682static struct lock_class_key cxl_port_key;
 683
 684static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
 685				       struct cxl_dport *parent_dport)
 686{
 687	struct cxl_root *cxl_root __free(kfree) = NULL;
 688	struct cxl_port *port, *_port __free(kfree) = NULL;
 689	struct device *dev;
 690	int rc;
 691
 692	/* No parent_dport, root cxl_port */
 693	if (!parent_dport) {
 694		cxl_root = kzalloc(sizeof(*cxl_root), GFP_KERNEL);
 695		if (!cxl_root)
 696			return ERR_PTR(-ENOMEM);
 697	} else {
 698		_port = kzalloc(sizeof(*port), GFP_KERNEL);
 699		if (!_port)
 700			return ERR_PTR(-ENOMEM);
 701	}
 702
 703	rc = ida_alloc(&cxl_port_ida, GFP_KERNEL);
 704	if (rc < 0)
 705		return ERR_PTR(rc);
 706
 707	if (cxl_root)
 708		port = &no_free_ptr(cxl_root)->port;
 709	else
 710		port = no_free_ptr(_port);
 711
 712	port->id = rc;
 713	port->uport_dev = uport_dev;
 714
 715	/*
 716	 * The top-level cxl_port "cxl_root" does not have a cxl_port as
 717	 * its parent and it does not have any corresponding component
 718	 * registers as its decode is described by a fixed platform
 719	 * description.
 720	 */
 721	dev = &port->dev;
 722	if (parent_dport) {
 723		struct cxl_port *parent_port = parent_dport->port;
 724		struct cxl_port *iter;
 725
 726		dev->parent = &parent_port->dev;
 727		get_device(dev->parent);
 728		port->depth = parent_port->depth + 1;
 729		port->parent_dport = parent_dport;
 730
 731		/*
 732		 * walk to the host bridge, or the first ancestor that knows
 733		 * the host bridge
 734		 */
 735		iter = port;
 736		while (!iter->host_bridge &&
 737		       !is_cxl_root(to_cxl_port(iter->dev.parent)))
 738			iter = to_cxl_port(iter->dev.parent);
 739		if (iter->host_bridge)
 740			port->host_bridge = iter->host_bridge;
 741		else if (parent_dport->rch)
 742			port->host_bridge = parent_dport->dport_dev;
 743		else
 744			port->host_bridge = iter->uport_dev;
 745		dev_dbg(uport_dev, "host-bridge: %s\n",
 746			dev_name(port->host_bridge));
 747	} else
 748		dev->parent = uport_dev;
 749
 750	ida_init(&port->decoder_ida);
 751	port->hdm_end = -1;
 752	port->commit_end = -1;
 753	xa_init(&port->dports);
 754	xa_init(&port->endpoints);
 755	xa_init(&port->regions);
 756	port->component_reg_phys = CXL_RESOURCE_NONE;
 757
 758	device_initialize(dev);
 759	lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);
 760	device_set_pm_not_required(dev);
 761	dev->bus = &cxl_bus_type;
 762	dev->type = &cxl_port_type;
 763
 764	return port;
 765}
 766
 767static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map,
 768			       resource_size_t component_reg_phys)
 769{
 770	*map = (struct cxl_register_map) {
 771		.host = host,
 772		.reg_type = CXL_REGLOC_RBI_EMPTY,
 773		.resource = component_reg_phys,
 774	};
 775
 776	if (component_reg_phys == CXL_RESOURCE_NONE)
 777		return 0;
 778
 779	map->reg_type = CXL_REGLOC_RBI_COMPONENT;
 780	map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
 781
 782	return cxl_setup_regs(map);
 783}
 784
 785static int cxl_port_setup_regs(struct cxl_port *port,
 786			resource_size_t component_reg_phys)
 787{
 788	if (dev_is_platform(port->uport_dev))
 789		return 0;
 790	return cxl_setup_comp_regs(&port->dev, &port->reg_map,
 791				   component_reg_phys);
 792}
 793
 794static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
 795				resource_size_t component_reg_phys)
 796{
 797	int rc;
 798
 799	if (dev_is_platform(dport->dport_dev))
 800		return 0;
 801
 802	/*
 803	 * use @dport->dport_dev for the context for error messages during
 804	 * register probing, and fixup @host after the fact, since @host may be
 805	 * NULL.
 806	 */
 807	rc = cxl_setup_comp_regs(dport->dport_dev, &dport->reg_map,
 808				 component_reg_phys);
 809	dport->reg_map.host = host;
 810	return rc;
 811}
 812
 813DEFINE_SHOW_ATTRIBUTE(einj_cxl_available_error_type);
 814
 815static int cxl_einj_inject(void *data, u64 type)
 816{
 817	struct cxl_dport *dport = data;
 818
 819	if (dport->rch)
 820		return einj_cxl_inject_rch_error(dport->rcrb.base, type);
 821
 822	return einj_cxl_inject_error(to_pci_dev(dport->dport_dev), type);
 823}
 824DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject,
 825			 "0x%llx\n");
 826
 827static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport)
 828{
 829	struct cxl_port *parent = parent_port_of(dport->port);
 830	struct dentry *dir;
 831
 832	if (!einj_cxl_is_initialized())
 833		return;
 834
 835	/*
 836	 * Protocol error injection is only available for CXL 2.0+ root ports
 837	 * and CXL 1.1 downstream ports
 838	 */
 839	if (!dport->rch &&
 840	    !(dev_is_pci(dport->dport_dev) && parent && is_cxl_root(parent)))
 841		return;
 842
 843	dir = cxl_debugfs_create_dir(dev_name(dport->dport_dev));
 844
 845	debugfs_create_file("einj_inject", 0200, dir, dport,
 846			    &cxl_einj_inject_fops);
 847}
 848
 849static int cxl_port_add(struct cxl_port *port,
 850			resource_size_t component_reg_phys,
 851			struct cxl_dport *parent_dport)
 852{
 853	struct device *dev __free(put_device) = &port->dev;
 854	int rc;
 855
 856	if (is_cxl_memdev(port->uport_dev)) {
 857		struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
 858		struct cxl_dev_state *cxlds = cxlmd->cxlds;
 859
 860		rc = dev_set_name(dev, "endpoint%d", port->id);
 861		if (rc)
 862			return rc;
 863
 864		/*
 865		 * The endpoint driver already enumerated the component and RAS
 866		 * registers. Reuse that enumeration while prepping them to be
 867		 * mapped by the cxl_port driver.
 868		 */
 869		port->reg_map = cxlds->reg_map;
 870		port->reg_map.host = &port->dev;
 871		cxlmd->endpoint = port;
 872	} else if (parent_dport) {
 873		rc = dev_set_name(dev, "port%d", port->id);
 874		if (rc)
 875			return rc;
 876
 877		port->component_reg_phys = component_reg_phys;
 878	} else {
 879		rc = dev_set_name(dev, "root%d", port->id);
 880		if (rc)
 881			return rc;
 882	}
 883
 884	rc = device_add(dev);
 885	if (rc)
 886		return rc;
 887
 888	/* Inhibit the cleanup function invoked */
 889	dev = NULL;
 890	return 0;
 891}
 892
 893static struct cxl_port *__devm_cxl_add_port(struct device *host,
 894					    struct device *uport_dev,
 895					    resource_size_t component_reg_phys,
 896					    struct cxl_dport *parent_dport)
 897{
 898	struct cxl_port *port;
 899	int rc;
 900
 901	port = cxl_port_alloc(uport_dev, parent_dport);
 902	if (IS_ERR(port))
 903		return port;
 904
 905	rc = cxl_port_add(port, component_reg_phys, parent_dport);
 906	if (rc)
 907		return ERR_PTR(rc);
 908
 909	rc = devm_add_action_or_reset(host, unregister_port, port);
 910	if (rc)
 911		return ERR_PTR(rc);
 912
 913	rc = devm_cxl_link_uport(host, port);
 914	if (rc)
 915		return ERR_PTR(rc);
 916
 917	rc = devm_cxl_link_parent_dport(host, port, parent_dport);
 918	if (rc)
 919		return ERR_PTR(rc);
 920
 921	if (parent_dport && dev_is_pci(uport_dev))
 922		port->pci_latency = cxl_pci_get_latency(to_pci_dev(uport_dev));
 923
 924	return port;
 925}
 926
 927/**
 928 * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
 929 * @host: host device for devm operations
 930 * @uport_dev: "physical" device implementing this upstream port
 931 * @component_reg_phys: (optional) for configurable cxl_port instances
 932 * @parent_dport: next hop up in the CXL memory decode hierarchy
 933 */
 934struct cxl_port *devm_cxl_add_port(struct device *host,
 935				   struct device *uport_dev,
 936				   resource_size_t component_reg_phys,
 937				   struct cxl_dport *parent_dport)
 938{
 939	struct cxl_port *port, *parent_port;
 940
 941	port = __devm_cxl_add_port(host, uport_dev, component_reg_phys,
 942				   parent_dport);
 943
 944	parent_port = parent_dport ? parent_dport->port : NULL;
 945	if (IS_ERR(port)) {
 946		dev_dbg(uport_dev, "Failed to add%s%s%s: %ld\n",
 947			parent_port ? " port to " : "",
 948			parent_port ? dev_name(&parent_port->dev) : "",
 949			parent_port ? "" : " root port",
 950			PTR_ERR(port));
 951	} else {
 952		dev_dbg(uport_dev, "%s added%s%s%s\n",
 953			dev_name(&port->dev),
 954			parent_port ? " to " : "",
 955			parent_port ? dev_name(&parent_port->dev) : "",
 956			parent_port ? "" : " (root port)");
 957	}
 958
 959	return port;
 960}
 961EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, "CXL");
 962
 963struct cxl_root *devm_cxl_add_root(struct device *host,
 964				   const struct cxl_root_ops *ops)
 965{
 966	struct cxl_root *cxl_root;
 967	struct cxl_port *port;
 968
 969	port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL);
 970	if (IS_ERR(port))
 971		return ERR_CAST(port);
 972
 973	cxl_root = to_cxl_root(port);
 974	cxl_root->ops = ops;
 975	return cxl_root;
 976}
 977EXPORT_SYMBOL_NS_GPL(devm_cxl_add_root, "CXL");
 978
 979struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port)
 980{
 981	/* There is no pci_bus associated with a CXL platform-root port */
 982	if (is_cxl_root(port))
 983		return NULL;
 984
 985	if (dev_is_pci(port->uport_dev)) {
 986		struct pci_dev *pdev = to_pci_dev(port->uport_dev);
 987
 988		return pdev->subordinate;
 989	}
 990
 991	return xa_load(&cxl_root_buses, (unsigned long)port->uport_dev);
 992}
 993EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, "CXL");
 994
 995static void unregister_pci_bus(void *uport_dev)
 996{
 997	xa_erase(&cxl_root_buses, (unsigned long)uport_dev);
 998}
 999
1000int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev,
1001			      struct pci_bus *bus)
1002{
1003	int rc;
1004
1005	if (dev_is_pci(uport_dev))
1006		return -EINVAL;
1007
1008	rc = xa_insert(&cxl_root_buses, (unsigned long)uport_dev, bus,
1009		       GFP_KERNEL);
1010	if (rc)
1011		return rc;
1012	return devm_add_action_or_reset(host, unregister_pci_bus, uport_dev);
1013}
1014EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, "CXL");
1015
1016static bool dev_is_cxl_root_child(struct device *dev)
1017{
1018	struct cxl_port *port, *parent;
1019
1020	if (!is_cxl_port(dev))
1021		return false;
1022
1023	port = to_cxl_port(dev);
1024	if (is_cxl_root(port))
1025		return false;
1026
1027	parent = to_cxl_port(port->dev.parent);
1028	if (is_cxl_root(parent))
1029		return true;
1030
1031	return false;
1032}
1033
1034struct cxl_root *find_cxl_root(struct cxl_port *port)
1035{
1036	struct cxl_port *iter = port;
1037
1038	while (iter && !is_cxl_root(iter))
1039		iter = to_cxl_port(iter->dev.parent);
1040
1041	if (!iter)
1042		return NULL;
1043	get_device(&iter->dev);
1044	return to_cxl_root(iter);
1045}
1046EXPORT_SYMBOL_NS_GPL(find_cxl_root, "CXL");
1047
1048static struct cxl_dport *find_dport(struct cxl_port *port, int id)
1049{
1050	struct cxl_dport *dport;
1051	unsigned long index;
1052
1053	device_lock_assert(&port->dev);
1054	xa_for_each(&port->dports, index, dport)
1055		if (dport->port_id == id)
1056			return dport;
1057	return NULL;
1058}
1059
1060static int add_dport(struct cxl_port *port, struct cxl_dport *dport)
1061{
1062	struct cxl_dport *dup;
1063	int rc;
1064
1065	device_lock_assert(&port->dev);
1066	dup = find_dport(port, dport->port_id);
1067	if (dup) {
1068		dev_err(&port->dev,
1069			"unable to add dport%d-%s non-unique port id (%s)\n",
1070			dport->port_id, dev_name(dport->dport_dev),
1071			dev_name(dup->dport_dev));
1072		return -EBUSY;
1073	}
1074
1075	rc = xa_insert(&port->dports, (unsigned long)dport->dport_dev, dport,
1076		       GFP_KERNEL);
1077	if (rc)
1078		return rc;
1079
1080	port->nr_dports++;
1081	return 0;
1082}
1083
1084/*
1085 * Since root-level CXL dports cannot be enumerated by PCI they are not
1086 * enumerated by the common port driver that acquires the port lock over
1087 * dport add/remove. Instead, root dports are manually added by a
1088 * platform driver and cond_cxl_root_lock() is used to take the missing
1089 * port lock in that case.
1090 */
1091static void cond_cxl_root_lock(struct cxl_port *port)
1092{
1093	if (is_cxl_root(port))
1094		device_lock(&port->dev);
1095}
1096
1097static void cond_cxl_root_unlock(struct cxl_port *port)
1098{
1099	if (is_cxl_root(port))
1100		device_unlock(&port->dev);
1101}
1102
1103static void cxl_dport_remove(void *data)
1104{
1105	struct cxl_dport *dport = data;
1106	struct cxl_port *port = dport->port;
1107
1108	xa_erase(&port->dports, (unsigned long) dport->dport_dev);
1109	put_device(dport->dport_dev);
1110}
1111
1112static void cxl_dport_unlink(void *data)
1113{
1114	struct cxl_dport *dport = data;
1115	struct cxl_port *port = dport->port;
1116	char link_name[CXL_TARGET_STRLEN];
1117
1118	sprintf(link_name, "dport%d", dport->port_id);
1119	sysfs_remove_link(&port->dev.kobj, link_name);
1120}
1121
1122static struct cxl_dport *
1123__devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
1124		     int port_id, resource_size_t component_reg_phys,
1125		     resource_size_t rcrb)
1126{
1127	char link_name[CXL_TARGET_STRLEN];
1128	struct cxl_dport *dport;
1129	struct device *host;
1130	int rc;
1131
1132	if (is_cxl_root(port))
1133		host = port->uport_dev;
1134	else
1135		host = &port->dev;
1136
1137	if (!host->driver) {
1138		dev_WARN_ONCE(&port->dev, 1, "dport:%s bad devm context\n",
1139			      dev_name(dport_dev));
1140		return ERR_PTR(-ENXIO);
1141	}
1142
1143	if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >=
1144	    CXL_TARGET_STRLEN)
1145		return ERR_PTR(-EINVAL);
1146
1147	dport = devm_kzalloc(host, sizeof(*dport), GFP_KERNEL);
1148	if (!dport)
1149		return ERR_PTR(-ENOMEM);
1150
1151	dport->dport_dev = dport_dev;
1152	dport->port_id = port_id;
1153	dport->port = port;
1154
1155	if (rcrb == CXL_RESOURCE_NONE) {
1156		rc = cxl_dport_setup_regs(&port->dev, dport,
1157					  component_reg_phys);
1158		if (rc)
1159			return ERR_PTR(rc);
1160	} else {
1161		dport->rcrb.base = rcrb;
1162		component_reg_phys = __rcrb_to_component(dport_dev, &dport->rcrb,
1163							 CXL_RCRB_DOWNSTREAM);
1164		if (component_reg_phys == CXL_RESOURCE_NONE) {
1165			dev_warn(dport_dev, "Invalid Component Registers in RCRB");
1166			return ERR_PTR(-ENXIO);
1167		}
1168
1169		/*
1170		 * RCH @dport is not ready to map until associated with its
1171		 * memdev
1172		 */
1173		rc = cxl_dport_setup_regs(NULL, dport, component_reg_phys);
1174		if (rc)
1175			return ERR_PTR(rc);
1176
1177		dport->rch = true;
1178	}
1179
1180	if (component_reg_phys != CXL_RESOURCE_NONE)
1181		dev_dbg(dport_dev, "Component Registers found for dport: %pa\n",
1182			&component_reg_phys);
1183
1184	cond_cxl_root_lock(port);
1185	rc = add_dport(port, dport);
1186	cond_cxl_root_unlock(port);
1187	if (rc)
1188		return ERR_PTR(rc);
1189
1190	/*
1191	 * Setup port register if this is the first dport showed up. Having
1192	 * a dport also means that there is at least 1 active link.
1193	 */
1194	if (port->nr_dports == 1 &&
1195	    port->component_reg_phys != CXL_RESOURCE_NONE) {
1196		rc = cxl_port_setup_regs(port, port->component_reg_phys);
1197		if (rc) {
1198			xa_erase(&port->dports, (unsigned long)dport->dport_dev);
1199			return ERR_PTR(rc);
1200		}
1201		port->component_reg_phys = CXL_RESOURCE_NONE;
1202	}
1203
1204	get_device(dport_dev);
1205	rc = devm_add_action_or_reset(host, cxl_dport_remove, dport);
1206	if (rc)
1207		return ERR_PTR(rc);
1208
1209	rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name);
1210	if (rc)
1211		return ERR_PTR(rc);
1212
1213	rc = devm_add_action_or_reset(host, cxl_dport_unlink, dport);
1214	if (rc)
1215		return ERR_PTR(rc);
1216
1217	if (dev_is_pci(dport_dev))
1218		dport->link_latency = cxl_pci_get_latency(to_pci_dev(dport_dev));
1219
1220	cxl_debugfs_create_dport_dir(dport);
1221
1222	return dport;
1223}
1224
1225/**
1226 * devm_cxl_add_dport - append VH downstream port data to a cxl_port
1227 * @port: the cxl_port that references this dport
1228 * @dport_dev: firmware or PCI device representing the dport
1229 * @port_id: identifier for this dport in a decoder's target list
1230 * @component_reg_phys: optional location of CXL component registers
1231 *
1232 * Note that dports are appended to the devm release action's of the
1233 * either the port's host (for root ports), or the port itself (for
1234 * switch ports)
1235 */
1236struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
1237				     struct device *dport_dev, int port_id,
1238				     resource_size_t component_reg_phys)
1239{
1240	struct cxl_dport *dport;
1241
1242	dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1243				     component_reg_phys, CXL_RESOURCE_NONE);
1244	if (IS_ERR(dport)) {
1245		dev_dbg(dport_dev, "failed to add dport to %s: %ld\n",
1246			dev_name(&port->dev), PTR_ERR(dport));
1247	} else {
1248		dev_dbg(dport_dev, "dport added to %s\n",
1249			dev_name(&port->dev));
1250	}
1251
1252	return dport;
1253}
1254EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, "CXL");
1255
1256/**
1257 * devm_cxl_add_rch_dport - append RCH downstream port data to a cxl_port
1258 * @port: the cxl_port that references this dport
1259 * @dport_dev: firmware or PCI device representing the dport
1260 * @port_id: identifier for this dport in a decoder's target list
1261 * @rcrb: mandatory location of a Root Complex Register Block
1262 *
1263 * See CXL 3.0 9.11.8 CXL Devices Attached to an RCH
1264 */
1265struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
1266					 struct device *dport_dev, int port_id,
1267					 resource_size_t rcrb)
1268{
1269	struct cxl_dport *dport;
1270
1271	if (rcrb == CXL_RESOURCE_NONE) {
1272		dev_dbg(&port->dev, "failed to add RCH dport, missing RCRB\n");
1273		return ERR_PTR(-EINVAL);
1274	}
1275
1276	dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1277				     CXL_RESOURCE_NONE, rcrb);
1278	if (IS_ERR(dport)) {
1279		dev_dbg(dport_dev, "failed to add RCH dport to %s: %ld\n",
1280			dev_name(&port->dev), PTR_ERR(dport));
1281	} else {
1282		dev_dbg(dport_dev, "RCH dport added to %s\n",
1283			dev_name(&port->dev));
1284	}
1285
1286	return dport;
1287}
1288EXPORT_SYMBOL_NS_GPL(devm_cxl_add_rch_dport, "CXL");
1289
1290static int add_ep(struct cxl_ep *new)
1291{
1292	struct cxl_port *port = new->dport->port;
1293
1294	guard(device)(&port->dev);
1295	if (port->dead)
1296		return -ENXIO;
1297
1298	return xa_insert(&port->endpoints, (unsigned long)new->ep,
1299			 new, GFP_KERNEL);
1300}
1301
1302/**
1303 * cxl_add_ep - register an endpoint's interest in a port
1304 * @dport: the dport that routes to @ep_dev
1305 * @ep_dev: device representing the endpoint
1306 *
1307 * Intermediate CXL ports are scanned based on the arrival of endpoints.
1308 * When those endpoints depart the port can be destroyed once all
1309 * endpoints that care about that port have been removed.
1310 */
1311static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev)
1312{
1313	struct cxl_ep *ep;
1314	int rc;
1315
1316	ep = kzalloc(sizeof(*ep), GFP_KERNEL);
1317	if (!ep)
1318		return -ENOMEM;
1319
1320	ep->ep = get_device(ep_dev);
1321	ep->dport = dport;
1322
1323	rc = add_ep(ep);
1324	if (rc)
1325		cxl_ep_release(ep);
1326	return rc;
1327}
1328
1329struct cxl_find_port_ctx {
1330	const struct device *dport_dev;
1331	const struct cxl_port *parent_port;
1332	struct cxl_dport **dport;
1333};
1334
1335static int match_port_by_dport(struct device *dev, const void *data)
1336{
1337	const struct cxl_find_port_ctx *ctx = data;
1338	struct cxl_dport *dport;
1339	struct cxl_port *port;
1340
1341	if (!is_cxl_port(dev))
1342		return 0;
1343	if (ctx->parent_port && dev->parent != &ctx->parent_port->dev)
1344		return 0;
1345
1346	port = to_cxl_port(dev);
1347	dport = cxl_find_dport_by_dev(port, ctx->dport_dev);
1348	if (ctx->dport)
1349		*ctx->dport = dport;
1350	return dport != NULL;
1351}
1352
1353static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx)
1354{
1355	struct device *dev;
1356
1357	if (!ctx->dport_dev)
1358		return NULL;
1359
1360	dev = bus_find_device(&cxl_bus_type, NULL, ctx, match_port_by_dport);
1361	if (dev)
1362		return to_cxl_port(dev);
1363	return NULL;
1364}
1365
1366static struct cxl_port *find_cxl_port(struct device *dport_dev,
1367				      struct cxl_dport **dport)
1368{
1369	struct cxl_find_port_ctx ctx = {
1370		.dport_dev = dport_dev,
1371		.dport = dport,
1372	};
1373	struct cxl_port *port;
1374
1375	port = __find_cxl_port(&ctx);
1376	return port;
1377}
1378
1379/*
1380 * All users of grandparent() are using it to walk PCIe-like switch port
1381 * hierarchy. A PCIe switch is comprised of a bridge device representing the
1382 * upstream switch port and N bridges representing downstream switch ports. When
1383 * bridges stack the grand-parent of a downstream switch port is another
1384 * downstream switch port in the immediate ancestor switch.
1385 */
1386static struct device *grandparent(struct device *dev)
1387{
1388	if (dev && dev->parent)
1389		return dev->parent->parent;
1390	return NULL;
1391}
1392
1393static struct device *endpoint_host(struct cxl_port *endpoint)
1394{
1395	struct cxl_port *port = to_cxl_port(endpoint->dev.parent);
1396
1397	if (is_cxl_root(port))
1398		return port->uport_dev;
1399	return &port->dev;
1400}
1401
1402static void delete_endpoint(void *data)
1403{
1404	struct cxl_memdev *cxlmd = data;
1405	struct cxl_port *endpoint = cxlmd->endpoint;
1406	struct device *host = endpoint_host(endpoint);
1407
1408	scoped_guard(device, host) {
1409		if (host->driver && !endpoint->dead) {
1410			devm_release_action(host, cxl_unlink_parent_dport, endpoint);
1411			devm_release_action(host, cxl_unlink_uport, endpoint);
1412			devm_release_action(host, unregister_port, endpoint);
1413		}
1414		cxlmd->endpoint = NULL;
1415	}
1416	put_device(&endpoint->dev);
1417	put_device(host);
1418}
1419
1420int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
1421{
1422	struct device *host = endpoint_host(endpoint);
1423	struct device *dev = &cxlmd->dev;
1424
1425	get_device(host);
1426	get_device(&endpoint->dev);
1427	cxlmd->depth = endpoint->depth;
1428	return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
1429}
1430EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, "CXL");
1431
1432/*
1433 * The natural end of life of a non-root 'cxl_port' is when its parent port goes
1434 * through a ->remove() event ("top-down" unregistration). The unnatural trigger
1435 * for a port to be unregistered is when all memdevs beneath that port have gone
1436 * through ->remove(). This "bottom-up" removal selectively removes individual
1437 * child ports manually. This depends on devm_cxl_add_port() to not change is
1438 * devm action registration order, and for dports to have already been
1439 * destroyed by del_dports().
1440 */
1441static void delete_switch_port(struct cxl_port *port)
1442{
1443	devm_release_action(port->dev.parent, cxl_unlink_parent_dport, port);
1444	devm_release_action(port->dev.parent, cxl_unlink_uport, port);
1445	devm_release_action(port->dev.parent, unregister_port, port);
1446}
1447
1448static void del_dport(struct cxl_dport *dport)
1449{
1450	struct cxl_port *port = dport->port;
1451
1452	devm_release_action(&port->dev, cxl_dport_unlink, dport);
1453	devm_release_action(&port->dev, cxl_dport_remove, dport);
1454	devm_kfree(&port->dev, dport);
1455}
1456
1457static void del_dports(struct cxl_port *port)
1458{
1459	struct cxl_dport *dport;
1460	unsigned long index;
1461
1462	device_lock_assert(&port->dev);
1463
1464	xa_for_each(&port->dports, index, dport)
1465		del_dport(dport);
1466}
1467
1468struct detach_ctx {
1469	struct cxl_memdev *cxlmd;
1470	int depth;
1471};
1472
1473static int port_has_memdev(struct device *dev, const void *data)
1474{
1475	const struct detach_ctx *ctx = data;
1476	struct cxl_port *port;
1477
1478	if (!is_cxl_port(dev))
1479		return 0;
1480
1481	port = to_cxl_port(dev);
1482	if (port->depth != ctx->depth)
1483		return 0;
1484
1485	return !!cxl_ep_load(port, ctx->cxlmd);
1486}
1487
1488static void cxl_detach_ep(void *data)
1489{
1490	struct cxl_memdev *cxlmd = data;
1491
1492	for (int i = cxlmd->depth - 1; i >= 1; i--) {
1493		struct cxl_port *port, *parent_port;
1494		struct detach_ctx ctx = {
1495			.cxlmd = cxlmd,
1496			.depth = i,
1497		};
1498		struct cxl_ep *ep;
1499		bool died = false;
1500
1501		struct device *dev __free(put_device) =
1502			bus_find_device(&cxl_bus_type, NULL, &ctx, port_has_memdev);
1503		if (!dev)
1504			continue;
1505		port = to_cxl_port(dev);
1506
1507		parent_port = to_cxl_port(port->dev.parent);
1508		device_lock(&parent_port->dev);
1509		device_lock(&port->dev);
1510		ep = cxl_ep_load(port, cxlmd);
1511		dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",
1512			ep ? dev_name(ep->ep) : "", dev_name(&port->dev));
1513		cxl_ep_remove(port, ep);
1514		if (ep && !port->dead && xa_empty(&port->endpoints) &&
1515		    !is_cxl_root(parent_port) && parent_port->dev.driver) {
1516			/*
1517			 * This was the last ep attached to a dynamically
1518			 * enumerated port. Block new cxl_add_ep() and garbage
1519			 * collect the port.
1520			 */
1521			died = true;
1522			port->dead = true;
1523			del_dports(port);
1524		}
1525		device_unlock(&port->dev);
1526
1527		if (died) {
1528			dev_dbg(&cxlmd->dev, "delete %s\n",
1529				dev_name(&port->dev));
1530			delete_switch_port(port);
1531		}
1532		device_unlock(&parent_port->dev);
1533	}
1534}
1535
1536static resource_size_t find_component_registers(struct device *dev)
1537{
1538	struct cxl_register_map map;
1539	struct pci_dev *pdev;
1540
1541	/*
1542	 * Theoretically, CXL component registers can be hosted on a
1543	 * non-PCI device, in practice, only cxl_test hits this case.
1544	 */
1545	if (!dev_is_pci(dev))
1546		return CXL_RESOURCE_NONE;
1547
1548	pdev = to_pci_dev(dev);
1549
1550	cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
1551	return map.resource;
1552}
1553
1554static int match_port_by_uport(struct device *dev, const void *data)
1555{
1556	const struct device *uport_dev = data;
1557	struct cxl_port *port;
1558
1559	if (!is_cxl_port(dev))
1560		return 0;
1561
1562	port = to_cxl_port(dev);
1563	return uport_dev == port->uport_dev;
1564}
1565
1566/*
1567 * Function takes a device reference on the port device. Caller should do a
1568 * put_device() when done.
1569 */
1570static struct cxl_port *find_cxl_port_by_uport(struct device *uport_dev)
1571{
1572	struct device *dev;
1573
1574	dev = bus_find_device(&cxl_bus_type, NULL, uport_dev, match_port_by_uport);
1575	if (dev)
1576		return to_cxl_port(dev);
1577	return NULL;
1578}
1579
1580static int update_decoder_targets(struct device *dev, void *data)
1581{
1582	struct cxl_dport *dport = data;
1583	struct cxl_switch_decoder *cxlsd;
1584	struct cxl_decoder *cxld;
1585	int i;
1586
1587	if (!is_switch_decoder(dev))
1588		return 0;
1589
1590	cxlsd = to_cxl_switch_decoder(dev);
1591	cxld = &cxlsd->cxld;
1592	guard(rwsem_write)(&cxl_rwsem.region);
1593
1594	for (i = 0; i < cxld->interleave_ways; i++) {
1595		if (cxld->target_map[i] == dport->port_id) {
1596			cxlsd->target[i] = dport;
1597			dev_dbg(dev, "dport%d found in target list, index %d\n",
1598				dport->port_id, i);
1599			return 0;
1600		}
1601	}
1602
1603	return 0;
1604}
1605
1606DEFINE_FREE(del_cxl_dport, struct cxl_dport *, if (!IS_ERR_OR_NULL(_T)) del_dport(_T))
1607static struct cxl_dport *cxl_port_add_dport(struct cxl_port *port,
1608					    struct device *dport_dev)
1609{
1610	struct cxl_dport *dport;
1611	int rc;
1612
1613	device_lock_assert(&port->dev);
1614	if (!port->dev.driver)
1615		return ERR_PTR(-ENXIO);
1616
1617	dport = cxl_find_dport_by_dev(port, dport_dev);
1618	if (dport) {
1619		dev_dbg(&port->dev, "dport%d:%s already exists\n",
1620			dport->port_id, dev_name(dport_dev));
1621		return ERR_PTR(-EBUSY);
1622	}
1623
1624	struct cxl_dport *new_dport __free(del_cxl_dport) =
1625		devm_cxl_add_dport_by_dev(port, dport_dev);
1626	if (IS_ERR(new_dport))
1627		return new_dport;
1628
1629	cxl_switch_parse_cdat(new_dport);
1630
1631	if (ida_is_empty(&port->decoder_ida)) {
1632		rc = devm_cxl_switch_port_decoders_setup(port);
1633		if (rc)
1634			return ERR_PTR(rc);
1635		dev_dbg(&port->dev, "first dport%d:%s added with decoders\n",
1636			new_dport->port_id, dev_name(dport_dev));
1637		return no_free_ptr(new_dport);
1638	}
1639
1640	/* New dport added, update the decoder targets */
1641	device_for_each_child(&port->dev, new_dport, update_decoder_targets);
1642
1643	dev_dbg(&port->dev, "dport%d:%s added\n", new_dport->port_id,
1644		dev_name(dport_dev));
1645
1646	return no_free_ptr(new_dport);
1647}
1648
1649static struct cxl_dport *devm_cxl_create_port(struct device *ep_dev,
1650					      struct cxl_port *parent_port,
1651					      struct cxl_dport *parent_dport,
1652					      struct device *uport_dev,
1653					      struct device *dport_dev)
1654{
1655	resource_size_t component_reg_phys;
1656
1657	device_lock_assert(&parent_port->dev);
1658	if (!parent_port->dev.driver) {
1659		dev_warn(ep_dev,
1660			 "port %s:%s:%s disabled, failed to enumerate CXL.mem\n",
1661			 dev_name(&parent_port->dev), dev_name(uport_dev),
1662			 dev_name(dport_dev));
1663	}
1664
1665	struct cxl_port *port __free(put_cxl_port) =
1666		find_cxl_port_by_uport(uport_dev);
1667	if (!port) {
1668		component_reg_phys = find_component_registers(uport_dev);
1669		port = devm_cxl_add_port(&parent_port->dev, uport_dev,
1670					 component_reg_phys, parent_dport);
1671		if (IS_ERR(port))
1672			return ERR_CAST(port);
1673
1674		/*
1675		 * retry to make sure a port is found. a port device
1676		 * reference is taken.
1677		 */
1678		port = find_cxl_port_by_uport(uport_dev);
1679		if (!port)
1680			return ERR_PTR(-ENODEV);
1681
1682		dev_dbg(ep_dev, "created port %s:%s\n",
1683			dev_name(&port->dev), dev_name(port->uport_dev));
1684	} else {
1685		/*
1686		 * Port was created before right before this function is
1687		 * called. Signal the caller to deal with it.
1688		 */
1689		return ERR_PTR(-EAGAIN);
1690	}
1691
1692	guard(device)(&port->dev);
1693	return cxl_port_add_dport(port, dport_dev);
1694}
1695
1696static int add_port_attach_ep(struct cxl_memdev *cxlmd,
1697			      struct device *uport_dev,
1698			      struct device *dport_dev)
1699{
1700	struct device *dparent = grandparent(dport_dev);
1701	struct cxl_dport *dport, *parent_dport;
1702	int rc;
1703
1704	if (is_cxl_host_bridge(dparent)) {
1705		/*
1706		 * The iteration reached the topology root without finding the
1707		 * CXL-root 'cxl_port' on a previous iteration, fail for now to
1708		 * be re-probed after platform driver attaches.
1709		 */
1710		dev_dbg(&cxlmd->dev, "%s is a root dport\n",
1711			dev_name(dport_dev));
1712		return -ENXIO;
1713	}
1714
1715	struct cxl_port *parent_port __free(put_cxl_port) =
1716		find_cxl_port_by_uport(dparent->parent);
1717	if (!parent_port) {
1718		/* iterate to create this parent_port */
1719		return -EAGAIN;
1720	}
1721
1722	scoped_guard(device, &parent_port->dev) {
1723		parent_dport = cxl_find_dport_by_dev(parent_port, dparent);
1724		if (!parent_dport) {
1725			parent_dport = cxl_port_add_dport(parent_port, dparent);
1726			if (IS_ERR(parent_dport))
1727				return PTR_ERR(parent_dport);
1728		}
1729
1730		dport = devm_cxl_create_port(&cxlmd->dev, parent_port,
1731					     parent_dport, uport_dev,
1732					     dport_dev);
1733		if (IS_ERR(dport)) {
1734			/* Port already exists, restart iteration */
1735			if (PTR_ERR(dport) == -EAGAIN)
1736				return 0;
1737			return PTR_ERR(dport);
1738		}
1739	}
1740
1741	rc = cxl_add_ep(dport, &cxlmd->dev);
1742	if (rc == -EBUSY) {
1743		/*
1744		 * "can't" happen, but this error code means
1745		 * something to the caller, so translate it.
1746		 */
1747		rc = -ENXIO;
1748	}
1749
1750	return rc;
1751}
1752
1753static struct cxl_dport *find_or_add_dport(struct cxl_port *port,
1754					   struct device *dport_dev)
1755{
1756	struct cxl_dport *dport;
1757
1758	device_lock_assert(&port->dev);
1759	dport = cxl_find_dport_by_dev(port, dport_dev);
1760	if (!dport) {
1761		dport = cxl_port_add_dport(port, dport_dev);
1762		if (IS_ERR(dport))
1763			return dport;
1764
1765		/* New dport added, restart iteration */
1766		return ERR_PTR(-EAGAIN);
1767	}
1768
1769	return dport;
1770}
1771
1772int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
1773{
1774	struct device *dev = &cxlmd->dev;
1775	struct device *iter;
1776	int rc;
1777
1778	/*
1779	 * Skip intermediate port enumeration in the RCH case, there
1780	 * are no ports in between a host bridge and an endpoint.
1781	 */
1782	if (cxlmd->cxlds->rcd)
1783		return 0;
1784
1785	rc = devm_add_action_or_reset(&cxlmd->dev, cxl_detach_ep, cxlmd);
1786	if (rc)
1787		return rc;
1788
1789	/*
1790	 * Scan for and add all cxl_ports in this device's ancestry.
1791	 * Repeat until no more ports are added. Abort if a port add
1792	 * attempt fails.
1793	 */
1794retry:
1795	for (iter = dev; iter; iter = grandparent(iter)) {
1796		struct device *dport_dev = grandparent(iter);
1797		struct device *uport_dev;
1798		struct cxl_dport *dport;
1799
1800		if (is_cxl_host_bridge(dport_dev))
1801			return 0;
1802
1803		uport_dev = dport_dev->parent;
1804		if (!uport_dev) {
1805			dev_warn(dev, "at %s no parent for dport: %s\n",
1806				 dev_name(iter), dev_name(dport_dev));
1807			return -ENXIO;
1808		}
1809
1810		dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",
1811			dev_name(iter), dev_name(dport_dev),
1812			dev_name(uport_dev));
1813		struct cxl_port *port __free(put_cxl_port) =
1814			find_cxl_port_by_uport(uport_dev);
1815		if (port) {
1816			dev_dbg(&cxlmd->dev,
1817				"found already registered port %s:%s\n",
1818				dev_name(&port->dev),
1819				dev_name(port->uport_dev));
1820
1821			/*
1822			 * RP port enumerated by cxl_acpi without dport will
1823			 * have the dport added here.
1824			 */
1825			scoped_guard(device, &port->dev) {
1826				dport = find_or_add_dport(port, dport_dev);
1827				if (IS_ERR(dport)) {
1828					if (PTR_ERR(dport) == -EAGAIN)
1829						goto retry;
1830					return PTR_ERR(dport);
1831				}
1832			}
1833
1834			rc = cxl_add_ep(dport, &cxlmd->dev);
1835
1836			/*
1837			 * If the endpoint already exists in the port's list,
1838			 * that's ok, it was added on a previous pass.
1839			 * Otherwise, retry in add_port_attach_ep() after taking
1840			 * the parent_port lock as the current port may be being
1841			 * reaped.
1842			 */
1843			if (rc && rc != -EBUSY)
1844				return rc;
1845
1846			cxl_gpf_port_setup(dport);
1847
1848			/* Any more ports to add between this one and the root? */
1849			if (!dev_is_cxl_root_child(&port->dev))
1850				continue;
1851
1852			return 0;
1853		}
1854
1855		rc = add_port_attach_ep(cxlmd, uport_dev, dport_dev);
1856		/* port missing, try to add parent */
1857		if (rc == -EAGAIN)
1858			continue;
1859		/* failed to add ep or port */
1860		if (rc)
1861			return rc;
1862		/* port added, new descendants possible, start over */
1863		goto retry;
1864	}
1865
1866	return 0;
1867}
1868EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, "CXL");
1869
1870struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
1871				   struct cxl_dport **dport)
1872{
1873	return find_cxl_port(pdev->dev.parent, dport);
1874}
1875EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, "CXL");
1876
1877struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
1878				   struct cxl_dport **dport)
1879{
1880	return find_cxl_port(grandparent(&cxlmd->dev), dport);
1881}
1882EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, "CXL");
1883
1884static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
1885				    struct cxl_port *port)
1886{
1887	struct cxl_decoder *cxld = &cxlsd->cxld;
1888	int i;
1889
1890	device_lock_assert(&port->dev);
1891
1892	if (xa_empty(&port->dports))
1893		return 0;
1894
1895	guard(rwsem_write)(&cxl_rwsem.region);
1896	for (i = 0; i < cxlsd->cxld.interleave_ways; i++) {
1897		struct cxl_dport *dport = find_dport(port, cxld->target_map[i]);
1898
1899		if (!dport) {
1900			/* dport may be activated later */
1901			continue;
1902		}
1903		cxlsd->target[i] = dport;
1904	}
1905
1906	return 0;
1907}
1908
1909static struct lock_class_key cxl_decoder_key;
1910
1911/**
1912 * cxl_decoder_init - Common decoder setup / initialization
1913 * @port: owning port of this decoder
1914 * @cxld: common decoder properties to initialize
1915 *
1916 * A port may contain one or more decoders. Each of those decoders
1917 * enable some address space for CXL.mem utilization. A decoder is
1918 * expected to be configured by the caller before registering via
1919 * cxl_decoder_add()
1920 */
1921static int cxl_decoder_init(struct cxl_port *port, struct cxl_decoder *cxld)
1922{
1923	struct device *dev;
1924	int rc;
1925
1926	rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
1927	if (rc < 0)
1928		return rc;
1929
1930	/* need parent to stick around to release the id */
1931	get_device(&port->dev);
1932	cxld->id = rc;
1933
1934	dev = &cxld->dev;
1935	device_initialize(dev);
1936	lockdep_set_class(&dev->mutex, &cxl_decoder_key);
1937	device_set_pm_not_required(dev);
1938	dev->parent = &port->dev;
1939	dev->bus = &cxl_bus_type;
1940
1941	/* Pre initialize an "empty" decoder */
1942	cxld->interleave_ways = 1;
1943	cxld->interleave_granularity = PAGE_SIZE;
1944	cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1945	cxld->hpa_range = (struct range) {
1946		.start = 0,
1947		.end = -1,
1948	};
1949
1950	return 0;
1951}
1952
1953static int cxl_switch_decoder_init(struct cxl_port *port,
1954				   struct cxl_switch_decoder *cxlsd,
1955				   int nr_targets)
1956{
1957	if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
1958		return -EINVAL;
1959
1960	cxlsd->nr_targets = nr_targets;
1961	return cxl_decoder_init(port, &cxlsd->cxld);
1962}
1963
1964/**
1965 * cxl_root_decoder_alloc - Allocate a root level decoder
1966 * @port: owning CXL root of this decoder
1967 * @nr_targets: static number of downstream targets
1968 *
1969 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1970 * 'CXL root' decoder is one that decodes from a top-level / static platform
1971 * firmware description of CXL resources into a CXL standard decode
1972 * topology.
1973 */
1974struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
1975						unsigned int nr_targets)
1976{
1977	struct cxl_root_decoder *cxlrd;
1978	struct cxl_switch_decoder *cxlsd;
1979	struct cxl_decoder *cxld;
1980	int rc;
1981
1982	if (!is_cxl_root(port))
1983		return ERR_PTR(-EINVAL);
1984
1985	cxlrd = kzalloc(struct_size(cxlrd, cxlsd.target, nr_targets),
1986			GFP_KERNEL);
1987	if (!cxlrd)
1988		return ERR_PTR(-ENOMEM);
1989
1990	cxlsd = &cxlrd->cxlsd;
1991	rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1992	if (rc) {
1993		kfree(cxlrd);
1994		return ERR_PTR(rc);
1995	}
1996
1997	mutex_init(&cxlrd->range_lock);
1998
1999	cxld = &cxlsd->cxld;
2000	cxld->dev.type = &cxl_decoder_root_type;
2001	/*
2002	 * cxl_root_decoder_release() special cases negative ids to
2003	 * detect memregion_alloc() failures.
2004	 */
2005	atomic_set(&cxlrd->region_id, -1);
2006	rc = memregion_alloc(GFP_KERNEL);
2007	if (rc < 0) {
2008		put_device(&cxld->dev);
2009		return ERR_PTR(rc);
2010	}
2011
2012	atomic_set(&cxlrd->region_id, rc);
2013	cxlrd->qos_class = CXL_QOS_CLASS_INVALID;
2014	return cxlrd;
2015}
2016EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, "CXL");
2017
2018/**
2019 * cxl_switch_decoder_alloc - Allocate a switch level decoder
2020 * @port: owning CXL switch port of this decoder
2021 * @nr_targets: max number of dynamically addressable downstream targets
2022 *
2023 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
2024 * 'switch' decoder is any decoder that can be enumerated by PCIe
2025 * topology and the HDM Decoder Capability. This includes the decoders
2026 * that sit between Switch Upstream Ports / Switch Downstream Ports and
2027 * Host Bridges / Root Ports.
2028 */
2029struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
2030						    unsigned int nr_targets)
2031{
2032	struct cxl_switch_decoder *cxlsd;
2033	struct cxl_decoder *cxld;
2034	int rc;
2035
2036	if (is_cxl_root(port) || is_cxl_endpoint(port))
2037		return ERR_PTR(-EINVAL);
2038
2039	cxlsd = kzalloc(struct_size(cxlsd, target, nr_targets), GFP_KERNEL);
2040	if (!cxlsd)
2041		return ERR_PTR(-ENOMEM);
2042
2043	rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
2044	if (rc) {
2045		kfree(cxlsd);
2046		return ERR_PTR(rc);
2047	}
2048
2049	cxld = &cxlsd->cxld;
2050	cxld->dev.type = &cxl_decoder_switch_type;
2051	return cxlsd;
2052}
2053EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, "CXL");
2054
2055/**
2056 * cxl_endpoint_decoder_alloc - Allocate an endpoint decoder
2057 * @port: owning port of this decoder
2058 *
2059 * Return: A new cxl decoder to be registered by cxl_decoder_add()
2060 */
2061struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port)
2062{
2063	struct cxl_endpoint_decoder *cxled;
2064	struct cxl_decoder *cxld;
2065	int rc;
2066
2067	if (!is_cxl_endpoint(port))
2068		return ERR_PTR(-EINVAL);
2069
2070	cxled = kzalloc(sizeof(*cxled), GFP_KERNEL);
2071	if (!cxled)
2072		return ERR_PTR(-ENOMEM);
2073
2074	cxled->pos = -1;
2075	cxled->part = -1;
2076	cxld = &cxled->cxld;
2077	rc = cxl_decoder_init(port, cxld);
2078	if (rc)	 {
2079		kfree(cxled);
2080		return ERR_PTR(rc);
2081	}
2082
2083	cxld->dev.type = &cxl_decoder_endpoint_type;
2084	return cxled;
2085}
2086EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, "CXL");
2087
2088/**
2089 * cxl_decoder_add_locked - Add a decoder with targets
2090 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
2091 *
2092 * Certain types of decoders may not have any targets. The main example of this
2093 * is an endpoint device. A more awkward example is a hostbridge whose root
2094 * ports get hot added (technically possible, though unlikely).
2095 *
2096 * This is the locked variant of cxl_decoder_add().
2097 *
2098 * Context: Process context. Expects the device lock of the port that owns the
2099 *	    @cxld to be held.
2100 *
2101 * Return: Negative error code if the decoder wasn't properly configured; else
2102 *	   returns 0.
2103 */
2104int cxl_decoder_add_locked(struct cxl_decoder *cxld)
2105{
2106	struct cxl_port *port;
2107	struct device *dev;
2108	int rc;
2109
2110	if (WARN_ON_ONCE(!cxld))
2111		return -EINVAL;
2112
2113	if (WARN_ON_ONCE(IS_ERR(cxld)))
2114		return PTR_ERR(cxld);
2115
2116	if (cxld->interleave_ways < 1)
2117		return -EINVAL;
2118
2119	dev = &cxld->dev;
2120
2121	port = to_cxl_port(cxld->dev.parent);
2122	if (!is_endpoint_decoder(dev)) {
2123		struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
2124
2125		rc = decoder_populate_targets(cxlsd, port);
2126		if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) {
2127			dev_err(&port->dev,
2128				"Failed to populate active decoder targets\n");
2129			return rc;
2130		}
2131	}
2132
2133	rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);
2134	if (rc)
2135		return rc;
2136
2137	return device_add(dev);
2138}
2139EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, "CXL");
2140
2141/**
2142 * cxl_decoder_add - Add a decoder with targets
2143 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
2144 *
2145 * This is the unlocked variant of cxl_decoder_add_locked().
2146 * See cxl_decoder_add_locked().
2147 *
2148 * Context: Process context. Takes and releases the device lock of the port that
2149 *	    owns the @cxld.
2150 */
2151int cxl_decoder_add(struct cxl_decoder *cxld)
2152{
2153	struct cxl_port *port;
2154
2155	if (WARN_ON_ONCE(!cxld))
2156		return -EINVAL;
2157
2158	if (WARN_ON_ONCE(IS_ERR(cxld)))
2159		return PTR_ERR(cxld);
2160
2161	port = to_cxl_port(cxld->dev.parent);
2162
2163	guard(device)(&port->dev);
2164	return cxl_decoder_add_locked(cxld);
2165}
2166EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, "CXL");
2167
2168static void cxld_unregister(void *dev)
2169{
2170	if (is_endpoint_decoder(dev))
2171		cxl_decoder_detach(NULL, to_cxl_endpoint_decoder(dev), -1,
2172				   DETACH_INVALIDATE);
2173
2174	device_unregister(dev);
2175}
2176
2177int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
2178{
2179	return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev);
2180}
2181EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, "CXL");
2182
2183/**
2184 * __cxl_driver_register - register a driver for the cxl bus
2185 * @cxl_drv: cxl driver structure to attach
2186 * @owner: owning module/driver
2187 * @modname: KBUILD_MODNAME for parent driver
2188 */
2189int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
2190			  const char *modname)
2191{
2192	if (!cxl_drv->probe) {
2193		pr_debug("%s ->probe() must be specified\n", modname);
2194		return -EINVAL;
2195	}
2196
2197	if (!cxl_drv->name) {
2198		pr_debug("%s ->name must be specified\n", modname);
2199		return -EINVAL;
2200	}
2201
2202	if (!cxl_drv->id) {
2203		pr_debug("%s ->id must be specified\n", modname);
2204		return -EINVAL;
2205	}
2206
2207	cxl_drv->drv.bus = &cxl_bus_type;
2208	cxl_drv->drv.owner = owner;
2209	cxl_drv->drv.mod_name = modname;
2210	cxl_drv->drv.name = cxl_drv->name;
2211
2212	return driver_register(&cxl_drv->drv);
2213}
2214EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, "CXL");
2215
2216void cxl_driver_unregister(struct cxl_driver *cxl_drv)
2217{
2218	driver_unregister(&cxl_drv->drv);
2219}
2220EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, "CXL");
2221
2222static int cxl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
2223{
2224	return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,
2225			      cxl_device_id(dev));
2226}
2227
2228static int cxl_bus_match(struct device *dev, const struct device_driver *drv)
2229{
2230	return cxl_device_id(dev) == to_cxl_drv(drv)->id;
2231}
2232
2233static int cxl_bus_probe(struct device *dev)
2234{
2235	int rc;
2236
2237	rc = to_cxl_drv(dev->driver)->probe(dev);
2238	dev_dbg(dev, "probe: %d\n", rc);
2239	return rc;
2240}
2241
2242static void cxl_bus_remove(struct device *dev)
2243{
2244	struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
2245
2246	if (cxl_drv->remove)
2247		cxl_drv->remove(dev);
2248}
2249
2250static struct workqueue_struct *cxl_bus_wq;
2251
2252static int cxl_rescan_attach(struct device *dev, void *data)
2253{
2254	int rc = device_attach(dev);
2255
2256	dev_vdbg(dev, "rescan: %s\n", rc ? "attach" : "detached");
2257
2258	return 0;
2259}
2260
2261static void cxl_bus_rescan_queue(struct work_struct *w)
2262{
2263	bus_for_each_dev(&cxl_bus_type, NULL, NULL, cxl_rescan_attach);
2264}
2265
2266void cxl_bus_rescan(void)
2267{
2268	static DECLARE_WORK(rescan_work, cxl_bus_rescan_queue);
2269
2270	queue_work(cxl_bus_wq, &rescan_work);
2271}
2272EXPORT_SYMBOL_NS_GPL(cxl_bus_rescan, "CXL");
2273
2274void cxl_bus_drain(void)
2275{
2276	drain_workqueue(cxl_bus_wq);
2277}
2278EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, "CXL");
2279
2280bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
2281{
2282	return queue_work(cxl_bus_wq, &cxlmd->detach_work);
2283}
2284EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, "CXL");
2285
2286static void add_latency(struct access_coordinate *c, long latency)
2287{
2288	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2289		c[i].write_latency += latency;
2290		c[i].read_latency += latency;
2291	}
2292}
2293
2294static bool coordinates_valid(struct access_coordinate *c)
2295{
2296	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2297		if (c[i].read_bandwidth && c[i].write_bandwidth &&
2298		    c[i].read_latency && c[i].write_latency)
2299			continue;
2300		return false;
2301	}
2302
2303	return true;
2304}
2305
2306static void set_min_bandwidth(struct access_coordinate *c, unsigned int bw)
2307{
2308	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2309		c[i].write_bandwidth = min(c[i].write_bandwidth, bw);
2310		c[i].read_bandwidth = min(c[i].read_bandwidth, bw);
2311	}
2312}
2313
2314static void set_access_coordinates(struct access_coordinate *out,
2315				   struct access_coordinate *in)
2316{
2317	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++)
2318		out[i] = in[i];
2319}
2320
2321static bool parent_port_is_cxl_root(struct cxl_port *port)
2322{
2323	return is_cxl_root(to_cxl_port(port->dev.parent));
2324}
2325
2326/**
2327 * cxl_endpoint_get_perf_coordinates - Retrieve performance numbers stored in dports
2328 *				   of CXL path
2329 * @port: endpoint cxl_port
2330 * @coord: output performance data
2331 *
2332 * Return: errno on failure, 0 on success.
2333 */
2334int cxl_endpoint_get_perf_coordinates(struct cxl_port *port,
2335				      struct access_coordinate *coord)
2336{
2337	struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
2338	struct access_coordinate c[] = {
2339		{
2340			.read_bandwidth = UINT_MAX,
2341			.write_bandwidth = UINT_MAX,
2342		},
2343		{
2344			.read_bandwidth = UINT_MAX,
2345			.write_bandwidth = UINT_MAX,
2346		},
2347	};
2348	struct cxl_port *iter = port;
2349	struct cxl_dport *dport;
2350	struct pci_dev *pdev;
2351	struct device *dev;
2352	unsigned int bw;
2353	bool is_cxl_root;
2354
2355	if (!is_cxl_endpoint(port))
2356		return -EINVAL;
2357
2358	/*
2359	 * Skip calculation for RCD. Expectation is HMAT already covers RCD case
2360	 * since RCH does not support hotplug.
2361	 */
2362	if (cxlmd->cxlds->rcd)
2363		return 0;
2364
2365	/*
2366	 * Exit the loop when the parent port of the current iter port is cxl
2367	 * root. The iterative loop starts at the endpoint and gathers the
2368	 * latency of the CXL link from the current device/port to the connected
2369	 * downstream port each iteration.
2370	 */
2371	do {
2372		dport = iter->parent_dport;
2373		iter = to_cxl_port(iter->dev.parent);
2374		is_cxl_root = parent_port_is_cxl_root(iter);
2375
2376		/*
2377		 * There's no valid access_coordinate for a root port since RPs do not
2378		 * have CDAT and therefore needs to be skipped.
2379		 */
2380		if (!is_cxl_root) {
2381			if (!coordinates_valid(dport->coord))
2382				return -EINVAL;
2383			cxl_coordinates_combine(c, c, dport->coord);
2384		}
2385		add_latency(c, dport->link_latency);
2386	} while (!is_cxl_root);
2387
2388	dport = iter->parent_dport;
2389	/* Retrieve HB coords */
2390	if (!coordinates_valid(dport->coord))
2391		return -EINVAL;
2392	cxl_coordinates_combine(c, c, dport->coord);
2393
2394	dev = port->uport_dev->parent;
2395	if (!dev_is_pci(dev))
2396		return -ENODEV;
2397
2398	/* Get the calculated PCI paths bandwidth */
2399	pdev = to_pci_dev(dev);
2400	bw = pcie_bandwidth_available(pdev, NULL, NULL, NULL);
2401	if (bw == 0)
2402		return -ENXIO;
2403	bw /= BITS_PER_BYTE;
2404
2405	set_min_bandwidth(c, bw);
2406	set_access_coordinates(coord, c);
2407
2408	return 0;
2409}
2410EXPORT_SYMBOL_NS_GPL(cxl_endpoint_get_perf_coordinates, "CXL");
2411
2412int cxl_port_get_switch_dport_bandwidth(struct cxl_port *port,
2413					struct access_coordinate *c)
2414{
2415	struct cxl_dport *dport = port->parent_dport;
2416
2417	/* Check this port is connected to a switch DSP and not an RP */
2418	if (parent_port_is_cxl_root(to_cxl_port(port->dev.parent)))
2419		return -ENODEV;
2420
2421	if (!coordinates_valid(dport->coord))
2422		return -EINVAL;
2423
2424	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2425		c[i].read_bandwidth = dport->coord[i].read_bandwidth;
2426		c[i].write_bandwidth = dport->coord[i].write_bandwidth;
2427	}
2428
2429	return 0;
2430}
2431
2432/* for user tooling to ensure port disable work has completed */
2433static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count)
2434{
2435	if (sysfs_streq(buf, "1")) {
2436		flush_workqueue(cxl_bus_wq);
2437		return count;
2438	}
2439
2440	return -EINVAL;
2441}
2442
2443static BUS_ATTR_WO(flush);
2444
2445static struct attribute *cxl_bus_attributes[] = {
2446	&bus_attr_flush.attr,
2447	NULL,
2448};
2449
2450static struct attribute_group cxl_bus_attribute_group = {
2451	.attrs = cxl_bus_attributes,
2452};
2453
2454static const struct attribute_group *cxl_bus_attribute_groups[] = {
2455	&cxl_bus_attribute_group,
2456	NULL,
2457};
2458
2459const struct bus_type cxl_bus_type = {
2460	.name = "cxl",
2461	.uevent = cxl_bus_uevent,
2462	.match = cxl_bus_match,
2463	.probe = cxl_bus_probe,
2464	.remove = cxl_bus_remove,
2465	.bus_groups = cxl_bus_attribute_groups,
2466};
2467EXPORT_SYMBOL_NS_GPL(cxl_bus_type, "CXL");
2468
2469static struct dentry *cxl_debugfs;
2470
2471struct dentry *cxl_debugfs_create_dir(const char *dir)
2472{
2473	return debugfs_create_dir(dir, cxl_debugfs);
2474}
2475EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, "CXL");
2476
2477static __init int cxl_core_init(void)
2478{
2479	int rc;
2480
2481	cxl_debugfs = debugfs_create_dir("cxl", NULL);
2482
2483	if (einj_cxl_is_initialized())
2484		debugfs_create_file("einj_types", 0400, cxl_debugfs, NULL,
2485				    &einj_cxl_available_error_type_fops);
2486
2487	cxl_mbox_init();
2488
2489	rc = cxl_memdev_init();
2490	if (rc)
2491		return rc;
2492
2493	cxl_bus_wq = alloc_ordered_workqueue("cxl_port", 0);
2494	if (!cxl_bus_wq) {
2495		rc = -ENOMEM;
2496		goto err_wq;
2497	}
2498
2499	rc = bus_register(&cxl_bus_type);
2500	if (rc)
2501		goto err_bus;
2502
2503	rc = cxl_region_init();
2504	if (rc)
2505		goto err_region;
2506
2507	rc = cxl_ras_init();
2508	if (rc)
2509		goto err_ras;
2510
2511	return 0;
2512
2513err_ras:
2514	cxl_region_exit();
2515err_region:
2516	bus_unregister(&cxl_bus_type);
2517err_bus:
2518	destroy_workqueue(cxl_bus_wq);
2519err_wq:
2520	cxl_memdev_exit();
2521	return rc;
2522}
2523
2524static void cxl_core_exit(void)
2525{
2526	cxl_ras_exit();
2527	cxl_region_exit();
2528	bus_unregister(&cxl_bus_type);
2529	destroy_workqueue(cxl_bus_wq);
2530	cxl_memdev_exit();
2531	debugfs_remove_recursive(cxl_debugfs);
2532}
2533
2534subsys_initcall(cxl_core_init);
2535module_exit(cxl_core_exit);
2536MODULE_DESCRIPTION("CXL: Core Compute Express Link support");
2537MODULE_LICENSE("GPL v2");
2538MODULE_IMPORT_NS("CXL");