pci.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright IBM Corp. 2012
   4 *
   5 * Author(s):
   6 *   Jan Glauber <jang@linux.vnet.ibm.com>
   7 *
   8 * The System z PCI code is a rewrite from a prototype by
   9 * the following people (Kudoz!):
  10 *   Alexander Schmidt
  11 *   Christoph Raisch
  12 *   Hannes Hering
  13 *   Hoang-Nam Nguyen
  14 *   Jan-Bernd Themann
  15 *   Stefan Roscher
  16 *   Thomas Klein
  17 */
  18
  19#define pr_fmt(fmt) "zpci: " fmt
  20
  21#include <linux/kernel.h>
  22#include <linux/slab.h>
  23#include <linux/err.h>
  24#include <linux/export.h>
  25#include <linux/delay.h>
  26#include <linux/seq_file.h>
  27#include <linux/jump_label.h>
  28#include <linux/pci.h>
  29#include <linux/printk.h>
  30#include <linux/lockdep.h>
  31#include <linux/list_sort.h>
  32
  33#include <asm/machine.h>
  34#include <asm/isc.h>
  35#include <asm/airq.h>
  36#include <asm/facility.h>
  37#include <asm/pci_insn.h>
  38#include <asm/pci_clp.h>
  39#include <asm/pci_dma.h>
  40
  41#include "pci_bus.h"
  42#include "pci_iov.h"
  43
  44/* list of all detected zpci devices */
  45static LIST_HEAD(zpci_list);
  46static DEFINE_SPINLOCK(zpci_list_lock);
  47static DEFINE_MUTEX(zpci_add_remove_lock);
  48
  49static DECLARE_BITMAP(zpci_domain, ZPCI_DOMAIN_BITMAP_SIZE);
  50static DEFINE_SPINLOCK(zpci_domain_lock);
  51
  52#define ZPCI_IOMAP_ENTRIES						\
  53	min(((unsigned long) ZPCI_NR_DEVICES * PCI_STD_NUM_BARS / 2),	\
  54	    ZPCI_IOMAP_MAX_ENTRIES)
  55
  56unsigned int s390_pci_no_rid;
  57
  58static DEFINE_SPINLOCK(zpci_iomap_lock);
  59static unsigned long *zpci_iomap_bitmap;
  60struct zpci_iomap_entry *zpci_iomap_start;
  61EXPORT_SYMBOL_GPL(zpci_iomap_start);
  62
  63DEFINE_STATIC_KEY_FALSE(have_mio);
  64
  65static struct kmem_cache *zdev_fmb_cache;
  66
  67/* AEN structures that must be preserved over KVM module re-insertion */
  68union zpci_sic_iib *zpci_aipb;
  69EXPORT_SYMBOL_GPL(zpci_aipb);
  70struct airq_iv *zpci_aif_sbv;
  71EXPORT_SYMBOL_GPL(zpci_aif_sbv);
  72
  73void zpci_zdev_put(struct zpci_dev *zdev)
  74{
  75	if (!zdev)
  76		return;
  77	mutex_lock(&zpci_add_remove_lock);
  78	kref_put_lock(&zdev->kref, zpci_release_device, &zpci_list_lock);
  79	mutex_unlock(&zpci_add_remove_lock);
  80}
  81
  82struct zpci_dev *get_zdev_by_fid(u32 fid)
  83{
  84	struct zpci_dev *tmp, *zdev = NULL;
  85
  86	spin_lock(&zpci_list_lock);
  87	list_for_each_entry(tmp, &zpci_list, entry) {
  88		if (tmp->fid == fid) {
  89			zdev = tmp;
  90			zpci_zdev_get(zdev);
  91			break;
  92		}
  93	}
  94	spin_unlock(&zpci_list_lock);
  95	return zdev;
  96}
  97
  98void zpci_remove_reserved_devices(void)
  99{
 100	struct zpci_dev *tmp, *zdev;
 101	enum zpci_state state;
 102	LIST_HEAD(remove);
 103
 104	spin_lock(&zpci_list_lock);
 105	list_for_each_entry_safe(zdev, tmp, &zpci_list, entry) {
 106		if (zdev->state == ZPCI_FN_STATE_STANDBY &&
 107		    !clp_get_state(zdev->fid, &state) &&
 108		    state == ZPCI_FN_STATE_RESERVED)
 109			list_move_tail(&zdev->entry, &remove);
 110	}
 111	spin_unlock(&zpci_list_lock);
 112
 113	list_for_each_entry_safe(zdev, tmp, &remove, entry)
 114		zpci_device_reserved(zdev);
 115}
 116
 117int pci_domain_nr(struct pci_bus *bus)
 118{
 119	return ((struct zpci_bus *) bus->sysdata)->domain_nr;
 120}
 121EXPORT_SYMBOL_GPL(pci_domain_nr);
 122
 123int pci_proc_domain(struct pci_bus *bus)
 124{
 125	return pci_domain_nr(bus);
 126}
 127EXPORT_SYMBOL_GPL(pci_proc_domain);
 128
 129/* Modify PCI: Register I/O address translation parameters */
 130int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
 131		       u64 base, u64 limit, u64 iota, u8 *status)
 132{
 133	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT);
 134	struct zpci_fib fib = {0};
 135	u8 cc;
 136
 137	fib.pba = base;
 138	/* Work around off by one in ISM virt device */
 139	if (zdev->pft == PCI_FUNC_TYPE_ISM && limit > base)
 140		fib.pal = limit + (1 << 12);
 141	else
 142		fib.pal = limit;
 143	fib.iota = iota;
 144	fib.gd = zdev->gisa;
 145	cc = zpci_mod_fc(req, &fib, status);
 146	if (cc)
 147		zpci_dbg(3, "reg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, *status);
 148	return cc;
 149}
 150EXPORT_SYMBOL_GPL(zpci_register_ioat);
 151
 152/* Modify PCI: Unregister I/O address translation parameters */
 153int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas)
 154{
 155	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_DEREG_IOAT);
 156	struct zpci_fib fib = {0};
 157	u8 cc, status;
 158
 159	fib.gd = zdev->gisa;
 160
 161	cc = zpci_mod_fc(req, &fib, &status);
 162	if (cc)
 163		zpci_dbg(3, "unreg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status);
 164	return cc;
 165}
 166
 167/* Modify PCI: Set PCI function measurement parameters */
 168int zpci_fmb_enable_device(struct zpci_dev *zdev)
 169{
 170	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
 171	struct zpci_iommu_ctrs *ctrs;
 172	struct zpci_fib fib = {0};
 173	unsigned long flags;
 174	u8 cc, status;
 175
 176	if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length)
 177		return -EINVAL;
 178
 179	zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL);
 180	if (!zdev->fmb)
 181		return -ENOMEM;
 182	WARN_ON((u64) zdev->fmb & 0xf);
 183
 184	/* reset software counters */
 185	spin_lock_irqsave(&zdev->dom_lock, flags);
 186	ctrs = zpci_get_iommu_ctrs(zdev);
 187	if (ctrs) {
 188		atomic64_set(&ctrs->mapped_pages, 0);
 189		atomic64_set(&ctrs->unmapped_pages, 0);
 190		atomic64_set(&ctrs->global_rpcits, 0);
 191		atomic64_set(&ctrs->sync_map_rpcits, 0);
 192		atomic64_set(&ctrs->sync_rpcits, 0);
 193	}
 194	spin_unlock_irqrestore(&zdev->dom_lock, flags);
 195
 196
 197	fib.fmb_addr = virt_to_phys(zdev->fmb);
 198	fib.gd = zdev->gisa;
 199	cc = zpci_mod_fc(req, &fib, &status);
 200	if (cc) {
 201		kmem_cache_free(zdev_fmb_cache, zdev->fmb);
 202		zdev->fmb = NULL;
 203	}
 204	return cc ? -EIO : 0;
 205}
 206
 207/* Modify PCI: Disable PCI function measurement */
 208int zpci_fmb_disable_device(struct zpci_dev *zdev)
 209{
 210	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
 211	struct zpci_fib fib = {0};
 212	u8 cc, status;
 213
 214	if (!zdev->fmb)
 215		return -EINVAL;
 216
 217	fib.gd = zdev->gisa;
 218
 219	/* Function measurement is disabled if fmb address is zero */
 220	cc = zpci_mod_fc(req, &fib, &status);
 221	if (cc == 3) /* Function already gone. */
 222		cc = 0;
 223
 224	if (!cc) {
 225		kmem_cache_free(zdev_fmb_cache, zdev->fmb);
 226		zdev->fmb = NULL;
 227	}
 228	return cc ? -EIO : 0;
 229}
 230
 231static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len)
 232{
 233	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
 234	int rc = -ENODEV;
 235	u64 data;
 236
 237	if (!zdev_enabled(zdev))
 238		goto out_err;
 239
 240	rc = __zpci_load(&data, req, offset);
 241	if (rc)
 242		goto out_err;
 243	data = le64_to_cpu((__force __le64)data);
 244	data >>= (8 - len) * 8;
 245	*val = (u32)data;
 246	return 0;
 247
 248out_err:
 249	PCI_SET_ERROR_RESPONSE(val);
 250	return rc;
 251}
 252
 253static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len)
 254{
 255	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
 256	int rc = -ENODEV;
 257	u64 data = val;
 258
 259	if (!zdev_enabled(zdev))
 260		return rc;
 261
 262	data <<= (8 - len) * 8;
 263	data = (__force u64) cpu_to_le64(data);
 264	rc = __zpci_store(data, req, offset);
 265	return rc;
 266}
 267
 268resource_size_t pcibios_align_resource(void *data, const struct resource *res,
 269				       resource_size_t size,
 270				       resource_size_t align)
 271{
 272	return 0;
 273}
 274
 275void __iomem *ioremap_prot(phys_addr_t phys_addr, size_t size,
 276			   pgprot_t prot)
 277{
 278	/*
 279	 * When PCI MIO instructions are unavailable the "physical" address
 280	 * encodes a hint for accessing the PCI memory space it represents.
 281	 * Just pass it unchanged such that ioread/iowrite can decode it.
 282	 */
 283	if (!static_branch_unlikely(&have_mio))
 284		return (void __iomem *)phys_addr;
 285
 286	return generic_ioremap_prot(phys_addr, size, prot);
 287}
 288EXPORT_SYMBOL(ioremap_prot);
 289
 290void iounmap(volatile void __iomem *addr)
 291{
 292	if (static_branch_likely(&have_mio))
 293		generic_iounmap(addr);
 294}
 295EXPORT_SYMBOL(iounmap);
 296
 297/* Create a virtual mapping cookie for a PCI BAR */
 298static void __iomem *pci_iomap_range_fh(struct pci_dev *pdev, int bar,
 299					unsigned long offset, unsigned long max)
 300{
 301	struct zpci_dev *zdev =	to_zpci(pdev);
 302	int idx;
 303
 304	idx = zdev->bars[bar].map_idx;
 305	spin_lock(&zpci_iomap_lock);
 306	/* Detect overrun */
 307	WARN_ON(!++zpci_iomap_start[idx].count);
 308	zpci_iomap_start[idx].fh = zdev->fh;
 309	zpci_iomap_start[idx].bar = bar;
 310	spin_unlock(&zpci_iomap_lock);
 311
 312	return (void __iomem *) ZPCI_ADDR(idx) + offset;
 313}
 314
 315static void __iomem *pci_iomap_range_mio(struct pci_dev *pdev, int bar,
 316					 unsigned long offset,
 317					 unsigned long max)
 318{
 319	unsigned long barsize = pci_resource_len(pdev, bar);
 320	struct zpci_dev *zdev = to_zpci(pdev);
 321	void __iomem *iova;
 322
 323	iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize);
 324	return iova ? iova + offset : iova;
 325}
 326
 327void __iomem *pci_iomap_range(struct pci_dev *pdev, int bar,
 328			      unsigned long offset, unsigned long max)
 329{
 330	if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar))
 331		return NULL;
 332
 333	if (static_branch_likely(&have_mio))
 334		return pci_iomap_range_mio(pdev, bar, offset, max);
 335	else
 336		return pci_iomap_range_fh(pdev, bar, offset, max);
 337}
 338EXPORT_SYMBOL(pci_iomap_range);
 339
 340void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen)
 341{
 342	return pci_iomap_range(dev, bar, 0, maxlen);
 343}
 344EXPORT_SYMBOL(pci_iomap);
 345
 346static void __iomem *pci_iomap_wc_range_mio(struct pci_dev *pdev, int bar,
 347					    unsigned long offset, unsigned long max)
 348{
 349	unsigned long barsize = pci_resource_len(pdev, bar);
 350	struct zpci_dev *zdev = to_zpci(pdev);
 351	void __iomem *iova;
 352
 353	iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize);
 354	return iova ? iova + offset : iova;
 355}
 356
 357void __iomem *pci_iomap_wc_range(struct pci_dev *pdev, int bar,
 358				 unsigned long offset, unsigned long max)
 359{
 360	if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar))
 361		return NULL;
 362
 363	if (static_branch_likely(&have_mio))
 364		return pci_iomap_wc_range_mio(pdev, bar, offset, max);
 365	else
 366		return pci_iomap_range_fh(pdev, bar, offset, max);
 367}
 368EXPORT_SYMBOL(pci_iomap_wc_range);
 369
 370void __iomem *pci_iomap_wc(struct pci_dev *dev, int bar, unsigned long maxlen)
 371{
 372	return pci_iomap_wc_range(dev, bar, 0, maxlen);
 373}
 374EXPORT_SYMBOL(pci_iomap_wc);
 375
 376static void pci_iounmap_fh(struct pci_dev *pdev, void __iomem *addr)
 377{
 378	unsigned int idx = ZPCI_IDX(addr);
 379
 380	spin_lock(&zpci_iomap_lock);
 381	/* Detect underrun */
 382	WARN_ON(!zpci_iomap_start[idx].count);
 383	if (!--zpci_iomap_start[idx].count) {
 384		zpci_iomap_start[idx].fh = 0;
 385		zpci_iomap_start[idx].bar = 0;
 386	}
 387	spin_unlock(&zpci_iomap_lock);
 388}
 389
 390static void pci_iounmap_mio(struct pci_dev *pdev, void __iomem *addr)
 391{
 392	iounmap(addr);
 393}
 394
 395void pci_iounmap(struct pci_dev *pdev, void __iomem *addr)
 396{
 397	if (static_branch_likely(&have_mio))
 398		pci_iounmap_mio(pdev, addr);
 399	else
 400		pci_iounmap_fh(pdev, addr);
 401}
 402EXPORT_SYMBOL(pci_iounmap);
 403
 404static int pci_read(struct pci_bus *bus, unsigned int devfn, int where,
 405		    int size, u32 *val)
 406{
 407	struct zpci_dev *zdev = zdev_from_bus(bus, devfn);
 408
 409	return (zdev) ? zpci_cfg_load(zdev, where, val, size) : -ENODEV;
 410}
 411
 412static int pci_write(struct pci_bus *bus, unsigned int devfn, int where,
 413		     int size, u32 val)
 414{
 415	struct zpci_dev *zdev = zdev_from_bus(bus, devfn);
 416
 417	return (zdev) ? zpci_cfg_store(zdev, where, val, size) : -ENODEV;
 418}
 419
 420static struct pci_ops pci_root_ops = {
 421	.read = pci_read,
 422	.write = pci_write,
 423};
 424
 425static void zpci_map_resources(struct pci_dev *pdev)
 426{
 427	struct zpci_dev *zdev = to_zpci(pdev);
 428	resource_size_t len;
 429	int i;
 430
 431	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 432		len = pci_resource_len(pdev, i);
 433		if (!len)
 434			continue;
 435
 436		if (zpci_use_mio(zdev))
 437			pdev->resource[i].start =
 438				(resource_size_t __force) zdev->bars[i].mio_wt;
 439		else
 440			pdev->resource[i].start = (resource_size_t __force)
 441				pci_iomap_range_fh(pdev, i, 0, 0);
 442		pdev->resource[i].end = pdev->resource[i].start + len - 1;
 443	}
 444
 445	zpci_iov_map_resources(pdev);
 446}
 447
 448static void zpci_unmap_resources(struct pci_dev *pdev)
 449{
 450	struct zpci_dev *zdev = to_zpci(pdev);
 451	resource_size_t len;
 452	int i;
 453
 454	if (zpci_use_mio(zdev))
 455		return;
 456
 457	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 458		len = pci_resource_len(pdev, i);
 459		if (!len)
 460			continue;
 461		pci_iounmap_fh(pdev, (void __iomem __force *)
 462			       pdev->resource[i].start);
 463	}
 464}
 465
 466static int zpci_alloc_iomap(struct zpci_dev *zdev)
 467{
 468	unsigned long entry;
 469
 470	spin_lock(&zpci_iomap_lock);
 471	entry = find_first_zero_bit(zpci_iomap_bitmap, ZPCI_IOMAP_ENTRIES);
 472	if (entry == ZPCI_IOMAP_ENTRIES) {
 473		spin_unlock(&zpci_iomap_lock);
 474		return -ENOSPC;
 475	}
 476	set_bit(entry, zpci_iomap_bitmap);
 477	spin_unlock(&zpci_iomap_lock);
 478	return entry;
 479}
 480
 481static void zpci_free_iomap(struct zpci_dev *zdev, int entry)
 482{
 483	spin_lock(&zpci_iomap_lock);
 484	memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry));
 485	clear_bit(entry, zpci_iomap_bitmap);
 486	spin_unlock(&zpci_iomap_lock);
 487}
 488
 489static void zpci_do_update_iomap_fh(struct zpci_dev *zdev, u32 fh)
 490{
 491	int bar, idx;
 492
 493	spin_lock(&zpci_iomap_lock);
 494	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
 495		if (!zdev->bars[bar].size)
 496			continue;
 497		idx = zdev->bars[bar].map_idx;
 498		if (!zpci_iomap_start[idx].count)
 499			continue;
 500		WRITE_ONCE(zpci_iomap_start[idx].fh, zdev->fh);
 501	}
 502	spin_unlock(&zpci_iomap_lock);
 503}
 504
 505void zpci_update_fh(struct zpci_dev *zdev, u32 fh)
 506{
 507	if (!fh || zdev->fh == fh)
 508		return;
 509
 510	zdev->fh = fh;
 511	if (zpci_use_mio(zdev))
 512		return;
 513	if (zdev->has_resources && zdev_enabled(zdev))
 514		zpci_do_update_iomap_fh(zdev, fh);
 515}
 516
 517static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start,
 518				    unsigned long size, unsigned long flags)
 519{
 520	struct resource *r;
 521
 522	r = kzalloc(sizeof(*r), GFP_KERNEL);
 523	if (!r)
 524		return NULL;
 525
 526	r->start = start;
 527	r->end = r->start + size - 1;
 528	r->flags = flags;
 529	r->name = zdev->res_name;
 530
 531	if (request_resource(&iomem_resource, r)) {
 532		kfree(r);
 533		return NULL;
 534	}
 535	return r;
 536}
 537
 538int zpci_setup_bus_resources(struct zpci_dev *zdev)
 539{
 540	unsigned long addr, size, flags;
 541	struct resource *res;
 542	int i, entry;
 543
 544	snprintf(zdev->res_name, sizeof(zdev->res_name),
 545		 "PCI Bus %04x:%02x", zdev->uid, ZPCI_BUS_NR);
 546
 547	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 548		if (!zdev->bars[i].size)
 549			continue;
 550		entry = zpci_alloc_iomap(zdev);
 551		if (entry < 0)
 552			return entry;
 553		zdev->bars[i].map_idx = entry;
 554
 555		/* only MMIO is supported */
 556		flags = IORESOURCE_MEM;
 557		if (zdev->bars[i].val & 8)
 558			flags |= IORESOURCE_PREFETCH;
 559		if (zdev->bars[i].val & 4)
 560			flags |= IORESOURCE_MEM_64;
 561
 562		if (zpci_use_mio(zdev))
 563			addr = (unsigned long) zdev->bars[i].mio_wt;
 564		else
 565			addr = ZPCI_ADDR(entry);
 566		size = 1UL << zdev->bars[i].size;
 567
 568		res = __alloc_res(zdev, addr, size, flags);
 569		if (!res) {
 570			zpci_free_iomap(zdev, entry);
 571			return -ENOMEM;
 572		}
 573		zdev->bars[i].res = res;
 574	}
 575	zdev->has_resources = 1;
 576
 577	return 0;
 578}
 579
 580static void zpci_cleanup_bus_resources(struct zpci_dev *zdev)
 581{
 582	struct resource *res;
 583	int i;
 584
 585	pci_lock_rescan_remove();
 586	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 587		res = zdev->bars[i].res;
 588		if (!res)
 589			continue;
 590
 591		release_resource(res);
 592		pci_bus_remove_resource(zdev->zbus->bus, res);
 593		zpci_free_iomap(zdev, zdev->bars[i].map_idx);
 594		zdev->bars[i].res = NULL;
 595		kfree(res);
 596	}
 597	zdev->has_resources = 0;
 598	pci_unlock_rescan_remove();
 599}
 600
 601int pcibios_device_add(struct pci_dev *pdev)
 602{
 603	struct zpci_dev *zdev = to_zpci(pdev);
 604	struct resource *res;
 605	int i;
 606
 607	/* The pdev has a reference to the zdev via its bus */
 608	zpci_zdev_get(zdev);
 609	if (pdev->is_physfn)
 610		pdev->no_vf_scan = 1;
 611
 612	zpci_map_resources(pdev);
 613
 614	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 615		res = &pdev->resource[i];
 616		if (res->parent || !res->flags)
 617			continue;
 618		pci_claim_resource(pdev, i);
 619	}
 620
 621	return 0;
 622}
 623
 624void pcibios_release_device(struct pci_dev *pdev)
 625{
 626	struct zpci_dev *zdev = to_zpci(pdev);
 627
 628	zpci_unmap_resources(pdev);
 629	zpci_zdev_put(zdev);
 630}
 631
 632int pcibios_enable_device(struct pci_dev *pdev, int mask)
 633{
 634	struct zpci_dev *zdev = to_zpci(pdev);
 635
 636	zpci_debug_init_device(zdev, dev_name(&pdev->dev));
 637	zpci_fmb_enable_device(zdev);
 638
 639	return pci_enable_resources(pdev, mask);
 640}
 641
 642void pcibios_disable_device(struct pci_dev *pdev)
 643{
 644	struct zpci_dev *zdev = to_zpci(pdev);
 645
 646	zpci_fmb_disable_device(zdev);
 647	zpci_debug_exit_device(zdev);
 648}
 649
 650static int __zpci_register_domain(int domain)
 651{
 652	spin_lock(&zpci_domain_lock);
 653	if (test_bit(domain, zpci_domain)) {
 654		spin_unlock(&zpci_domain_lock);
 655		pr_err("Domain %04x is already assigned\n", domain);
 656		return -EEXIST;
 657	}
 658	set_bit(domain, zpci_domain);
 659	spin_unlock(&zpci_domain_lock);
 660	return domain;
 661}
 662
 663static int __zpci_alloc_domain(void)
 664{
 665	int domain;
 666
 667	spin_lock(&zpci_domain_lock);
 668	/*
 669	 * We can always auto allocate domains below ZPCI_NR_DEVICES.
 670	 * There is either a free domain or we have reached the maximum in
 671	 * which case we would have bailed earlier.
 672	 */
 673	domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES);
 674	set_bit(domain, zpci_domain);
 675	spin_unlock(&zpci_domain_lock);
 676	return domain;
 677}
 678
 679int zpci_alloc_domain(int domain)
 680{
 681	if (zpci_unique_uid) {
 682		if (domain)
 683			return __zpci_register_domain(domain);
 684		pr_warn("UID checking was active but no UID is provided: switching to automatic domain allocation\n");
 685		update_uid_checking(false);
 686	}
 687	return __zpci_alloc_domain();
 688}
 689
 690void zpci_free_domain(int domain)
 691{
 692	spin_lock(&zpci_domain_lock);
 693	clear_bit(domain, zpci_domain);
 694	spin_unlock(&zpci_domain_lock);
 695}
 696
 697
 698int zpci_enable_device(struct zpci_dev *zdev)
 699{
 700	u32 fh = zdev->fh;
 701	int rc = 0;
 702
 703	if (clp_enable_fh(zdev, &fh, ZPCI_NR_DMA_SPACES))
 704		rc = -EIO;
 705	else
 706		zpci_update_fh(zdev, fh);
 707	return rc;
 708}
 709EXPORT_SYMBOL_GPL(zpci_enable_device);
 710
 711int zpci_reenable_device(struct zpci_dev *zdev)
 712{
 713	u8 status;
 714	int rc;
 715
 716	rc = zpci_enable_device(zdev);
 717	if (rc)
 718		return rc;
 719
 720	if (zdev->msi_nr_irqs > 0) {
 721		rc = zpci_set_irq(zdev);
 722		if (rc)
 723			return rc;
 724	}
 725
 726	rc = zpci_iommu_register_ioat(zdev, &status);
 727	if (rc)
 728		zpci_disable_device(zdev);
 729
 730	return rc;
 731}
 732EXPORT_SYMBOL_GPL(zpci_reenable_device);
 733
 734int zpci_disable_device(struct zpci_dev *zdev)
 735{
 736	u32 fh = zdev->fh;
 737	int cc, rc = 0;
 738
 739	cc = clp_disable_fh(zdev, &fh);
 740	if (!cc) {
 741		zpci_update_fh(zdev, fh);
 742	} else if (cc == CLP_RC_SETPCIFN_ALRDY) {
 743		pr_info("Disabling PCI function %08x had no effect as it was already disabled\n",
 744			zdev->fid);
 745		/* Function is already disabled - update handle */
 746		rc = clp_refresh_fh(zdev->fid, &fh);
 747		if (!rc) {
 748			zpci_update_fh(zdev, fh);
 749			rc = -EINVAL;
 750		}
 751	} else {
 752		rc = -EIO;
 753	}
 754	return rc;
 755}
 756EXPORT_SYMBOL_GPL(zpci_disable_device);
 757
 758/**
 759 * zpci_hot_reset_device - perform a reset of the given zPCI function
 760 * @zdev: the slot which should be reset
 761 *
 762 * Performs a low level reset of the zPCI function. The reset is low level in
 763 * the sense that the zPCI function can be reset without detaching it from the
 764 * common PCI subsystem. The reset may be performed while under control of
 765 * either DMA or IOMMU APIs in which case the existing DMA/IOMMU translation
 766 * table is reinstated at the end of the reset.
 767 *
 768 * After the reset the functions internal state is reset to an initial state
 769 * equivalent to its state during boot when first probing a driver.
 770 * Consequently after reset the PCI function requires re-initialization via the
 771 * common PCI code including re-enabling IRQs via pci_alloc_irq_vectors()
 772 * and enabling the function via e.g. pci_enable_device_flags(). The caller
 773 * must guard against concurrent reset attempts.
 774 *
 775 * In most cases this function should not be called directly but through
 776 * pci_reset_function() or pci_reset_bus() which handle the save/restore and
 777 * locking - asserted by lockdep.
 778 *
 779 * Return: 0 on success and an error value otherwise
 780 */
 781int zpci_hot_reset_device(struct zpci_dev *zdev)
 782{
 783	int rc;
 784
 785	lockdep_assert_held(&zdev->state_lock);
 786	zpci_dbg(3, "rst fid:%x, fh:%x\n", zdev->fid, zdev->fh);
 787	if (zdev_enabled(zdev)) {
 788		/* Disables device access, DMAs and IRQs (reset state) */
 789		rc = zpci_disable_device(zdev);
 790		/*
 791		 * Due to a z/VM vs LPAR inconsistency in the error state the
 792		 * FH may indicate an enabled device but disable says the
 793		 * device is already disabled don't treat it as an error here.
 794		 */
 795		if (rc == -EINVAL)
 796			rc = 0;
 797		if (rc)
 798			return rc;
 799	}
 800
 801	rc = zpci_reenable_device(zdev);
 802
 803	return rc;
 804}
 805
 806/**
 807 * zpci_create_device() - Create a new zpci_dev and add it to the zbus
 808 * @fid: Function ID of the device to be created
 809 * @fh: Current Function Handle of the device to be created
 810 * @state: Initial state after creation either Standby or Configured
 811 *
 812 * Allocates a new struct zpci_dev and queries the platform for its details.
 813 * If successful the device can subsequently be added to the zPCI subsystem
 814 * using zpci_add_device().
 815 *
 816 * Returns: the zdev on success or an error pointer otherwise
 817 */
 818struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state)
 819{
 820	struct zpci_dev *zdev;
 821	int rc;
 822
 823	zdev = kzalloc(sizeof(*zdev), GFP_KERNEL);
 824	if (!zdev)
 825		return ERR_PTR(-ENOMEM);
 826
 827	/* FID and Function Handle are the static/dynamic identifiers */
 828	zdev->fid = fid;
 829	zdev->fh = fh;
 830
 831	/* Query function properties and update zdev */
 832	rc = clp_query_pci_fn(zdev);
 833	if (rc)
 834		goto error;
 835	zdev->state =  state;
 836
 837	mutex_init(&zdev->state_lock);
 838	mutex_init(&zdev->fmb_lock);
 839	mutex_init(&zdev->kzdev_lock);
 840
 841	return zdev;
 842
 843error:
 844	zpci_dbg(0, "crt fid:%x, rc:%d\n", fid, rc);
 845	kfree(zdev);
 846	return ERR_PTR(rc);
 847}
 848
 849/**
 850 * zpci_add_device() - Add a previously created zPCI device to the zPCI subsystem
 851 * @zdev: The zPCI device to be added
 852 *
 853 * A struct zpci_dev is added to the zPCI subsystem and to a virtual PCI bus creating
 854 * a new one as necessary. A hotplug slot is created and events start to be handled.
 855 * If successful from this point on zpci_zdev_get() and zpci_zdev_put() must be used.
 856 * If adding the struct zpci_dev fails the device was not added and should be freed.
 857 *
 858 * Return: 0 on success, or an error code otherwise
 859 */
 860int zpci_add_device(struct zpci_dev *zdev)
 861{
 862	int rc;
 863
 864	mutex_lock(&zpci_add_remove_lock);
 865	zpci_dbg(1, "add fid:%x, fh:%x, c:%d\n", zdev->fid, zdev->fh, zdev->state);
 866	rc = zpci_init_iommu(zdev);
 867	if (rc)
 868		goto error;
 869
 870	rc = zpci_bus_device_register(zdev, &pci_root_ops);
 871	if (rc)
 872		goto error_destroy_iommu;
 873
 874	kref_init(&zdev->kref);
 875	spin_lock(&zpci_list_lock);
 876	list_add_tail(&zdev->entry, &zpci_list);
 877	spin_unlock(&zpci_list_lock);
 878	mutex_unlock(&zpci_add_remove_lock);
 879	return 0;
 880
 881error_destroy_iommu:
 882	zpci_destroy_iommu(zdev);
 883error:
 884	zpci_dbg(0, "add fid:%x, rc:%d\n", zdev->fid, rc);
 885	mutex_unlock(&zpci_add_remove_lock);
 886	return rc;
 887}
 888
 889bool zpci_is_device_configured(struct zpci_dev *zdev)
 890{
 891	enum zpci_state state = zdev->state;
 892
 893	return state != ZPCI_FN_STATE_RESERVED &&
 894		state != ZPCI_FN_STATE_STANDBY;
 895}
 896
 897/**
 898 * zpci_scan_configured_device() - Scan a freshly configured zpci_dev
 899 * @zdev: The zpci_dev to be configured
 900 * @fh: The general function handle supplied by the platform
 901 *
 902 * Given a device in the configuration state Configured, enables, scans and
 903 * adds it to the common code PCI subsystem if possible. If any failure occurs,
 904 * the zpci_dev is left disabled.
 905 *
 906 * Return: 0 on success, or an error code otherwise
 907 */
 908int zpci_scan_configured_device(struct zpci_dev *zdev, u32 fh)
 909{
 910	zpci_update_fh(zdev, fh);
 911	return zpci_bus_scan_device(zdev);
 912}
 913
 914/**
 915 * zpci_deconfigure_device() - Deconfigure a zpci_dev
 916 * @zdev: The zpci_dev to configure
 917 *
 918 * Deconfigure a zPCI function that is currently configured and possibly known
 919 * to the common code PCI subsystem.
 920 * If any failure occurs the device is left as is.
 921 *
 922 * Return: 0 on success, or an error code otherwise
 923 */
 924int zpci_deconfigure_device(struct zpci_dev *zdev)
 925{
 926	int rc;
 927
 928	lockdep_assert_held(&zdev->state_lock);
 929	if (zdev->state != ZPCI_FN_STATE_CONFIGURED)
 930		return 0;
 931
 932	if (zdev->zbus->bus)
 933		zpci_bus_remove_device(zdev, false);
 934
 935	if (zdev_enabled(zdev)) {
 936		rc = zpci_disable_device(zdev);
 937		if (rc)
 938			return rc;
 939	}
 940
 941	rc = sclp_pci_deconfigure(zdev->fid);
 942	zpci_dbg(3, "deconf fid:%x, rc:%d\n", zdev->fid, rc);
 943	if (rc)
 944		return rc;
 945	zdev->state = ZPCI_FN_STATE_STANDBY;
 946
 947	return 0;
 948}
 949
 950/**
 951 * zpci_device_reserved() - Mark device as reserved
 952 * @zdev: the zpci_dev that was reserved
 953 *
 954 * Handle the case that a given zPCI function was reserved by another system.
 955 */
 956void zpci_device_reserved(struct zpci_dev *zdev)
 957{
 958	lockdep_assert_held(&zdev->state_lock);
 959	/* We may declare the device reserved multiple times */
 960	if (zdev->state == ZPCI_FN_STATE_RESERVED)
 961		return;
 962	zdev->state = ZPCI_FN_STATE_RESERVED;
 963	zpci_dbg(3, "rsv fid:%x\n", zdev->fid);
 964	/*
 965	 * The underlying device is gone. Allow the zdev to be freed
 966	 * as soon as all other references are gone by accounting for
 967	 * the removal as a dropped reference.
 968	 */
 969	zpci_zdev_put(zdev);
 970}
 971
 972void zpci_release_device(struct kref *kref)
 973	__releases(&zpci_list_lock)
 974{
 975	struct zpci_dev *zdev = container_of(kref, struct zpci_dev, kref);
 976
 977	lockdep_assert_held(&zpci_add_remove_lock);
 978	WARN_ON(zdev->state != ZPCI_FN_STATE_RESERVED);
 979	/*
 980	 * We already hold zpci_list_lock thanks to kref_put_lock().
 981	 * This makes sure no new reference can be taken from the list.
 982	 */
 983	list_del(&zdev->entry);
 984	spin_unlock(&zpci_list_lock);
 985
 986	if (zdev->has_hp_slot)
 987		zpci_exit_slot(zdev);
 988
 989	if (zdev->has_resources)
 990		zpci_cleanup_bus_resources(zdev);
 991
 992	zpci_bus_device_unregister(zdev);
 993	zpci_destroy_iommu(zdev);
 994	zpci_dbg(3, "rem fid:%x\n", zdev->fid);
 995	kfree_rcu(zdev, rcu);
 996}
 997
 998int zpci_report_error(struct pci_dev *pdev,
 999		      struct zpci_report_error_header *report)
1000{
1001	struct zpci_dev *zdev = to_zpci(pdev);
1002
1003	return sclp_pci_report(report, zdev->fh, zdev->fid);
1004}
1005EXPORT_SYMBOL(zpci_report_error);
1006
1007/**
1008 * zpci_clear_error_state() - Clears the zPCI error state of the device
1009 * @zdev: The zdev for which the zPCI error state should be reset
1010 *
1011 * Clear the zPCI error state of the device. If clearing the zPCI error state
1012 * fails the device is left in the error state. In this case it may make sense
1013 * to call zpci_io_perm_failure() on the associated pdev if it exists.
1014 *
1015 * Returns: 0 on success, -EIO otherwise
1016 */
1017int zpci_clear_error_state(struct zpci_dev *zdev)
1018{
1019	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_ERROR);
1020	struct zpci_fib fib = {0};
1021	u8 status;
1022	int cc;
1023
1024	cc = zpci_mod_fc(req, &fib, &status);
1025	if (cc) {
1026		zpci_dbg(3, "ces fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status);
1027		return -EIO;
1028	}
1029
1030	return 0;
1031}
1032
1033/**
1034 * zpci_reset_load_store_blocked() - Re-enables L/S from error state
1035 * @zdev: The zdev for which to unblock load/store access
1036 *
1037 * Re-enables load/store access for a PCI function in the error state while
1038 * keeping DMA blocked. In this state drivers can poke MMIO space to determine
1039 * if error recovery is possible while catching any rogue DMA access from the
1040 * device.
1041 *
1042 * Returns: 0 on success, -EIO otherwise
1043 */
1044int zpci_reset_load_store_blocked(struct zpci_dev *zdev)
1045{
1046	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_BLOCK);
1047	struct zpci_fib fib = {0};
1048	u8 status;
1049	int cc;
1050
1051	cc = zpci_mod_fc(req, &fib, &status);
1052	if (cc) {
1053		zpci_dbg(3, "rls fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status);
1054		return -EIO;
1055	}
1056
1057	return 0;
1058}
1059
1060static int zpci_mem_init(void)
1061{
1062	BUILD_BUG_ON(!is_power_of_2(__alignof__(struct zpci_fmb)) ||
1063		     __alignof__(struct zpci_fmb) < sizeof(struct zpci_fmb));
1064
1065	zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb),
1066					   __alignof__(struct zpci_fmb), 0, NULL);
1067	if (!zdev_fmb_cache)
1068		goto error_fmb;
1069
1070	zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES,
1071				   sizeof(*zpci_iomap_start), GFP_KERNEL);
1072	if (!zpci_iomap_start)
1073		goto error_iomap;
1074
1075	zpci_iomap_bitmap = kcalloc(BITS_TO_LONGS(ZPCI_IOMAP_ENTRIES),
1076				    sizeof(*zpci_iomap_bitmap), GFP_KERNEL);
1077	if (!zpci_iomap_bitmap)
1078		goto error_iomap_bitmap;
1079
1080	if (static_branch_likely(&have_mio))
1081		clp_setup_writeback_mio();
1082
1083	return 0;
1084error_iomap_bitmap:
1085	kfree(zpci_iomap_start);
1086error_iomap:
1087	kmem_cache_destroy(zdev_fmb_cache);
1088error_fmb:
1089	return -ENOMEM;
1090}
1091
1092static void zpci_mem_exit(void)
1093{
1094	kfree(zpci_iomap_bitmap);
1095	kfree(zpci_iomap_start);
1096	kmem_cache_destroy(zdev_fmb_cache);
1097}
1098
1099static unsigned int s390_pci_probe __initdata = 1;
1100unsigned int s390_pci_force_floating __initdata;
1101static unsigned int s390_pci_initialized;
1102
1103char * __init pcibios_setup(char *str)
1104{
1105	if (!strcmp(str, "off")) {
1106		s390_pci_probe = 0;
1107		return NULL;
1108	}
1109	if (!strcmp(str, "nomio")) {
1110		clear_machine_feature(MFEATURE_PCI_MIO);
1111		return NULL;
1112	}
1113	if (!strcmp(str, "force_floating")) {
1114		s390_pci_force_floating = 1;
1115		return NULL;
1116	}
1117	if (!strcmp(str, "norid")) {
1118		s390_pci_no_rid = 1;
1119		return NULL;
1120	}
1121	return str;
1122}
1123
1124bool zpci_is_enabled(void)
1125{
1126	return s390_pci_initialized;
1127}
1128
1129static int zpci_cmp_rid(void *priv, const struct list_head *a,
1130			const struct list_head *b)
1131{
1132	struct zpci_dev *za = container_of(a, struct zpci_dev, entry);
1133	struct zpci_dev *zb = container_of(b, struct zpci_dev, entry);
1134
1135	/*
1136	 * PCI functions without RID available maintain original order
1137	 * between themselves but sort before those with RID.
1138	 */
1139	if (za->rid == zb->rid)
1140		return za->rid_available > zb->rid_available;
1141	/*
1142	 * PCI functions with RID sort by RID ascending.
1143	 */
1144	return za->rid > zb->rid;
1145}
1146
1147static void zpci_add_devices(struct list_head *scan_list)
1148{
1149	struct zpci_dev *zdev, *tmp;
1150
1151	list_sort(NULL, scan_list, &zpci_cmp_rid);
1152	list_for_each_entry_safe(zdev, tmp, scan_list, entry) {
1153		list_del_init(&zdev->entry);
1154		if (zpci_add_device(zdev))
1155			kfree(zdev);
1156	}
1157}
1158
1159int zpci_scan_devices(void)
1160{
1161	struct zpci_bus *zbus;
1162	LIST_HEAD(scan_list);
1163	int rc;
1164
1165	rc = clp_scan_pci_devices(&scan_list);
1166	if (rc)
1167		return rc;
1168
1169	zpci_add_devices(&scan_list);
1170	zpci_bus_for_each(zbus) {
1171		zpci_bus_scan_bus(zbus);
1172		cond_resched();
1173	}
1174	return 0;
1175}
1176
1177static int __init pci_base_init(void)
1178{
1179	int rc;
1180
1181	if (!s390_pci_probe)
1182		return 0;
1183
1184	if (!test_facility(69) || !test_facility(71)) {
1185		pr_info("PCI is not supported because CPU facilities 69 or 71 are not available\n");
1186		return 0;
1187	}
1188
1189	if (test_machine_feature(MFEATURE_PCI_MIO)) {
1190		static_branch_enable(&have_mio);
1191		system_ctl_set_bit(2, CR2_MIO_ADDRESSING_BIT);
1192	}
1193
1194	rc = zpci_debug_init();
1195	if (rc)
1196		goto out;
1197
1198	rc = zpci_mem_init();
1199	if (rc)
1200		goto out_mem;
1201
1202	rc = zpci_irq_init();
1203	if (rc)
1204		goto out_irq;
1205
1206	rc = zpci_scan_devices();
1207	if (rc)
1208		goto out_find;
1209
1210	rc = zpci_fw_sysfs_init();
1211	if (rc)
1212		goto out_find;
1213
1214	s390_pci_initialized = 1;
1215	return 0;
1216
1217out_find:
1218	zpci_irq_exit();
1219out_irq:
1220	zpci_mem_exit();
1221out_mem:
1222	zpci_debug_exit();
1223out:
1224	return rc;
1225}
1226subsys_initcall_sync(pci_base_init);