pci_bus.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright IBM Corp. 2020
  4 *
  5 * Author(s):
  6 *   Pierre Morel <pmorel@linux.ibm.com>
  7 *
  8 */
  9
 10#define pr_fmt(fmt) "zpci: " fmt
 11
 12#include <linux/kernel.h>
 13#include <linux/slab.h>
 14#include <linux/err.h>
 15#include <linux/delay.h>
 16#include <linux/seq_file.h>
 17#include <linux/irqdomain.h>
 18#include <linux/jump_label.h>
 19#include <linux/pci.h>
 20#include <linux/printk.h>
 21#include <linux/dma-direct.h>
 22
 23#include <asm/pci_clp.h>
 24#include <asm/pci_dma.h>
 25
 26#include "pci_bus.h"
 27#include "pci_iov.h"
 28
 29static LIST_HEAD(zbus_list);
 30static DEFINE_MUTEX(zbus_list_lock);
 31static int zpci_nb_devices;
 32
 33/* zpci_bus_prepare_device - Prepare a zPCI function for scanning
 34 * @zdev: the zPCI function to be prepared
 35 *
 36 * The PCI resources for the function are set up and added to its zbus and the
 37 * function is enabled. The function must be added to a zbus which must have
 38 * a PCI bus created. If an error occurs the zPCI function is not enabled.
 39 *
 40 * Return: 0 on success, an error code otherwise
 41 */
 42static int zpci_bus_prepare_device(struct zpci_dev *zdev)
 43{
 44	int rc, i;
 45
 46	if (!zdev_enabled(zdev)) {
 47		rc = zpci_enable_device(zdev);
 48		if (rc) {
 49			pr_err("Enabling PCI function %08x failed\n", zdev->fid);
 50			return rc;
 51		}
 52	}
 53
 54	if (!zdev->has_resources) {
 55		zpci_setup_bus_resources(zdev);
 56		for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 57			if (zdev->bars[i].res)
 58				pci_bus_add_resource(zdev->zbus->bus, zdev->bars[i].res);
 59		}
 60	}
 61
 62	return 0;
 63}
 64
 65/* zpci_bus_scan_device - Scan a single device adding it to the PCI core
 66 * @zdev: the zdev to be scanned
 67 *
 68 * Scans the PCI function making it available to the common PCI code.
 69 *
 70 * Return: 0 on success, an error value otherwise
 71 */
 72int zpci_bus_scan_device(struct zpci_dev *zdev)
 73{
 74	struct pci_dev *pdev;
 75	int rc;
 76
 77	rc = zpci_bus_prepare_device(zdev);
 78	if (rc)
 79		return rc;
 80
 81	pdev = pci_scan_single_device(zdev->zbus->bus, zdev->devfn);
 82	if (!pdev)
 83		return -ENODEV;
 84
 85	pci_lock_rescan_remove();
 86	pci_bus_add_device(pdev);
 87	pci_unlock_rescan_remove();
 88
 89	return 0;
 90}
 91
 92/* zpci_bus_remove_device - Removes the given zdev from the PCI core
 93 * @zdev: the zdev to be removed from the PCI core
 94 * @set_error: if true the device's error state is set to permanent failure
 95 *
 96 * Sets a zPCI device to a configured but offline state; the zPCI
 97 * device is still accessible through its hotplug slot and the zPCI
 98 * API but is removed from the common code PCI bus, making it
 99 * no longer available to drivers.
100 */
101void zpci_bus_remove_device(struct zpci_dev *zdev, bool set_error)
102{
103	struct zpci_bus *zbus = zdev->zbus;
104	struct pci_dev *pdev;
105
106	if (!zdev->zbus->bus)
107		return;
108
109	pdev = pci_get_slot(zbus->bus, zdev->devfn);
110	if (pdev) {
111		if (set_error)
112			pdev->error_state = pci_channel_io_perm_failure;
113		if (pdev->is_virtfn) {
114			zpci_iov_remove_virtfn(pdev, zdev->vfn);
115			/* balance pci_get_slot */
116			pci_dev_put(pdev);
117			return;
118		}
119		pci_stop_and_remove_bus_device_locked(pdev);
120		/* balance pci_get_slot */
121		pci_dev_put(pdev);
122	}
123}
124
125/* zpci_bus_scan_bus - Scan all configured zPCI functions on the bus
126 * @zbus: the zbus to be scanned
127 *
128 * Enables and scans all PCI functions on the bus making them available to the
129 * common PCI code. If a PCI function fails to be initialized an error will be
130 * returned but attempts will still be made for all other functions on the bus.
131 *
132 * Return: 0 on success, an error value otherwise
133 */
134int zpci_bus_scan_bus(struct zpci_bus *zbus)
135{
136	struct zpci_dev *zdev;
137	int devfn, rc, ret = 0;
138
139	for (devfn = 0; devfn < ZPCI_FUNCTIONS_PER_BUS; devfn++) {
140		zdev = zbus->function[devfn];
141		if (zdev && zdev->state == ZPCI_FN_STATE_CONFIGURED) {
142			rc = zpci_bus_prepare_device(zdev);
143			if (rc)
144				ret = -EIO;
145		}
146	}
147
148	pci_lock_rescan_remove();
149	pci_scan_child_bus(zbus->bus);
150	pci_bus_add_devices(zbus->bus);
151	pci_unlock_rescan_remove();
152
153	return ret;
154}
155
156static bool zpci_bus_is_multifunction_root(struct zpci_dev *zdev)
157{
158	return !s390_pci_no_rid && zdev->rid_available &&
159		!zdev->vfn;
160}
161
162/* zpci_bus_create_pci_bus - Create the PCI bus associated with this zbus
163 * @zbus: the zbus holding the zdevices
164 * @fr: PCI root function that will determine the bus's domain, and bus speed
165 * @ops: the pci operations
166 *
167 * The PCI function @fr determines the domain (its UID), multifunction property
168 * and maximum bus speed of the entire bus.
169 *
170 * Return: 0 on success, an error code otherwise
171 */
172static int zpci_bus_create_pci_bus(struct zpci_bus *zbus, struct zpci_dev *fr, struct pci_ops *ops)
173{
174	struct pci_bus *bus;
175	int domain;
176
177	domain = zpci_alloc_domain((u16)fr->uid);
178	if (domain < 0)
179		return domain;
180
181	zbus->domain_nr = domain;
182	zbus->multifunction = zpci_bus_is_multifunction_root(fr);
183	zbus->max_bus_speed = fr->max_bus_speed;
184
185	if (zpci_create_parent_msi_domain(zbus))
186		goto out_free_domain;
187
188	/*
189	 * Note that the zbus->resources are taken over and zbus->resources
190	 * is empty after a successful call
191	 */
192	bus = pci_create_root_bus(NULL, ZPCI_BUS_NR, ops, zbus, &zbus->resources);
193	if (!bus)
194		goto out_remove_msi_domain;
195
196	zbus->bus = bus;
197	dev_set_msi_domain(&zbus->bus->dev, zbus->msi_parent_domain);
198
199	return 0;
200
201out_remove_msi_domain:
202	zpci_remove_parent_msi_domain(zbus);
203out_free_domain:
204	zpci_free_domain(zbus->domain_nr);
205	return -ENOMEM;
206}
207
208/**
209 * zpci_bus_release - Un-initialize resources associated with the zbus and
210 *		      free memory
211 * @kref:	refcount * that is part of struct zpci_bus
212 *
213 * MUST be called with `zbus_list_lock` held, but the lock is released during
214 * run of the function.
215 */
216static inline void zpci_bus_release(struct kref *kref)
217	__releases(&zbus_list_lock)
218{
219	struct zpci_bus *zbus = container_of(kref, struct zpci_bus, kref);
220
221	lockdep_assert_held(&zbus_list_lock);
222
223	list_del(&zbus->bus_next);
224	mutex_unlock(&zbus_list_lock);
225
226	/*
227	 * At this point no-one should see this object, or be able to get a new
228	 * reference to it.
229	 */
230
231	if (zbus->bus) {
232		pci_lock_rescan_remove();
233		pci_stop_root_bus(zbus->bus);
234
235		zpci_free_domain(zbus->domain_nr);
236		pci_free_resource_list(&zbus->resources);
237
238		pci_remove_root_bus(zbus->bus);
239		pci_unlock_rescan_remove();
240	}
241
242	zpci_remove_parent_msi_domain(zbus);
243	kfree(zbus);
244}
245
246static inline void __zpci_bus_get(struct zpci_bus *zbus)
247{
248	lockdep_assert_held(&zbus_list_lock);
249	kref_get(&zbus->kref);
250}
251
252static inline void zpci_bus_put(struct zpci_bus *zbus)
253{
254	kref_put_mutex(&zbus->kref, zpci_bus_release, &zbus_list_lock);
255}
256
257static struct zpci_bus *zpci_bus_get(int topo, bool topo_is_tid)
258{
259	struct zpci_bus *zbus;
260
261	mutex_lock(&zbus_list_lock);
262	list_for_each_entry(zbus, &zbus_list, bus_next) {
263		if (!zbus->multifunction)
264			continue;
265		if (topo_is_tid == zbus->topo_is_tid && topo == zbus->topo) {
266			__zpci_bus_get(zbus);
267			goto out_unlock;
268		}
269	}
270	zbus = NULL;
271out_unlock:
272	mutex_unlock(&zbus_list_lock);
273	return zbus;
274}
275
276/**
277 * zpci_bus_get_next - get the next zbus object from given position in the list
278 * @pos:	current position/cursor in the global zbus list
279 *
280 * Acquires and releases references as the cursor iterates (might also free/
281 * release the cursor). Is tolerant of concurrent operations on the list.
282 *
283 * To begin the iteration, set *@pos to %NULL before calling the function.
284 *
285 * *@pos is set to %NULL in cases where either the list is empty, or *@pos is
286 * the last element in the list.
287 *
288 * Context: Process context. May sleep.
289 */
290void zpci_bus_get_next(struct zpci_bus **pos)
291{
292	struct zpci_bus *curp = *pos, *next = NULL;
293
294	mutex_lock(&zbus_list_lock);
295	if (curp)
296		next = list_next_entry(curp, bus_next);
297	else
298		next = list_first_entry(&zbus_list, typeof(*curp), bus_next);
299
300	if (list_entry_is_head(next, &zbus_list, bus_next))
301		next = NULL;
302
303	if (next)
304		__zpci_bus_get(next);
305
306	*pos = next;
307	mutex_unlock(&zbus_list_lock);
308
309	/* zpci_bus_put() might drop refcount to 0 and locks zbus_list_lock */
310	if (curp)
311		zpci_bus_put(curp);
312}
313
314static struct zpci_bus *zpci_bus_alloc(int topo, bool topo_is_tid)
315{
316	struct zpci_bus *zbus;
317
318	zbus = kzalloc(sizeof(*zbus), GFP_KERNEL);
319	if (!zbus)
320		return NULL;
321
322	zbus->topo = topo;
323	zbus->topo_is_tid = topo_is_tid;
324	INIT_LIST_HEAD(&zbus->bus_next);
325
326	kref_init(&zbus->kref);
327	INIT_LIST_HEAD(&zbus->resources);
328
329	zbus->bus_resource.start = 0;
330	zbus->bus_resource.end = ZPCI_BUS_NR;
331	zbus->bus_resource.flags = IORESOURCE_BUS;
332	pci_add_resource(&zbus->resources, &zbus->bus_resource);
333
334	mutex_lock(&zbus_list_lock);
335	list_add_tail(&zbus->bus_next, &zbus_list);
336	mutex_unlock(&zbus_list_lock);
337
338	return zbus;
339}
340
341static void pci_dma_range_setup(struct pci_dev *pdev)
342{
343	struct zpci_dev *zdev = to_zpci(pdev);
344	u64 aligned_end, size;
345	dma_addr_t dma_start;
346	int ret;
347
348	dma_start = PAGE_ALIGN(zdev->start_dma);
349	aligned_end = PAGE_ALIGN_DOWN(zdev->end_dma + 1);
350	if (aligned_end >= dma_start)
351		size = aligned_end - dma_start;
352	else
353		size = 0;
354	WARN_ON_ONCE(size == 0);
355
356	ret = dma_direct_set_offset(&pdev->dev, 0, dma_start, size);
357	if (ret)
358		pr_err("Failed to allocate DMA range map for %s\n", pci_name(pdev));
359}
360
361void pcibios_bus_add_device(struct pci_dev *pdev)
362{
363	struct zpci_dev *zdev = to_zpci(pdev);
364
365	pci_dma_range_setup(pdev);
366
367	/*
368	 * With pdev->no_vf_scan the common PCI probing code does not
369	 * perform PF/VF linking.
370	 */
371	if (zdev->vfn) {
372		zpci_iov_setup_virtfn(zdev->zbus, pdev, zdev->vfn);
373		pdev->no_command_memory = 1;
374	}
375}
376
377static int zpci_bus_add_device(struct zpci_bus *zbus, struct zpci_dev *zdev)
378{
379	int rc = -EINVAL;
380
381	if (zbus->multifunction) {
382		if (!zdev->rid_available) {
383			WARN_ONCE(1, "rid_available not set for multifunction\n");
384			return rc;
385		}
386		zdev->devfn = zdev->rid & ZPCI_RID_MASK_DEVFN;
387	}
388
389	if (zbus->function[zdev->devfn]) {
390		pr_err("devfn %04x is already assigned\n", zdev->devfn);
391		return rc;
392	}
393	zdev->zbus = zbus;
394	zbus->function[zdev->devfn] = zdev;
395	zpci_nb_devices++;
396
397	rc = zpci_init_slot(zdev);
398	if (rc)
399		goto error;
400	zdev->has_hp_slot = 1;
401
402	return 0;
403
404error:
405	zbus->function[zdev->devfn] = NULL;
406	zdev->zbus = NULL;
407	zpci_nb_devices--;
408	return rc;
409}
410
411static bool zpci_bus_is_isolated_vf(struct zpci_bus *zbus, struct zpci_dev *zdev)
412{
413	struct pci_dev *pdev;
414
415	if (!zdev->vfn)
416		return false;
417
418	pdev = zpci_iov_find_parent_pf(zbus, zdev);
419	if (!pdev)
420		return true;
421	pci_dev_put(pdev);
422	return false;
423}
424
425int zpci_bus_device_register(struct zpci_dev *zdev, struct pci_ops *ops)
426{
427	bool topo_is_tid = zdev->tid_avail;
428	struct zpci_bus *zbus = NULL;
429	int topo, rc = -EBADF;
430
431	if (zpci_nb_devices == ZPCI_NR_DEVICES) {
432		pr_warn("Adding PCI function %08x failed because the configured limit of %d is reached\n",
433			zdev->fid, ZPCI_NR_DEVICES);
434		return -ENOSPC;
435	}
436
437	topo = topo_is_tid ? zdev->tid : zdev->pchid;
438	zbus = zpci_bus_get(topo, topo_is_tid);
439	/*
440	 * An isolated VF gets its own domain/bus even if there exists
441	 * a matching domain/bus already
442	 */
443	if (zbus && zpci_bus_is_isolated_vf(zbus, zdev)) {
444		zpci_bus_put(zbus);
445		zbus = NULL;
446	}
447
448	if (!zbus) {
449		zbus = zpci_bus_alloc(topo, topo_is_tid);
450		if (!zbus)
451			return -ENOMEM;
452	}
453
454	if (!zbus->bus) {
455		/* The UID of the first PCI function registered with a zpci_bus
456		 * is used as the domain number for that bus. Currently there
457		 * is exactly one zpci_bus per domain.
458		 */
459		rc = zpci_bus_create_pci_bus(zbus, zdev, ops);
460		if (rc)
461			goto error;
462	}
463
464	rc = zpci_bus_add_device(zbus, zdev);
465	if (rc)
466		goto error;
467
468	return 0;
469
470error:
471	pr_err("Adding PCI function %08x failed\n", zdev->fid);
472	zpci_bus_put(zbus);
473	return rc;
474}
475
476void zpci_bus_device_unregister(struct zpci_dev *zdev)
477{
478	struct zpci_bus *zbus = zdev->zbus;
479
480	zpci_nb_devices--;
481	zbus->function[zdev->devfn] = NULL;
482	zpci_bus_put(zbus);
483}