pci-epf-core.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * PCI Endpoint *Function* (EPF) library
  4 *
  5 * Copyright (C) 2017 Texas Instruments
  6 * Author: Kishon Vijay Abraham I <kishon@ti.com>
  7 */
  8
  9#include <linux/device.h>
 10#include <linux/dma-mapping.h>
 11#include <linux/slab.h>
 12#include <linux/module.h>
 13
 14#include <linux/pci-epc.h>
 15#include <linux/pci-epf.h>
 16#include <linux/pci-ep-cfs.h>
 17
 18static DEFINE_MUTEX(pci_epf_mutex);
 19
 20static const struct bus_type pci_epf_bus_type;
 21static const struct device_type pci_epf_type;
 22
 23/**
 24 * pci_epf_unbind() - Notify the function driver that the binding between the
 25 *		      EPF device and EPC device has been lost
 26 * @epf: the EPF device which has lost the binding with the EPC device
 27 *
 28 * Invoke to notify the function driver that the binding between the EPF device
 29 * and EPC device has been lost.
 30 */
 31void pci_epf_unbind(struct pci_epf *epf)
 32{
 33	struct pci_epf *epf_vf;
 34
 35	if (!epf->driver) {
 36		dev_WARN(&epf->dev, "epf device not bound to driver\n");
 37		return;
 38	}
 39
 40	mutex_lock(&epf->lock);
 41	list_for_each_entry(epf_vf, &epf->pci_vepf, list) {
 42		if (epf_vf->is_bound)
 43			epf_vf->driver->ops->unbind(epf_vf);
 44	}
 45	if (epf->is_bound)
 46		epf->driver->ops->unbind(epf);
 47	mutex_unlock(&epf->lock);
 48	module_put(epf->driver->owner);
 49}
 50EXPORT_SYMBOL_GPL(pci_epf_unbind);
 51
 52/**
 53 * pci_epf_bind() - Notify the function driver that the EPF device has been
 54 *		    bound to a EPC device
 55 * @epf: the EPF device which has been bound to the EPC device
 56 *
 57 * Invoke to notify the function driver that it has been bound to a EPC device
 58 */
 59int pci_epf_bind(struct pci_epf *epf)
 60{
 61	struct device *dev = &epf->dev;
 62	struct pci_epf *epf_vf;
 63	u8 func_no, vfunc_no;
 64	struct pci_epc *epc;
 65	int ret;
 66
 67	if (!epf->driver) {
 68		dev_WARN(dev, "epf device not bound to driver\n");
 69		return -EINVAL;
 70	}
 71
 72	if (!try_module_get(epf->driver->owner))
 73		return -EAGAIN;
 74
 75	mutex_lock(&epf->lock);
 76	list_for_each_entry(epf_vf, &epf->pci_vepf, list) {
 77		vfunc_no = epf_vf->vfunc_no;
 78
 79		if (vfunc_no < 1) {
 80			dev_err(dev, "Invalid virtual function number\n");
 81			ret = -EINVAL;
 82			goto ret;
 83		}
 84
 85		epc = epf->epc;
 86		func_no = epf->func_no;
 87		if (!IS_ERR_OR_NULL(epc)) {
 88			if (!epc->max_vfs) {
 89				dev_err(dev, "No support for virt function\n");
 90				ret = -EINVAL;
 91				goto ret;
 92			}
 93
 94			if (vfunc_no > epc->max_vfs[func_no]) {
 95				dev_err(dev, "PF%d: Exceeds max vfunc number\n",
 96					func_no);
 97				ret = -EINVAL;
 98				goto ret;
 99			}
100		}
101
102		epc = epf->sec_epc;
103		func_no = epf->sec_epc_func_no;
104		if (!IS_ERR_OR_NULL(epc)) {
105			if (!epc->max_vfs) {
106				dev_err(dev, "No support for virt function\n");
107				ret = -EINVAL;
108				goto ret;
109			}
110
111			if (vfunc_no > epc->max_vfs[func_no]) {
112				dev_err(dev, "PF%d: Exceeds max vfunc number\n",
113					func_no);
114				ret = -EINVAL;
115				goto ret;
116			}
117		}
118
119		epf_vf->func_no = epf->func_no;
120		epf_vf->sec_epc_func_no = epf->sec_epc_func_no;
121		epf_vf->epc = epf->epc;
122		epf_vf->sec_epc = epf->sec_epc;
123		ret = epf_vf->driver->ops->bind(epf_vf);
124		if (ret)
125			goto ret;
126		epf_vf->is_bound = true;
127	}
128
129	ret = epf->driver->ops->bind(epf);
130	if (ret)
131		goto ret;
132	epf->is_bound = true;
133
134	mutex_unlock(&epf->lock);
135	return 0;
136
137ret:
138	mutex_unlock(&epf->lock);
139	pci_epf_unbind(epf);
140
141	return ret;
142}
143EXPORT_SYMBOL_GPL(pci_epf_bind);
144
145/**
146 * pci_epf_add_vepf() - associate virtual EP function to physical EP function
147 * @epf_pf: the physical EP function to which the virtual EP function should be
148 *   associated
149 * @epf_vf: the virtual EP function to be added
150 *
151 * A physical endpoint function can be associated with multiple virtual
152 * endpoint functions. Invoke pci_epf_add_epf() to add a virtual PCI endpoint
153 * function to a physical PCI endpoint function.
154 */
155int pci_epf_add_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf)
156{
157	u32 vfunc_no;
158
159	if (IS_ERR_OR_NULL(epf_pf) || IS_ERR_OR_NULL(epf_vf))
160		return -EINVAL;
161
162	if (epf_pf->epc || epf_vf->epc || epf_vf->epf_pf)
163		return -EBUSY;
164
165	if (epf_pf->sec_epc || epf_vf->sec_epc)
166		return -EBUSY;
167
168	mutex_lock(&epf_pf->lock);
169	vfunc_no = find_first_zero_bit(&epf_pf->vfunction_num_map,
170				       BITS_PER_LONG);
171	if (vfunc_no >= BITS_PER_LONG) {
172		mutex_unlock(&epf_pf->lock);
173		return -EINVAL;
174	}
175
176	set_bit(vfunc_no, &epf_pf->vfunction_num_map);
177	epf_vf->vfunc_no = vfunc_no;
178
179	epf_vf->epf_pf = epf_pf;
180	epf_vf->is_vf = true;
181
182	list_add_tail(&epf_vf->list, &epf_pf->pci_vepf);
183	mutex_unlock(&epf_pf->lock);
184
185	return 0;
186}
187EXPORT_SYMBOL_GPL(pci_epf_add_vepf);
188
189/**
190 * pci_epf_remove_vepf() - remove virtual EP function from physical EP function
191 * @epf_pf: the physical EP function from which the virtual EP function should
192 *   be removed
193 * @epf_vf: the virtual EP function to be removed
194 *
195 * Invoke to remove a virtual endpoint function from the physical endpoint
196 * function.
197 */
198void pci_epf_remove_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf)
199{
200	if (IS_ERR_OR_NULL(epf_pf) || IS_ERR_OR_NULL(epf_vf))
201		return;
202
203	mutex_lock(&epf_pf->lock);
204	clear_bit(epf_vf->vfunc_no, &epf_pf->vfunction_num_map);
205	epf_vf->epf_pf = NULL;
206	list_del(&epf_vf->list);
207	mutex_unlock(&epf_pf->lock);
208}
209EXPORT_SYMBOL_GPL(pci_epf_remove_vepf);
210
211static int pci_epf_get_required_bar_size(struct pci_epf *epf, size_t *bar_size,
212				size_t *aligned_mem_size,
213				enum pci_barno bar,
214				const struct pci_epc_features *epc_features,
215				enum pci_epc_interface_type type)
216{
217	u64 bar_fixed_size = epc_features->bar[bar].fixed_size;
218	size_t align = epc_features->align;
219	size_t size = *bar_size;
220
221	if (size < 128)
222		size = 128;
223
224	/* According to PCIe base spec, min size for a resizable BAR is 1 MB. */
225	if (epc_features->bar[bar].type == BAR_RESIZABLE && size < SZ_1M)
226		size = SZ_1M;
227
228	if (epc_features->bar[bar].type == BAR_FIXED && bar_fixed_size) {
229		if (size > bar_fixed_size) {
230			dev_err(&epf->dev,
231				"requested BAR size is larger than fixed size\n");
232			return -ENOMEM;
233		}
234		size = bar_fixed_size;
235	} else {
236		/* BAR size must be power of two */
237		size = roundup_pow_of_two(size);
238	}
239
240	*bar_size = size;
241
242	/*
243	 * The EPC's BAR start address must meet alignment requirements. In most
244	 * cases, the alignment will match the BAR size. However, differences
245	 * can occur—for example, when the fixed BAR size (e.g., 128 bytes) is
246	 * smaller than the required alignment (e.g., 4 KB).
247	 */
248	*aligned_mem_size = align ? ALIGN(size, align) : size;
249
250	return 0;
251}
252
253/**
254 * pci_epf_free_space() - free the allocated PCI EPF register space
255 * @epf: the EPF device from whom to free the memory
256 * @addr: the virtual address of the PCI EPF register space
257 * @bar: the BAR number corresponding to the register space
258 * @type: Identifies if the allocated space is for primary EPC or secondary EPC
259 *
260 * Invoke to free the allocated PCI EPF register space.
261 */
262void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar,
263			enum pci_epc_interface_type type)
264{
265	struct device *dev;
266	struct pci_epf_bar *epf_bar;
267	struct pci_epc *epc;
268
269	if (!addr)
270		return;
271
272	if (type == PRIMARY_INTERFACE) {
273		epc = epf->epc;
274		epf_bar = epf->bar;
275	} else {
276		epc = epf->sec_epc;
277		epf_bar = epf->sec_epc_bar;
278	}
279
280	dev = epc->dev.parent;
281	dma_free_coherent(dev, epf_bar[bar].mem_size, addr,
282			  epf_bar[bar].phys_addr);
283
284	epf_bar[bar].phys_addr = 0;
285	epf_bar[bar].addr = NULL;
286	epf_bar[bar].size = 0;
287	epf_bar[bar].mem_size = 0;
288	epf_bar[bar].barno = 0;
289	epf_bar[bar].flags = 0;
290}
291EXPORT_SYMBOL_GPL(pci_epf_free_space);
292
293/**
294 * pci_epf_alloc_space() - allocate memory for the PCI EPF register space
295 * @epf: the EPF device to whom allocate the memory
296 * @size: the size of the memory that has to be allocated
297 * @bar: the BAR number corresponding to the allocated register space
298 * @epc_features: the features provided by the EPC specific to this EPF
299 * @type: Identifies if the allocation is for primary EPC or secondary EPC
300 *
301 * Invoke to allocate memory for the PCI EPF register space.
302 * Flag PCI_BASE_ADDRESS_MEM_TYPE_64 will automatically get set if the BAR
303 * can only be a 64-bit BAR, or if the requested size is larger than 2 GB.
304 */
305void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar,
306			  const struct pci_epc_features *epc_features,
307			  enum pci_epc_interface_type type)
308{
309	struct pci_epf_bar *epf_bar;
310	dma_addr_t phys_addr;
311	struct pci_epc *epc;
312	struct device *dev;
313	size_t mem_size;
314	void *space;
315
316	if (pci_epf_get_required_bar_size(epf, &size, &mem_size, bar,
317					  epc_features, type))
318		return NULL;
319
320	if (type == PRIMARY_INTERFACE) {
321		epc = epf->epc;
322		epf_bar = epf->bar;
323	} else {
324		epc = epf->sec_epc;
325		epf_bar = epf->sec_epc_bar;
326	}
327
328	dev = epc->dev.parent;
329	space = dma_alloc_coherent(dev, mem_size, &phys_addr, GFP_KERNEL);
330	if (!space) {
331		dev_err(dev, "failed to allocate mem space\n");
332		return NULL;
333	}
334
335	epf_bar[bar].phys_addr = phys_addr;
336	epf_bar[bar].addr = space;
337	epf_bar[bar].size = size;
338	epf_bar[bar].mem_size = mem_size;
339	epf_bar[bar].barno = bar;
340	if (upper_32_bits(size) || epc_features->bar[bar].only_64bit)
341		epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_64;
342	else
343		epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_32;
344
345	return space;
346}
347EXPORT_SYMBOL_GPL(pci_epf_alloc_space);
348
349/**
350 * pci_epf_assign_bar_space() - Assign PCI EPF BAR space
351 * @epf: EPF device to assign the BAR memory
352 * @size: Size of the memory that has to be assigned
353 * @bar: BAR number for which the memory is assigned
354 * @epc_features: Features provided by the EPC specific to this EPF
355 * @type: Identifies if the assignment is for primary EPC or secondary EPC
356 * @bar_addr: Address to be assigned for the @bar
357 *
358 * Invoke to assign memory for the PCI EPF BAR.
359 * Flag PCI_BASE_ADDRESS_MEM_TYPE_64 will automatically get set if the BAR
360 * can only be a 64-bit BAR, or if the requested size is larger than 2 GB.
361 */
362int pci_epf_assign_bar_space(struct pci_epf *epf, size_t size,
363			     enum pci_barno bar,
364			     const struct pci_epc_features *epc_features,
365			     enum pci_epc_interface_type type,
366			     dma_addr_t bar_addr)
367{
368	size_t bar_size, aligned_mem_size;
369	struct pci_epf_bar *epf_bar;
370	dma_addr_t limit;
371	int pos;
372
373	if (!size)
374		return -EINVAL;
375
376	limit = bar_addr + size - 1;
377
378	/*
379	 *  Bits:		15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
380	 *  bar_addr:		U  U  U  U  U  U  0 X X X X X X X X X
381	 *  limit:		U  U  U  U  U  U  1 X X X X X X X X X
382	 *
383	 *  bar_addr^limit	0  0  0  0  0  0  1 X X X X X X X X X
384	 *
385	 *  U: unchanged address bits in range [bar_addr, limit]
386	 *  X: bit 0 or 1
387	 *
388	 *  (bar_addr^limit) & BIT_ULL(pos) will find the first set bit from MSB
389	 *  (pos). And value of (2 ^ pos) should be able to cover the BAR range.
390	 */
391	for (pos = 8 * sizeof(dma_addr_t) - 1; pos > 0; pos--)
392		if ((limit ^ bar_addr) & BIT_ULL(pos))
393			break;
394
395	if (pos == 8 * sizeof(dma_addr_t) - 1)
396		return -EINVAL;
397
398	bar_size = BIT_ULL(pos + 1);
399	if (pci_epf_get_required_bar_size(epf, &bar_size, &aligned_mem_size,
400					  bar, epc_features, type))
401		return -ENOMEM;
402
403	if (type == PRIMARY_INTERFACE)
404		epf_bar = epf->bar;
405	else
406		epf_bar = epf->sec_epc_bar;
407
408	epf_bar[bar].phys_addr = ALIGN_DOWN(bar_addr, aligned_mem_size);
409
410	if (epf_bar[bar].phys_addr + bar_size < limit)
411		return -ENOMEM;
412
413	epf_bar[bar].addr = NULL;
414	epf_bar[bar].size = bar_size;
415	epf_bar[bar].mem_size = aligned_mem_size;
416	epf_bar[bar].barno = bar;
417	if (upper_32_bits(size) || epc_features->bar[bar].only_64bit)
418		epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_64;
419	else
420		epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_32;
421
422	return 0;
423}
424EXPORT_SYMBOL_GPL(pci_epf_assign_bar_space);
425
426static void pci_epf_remove_cfs(struct pci_epf_driver *driver)
427{
428	struct config_group *group, *tmp;
429
430	if (!IS_ENABLED(CONFIG_PCI_ENDPOINT_CONFIGFS))
431		return;
432
433	mutex_lock(&pci_epf_mutex);
434	list_for_each_entry_safe(group, tmp, &driver->epf_group, group_entry)
435		pci_ep_cfs_remove_epf_group(group);
436	WARN_ON(!list_empty(&driver->epf_group));
437	mutex_unlock(&pci_epf_mutex);
438}
439
440/**
441 * pci_epf_unregister_driver() - unregister the PCI EPF driver
442 * @driver: the PCI EPF driver that has to be unregistered
443 *
444 * Invoke to unregister the PCI EPF driver.
445 */
446void pci_epf_unregister_driver(struct pci_epf_driver *driver)
447{
448	pci_epf_remove_cfs(driver);
449	driver_unregister(&driver->driver);
450}
451EXPORT_SYMBOL_GPL(pci_epf_unregister_driver);
452
453static int pci_epf_add_cfs(struct pci_epf_driver *driver)
454{
455	struct config_group *group;
456	const struct pci_epf_device_id *id;
457
458	if (!IS_ENABLED(CONFIG_PCI_ENDPOINT_CONFIGFS))
459		return 0;
460
461	INIT_LIST_HEAD(&driver->epf_group);
462
463	id = driver->id_table;
464	while (id->name[0]) {
465		group = pci_ep_cfs_add_epf_group(id->name);
466		if (IS_ERR(group)) {
467			pci_epf_remove_cfs(driver);
468			return PTR_ERR(group);
469		}
470
471		mutex_lock(&pci_epf_mutex);
472		list_add_tail(&group->group_entry, &driver->epf_group);
473		mutex_unlock(&pci_epf_mutex);
474		id++;
475	}
476
477	return 0;
478}
479
480/**
481 * __pci_epf_register_driver() - register a new PCI EPF driver
482 * @driver: structure representing PCI EPF driver
483 * @owner: the owner of the module that registers the PCI EPF driver
484 *
485 * Invoke to register a new PCI EPF driver.
486 */
487int __pci_epf_register_driver(struct pci_epf_driver *driver,
488			      struct module *owner)
489{
490	int ret;
491
492	if (!driver->ops)
493		return -EINVAL;
494
495	if (!driver->ops->bind || !driver->ops->unbind)
496		return -EINVAL;
497
498	driver->driver.bus = &pci_epf_bus_type;
499	driver->driver.owner = owner;
500
501	ret = driver_register(&driver->driver);
502	if (ret)
503		return ret;
504
505	pci_epf_add_cfs(driver);
506
507	return 0;
508}
509EXPORT_SYMBOL_GPL(__pci_epf_register_driver);
510
511/**
512 * pci_epf_destroy() - destroy the created PCI EPF device
513 * @epf: the PCI EPF device that has to be destroyed.
514 *
515 * Invoke to destroy the PCI EPF device created by invoking pci_epf_create().
516 */
517void pci_epf_destroy(struct pci_epf *epf)
518{
519	device_unregister(&epf->dev);
520}
521EXPORT_SYMBOL_GPL(pci_epf_destroy);
522
523/**
524 * pci_epf_create() - create a new PCI EPF device
525 * @name: the name of the PCI EPF device. This name will be used to bind the
526 *	  EPF device to a EPF driver
527 *
528 * Invoke to create a new PCI EPF device by providing the name of the function
529 * device.
530 */
531struct pci_epf *pci_epf_create(const char *name)
532{
533	int ret;
534	struct pci_epf *epf;
535	struct device *dev;
536	int len;
537
538	epf = kzalloc(sizeof(*epf), GFP_KERNEL);
539	if (!epf)
540		return ERR_PTR(-ENOMEM);
541
542	len = strchrnul(name, '.') - name;
543	epf->name = kstrndup(name, len, GFP_KERNEL);
544	if (!epf->name) {
545		kfree(epf);
546		return ERR_PTR(-ENOMEM);
547	}
548
549	/* VFs are numbered starting with 1. So set BIT(0) by default */
550	epf->vfunction_num_map = 1;
551	INIT_LIST_HEAD(&epf->pci_vepf);
552
553	dev = &epf->dev;
554	device_initialize(dev);
555	dev->bus = &pci_epf_bus_type;
556	dev->type = &pci_epf_type;
557	mutex_init(&epf->lock);
558
559	ret = dev_set_name(dev, "%s", name);
560	if (ret) {
561		put_device(dev);
562		return ERR_PTR(ret);
563	}
564
565	ret = device_add(dev);
566	if (ret) {
567		put_device(dev);
568		return ERR_PTR(ret);
569	}
570
571	return epf;
572}
573EXPORT_SYMBOL_GPL(pci_epf_create);
574
575/**
576 * pci_epf_align_inbound_addr() - Align the given address based on the BAR
577 *				  alignment requirement
578 * @epf: the EPF device
579 * @addr: inbound address to be aligned
580 * @bar: the BAR number corresponding to the given addr
581 * @base: base address matching the @bar alignment requirement
582 * @off: offset to be added to the @base address
583 *
584 * Helper function to align input @addr based on BAR's alignment requirement.
585 * The aligned base address and offset are returned via @base and @off.
586 *
587 * NOTE: The pci_epf_alloc_space() function already accounts for alignment.
588 * This API is primarily intended for use with other memory regions not
589 * allocated by pci_epf_alloc_space(), such as peripheral register spaces or
590 * the message address of a platform MSI controller.
591 *
592 * Return: 0 on success, errno otherwise.
593 */
594int pci_epf_align_inbound_addr(struct pci_epf *epf, enum pci_barno bar,
595			       u64 addr, dma_addr_t *base, size_t *off)
596{
597	/*
598	 * Most EP controllers require the BAR start address to be aligned to
599	 * the BAR size, because they mask off the lower bits.
600	 *
601	 * Alignment to BAR size also works for controllers that support
602	 * unaligned addresses.
603	 */
604	u64 align = epf->bar[bar].size;
605
606	*base = round_down(addr, align);
607	*off = addr & (align - 1);
608
609	return 0;
610}
611EXPORT_SYMBOL_GPL(pci_epf_align_inbound_addr);
612
613static void pci_epf_dev_release(struct device *dev)
614{
615	struct pci_epf *epf = to_pci_epf(dev);
616
617	kfree(epf->name);
618	kfree(epf);
619}
620
621static const struct device_type pci_epf_type = {
622	.release	= pci_epf_dev_release,
623};
624
625static const struct pci_epf_device_id *
626pci_epf_match_id(const struct pci_epf_device_id *id, const struct pci_epf *epf)
627{
628	while (id->name[0]) {
629		if (strcmp(epf->name, id->name) == 0)
630			return id;
631		id++;
632	}
633
634	return NULL;
635}
636
637static int pci_epf_device_match(struct device *dev, const struct device_driver *drv)
638{
639	struct pci_epf *epf = to_pci_epf(dev);
640	const struct pci_epf_driver *driver = to_pci_epf_driver(drv);
641
642	if (driver->id_table)
643		return !!pci_epf_match_id(driver->id_table, epf);
644
645	return !strcmp(epf->name, drv->name);
646}
647
648static int pci_epf_device_probe(struct device *dev)
649{
650	struct pci_epf *epf = to_pci_epf(dev);
651	struct pci_epf_driver *driver = to_pci_epf_driver(dev->driver);
652
653	if (!driver->probe)
654		return -ENODEV;
655
656	epf->driver = driver;
657
658	return driver->probe(epf, pci_epf_match_id(driver->id_table, epf));
659}
660
661static void pci_epf_device_remove(struct device *dev)
662{
663	struct pci_epf *epf = to_pci_epf(dev);
664	struct pci_epf_driver *driver = to_pci_epf_driver(dev->driver);
665
666	if (driver->remove)
667		driver->remove(epf);
668	epf->driver = NULL;
669}
670
671static const struct bus_type pci_epf_bus_type = {
672	.name		= "pci-epf",
673	.match		= pci_epf_device_match,
674	.probe		= pci_epf_device_probe,
675	.remove		= pci_epf_device_remove,
676};
677
678static int __init pci_epf_init(void)
679{
680	int ret;
681
682	ret = bus_register(&pci_epf_bus_type);
683	if (ret) {
684		pr_err("failed to register pci epf bus --> %d\n", ret);
685		return ret;
686	}
687
688	return 0;
689}
690module_init(pci_epf_init);
691
692static void __exit pci_epf_exit(void)
693{
694	bus_unregister(&pci_epf_bus_type);
695}
696module_exit(pci_epf_exit);
697
698MODULE_DESCRIPTION("PCI EPF Library");
699MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>");