dfl.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Driver for FPGA Device Feature List (DFL) Support
   4 *
   5 * Copyright (C) 2017-2018 Intel Corporation, Inc.
   6 *
   7 * Authors:
   8 *   Kang Luwei <luwei.kang@intel.com>
   9 *   Zhang Yi <yi.z.zhang@intel.com>
  10 *   Wu Hao <hao.wu@intel.com>
  11 *   Xiao Guangrong <guangrong.xiao@linux.intel.com>
  12 */
  13#include <linux/dfl.h>
  14#include <linux/fpga-dfl.h>
  15#include <linux/module.h>
  16#include <linux/overflow.h>
  17#include <linux/uaccess.h>
  18
  19#include "dfl.h"
  20
  21static DEFINE_MUTEX(dfl_id_mutex);
  22
  23/*
  24 * when adding a new feature dev support in DFL framework, it's required to
  25 * add a new item in enum dfl_id_type and provide related information in below
  26 * dfl_devs table which is indexed by dfl_id_type, e.g. name string used for
  27 * platform device creation (define name strings in dfl.h, as they could be
  28 * reused by platform device drivers).
  29 *
  30 * if the new feature dev needs chardev support, then it's required to add
  31 * a new item in dfl_chardevs table and configure dfl_devs[i].devt_type as
  32 * index to dfl_chardevs table. If no chardev support just set devt_type
  33 * as one invalid index (DFL_FPGA_DEVT_MAX).
  34 */
  35enum dfl_fpga_devt_type {
  36	DFL_FPGA_DEVT_FME,
  37	DFL_FPGA_DEVT_PORT,
  38	DFL_FPGA_DEVT_MAX,
  39};
  40
  41static struct lock_class_key dfl_pdata_keys[DFL_ID_MAX];
  42
  43static const char *dfl_pdata_key_strings[DFL_ID_MAX] = {
  44	"dfl-fme-pdata",
  45	"dfl-port-pdata",
  46};
  47
  48/**
  49 * struct dfl_dev_info - dfl feature device information.
  50 * @name: name string of the feature platform device.
  51 * @dfh_id: id value in Device Feature Header (DFH) register by DFL spec.
  52 * @id: idr id of the feature dev.
  53 * @devt_type: index to dfl_chrdevs[].
  54 */
  55struct dfl_dev_info {
  56	const char *name;
  57	u16 dfh_id;
  58	struct idr id;
  59	enum dfl_fpga_devt_type devt_type;
  60};
  61
  62/* it is indexed by dfl_id_type */
  63static struct dfl_dev_info dfl_devs[] = {
  64	{.name = DFL_FPGA_FEATURE_DEV_FME, .dfh_id = DFH_ID_FIU_FME,
  65	 .devt_type = DFL_FPGA_DEVT_FME},
  66	{.name = DFL_FPGA_FEATURE_DEV_PORT, .dfh_id = DFH_ID_FIU_PORT,
  67	 .devt_type = DFL_FPGA_DEVT_PORT},
  68};
  69
  70/**
  71 * struct dfl_chardev_info - chardev information of dfl feature device
  72 * @name: nmae string of the char device.
  73 * @devt: devt of the char device.
  74 */
  75struct dfl_chardev_info {
  76	const char *name;
  77	dev_t devt;
  78};
  79
  80/* indexed by enum dfl_fpga_devt_type */
  81static struct dfl_chardev_info dfl_chrdevs[] = {
  82	{.name = DFL_FPGA_FEATURE_DEV_FME},
  83	{.name = DFL_FPGA_FEATURE_DEV_PORT},
  84};
  85
  86static void dfl_ids_init(void)
  87{
  88	int i;
  89
  90	for (i = 0; i < ARRAY_SIZE(dfl_devs); i++)
  91		idr_init(&dfl_devs[i].id);
  92}
  93
  94static void dfl_ids_destroy(void)
  95{
  96	int i;
  97
  98	for (i = 0; i < ARRAY_SIZE(dfl_devs); i++)
  99		idr_destroy(&dfl_devs[i].id);
 100}
 101
 102static int dfl_id_alloc(enum dfl_id_type type, struct device *dev)
 103{
 104	int id;
 105
 106	WARN_ON(type >= DFL_ID_MAX);
 107	mutex_lock(&dfl_id_mutex);
 108	id = idr_alloc(&dfl_devs[type].id, dev, 0, 0, GFP_KERNEL);
 109	mutex_unlock(&dfl_id_mutex);
 110
 111	return id;
 112}
 113
 114static void dfl_id_free(enum dfl_id_type type, int id)
 115{
 116	WARN_ON(type >= DFL_ID_MAX);
 117	mutex_lock(&dfl_id_mutex);
 118	idr_remove(&dfl_devs[type].id, id);
 119	mutex_unlock(&dfl_id_mutex);
 120}
 121
 122static enum dfl_id_type dfh_id_to_type(u16 id)
 123{
 124	int i;
 125
 126	for (i = 0; i < ARRAY_SIZE(dfl_devs); i++)
 127		if (dfl_devs[i].dfh_id == id)
 128			return i;
 129
 130	return DFL_ID_MAX;
 131}
 132
 133/*
 134 * introduce a global port_ops list, it allows port drivers to register ops
 135 * in such list, then other feature devices (e.g. FME), could use the port
 136 * functions even related port platform device is hidden. Below is one example,
 137 * in virtualization case of PCIe-based FPGA DFL device, when SRIOV is
 138 * enabled, port (and it's AFU) is turned into VF and port platform device
 139 * is hidden from system but it's still required to access port to finish FPGA
 140 * reconfiguration function in FME.
 141 */
 142
 143static DEFINE_MUTEX(dfl_port_ops_mutex);
 144static LIST_HEAD(dfl_port_ops_list);
 145
 146/**
 147 * dfl_fpga_port_ops_get - get matched port ops from the global list
 148 * @fdata: feature dev data to match with associated port ops.
 149 * Return: matched port ops on success, NULL otherwise.
 150 *
 151 * Please note that must dfl_fpga_port_ops_put after use the port_ops.
 152 */
 153struct dfl_fpga_port_ops *dfl_fpga_port_ops_get(struct dfl_feature_dev_data *fdata)
 154{
 155	struct dfl_fpga_port_ops *ops = NULL;
 156
 157	mutex_lock(&dfl_port_ops_mutex);
 158	if (list_empty(&dfl_port_ops_list))
 159		goto done;
 160
 161	list_for_each_entry(ops, &dfl_port_ops_list, node) {
 162		/* match port_ops using the name of platform device */
 163		if (!strcmp(fdata->pdev_name, ops->name)) {
 164			if (!try_module_get(ops->owner))
 165				ops = NULL;
 166			goto done;
 167		}
 168	}
 169
 170	ops = NULL;
 171done:
 172	mutex_unlock(&dfl_port_ops_mutex);
 173	return ops;
 174}
 175EXPORT_SYMBOL_GPL(dfl_fpga_port_ops_get);
 176
 177/**
 178 * dfl_fpga_port_ops_put - put port ops
 179 * @ops: port ops.
 180 */
 181void dfl_fpga_port_ops_put(struct dfl_fpga_port_ops *ops)
 182{
 183	if (ops && ops->owner)
 184		module_put(ops->owner);
 185}
 186EXPORT_SYMBOL_GPL(dfl_fpga_port_ops_put);
 187
 188/**
 189 * dfl_fpga_port_ops_add - add port_ops to global list
 190 * @ops: port ops to add.
 191 */
 192void dfl_fpga_port_ops_add(struct dfl_fpga_port_ops *ops)
 193{
 194	mutex_lock(&dfl_port_ops_mutex);
 195	list_add_tail(&ops->node, &dfl_port_ops_list);
 196	mutex_unlock(&dfl_port_ops_mutex);
 197}
 198EXPORT_SYMBOL_GPL(dfl_fpga_port_ops_add);
 199
 200/**
 201 * dfl_fpga_port_ops_del - remove port_ops from global list
 202 * @ops: port ops to del.
 203 */
 204void dfl_fpga_port_ops_del(struct dfl_fpga_port_ops *ops)
 205{
 206	mutex_lock(&dfl_port_ops_mutex);
 207	list_del(&ops->node);
 208	mutex_unlock(&dfl_port_ops_mutex);
 209}
 210EXPORT_SYMBOL_GPL(dfl_fpga_port_ops_del);
 211
 212/**
 213 * dfl_fpga_check_port_id - check the port id
 214 * @fdata: port feature dev data.
 215 * @pport_id: port id to compare.
 216 *
 217 * Return: 1 if port device matches with given port id, otherwise 0.
 218 */
 219int dfl_fpga_check_port_id(struct dfl_feature_dev_data *fdata, void *pport_id)
 220{
 221	struct dfl_fpga_port_ops *port_ops;
 222
 223	if (fdata->id != FEATURE_DEV_ID_UNUSED)
 224		return fdata->id == *(int *)pport_id;
 225
 226	port_ops = dfl_fpga_port_ops_get(fdata);
 227	if (!port_ops || !port_ops->get_id)
 228		return 0;
 229
 230	fdata->id = port_ops->get_id(fdata);
 231	dfl_fpga_port_ops_put(port_ops);
 232
 233	return fdata->id == *(int *)pport_id;
 234}
 235EXPORT_SYMBOL_GPL(dfl_fpga_check_port_id);
 236
 237static DEFINE_IDA(dfl_device_ida);
 238
 239static const struct dfl_device_id *
 240dfl_match_one_device(const struct dfl_device_id *id, struct dfl_device *ddev)
 241{
 242	if (id->type == ddev->type && id->feature_id == ddev->feature_id)
 243		return id;
 244
 245	return NULL;
 246}
 247
 248static int dfl_bus_match(struct device *dev, const struct device_driver *drv)
 249{
 250	struct dfl_device *ddev = to_dfl_dev(dev);
 251	const struct dfl_driver *ddrv = to_dfl_drv(drv);
 252	const struct dfl_device_id *id_entry;
 253
 254	id_entry = ddrv->id_table;
 255	if (id_entry) {
 256		while (id_entry->feature_id) {
 257			if (dfl_match_one_device(id_entry, ddev)) {
 258				ddev->id_entry = id_entry;
 259				return 1;
 260			}
 261			id_entry++;
 262		}
 263	}
 264
 265	return 0;
 266}
 267
 268static int dfl_bus_probe(struct device *dev)
 269{
 270	struct dfl_driver *ddrv = to_dfl_drv(dev->driver);
 271	struct dfl_device *ddev = to_dfl_dev(dev);
 272
 273	return ddrv->probe(ddev);
 274}
 275
 276static void dfl_bus_remove(struct device *dev)
 277{
 278	struct dfl_driver *ddrv = to_dfl_drv(dev->driver);
 279	struct dfl_device *ddev = to_dfl_dev(dev);
 280
 281	if (ddrv->remove)
 282		ddrv->remove(ddev);
 283}
 284
 285static int dfl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
 286{
 287	const struct dfl_device *ddev = to_dfl_dev(dev);
 288
 289	return add_uevent_var(env, "MODALIAS=dfl:t%04Xf%04X",
 290			      ddev->type, ddev->feature_id);
 291}
 292
 293static ssize_t
 294type_show(struct device *dev, struct device_attribute *attr, char *buf)
 295{
 296	struct dfl_device *ddev = to_dfl_dev(dev);
 297
 298	return sprintf(buf, "0x%x\n", ddev->type);
 299}
 300static DEVICE_ATTR_RO(type);
 301
 302static ssize_t
 303feature_id_show(struct device *dev, struct device_attribute *attr, char *buf)
 304{
 305	struct dfl_device *ddev = to_dfl_dev(dev);
 306
 307	return sprintf(buf, "0x%x\n", ddev->feature_id);
 308}
 309static DEVICE_ATTR_RO(feature_id);
 310
 311static struct attribute *dfl_dev_attrs[] = {
 312	&dev_attr_type.attr,
 313	&dev_attr_feature_id.attr,
 314	NULL,
 315};
 316ATTRIBUTE_GROUPS(dfl_dev);
 317
 318static const struct bus_type dfl_bus_type = {
 319	.name		= "dfl",
 320	.match		= dfl_bus_match,
 321	.probe		= dfl_bus_probe,
 322	.remove		= dfl_bus_remove,
 323	.uevent		= dfl_bus_uevent,
 324	.dev_groups	= dfl_dev_groups,
 325};
 326
 327static void release_dfl_dev(struct device *dev)
 328{
 329	struct dfl_device *ddev = to_dfl_dev(dev);
 330
 331	if (ddev->mmio_res.parent)
 332		release_resource(&ddev->mmio_res);
 333
 334	kfree(ddev->params);
 335
 336	ida_free(&dfl_device_ida, ddev->id);
 337	kfree(ddev->irqs);
 338	kfree(ddev);
 339}
 340
 341static struct dfl_device *
 342dfl_dev_add(struct dfl_feature_dev_data *fdata,
 343	    struct dfl_feature *feature)
 344{
 345	struct platform_device *pdev = fdata->dev;
 346	struct resource *parent_res;
 347	struct dfl_device *ddev;
 348	int id, i, ret;
 349
 350	ddev = kzalloc(sizeof(*ddev), GFP_KERNEL);
 351	if (!ddev)
 352		return ERR_PTR(-ENOMEM);
 353
 354	id = ida_alloc(&dfl_device_ida, GFP_KERNEL);
 355	if (id < 0) {
 356		dev_err(&pdev->dev, "unable to get id\n");
 357		kfree(ddev);
 358		return ERR_PTR(id);
 359	}
 360
 361	/* freeing resources by put_device() after device_initialize() */
 362	device_initialize(&ddev->dev);
 363	ddev->dev.parent = &pdev->dev;
 364	ddev->dev.bus = &dfl_bus_type;
 365	ddev->dev.release = release_dfl_dev;
 366	ddev->id = id;
 367	ret = dev_set_name(&ddev->dev, "dfl_dev.%d", id);
 368	if (ret)
 369		goto put_dev;
 370
 371	ddev->type = fdata->type;
 372	ddev->feature_id = feature->id;
 373	ddev->revision = feature->revision;
 374	ddev->dfh_version = feature->dfh_version;
 375	ddev->cdev = fdata->dfl_cdev;
 376	if (feature->param_size) {
 377		ddev->params = kmemdup(feature->params, feature->param_size, GFP_KERNEL);
 378		if (!ddev->params) {
 379			ret = -ENOMEM;
 380			goto put_dev;
 381		}
 382		ddev->param_size = feature->param_size;
 383	}
 384
 385	/* add mmio resource */
 386	parent_res = &pdev->resource[feature->resource_index];
 387	ddev->mmio_res.flags = IORESOURCE_MEM;
 388	ddev->mmio_res.start = parent_res->start;
 389	ddev->mmio_res.end = parent_res->end;
 390	ddev->mmio_res.name = dev_name(&ddev->dev);
 391	ret = insert_resource(parent_res, &ddev->mmio_res);
 392	if (ret) {
 393		dev_err(&pdev->dev, "%s failed to claim resource: %pR\n",
 394			dev_name(&ddev->dev), &ddev->mmio_res);
 395		goto put_dev;
 396	}
 397
 398	/* then add irq resource */
 399	if (feature->nr_irqs) {
 400		ddev->irqs = kcalloc(feature->nr_irqs,
 401				     sizeof(*ddev->irqs), GFP_KERNEL);
 402		if (!ddev->irqs) {
 403			ret = -ENOMEM;
 404			goto put_dev;
 405		}
 406
 407		for (i = 0; i < feature->nr_irqs; i++)
 408			ddev->irqs[i] = feature->irq_ctx[i].irq;
 409
 410		ddev->num_irqs = feature->nr_irqs;
 411	}
 412
 413	ret = device_add(&ddev->dev);
 414	if (ret)
 415		goto put_dev;
 416
 417	dev_dbg(&pdev->dev, "add dfl_dev: %s\n", dev_name(&ddev->dev));
 418	return ddev;
 419
 420put_dev:
 421	/* calls release_dfl_dev() which does the clean up  */
 422	put_device(&ddev->dev);
 423	return ERR_PTR(ret);
 424}
 425
 426static void dfl_devs_remove(struct dfl_feature_dev_data *fdata)
 427{
 428	struct dfl_feature *feature;
 429
 430	dfl_fpga_dev_for_each_feature(fdata, feature) {
 431		if (feature->ddev) {
 432			device_unregister(&feature->ddev->dev);
 433			feature->ddev = NULL;
 434		}
 435	}
 436}
 437
 438static int dfl_devs_add(struct dfl_feature_dev_data *fdata)
 439{
 440	struct dfl_feature *feature;
 441	struct dfl_device *ddev;
 442	int ret;
 443
 444	dfl_fpga_dev_for_each_feature(fdata, feature) {
 445		if (feature->ioaddr)
 446			continue;
 447
 448		if (feature->ddev) {
 449			ret = -EEXIST;
 450			goto err;
 451		}
 452
 453		ddev = dfl_dev_add(fdata, feature);
 454		if (IS_ERR(ddev)) {
 455			ret = PTR_ERR(ddev);
 456			goto err;
 457		}
 458
 459		feature->ddev = ddev;
 460	}
 461
 462	return 0;
 463
 464err:
 465	dfl_devs_remove(fdata);
 466	return ret;
 467}
 468
 469int __dfl_driver_register(struct dfl_driver *dfl_drv, struct module *owner)
 470{
 471	if (!dfl_drv || !dfl_drv->probe || !dfl_drv->id_table)
 472		return -EINVAL;
 473
 474	dfl_drv->drv.owner = owner;
 475	dfl_drv->drv.bus = &dfl_bus_type;
 476
 477	return driver_register(&dfl_drv->drv);
 478}
 479EXPORT_SYMBOL(__dfl_driver_register);
 480
 481void dfl_driver_unregister(struct dfl_driver *dfl_drv)
 482{
 483	driver_unregister(&dfl_drv->drv);
 484}
 485EXPORT_SYMBOL(dfl_driver_unregister);
 486
 487#define is_header_feature(feature) ((feature)->id == FEATURE_ID_FIU_HEADER)
 488
 489/**
 490 * dfl_fpga_dev_feature_uinit - uinit for sub features of dfl feature device
 491 * @pdev: feature device.
 492 */
 493void dfl_fpga_dev_feature_uinit(struct platform_device *pdev)
 494{
 495	struct dfl_feature_dev_data *fdata = to_dfl_feature_dev_data(&pdev->dev);
 496	struct dfl_feature *feature;
 497
 498	dfl_devs_remove(fdata);
 499
 500	dfl_fpga_dev_for_each_feature(fdata, feature) {
 501		if (feature->ops) {
 502			if (feature->ops->uinit)
 503				feature->ops->uinit(pdev, feature);
 504			feature->ops = NULL;
 505		}
 506	}
 507}
 508EXPORT_SYMBOL_GPL(dfl_fpga_dev_feature_uinit);
 509
 510static int dfl_feature_instance_init(struct platform_device *pdev,
 511				     struct dfl_feature *feature,
 512				     struct dfl_feature_driver *drv)
 513{
 514	void __iomem *base;
 515	int ret = 0;
 516
 517	if (!is_header_feature(feature)) {
 518		base = devm_platform_ioremap_resource(pdev,
 519						      feature->resource_index);
 520		if (IS_ERR(base)) {
 521			dev_err(&pdev->dev,
 522				"ioremap failed for feature 0x%x!\n",
 523				feature->id);
 524			return PTR_ERR(base);
 525		}
 526
 527		feature->ioaddr = base;
 528	}
 529
 530	if (drv->ops->init) {
 531		ret = drv->ops->init(pdev, feature);
 532		if (ret)
 533			return ret;
 534	}
 535
 536	feature->ops = drv->ops;
 537
 538	return ret;
 539}
 540
 541static bool dfl_feature_drv_match(struct dfl_feature *feature,
 542				  struct dfl_feature_driver *driver)
 543{
 544	const struct dfl_feature_id *ids = driver->id_table;
 545
 546	if (ids) {
 547		while (ids->id) {
 548			if (ids->id == feature->id)
 549				return true;
 550			ids++;
 551		}
 552	}
 553	return false;
 554}
 555
 556/**
 557 * dfl_fpga_dev_feature_init - init for sub features of dfl feature device
 558 * @pdev: feature device.
 559 * @feature_drvs: drvs for sub features.
 560 *
 561 * This function will match sub features with given feature drvs list and
 562 * use matched drv to init related sub feature.
 563 *
 564 * Return: 0 on success, negative error code otherwise.
 565 */
 566int dfl_fpga_dev_feature_init(struct platform_device *pdev,
 567			      struct dfl_feature_driver *feature_drvs)
 568{
 569	struct dfl_feature_dev_data *fdata = to_dfl_feature_dev_data(&pdev->dev);
 570	struct dfl_feature_driver *drv = feature_drvs;
 571	struct dfl_feature *feature;
 572	int ret;
 573
 574	while (drv->ops) {
 575		dfl_fpga_dev_for_each_feature(fdata, feature) {
 576			if (dfl_feature_drv_match(feature, drv)) {
 577				ret = dfl_feature_instance_init(pdev, feature, drv);
 578				if (ret)
 579					goto exit;
 580			}
 581		}
 582		drv++;
 583	}
 584
 585	ret = dfl_devs_add(fdata);
 586	if (ret)
 587		goto exit;
 588
 589	return 0;
 590exit:
 591	dfl_fpga_dev_feature_uinit(pdev);
 592	return ret;
 593}
 594EXPORT_SYMBOL_GPL(dfl_fpga_dev_feature_init);
 595
 596static void dfl_chardev_uinit(void)
 597{
 598	int i;
 599
 600	for (i = 0; i < DFL_FPGA_DEVT_MAX; i++)
 601		if (MAJOR(dfl_chrdevs[i].devt)) {
 602			unregister_chrdev_region(dfl_chrdevs[i].devt,
 603						 MINORMASK + 1);
 604			dfl_chrdevs[i].devt = MKDEV(0, 0);
 605		}
 606}
 607
 608static int dfl_chardev_init(void)
 609{
 610	int i, ret;
 611
 612	for (i = 0; i < DFL_FPGA_DEVT_MAX; i++) {
 613		ret = alloc_chrdev_region(&dfl_chrdevs[i].devt, 0,
 614					  MINORMASK + 1, dfl_chrdevs[i].name);
 615		if (ret)
 616			goto exit;
 617	}
 618
 619	return 0;
 620
 621exit:
 622	dfl_chardev_uinit();
 623	return ret;
 624}
 625
 626static dev_t dfl_get_devt(enum dfl_fpga_devt_type type, int id)
 627{
 628	if (type >= DFL_FPGA_DEVT_MAX)
 629		return 0;
 630
 631	return MKDEV(MAJOR(dfl_chrdevs[type].devt), id);
 632}
 633
 634/**
 635 * dfl_fpga_dev_ops_register - register cdev ops for feature dev
 636 *
 637 * @pdev: feature dev.
 638 * @fops: file operations for feature dev's cdev.
 639 * @owner: owning module/driver.
 640 *
 641 * Return: 0 on success, negative error code otherwise.
 642 */
 643int dfl_fpga_dev_ops_register(struct platform_device *pdev,
 644			      const struct file_operations *fops,
 645			      struct module *owner)
 646{
 647	struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev->dev);
 648
 649	cdev_init(&pdata->cdev, fops);
 650	pdata->cdev.owner = owner;
 651
 652	/*
 653	 * set parent to the feature device so that its refcount is
 654	 * decreased after the last refcount of cdev is gone, that
 655	 * makes sure the feature device is valid during device
 656	 * file's life-cycle.
 657	 */
 658	pdata->cdev.kobj.parent = &pdev->dev.kobj;
 659
 660	return cdev_add(&pdata->cdev, pdev->dev.devt, 1);
 661}
 662EXPORT_SYMBOL_GPL(dfl_fpga_dev_ops_register);
 663
 664/**
 665 * dfl_fpga_dev_ops_unregister - unregister cdev ops for feature dev
 666 * @pdev: feature dev.
 667 */
 668void dfl_fpga_dev_ops_unregister(struct platform_device *pdev)
 669{
 670	struct dfl_feature_platform_data *pdata = dev_get_platdata(&pdev->dev);
 671
 672	cdev_del(&pdata->cdev);
 673}
 674EXPORT_SYMBOL_GPL(dfl_fpga_dev_ops_unregister);
 675
 676/**
 677 * struct build_feature_devs_info - info collected during feature dev build.
 678 *
 679 * @dev: device to enumerate.
 680 * @cdev: the container device for all feature devices.
 681 * @nr_irqs: number of irqs for all feature devices.
 682 * @irq_table: Linux IRQ numbers for all irqs, indexed by local irq index of
 683 *	       this device.
 684 * @type: the current FIU type.
 685 * @ioaddr: header register region address of current FIU in enumeration.
 686 * @start: register resource start of current FIU.
 687 * @len: max register resource length of current FIU.
 688 * @sub_features: a sub features linked list for feature device in enumeration.
 689 * @feature_num: number of sub features for feature device in enumeration.
 690 */
 691struct build_feature_devs_info {
 692	struct device *dev;
 693	struct dfl_fpga_cdev *cdev;
 694	unsigned int nr_irqs;
 695	int *irq_table;
 696
 697	enum dfl_id_type type;
 698	void __iomem *ioaddr;
 699	resource_size_t start;
 700	resource_size_t len;
 701	struct list_head sub_features;
 702	int feature_num;
 703};
 704
 705/**
 706 * struct dfl_feature_info - sub feature info collected during feature dev build
 707 *
 708 * @fid: id of this sub feature.
 709 * @revision: revision of this sub feature
 710 * @dfh_version: version of Device Feature Header (DFH)
 711 * @mmio_res: mmio resource of this sub feature.
 712 * @ioaddr: mapped base address of mmio resource.
 713 * @node: node in sub_features linked list.
 714 * @irq_base: start of irq index in this sub feature.
 715 * @nr_irqs: number of irqs of this sub feature.
 716 * @param_size: size DFH parameters.
 717 * @params: DFH parameter data.
 718 */
 719struct dfl_feature_info {
 720	u16 fid;
 721	u8 revision;
 722	u8 dfh_version;
 723	struct resource mmio_res;
 724	void __iomem *ioaddr;
 725	struct list_head node;
 726	unsigned int irq_base;
 727	unsigned int nr_irqs;
 728	unsigned int param_size;
 729	u64 params[];
 730};
 731
 732static void dfl_fpga_cdev_add_port_data(struct dfl_fpga_cdev *cdev,
 733					struct dfl_feature_dev_data *fdata)
 734{
 735	mutex_lock(&cdev->lock);
 736	list_add(&fdata->node, &cdev->port_dev_list);
 737	mutex_unlock(&cdev->lock);
 738}
 739
 740static void dfl_id_free_action(void *arg)
 741{
 742	struct dfl_feature_dev_data *fdata = arg;
 743
 744	dfl_id_free(fdata->type, fdata->pdev_id);
 745}
 746
 747static struct dfl_feature_dev_data *
 748binfo_create_feature_dev_data(struct build_feature_devs_info *binfo)
 749{
 750	enum dfl_id_type type = binfo->type;
 751	struct dfl_feature_info *finfo, *p;
 752	struct dfl_feature_dev_data *fdata;
 753	int ret, index = 0, res_idx = 0;
 754
 755	if (WARN_ON_ONCE(type >= DFL_ID_MAX))
 756		return ERR_PTR(-EINVAL);
 757
 758	fdata = devm_kzalloc(binfo->dev, sizeof(*fdata), GFP_KERNEL);
 759	if (!fdata)
 760		return ERR_PTR(-ENOMEM);
 761
 762	fdata->features = devm_kcalloc(binfo->dev, binfo->feature_num,
 763				       sizeof(*fdata->features), GFP_KERNEL);
 764	if (!fdata->features)
 765		return ERR_PTR(-ENOMEM);
 766
 767	fdata->resources = devm_kcalloc(binfo->dev, binfo->feature_num,
 768					sizeof(*fdata->resources), GFP_KERNEL);
 769	if (!fdata->resources)
 770		return ERR_PTR(-ENOMEM);
 771
 772	fdata->type = type;
 773
 774	fdata->pdev_id = dfl_id_alloc(type, binfo->dev);
 775	if (fdata->pdev_id < 0)
 776		return ERR_PTR(fdata->pdev_id);
 777
 778	ret = devm_add_action_or_reset(binfo->dev, dfl_id_free_action, fdata);
 779	if (ret)
 780		return ERR_PTR(ret);
 781
 782	fdata->pdev_name = dfl_devs[type].name;
 783	fdata->num = binfo->feature_num;
 784	fdata->dfl_cdev = binfo->cdev;
 785	fdata->id = FEATURE_DEV_ID_UNUSED;
 786	mutex_init(&fdata->lock);
 787	lockdep_set_class_and_name(&fdata->lock, &dfl_pdata_keys[type],
 788				   dfl_pdata_key_strings[type]);
 789
 790	/*
 791	 * the count should be initialized to 0 to make sure
 792	 *__fpga_port_enable() following __fpga_port_disable()
 793	 * works properly for port device.
 794	 * and it should always be 0 for fme device.
 795	 */
 796	WARN_ON(fdata->disable_count);
 797
 798	/* fill features and resource information for feature dev */
 799	list_for_each_entry_safe(finfo, p, &binfo->sub_features, node) {
 800		struct dfl_feature *feature = &fdata->features[index++];
 801		struct dfl_feature_irq_ctx *ctx;
 802		unsigned int i;
 803
 804		/* save resource information for each feature */
 805		feature->id = finfo->fid;
 806		feature->revision = finfo->revision;
 807		feature->dfh_version = finfo->dfh_version;
 808
 809		if (finfo->param_size) {
 810			feature->params = devm_kmemdup(binfo->dev,
 811						       finfo->params, finfo->param_size,
 812						       GFP_KERNEL);
 813			if (!feature->params)
 814				return ERR_PTR(-ENOMEM);
 815
 816			feature->param_size = finfo->param_size;
 817		}
 818		/*
 819		 * the FIU header feature has some fundamental functions (sriov
 820		 * set, port enable/disable) needed for the dfl bus device and
 821		 * other sub features. So its mmio resource should be mapped by
 822		 * DFL bus device. And we should not assign it to feature
 823		 * devices (dfl-fme/afu) again.
 824		 */
 825		if (is_header_feature(feature)) {
 826			feature->resource_index = -1;
 827			feature->ioaddr =
 828				devm_ioremap_resource(binfo->dev,
 829						      &finfo->mmio_res);
 830			if (IS_ERR(feature->ioaddr))
 831				return ERR_CAST(feature->ioaddr);
 832		} else {
 833			feature->resource_index = res_idx;
 834			fdata->resources[res_idx++] = finfo->mmio_res;
 835		}
 836
 837		if (finfo->nr_irqs) {
 838			ctx = devm_kcalloc(binfo->dev, finfo->nr_irqs,
 839					   sizeof(*ctx), GFP_KERNEL);
 840			if (!ctx)
 841				return ERR_PTR(-ENOMEM);
 842
 843			for (i = 0; i < finfo->nr_irqs; i++)
 844				ctx[i].irq =
 845					binfo->irq_table[finfo->irq_base + i];
 846
 847			feature->irq_ctx = ctx;
 848			feature->nr_irqs = finfo->nr_irqs;
 849		}
 850
 851		list_del(&finfo->node);
 852		kfree(finfo);
 853	}
 854
 855	fdata->resource_num = res_idx;
 856
 857	return fdata;
 858}
 859
 860/*
 861 * register current feature device, it is called when we need to switch to
 862 * another feature parsing or we have parsed all features on given device
 863 * feature list.
 864 */
 865static int feature_dev_register(struct dfl_feature_dev_data *fdata)
 866{
 867	struct dfl_feature_platform_data pdata = {};
 868	struct platform_device *fdev;
 869	struct dfl_feature *feature;
 870	int ret;
 871
 872	fdev = platform_device_alloc(fdata->pdev_name, fdata->pdev_id);
 873	if (!fdev)
 874		return -ENOMEM;
 875
 876	fdata->dev = fdev;
 877
 878	fdev->dev.parent = &fdata->dfl_cdev->region->dev;
 879	fdev->dev.devt = dfl_get_devt(dfl_devs[fdata->type].devt_type, fdev->id);
 880
 881	dfl_fpga_dev_for_each_feature(fdata, feature)
 882		feature->dev = fdev;
 883
 884	ret = platform_device_add_resources(fdev, fdata->resources,
 885					    fdata->resource_num);
 886	if (ret)
 887		goto err_put_dev;
 888
 889	pdata.fdata = fdata;
 890	ret = platform_device_add_data(fdev, &pdata, sizeof(pdata));
 891	if (ret)
 892		goto err_put_dev;
 893
 894	ret = platform_device_add(fdev);
 895	if (ret)
 896		goto err_put_dev;
 897
 898	return 0;
 899
 900err_put_dev:
 901	platform_device_put(fdev);
 902
 903	fdata->dev = NULL;
 904
 905	dfl_fpga_dev_for_each_feature(fdata, feature)
 906		feature->dev = NULL;
 907
 908	return ret;
 909}
 910
 911static void feature_dev_unregister(struct dfl_feature_dev_data *fdata)
 912{
 913	struct dfl_feature *feature;
 914
 915	platform_device_unregister(fdata->dev);
 916
 917	fdata->dev = NULL;
 918
 919	dfl_fpga_dev_for_each_feature(fdata, feature)
 920		feature->dev = NULL;
 921}
 922
 923static int build_info_commit_dev(struct build_feature_devs_info *binfo)
 924{
 925	struct dfl_feature_dev_data *fdata;
 926	int ret;
 927
 928	fdata = binfo_create_feature_dev_data(binfo);
 929	if (IS_ERR(fdata))
 930		return PTR_ERR(fdata);
 931
 932	ret = feature_dev_register(fdata);
 933	if (ret)
 934		return ret;
 935
 936	if (binfo->type == PORT_ID)
 937		dfl_fpga_cdev_add_port_data(binfo->cdev, fdata);
 938	else
 939		binfo->cdev->fme_dev = get_device(&fdata->dev->dev);
 940
 941	/* reset the binfo for next FIU */
 942	binfo->type = DFL_ID_MAX;
 943
 944	return 0;
 945}
 946
 947static void build_info_free(struct build_feature_devs_info *binfo)
 948{
 949	struct dfl_feature_info *finfo, *p;
 950
 951	list_for_each_entry_safe(finfo, p, &binfo->sub_features, node) {
 952		list_del(&finfo->node);
 953		kfree(finfo);
 954	}
 955
 956	devm_kfree(binfo->dev, binfo);
 957}
 958
 959static inline u32 feature_size(u64 value)
 960{
 961	u32 ofst = FIELD_GET(DFH_NEXT_HDR_OFST, value);
 962	/* workaround for private features with invalid size, use 4K instead */
 963	return ofst ? ofst : 4096;
 964}
 965
 966static u16 feature_id(u64 value)
 967{
 968	u16 id = FIELD_GET(DFH_ID, value);
 969	u8 type = FIELD_GET(DFH_TYPE, value);
 970
 971	if (type == DFH_TYPE_FIU)
 972		return FEATURE_ID_FIU_HEADER;
 973	else if (type == DFH_TYPE_PRIVATE)
 974		return id;
 975	else if (type == DFH_TYPE_AFU)
 976		return FEATURE_ID_AFU;
 977
 978	WARN_ON(1);
 979	return 0;
 980}
 981
 982static u64 *find_param(u64 *params, resource_size_t max, int param_id)
 983{
 984	u64 *end = params + max / sizeof(u64);
 985	u64 v, next;
 986
 987	while (params < end) {
 988		v = *params;
 989		if (param_id == FIELD_GET(DFHv1_PARAM_HDR_ID, v))
 990			return params;
 991
 992		if (FIELD_GET(DFHv1_PARAM_HDR_NEXT_EOP, v))
 993			break;
 994
 995		next = FIELD_GET(DFHv1_PARAM_HDR_NEXT_OFFSET, v);
 996		params += next;
 997	}
 998
 999	return NULL;
1000}
1001
1002/**
1003 * dfh_find_param() - find parameter block for the given parameter id
1004 * @dfl_dev: dfl device
1005 * @param_id: id of dfl parameter
1006 * @psize: destination to store size of parameter data in bytes
1007 *
1008 * Return: pointer to start of parameter data, PTR_ERR otherwise.
1009 */
1010void *dfh_find_param(struct dfl_device *dfl_dev, int param_id, size_t *psize)
1011{
1012	u64 *phdr = find_param(dfl_dev->params, dfl_dev->param_size, param_id);
1013
1014	if (!phdr)
1015		return ERR_PTR(-ENOENT);
1016
1017	if (psize)
1018		*psize = (FIELD_GET(DFHv1_PARAM_HDR_NEXT_OFFSET, *phdr) - 1) * sizeof(u64);
1019
1020	return phdr + 1;
1021}
1022EXPORT_SYMBOL_GPL(dfh_find_param);
1023
1024static int parse_feature_irqs(struct build_feature_devs_info *binfo,
1025			      resource_size_t ofst, struct dfl_feature_info *finfo)
1026{
1027	void __iomem *base = binfo->ioaddr + ofst;
1028	unsigned int i, ibase, inr = 0;
1029	void *params = finfo->params;
1030	enum dfl_id_type type;
1031	u16 fid = finfo->fid;
1032	int virq;
1033	u64 *p;
1034	u64 v;
1035
1036	switch (finfo->dfh_version) {
1037	case 0:
1038		/*
1039		 * DFHv0 only provides MMIO resource information for each feature
1040		 * in the DFL header.  There is no generic interrupt information.
1041		 * Instead, features with interrupt functionality provide
1042		 * the information in feature specific registers.
1043		 */
1044		type = binfo->type;
1045		if (type == PORT_ID) {
1046			switch (fid) {
1047			case PORT_FEATURE_ID_UINT:
1048				v = readq(base + PORT_UINT_CAP);
1049				ibase = FIELD_GET(PORT_UINT_CAP_FST_VECT, v);
1050				inr = FIELD_GET(PORT_UINT_CAP_INT_NUM, v);
1051				break;
1052			case PORT_FEATURE_ID_ERROR:
1053				v = readq(base + PORT_ERROR_CAP);
1054				ibase = FIELD_GET(PORT_ERROR_CAP_INT_VECT, v);
1055				inr = FIELD_GET(PORT_ERROR_CAP_SUPP_INT, v);
1056				break;
1057			}
1058		} else if (type == FME_ID) {
1059			switch (fid) {
1060			case FME_FEATURE_ID_GLOBAL_ERR:
1061				v = readq(base + FME_ERROR_CAP);
1062				ibase = FIELD_GET(FME_ERROR_CAP_INT_VECT, v);
1063				inr = FIELD_GET(FME_ERROR_CAP_SUPP_INT, v);
1064				break;
1065			}
1066		}
1067		break;
1068
1069	case 1:
1070		/*
1071		 * DFHv1 provides interrupt resource information in DFHv1
1072		 * parameter blocks.
1073		 */
1074		p = find_param(params, finfo->param_size, DFHv1_PARAM_ID_MSI_X);
1075		if (!p)
1076			break;
1077
1078		p++;
1079		ibase = FIELD_GET(DFHv1_PARAM_MSI_X_STARTV, *p);
1080		inr = FIELD_GET(DFHv1_PARAM_MSI_X_NUMV, *p);
1081		break;
1082
1083	default:
1084		dev_warn(binfo->dev, "unexpected DFH version %d\n", finfo->dfh_version);
1085		break;
1086	}
1087
1088	if (!inr) {
1089		finfo->irq_base = 0;
1090		finfo->nr_irqs = 0;
1091		return 0;
1092	}
1093
1094	dev_dbg(binfo->dev, "feature: 0x%x, irq_base: %u, nr_irqs: %u\n",
1095		fid, ibase, inr);
1096
1097	if (ibase + inr > binfo->nr_irqs) {
1098		dev_err(binfo->dev,
1099			"Invalid interrupt number in feature 0x%x\n", fid);
1100		return -EINVAL;
1101	}
1102
1103	for (i = 0; i < inr; i++) {
1104		virq = binfo->irq_table[ibase + i];
1105		if (virq < 0 || virq > NR_IRQS) {
1106			dev_err(binfo->dev,
1107				"Invalid irq table entry for feature 0x%x\n",
1108				fid);
1109			return -EINVAL;
1110		}
1111	}
1112
1113	finfo->irq_base = ibase;
1114	finfo->nr_irqs = inr;
1115
1116	return 0;
1117}
1118
1119static int dfh_get_param_size(void __iomem *dfh_base, resource_size_t max)
1120{
1121	int size = 0;
1122	u64 v, next;
1123
1124	if (!FIELD_GET(DFHv1_CSR_SIZE_GRP_HAS_PARAMS,
1125		       readq(dfh_base + DFHv1_CSR_SIZE_GRP)))
1126		return 0;
1127
1128	while (size + DFHv1_PARAM_HDR < max) {
1129		v = readq(dfh_base + DFHv1_PARAM_HDR + size);
1130
1131		next = FIELD_GET(DFHv1_PARAM_HDR_NEXT_OFFSET, v);
1132		if (!next)
1133			return -EINVAL;
1134
1135		size += next * sizeof(u64);
1136
1137		if (FIELD_GET(DFHv1_PARAM_HDR_NEXT_EOP, v))
1138			return size;
1139	}
1140
1141	return -ENOENT;
1142}
1143
1144/*
1145 * when create sub feature instances, for private features, it doesn't need
1146 * to provide resource size and feature id as they could be read from DFH
1147 * register. For afu sub feature, its register region only contains user
1148 * defined registers, so never trust any information from it, just use the
1149 * resource size information provided by its parent FIU.
1150 */
1151static int
1152create_feature_instance(struct build_feature_devs_info *binfo,
1153			resource_size_t ofst, resource_size_t size, u16 fid)
1154{
1155	struct dfl_feature_info *finfo;
1156	resource_size_t start, end;
1157	int dfh_psize = 0;
1158	u8 revision = 0;
1159	u64 v, addr_off;
1160	u8 dfh_ver = 0;
1161	int ret;
1162
1163	if (fid != FEATURE_ID_AFU) {
1164		v = readq(binfo->ioaddr + ofst);
1165		revision = FIELD_GET(DFH_REVISION, v);
1166		dfh_ver = FIELD_GET(DFH_VERSION, v);
1167		/* read feature size and id if inputs are invalid */
1168		size = size ? size : feature_size(v);
1169		fid = fid ? fid : feature_id(v);
1170		if (dfh_ver == 1) {
1171			dfh_psize = dfh_get_param_size(binfo->ioaddr + ofst, size);
1172			if (dfh_psize < 0) {
1173				dev_err(binfo->dev,
1174					"failed to read size of DFHv1 parameters %d\n",
1175					dfh_psize);
1176				return dfh_psize;
1177			}
1178			dev_dbg(binfo->dev, "dfhv1_psize %d\n", dfh_psize);
1179		}
1180	}
1181
1182	if (binfo->len - ofst < size)
1183		return -EINVAL;
1184
1185	finfo = kzalloc(struct_size(finfo, params, dfh_psize / sizeof(u64)), GFP_KERNEL);
1186	if (!finfo)
1187		return -ENOMEM;
1188
1189	memcpy_fromio(finfo->params, binfo->ioaddr + ofst + DFHv1_PARAM_HDR, dfh_psize);
1190	finfo->param_size = dfh_psize;
1191
1192	finfo->fid = fid;
1193	finfo->revision = revision;
1194	finfo->dfh_version = dfh_ver;
1195	if (dfh_ver == 1) {
1196		v = readq(binfo->ioaddr + ofst + DFHv1_CSR_ADDR);
1197		addr_off = FIELD_GET(DFHv1_CSR_ADDR_MASK, v);
1198		if (FIELD_GET(DFHv1_CSR_ADDR_REL, v))
1199			start = addr_off << 1;
1200		else
1201			start = binfo->start + ofst + addr_off;
1202
1203		v = readq(binfo->ioaddr + ofst + DFHv1_CSR_SIZE_GRP);
1204		end = start + FIELD_GET(DFHv1_CSR_SIZE_GRP_SIZE, v) - 1;
1205	} else {
1206		start = binfo->start + ofst;
1207		end = start + size - 1;
1208	}
1209	finfo->mmio_res.flags = IORESOURCE_MEM;
1210	finfo->mmio_res.start = start;
1211	finfo->mmio_res.end = end;
1212
1213	ret = parse_feature_irqs(binfo, ofst, finfo);
1214	if (ret) {
1215		kfree(finfo);
1216		return ret;
1217	}
1218
1219	list_add_tail(&finfo->node, &binfo->sub_features);
1220	binfo->feature_num++;
1221
1222	return 0;
1223}
1224
1225static int parse_feature_port_afu(struct build_feature_devs_info *binfo,
1226				  resource_size_t ofst)
1227{
1228	u64 v = readq(binfo->ioaddr + PORT_HDR_CAP);
1229	u32 size = FIELD_GET(PORT_CAP_MMIO_SIZE, v) << 10;
1230
1231	WARN_ON(!size);
1232
1233	return create_feature_instance(binfo, ofst, size, FEATURE_ID_AFU);
1234}
1235
1236#define is_feature_dev_detected(binfo) ((binfo)->type != DFL_ID_MAX)
1237
1238static int parse_feature_afu(struct build_feature_devs_info *binfo,
1239			     resource_size_t ofst)
1240{
1241	if (!is_feature_dev_detected(binfo)) {
1242		dev_err(binfo->dev, "this AFU does not belong to any FIU.\n");
1243		return -EINVAL;
1244	}
1245
1246	switch (binfo->type) {
1247	case PORT_ID:
1248		return parse_feature_port_afu(binfo, ofst);
1249	default:
1250		dev_info(binfo->dev, "AFU belonging to FIU is not supported yet.\n");
1251	}
1252
1253	return 0;
1254}
1255
1256static int build_info_prepare(struct build_feature_devs_info *binfo,
1257			      resource_size_t start, resource_size_t len)
1258{
1259	struct device *dev = binfo->dev;
1260	void __iomem *ioaddr;
1261
1262	if (!devm_request_mem_region(dev, start, len, dev_name(dev))) {
1263		dev_err(dev, "request region fail, start:%pa, len:%pa\n",
1264			&start, &len);
1265		return -EBUSY;
1266	}
1267
1268	ioaddr = devm_ioremap(dev, start, len);
1269	if (!ioaddr) {
1270		dev_err(dev, "ioremap region fail, start:%pa, len:%pa\n",
1271			&start, &len);
1272		return -ENOMEM;
1273	}
1274
1275	binfo->start = start;
1276	binfo->len = len;
1277	binfo->ioaddr = ioaddr;
1278
1279	return 0;
1280}
1281
1282static void build_info_complete(struct build_feature_devs_info *binfo)
1283{
1284	devm_iounmap(binfo->dev, binfo->ioaddr);
1285	devm_release_mem_region(binfo->dev, binfo->start, binfo->len);
1286}
1287
1288static int parse_feature_fiu(struct build_feature_devs_info *binfo,
1289			     resource_size_t ofst)
1290{
1291	enum dfl_id_type type;
1292	int ret = 0;
1293	u32 offset;
1294	u16 id;
1295	u64 v;
1296
1297	if (is_feature_dev_detected(binfo)) {
1298		build_info_complete(binfo);
1299
1300		ret = build_info_commit_dev(binfo);
1301		if (ret)
1302			return ret;
1303
1304		ret = build_info_prepare(binfo, binfo->start + ofst,
1305					 binfo->len - ofst);
1306		if (ret)
1307			return ret;
1308	}
1309
1310	v = readq(binfo->ioaddr + DFH);
1311	id = FIELD_GET(DFH_ID, v);
1312
1313	type = dfh_id_to_type(id);
1314	if (type >= DFL_ID_MAX)
1315		return -EINVAL;
1316
1317	binfo->type = type;
1318	binfo->feature_num = 0;
1319	INIT_LIST_HEAD(&binfo->sub_features);
1320
1321	ret = create_feature_instance(binfo, 0, 0, 0);
1322	if (ret)
1323		return ret;
1324	/*
1325	 * find and parse FIU's child AFU via its NEXT_AFU register.
1326	 * please note that only Port has valid NEXT_AFU pointer per spec.
1327	 */
1328	v = readq(binfo->ioaddr + NEXT_AFU);
1329
1330	offset = FIELD_GET(NEXT_AFU_NEXT_DFH_OFST, v);
1331	if (offset)
1332		return parse_feature_afu(binfo, offset);
1333
1334	dev_dbg(binfo->dev, "No AFUs detected on FIU %d\n", id);
1335
1336	return ret;
1337}
1338
1339static int parse_feature_private(struct build_feature_devs_info *binfo,
1340				 resource_size_t ofst)
1341{
1342	if (!is_feature_dev_detected(binfo)) {
1343		dev_err(binfo->dev, "the private feature 0x%x does not belong to any AFU.\n",
1344			feature_id(readq(binfo->ioaddr + ofst)));
1345		return -EINVAL;
1346	}
1347
1348	return create_feature_instance(binfo, ofst, 0, 0);
1349}
1350
1351/**
1352 * parse_feature - parse a feature on given device feature list
1353 *
1354 * @binfo: build feature devices information.
1355 * @ofst: offset to current FIU header
1356 */
1357static int parse_feature(struct build_feature_devs_info *binfo,
1358			 resource_size_t ofst)
1359{
1360	u64 v;
1361	u32 type;
1362
1363	v = readq(binfo->ioaddr + ofst + DFH);
1364	type = FIELD_GET(DFH_TYPE, v);
1365
1366	switch (type) {
1367	case DFH_TYPE_AFU:
1368		return parse_feature_afu(binfo, ofst);
1369	case DFH_TYPE_PRIVATE:
1370		return parse_feature_private(binfo, ofst);
1371	case DFH_TYPE_FIU:
1372		return parse_feature_fiu(binfo, ofst);
1373	default:
1374		dev_info(binfo->dev,
1375			 "Feature Type %x is not supported.\n", type);
1376	}
1377
1378	return 0;
1379}
1380
1381static int parse_feature_list(struct build_feature_devs_info *binfo,
1382			      resource_size_t start, resource_size_t len)
1383{
1384	resource_size_t end = start + len;
1385	int ret = 0;
1386	u32 ofst = 0;
1387	u64 v;
1388
1389	ret = build_info_prepare(binfo, start, len);
1390	if (ret)
1391		return ret;
1392
1393	/* walk through the device feature list via DFH's next DFH pointer. */
1394	for (; start < end; start += ofst) {
1395		if (end - start < DFH_SIZE) {
1396			dev_err(binfo->dev, "The region is too small to contain a feature.\n");
1397			return -EINVAL;
1398		}
1399
1400		ret = parse_feature(binfo, start - binfo->start);
1401		if (ret)
1402			return ret;
1403
1404		v = readq(binfo->ioaddr + start - binfo->start + DFH);
1405		ofst = FIELD_GET(DFH_NEXT_HDR_OFST, v);
1406
1407		/* stop parsing if EOL(End of List) is set or offset is 0 */
1408		if ((v & DFH_EOL) || !ofst)
1409			break;
1410	}
1411
1412	/* commit current feature device when reach the end of list */
1413	build_info_complete(binfo);
1414
1415	if (is_feature_dev_detected(binfo))
1416		ret = build_info_commit_dev(binfo);
1417
1418	return ret;
1419}
1420
1421struct dfl_fpga_enum_info *dfl_fpga_enum_info_alloc(struct device *dev)
1422{
1423	struct dfl_fpga_enum_info *info;
1424
1425	get_device(dev);
1426
1427	info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);
1428	if (!info) {
1429		put_device(dev);
1430		return NULL;
1431	}
1432
1433	info->dev = dev;
1434	INIT_LIST_HEAD(&info->dfls);
1435
1436	return info;
1437}
1438EXPORT_SYMBOL_GPL(dfl_fpga_enum_info_alloc);
1439
1440void dfl_fpga_enum_info_free(struct dfl_fpga_enum_info *info)
1441{
1442	struct dfl_fpga_enum_dfl *tmp, *dfl;
1443	struct device *dev;
1444
1445	if (!info)
1446		return;
1447
1448	dev = info->dev;
1449
1450	/* remove all device feature lists in the list. */
1451	list_for_each_entry_safe(dfl, tmp, &info->dfls, node) {
1452		list_del(&dfl->node);
1453		devm_kfree(dev, dfl);
1454	}
1455
1456	/* remove irq table */
1457	if (info->irq_table)
1458		devm_kfree(dev, info->irq_table);
1459
1460	devm_kfree(dev, info);
1461	put_device(dev);
1462}
1463EXPORT_SYMBOL_GPL(dfl_fpga_enum_info_free);
1464
1465/**
1466 * dfl_fpga_enum_info_add_dfl - add info of a device feature list to enum info
1467 *
1468 * @info: ptr to dfl_fpga_enum_info
1469 * @start: mmio resource address of the device feature list.
1470 * @len: mmio resource length of the device feature list.
1471 *
1472 * One FPGA device may have one or more Device Feature Lists (DFLs), use this
1473 * function to add information of each DFL to common data structure for next
1474 * step enumeration.
1475 *
1476 * Return: 0 on success, negative error code otherwise.
1477 */
1478int dfl_fpga_enum_info_add_dfl(struct dfl_fpga_enum_info *info,
1479			       resource_size_t start, resource_size_t len)
1480{
1481	struct dfl_fpga_enum_dfl *dfl;
1482
1483	dfl = devm_kzalloc(info->dev, sizeof(*dfl), GFP_KERNEL);
1484	if (!dfl)
1485		return -ENOMEM;
1486
1487	dfl->start = start;
1488	dfl->len = len;
1489
1490	list_add_tail(&dfl->node, &info->dfls);
1491
1492	return 0;
1493}
1494EXPORT_SYMBOL_GPL(dfl_fpga_enum_info_add_dfl);
1495
1496/**
1497 * dfl_fpga_enum_info_add_irq - add irq table to enum info
1498 *
1499 * @info: ptr to dfl_fpga_enum_info
1500 * @nr_irqs: number of irqs of the DFL fpga device to be enumerated.
1501 * @irq_table: Linux IRQ numbers for all irqs, indexed by local irq index of
1502 *	       this device.
1503 *
1504 * One FPGA device may have several interrupts. This function adds irq
1505 * information of the DFL fpga device to enum info for next step enumeration.
1506 * This function should be called before dfl_fpga_feature_devs_enumerate().
1507 * As we only support one irq domain for all DFLs in the same enum info, adding
1508 * irq table a second time for the same enum info will return error.
1509 *
1510 * If we need to enumerate DFLs which belong to different irq domains, we
1511 * should fill more enum info and enumerate them one by one.
1512 *
1513 * Return: 0 on success, negative error code otherwise.
1514 */
1515int dfl_fpga_enum_info_add_irq(struct dfl_fpga_enum_info *info,
1516			       unsigned int nr_irqs, int *irq_table)
1517{
1518	if (!nr_irqs || !irq_table)
1519		return -EINVAL;
1520
1521	if (info->irq_table)
1522		return -EEXIST;
1523
1524	info->irq_table = devm_kmemdup(info->dev, irq_table,
1525				       sizeof(int) * nr_irqs, GFP_KERNEL);
1526	if (!info->irq_table)
1527		return -ENOMEM;
1528
1529	info->nr_irqs = nr_irqs;
1530
1531	return 0;
1532}
1533EXPORT_SYMBOL_GPL(dfl_fpga_enum_info_add_irq);
1534
1535static int remove_feature_dev(struct device *dev, void *data)
1536{
1537	struct dfl_feature_dev_data *fdata = to_dfl_feature_dev_data(dev);
1538
1539	feature_dev_unregister(fdata);
1540
1541	return 0;
1542}
1543
1544static void remove_feature_devs(struct dfl_fpga_cdev *cdev)
1545{
1546	device_for_each_child(&cdev->region->dev, NULL, remove_feature_dev);
1547}
1548
1549/**
1550 * dfl_fpga_feature_devs_enumerate - enumerate feature devices
1551 * @info: information for enumeration.
1552 *
1553 * This function creates a container device (base FPGA region), enumerates
1554 * feature devices based on the enumeration info and creates platform devices
1555 * under the container device.
1556 *
1557 * Return: dfl_fpga_cdev struct on success, -errno on failure
1558 */
1559struct dfl_fpga_cdev *
1560dfl_fpga_feature_devs_enumerate(struct dfl_fpga_enum_info *info)
1561{
1562	struct build_feature_devs_info *binfo;
1563	struct dfl_fpga_enum_dfl *dfl;
1564	struct dfl_fpga_cdev *cdev;
1565	int ret = 0;
1566
1567	if (!info->dev)
1568		return ERR_PTR(-ENODEV);
1569
1570	cdev = devm_kzalloc(info->dev, sizeof(*cdev), GFP_KERNEL);
1571	if (!cdev)
1572		return ERR_PTR(-ENOMEM);
1573
1574	cdev->parent = info->dev;
1575	mutex_init(&cdev->lock);
1576	INIT_LIST_HEAD(&cdev->port_dev_list);
1577
1578	cdev->region = fpga_region_register(info->dev, NULL, NULL);
1579	if (IS_ERR(cdev->region)) {
1580		ret = PTR_ERR(cdev->region);
1581		goto free_cdev_exit;
1582	}
1583
1584	/* create and init build info for enumeration */
1585	binfo = devm_kzalloc(info->dev, sizeof(*binfo), GFP_KERNEL);
1586	if (!binfo) {
1587		ret = -ENOMEM;
1588		goto unregister_region_exit;
1589	}
1590
1591	binfo->type = DFL_ID_MAX;
1592	binfo->dev = info->dev;
1593	binfo->cdev = cdev;
1594
1595	binfo->nr_irqs = info->nr_irqs;
1596	if (info->nr_irqs)
1597		binfo->irq_table = info->irq_table;
1598
1599	/*
1600	 * start enumeration for all feature devices based on Device Feature
1601	 * Lists.
1602	 */
1603	list_for_each_entry(dfl, &info->dfls, node) {
1604		ret = parse_feature_list(binfo, dfl->start, dfl->len);
1605		if (ret) {
1606			remove_feature_devs(cdev);
1607			build_info_free(binfo);
1608			goto unregister_region_exit;
1609		}
1610	}
1611
1612	build_info_free(binfo);
1613
1614	return cdev;
1615
1616unregister_region_exit:
1617	fpga_region_unregister(cdev->region);
1618free_cdev_exit:
1619	devm_kfree(info->dev, cdev);
1620	return ERR_PTR(ret);
1621}
1622EXPORT_SYMBOL_GPL(dfl_fpga_feature_devs_enumerate);
1623
1624/**
1625 * dfl_fpga_feature_devs_remove - remove all feature devices
1626 * @cdev: fpga container device.
1627 *
1628 * Remove the container device and all feature devices under given container
1629 * devices.
1630 */
1631void dfl_fpga_feature_devs_remove(struct dfl_fpga_cdev *cdev)
1632{
1633	mutex_lock(&cdev->lock);
1634	if (cdev->fme_dev)
1635		put_device(cdev->fme_dev);
1636
1637	mutex_unlock(&cdev->lock);
1638
1639	remove_feature_devs(cdev);
1640
1641	fpga_region_unregister(cdev->region);
1642	devm_kfree(cdev->parent, cdev);
1643}
1644EXPORT_SYMBOL_GPL(dfl_fpga_feature_devs_remove);
1645
1646/**
1647 * __dfl_fpga_cdev_find_port_data - find a port under given container device
1648 *
1649 * @cdev: container device
1650 * @data: data passed to match function
1651 * @match: match function used to find specific port from the port device list
1652 *
1653 * Find a port device under container device. This function needs to be
1654 * invoked with lock held.
1655 *
1656 * Return: pointer to port's platform device if successful, NULL otherwise.
1657 *
1658 * NOTE: you will need to drop the device reference with put_device() after use.
1659 */
1660struct dfl_feature_dev_data *
1661__dfl_fpga_cdev_find_port_data(struct dfl_fpga_cdev *cdev, void *data,
1662			       int (*match)(struct dfl_feature_dev_data *, void *))
1663{
1664	struct dfl_feature_dev_data *fdata;
1665
1666	list_for_each_entry(fdata, &cdev->port_dev_list, node) {
1667		if (match(fdata, data))
1668			return fdata;
1669	}
1670
1671	return NULL;
1672}
1673EXPORT_SYMBOL_GPL(__dfl_fpga_cdev_find_port_data);
1674
1675static int __init dfl_fpga_init(void)
1676{
1677	int ret;
1678
1679	ret = bus_register(&dfl_bus_type);
1680	if (ret)
1681		return ret;
1682
1683	dfl_ids_init();
1684
1685	ret = dfl_chardev_init();
1686	if (ret) {
1687		dfl_ids_destroy();
1688		bus_unregister(&dfl_bus_type);
1689	}
1690
1691	return ret;
1692}
1693
1694/**
1695 * dfl_fpga_cdev_release_port - release a port platform device
1696 *
1697 * @cdev: parent container device.
1698 * @port_id: id of the port platform device.
1699 *
1700 * This function allows user to release a port platform device. This is a
1701 * mandatory step before turn a port from PF into VF for SRIOV support.
1702 *
1703 * Return: 0 on success, negative error code otherwise.
1704 */
1705int dfl_fpga_cdev_release_port(struct dfl_fpga_cdev *cdev, int port_id)
1706{
1707	struct dfl_feature_dev_data *fdata;
1708	int ret = -ENODEV;
1709
1710	mutex_lock(&cdev->lock);
1711	fdata = __dfl_fpga_cdev_find_port_data(cdev, &port_id,
1712					       dfl_fpga_check_port_id);
1713	if (!fdata)
1714		goto unlock_exit;
1715
1716	if (!fdata->dev) {
1717		ret = -EBUSY;
1718		goto unlock_exit;
1719	}
1720
1721	mutex_lock(&fdata->lock);
1722	ret = dfl_feature_dev_use_begin(fdata, true);
1723	mutex_unlock(&fdata->lock);
1724	if (ret)
1725		goto unlock_exit;
1726
1727	feature_dev_unregister(fdata);
1728	cdev->released_port_num++;
1729unlock_exit:
1730	mutex_unlock(&cdev->lock);
1731	return ret;
1732}
1733EXPORT_SYMBOL_GPL(dfl_fpga_cdev_release_port);
1734
1735/**
1736 * dfl_fpga_cdev_assign_port - assign a port platform device back
1737 *
1738 * @cdev: parent container device.
1739 * @port_id: id of the port platform device.
1740 *
1741 * This function allows user to assign a port platform device back. This is
1742 * a mandatory step after disable SRIOV support.
1743 *
1744 * Return: 0 on success, negative error code otherwise.
1745 */
1746int dfl_fpga_cdev_assign_port(struct dfl_fpga_cdev *cdev, int port_id)
1747{
1748	struct dfl_feature_dev_data *fdata;
1749	int ret = -ENODEV;
1750
1751	mutex_lock(&cdev->lock);
1752	fdata = __dfl_fpga_cdev_find_port_data(cdev, &port_id,
1753					       dfl_fpga_check_port_id);
1754	if (!fdata)
1755		goto unlock_exit;
1756
1757	if (fdata->dev) {
1758		ret = -EBUSY;
1759		goto unlock_exit;
1760	}
1761
1762	ret = feature_dev_register(fdata);
1763	if (ret)
1764		goto unlock_exit;
1765
1766	mutex_lock(&fdata->lock);
1767	dfl_feature_dev_use_end(fdata);
1768	mutex_unlock(&fdata->lock);
1769
1770	cdev->released_port_num--;
1771unlock_exit:
1772	mutex_unlock(&cdev->lock);
1773	return ret;
1774}
1775EXPORT_SYMBOL_GPL(dfl_fpga_cdev_assign_port);
1776
1777static void config_port_access_mode(struct device *fme_dev, int port_id,
1778				    bool is_vf)
1779{
1780	struct dfl_feature_dev_data *fdata = to_dfl_feature_dev_data(fme_dev);
1781	void __iomem *base;
1782	u64 v;
1783
1784	base = dfl_get_feature_ioaddr_by_id(fdata, FME_FEATURE_ID_HEADER);
1785
1786	v = readq(base + FME_HDR_PORT_OFST(port_id));
1787
1788	v &= ~FME_PORT_OFST_ACC_CTRL;
1789	v |= FIELD_PREP(FME_PORT_OFST_ACC_CTRL,
1790			is_vf ? FME_PORT_OFST_ACC_VF : FME_PORT_OFST_ACC_PF);
1791
1792	writeq(v, base + FME_HDR_PORT_OFST(port_id));
1793}
1794
1795#define config_port_vf_mode(dev, id) config_port_access_mode(dev, id, true)
1796#define config_port_pf_mode(dev, id) config_port_access_mode(dev, id, false)
1797
1798/**
1799 * dfl_fpga_cdev_config_ports_pf - configure ports to PF access mode
1800 *
1801 * @cdev: parent container device.
1802 *
1803 * This function is needed in sriov configuration routine. It could be used to
1804 * configure the all released ports from VF access mode to PF.
1805 */
1806void dfl_fpga_cdev_config_ports_pf(struct dfl_fpga_cdev *cdev)
1807{
1808	struct dfl_feature_dev_data *fdata;
1809
1810	mutex_lock(&cdev->lock);
1811	list_for_each_entry(fdata, &cdev->port_dev_list, node) {
1812		if (fdata->dev)
1813			continue;
1814
1815		config_port_pf_mode(cdev->fme_dev, fdata->id);
1816	}
1817	mutex_unlock(&cdev->lock);
1818}
1819EXPORT_SYMBOL_GPL(dfl_fpga_cdev_config_ports_pf);
1820
1821/**
1822 * dfl_fpga_cdev_config_ports_vf - configure ports to VF access mode
1823 *
1824 * @cdev: parent container device.
1825 * @num_vfs: VF device number.
1826 *
1827 * This function is needed in sriov configuration routine. It could be used to
1828 * configure the released ports from PF access mode to VF.
1829 *
1830 * Return: 0 on success, negative error code otherwise.
1831 */
1832int dfl_fpga_cdev_config_ports_vf(struct dfl_fpga_cdev *cdev, int num_vfs)
1833{
1834	struct dfl_feature_dev_data *fdata;
1835	int ret = 0;
1836
1837	mutex_lock(&cdev->lock);
1838	/*
1839	 * can't turn multiple ports into 1 VF device, only 1 port for 1 VF
1840	 * device, so if released port number doesn't match VF device number,
1841	 * then reject the request with -EINVAL error code.
1842	 */
1843	if (cdev->released_port_num != num_vfs) {
1844		ret = -EINVAL;
1845		goto done;
1846	}
1847
1848	list_for_each_entry(fdata, &cdev->port_dev_list, node) {
1849		if (fdata->dev)
1850			continue;
1851
1852		config_port_vf_mode(cdev->fme_dev, fdata->id);
1853	}
1854done:
1855	mutex_unlock(&cdev->lock);
1856	return ret;
1857}
1858EXPORT_SYMBOL_GPL(dfl_fpga_cdev_config_ports_vf);
1859
1860static irqreturn_t dfl_irq_handler(int irq, void *arg)
1861{
1862	struct eventfd_ctx *trigger = arg;
1863
1864	eventfd_signal(trigger);
1865	return IRQ_HANDLED;
1866}
1867
1868static int do_set_irq_trigger(struct dfl_feature *feature, unsigned int idx,
1869			      int fd)
1870{
1871	struct platform_device *pdev = feature->dev;
1872	struct eventfd_ctx *trigger;
1873	int irq, ret;
1874
1875	irq = feature->irq_ctx[idx].irq;
1876
1877	if (feature->irq_ctx[idx].trigger) {
1878		free_irq(irq, feature->irq_ctx[idx].trigger);
1879		kfree(feature->irq_ctx[idx].name);
1880		eventfd_ctx_put(feature->irq_ctx[idx].trigger);
1881		feature->irq_ctx[idx].trigger = NULL;
1882	}
1883
1884	if (fd < 0)
1885		return 0;
1886
1887	feature->irq_ctx[idx].name =
1888		kasprintf(GFP_KERNEL, "fpga-irq[%u](%s-%x)", idx,
1889			  dev_name(&pdev->dev), feature->id);
1890	if (!feature->irq_ctx[idx].name)
1891		return -ENOMEM;
1892
1893	trigger = eventfd_ctx_fdget(fd);
1894	if (IS_ERR(trigger)) {
1895		ret = PTR_ERR(trigger);
1896		goto free_name;
1897	}
1898
1899	ret = request_irq(irq, dfl_irq_handler, 0,
1900			  feature->irq_ctx[idx].name, trigger);
1901	if (!ret) {
1902		feature->irq_ctx[idx].trigger = trigger;
1903		return ret;
1904	}
1905
1906	eventfd_ctx_put(trigger);
1907free_name:
1908	kfree(feature->irq_ctx[idx].name);
1909
1910	return ret;
1911}
1912
1913/**
1914 * dfl_fpga_set_irq_triggers - set eventfd triggers for dfl feature interrupts
1915 *
1916 * @feature: dfl sub feature.
1917 * @start: start of irq index in this dfl sub feature.
1918 * @count: number of irqs.
1919 * @fds: eventfds to bind with irqs. unbind related irq if fds[n] is negative.
1920 *	 unbind "count" specified number of irqs if fds ptr is NULL.
1921 *
1922 * Bind given eventfds with irqs in this dfl sub feature. Unbind related irq if
1923 * fds[n] is negative. Unbind "count" specified number of irqs if fds ptr is
1924 * NULL.
1925 *
1926 * Return: 0 on success, negative error code otherwise.
1927 */
1928int dfl_fpga_set_irq_triggers(struct dfl_feature *feature, unsigned int start,
1929			      unsigned int count, int32_t *fds)
1930{
1931	unsigned int i;
1932	int ret = 0;
1933
1934	/* overflow */
1935	if (unlikely(start + count < start))
1936		return -EINVAL;
1937
1938	/* exceeds nr_irqs */
1939	if (start + count > feature->nr_irqs)
1940		return -EINVAL;
1941
1942	for (i = 0; i < count; i++) {
1943		int fd = fds ? fds[i] : -1;
1944
1945		ret = do_set_irq_trigger(feature, start + i, fd);
1946		if (ret) {
1947			while (i--)
1948				do_set_irq_trigger(feature, start + i, -1);
1949			break;
1950		}
1951	}
1952
1953	return ret;
1954}
1955EXPORT_SYMBOL_GPL(dfl_fpga_set_irq_triggers);
1956
1957/**
1958 * dfl_feature_ioctl_get_num_irqs - dfl feature _GET_IRQ_NUM ioctl interface.
1959 * @pdev: the feature device which has the sub feature
1960 * @feature: the dfl sub feature
1961 * @arg: ioctl argument
1962 *
1963 * Return: 0 on success, negative error code otherwise.
1964 */
1965long dfl_feature_ioctl_get_num_irqs(struct platform_device *pdev,
1966				    struct dfl_feature *feature,
1967				    unsigned long arg)
1968{
1969	return put_user(feature->nr_irqs, (__u32 __user *)arg);
1970}
1971EXPORT_SYMBOL_GPL(dfl_feature_ioctl_get_num_irqs);
1972
1973/**
1974 * dfl_feature_ioctl_set_irq - dfl feature _SET_IRQ ioctl interface.
1975 * @pdev: the feature device which has the sub feature
1976 * @feature: the dfl sub feature
1977 * @arg: ioctl argument
1978 *
1979 * Return: 0 on success, negative error code otherwise.
1980 */
1981long dfl_feature_ioctl_set_irq(struct platform_device *pdev,
1982			       struct dfl_feature *feature,
1983			       unsigned long arg)
1984{
1985	struct dfl_feature_dev_data *fdata = to_dfl_feature_dev_data(&pdev->dev);
1986	struct dfl_fpga_irq_set hdr;
1987	s32 *fds;
1988	long ret;
1989
1990	if (!feature->nr_irqs)
1991		return -ENOENT;
1992
1993	if (copy_from_user(&hdr, (void __user *)arg, sizeof(hdr)))
1994		return -EFAULT;
1995
1996	if (!hdr.count || (hdr.start + hdr.count > feature->nr_irqs) ||
1997	    (hdr.start + hdr.count < hdr.start))
1998		return -EINVAL;
1999
2000	fds = memdup_array_user((void __user *)(arg + sizeof(hdr)),
2001				hdr.count, sizeof(s32));
2002	if (IS_ERR(fds))
2003		return PTR_ERR(fds);
2004
2005	mutex_lock(&fdata->lock);
2006	ret = dfl_fpga_set_irq_triggers(feature, hdr.start, hdr.count, fds);
2007	mutex_unlock(&fdata->lock);
2008
2009	kfree(fds);
2010	return ret;
2011}
2012EXPORT_SYMBOL_GPL(dfl_feature_ioctl_set_irq);
2013
2014static void __exit dfl_fpga_exit(void)
2015{
2016	dfl_chardev_uinit();
2017	dfl_ids_destroy();
2018	bus_unregister(&dfl_bus_type);
2019}
2020
2021subsys_initcall(dfl_fpga_init);
2022module_exit(dfl_fpga_exit);
2023
2024MODULE_DESCRIPTION("FPGA Device Feature List (DFL) Support");
2025MODULE_AUTHOR("Intel Corporation");
2026MODULE_LICENSE("GPL v2");