dd.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * drivers/base/dd.c - The core device/driver interactions.
   4 *
   5 * This file contains the (sometimes tricky) code that controls the
   6 * interactions between devices and drivers, which primarily includes
   7 * driver binding and unbinding.
   8 *
   9 * All of this code used to exist in drivers/base/bus.c, but was
  10 * relocated to here in the name of compartmentalization (since it wasn't
  11 * strictly code just for the 'struct bus_type'.
  12 *
  13 * Copyright (c) 2002-5 Patrick Mochel
  14 * Copyright (c) 2002-3 Open Source Development Labs
  15 * Copyright (c) 2007-2009 Greg Kroah-Hartman <gregkh@suse.de>
  16 * Copyright (c) 2007-2009 Novell Inc.
  17 */
  18
  19#include <linux/debugfs.h>
  20#include <linux/device.h>
  21#include <linux/delay.h>
  22#include <linux/dma-map-ops.h>
  23#include <linux/init.h>
  24#include <linux/module.h>
  25#include <linux/kthread.h>
  26#include <linux/wait.h>
  27#include <linux/async.h>
  28#include <linux/pm_domain.h>
  29#include <linux/pm_runtime.h>
  30#include <linux/pinctrl/devinfo.h>
  31#include <linux/slab.h>
  32
  33#include "base.h"
  34#include "power/power.h"
  35
  36/*
  37 * Deferred Probe infrastructure.
  38 *
  39 * Sometimes driver probe order matters, but the kernel doesn't always have
  40 * dependency information which means some drivers will get probed before a
  41 * resource it depends on is available.  For example, an SDHCI driver may
  42 * first need a GPIO line from an i2c GPIO controller before it can be
  43 * initialized.  If a required resource is not available yet, a driver can
  44 * request probing to be deferred by returning -EPROBE_DEFER from its probe hook
  45 *
  46 * Deferred probe maintains two lists of devices, a pending list and an active
  47 * list.  A driver returning -EPROBE_DEFER causes the device to be added to the
  48 * pending list.  A successful driver probe will trigger moving all devices
  49 * from the pending to the active list so that the workqueue will eventually
  50 * retry them.
  51 *
  52 * The deferred_probe_mutex must be held any time the deferred_probe_*_list
  53 * of the (struct device*)->p->deferred_probe pointers are manipulated
  54 */
  55static DEFINE_MUTEX(deferred_probe_mutex);
  56static LIST_HEAD(deferred_probe_pending_list);
  57static LIST_HEAD(deferred_probe_active_list);
  58static atomic_t deferred_trigger_count = ATOMIC_INIT(0);
  59static bool initcalls_done;
  60
  61/* Save the async probe drivers' name from kernel cmdline */
  62#define ASYNC_DRV_NAMES_MAX_LEN	256
  63static char async_probe_drv_names[ASYNC_DRV_NAMES_MAX_LEN];
  64static bool async_probe_default;
  65
  66/*
  67 * In some cases, like suspend to RAM or hibernation, It might be reasonable
  68 * to prohibit probing of devices as it could be unsafe.
  69 * Once defer_all_probes is true all drivers probes will be forcibly deferred.
  70 */
  71static bool defer_all_probes;
  72
  73static void __device_set_deferred_probe_reason(const struct device *dev, char *reason)
  74{
  75	kfree(dev->p->deferred_probe_reason);
  76	dev->p->deferred_probe_reason = reason;
  77}
  78
  79/*
  80 * deferred_probe_work_func() - Retry probing devices in the active list.
  81 */
  82static void deferred_probe_work_func(struct work_struct *work)
  83{
  84	struct device *dev;
  85	struct device_private *private;
  86	/*
  87	 * This block processes every device in the deferred 'active' list.
  88	 * Each device is removed from the active list and passed to
  89	 * bus_probe_device() to re-attempt the probe.  The loop continues
  90	 * until every device in the active list is removed and retried.
  91	 *
  92	 * Note: Once the device is removed from the list and the mutex is
  93	 * released, it is possible for the device get freed by another thread
  94	 * and cause a illegal pointer dereference.  This code uses
  95	 * get/put_device() to ensure the device structure cannot disappear
  96	 * from under our feet.
  97	 */
  98	mutex_lock(&deferred_probe_mutex);
  99	while (!list_empty(&deferred_probe_active_list)) {
 100		private = list_first_entry(&deferred_probe_active_list,
 101					typeof(*dev->p), deferred_probe);
 102		dev = private->device;
 103		list_del_init(&private->deferred_probe);
 104
 105		get_device(dev);
 106
 107		__device_set_deferred_probe_reason(dev, NULL);
 108
 109		/*
 110		 * Drop the mutex while probing each device; the probe path may
 111		 * manipulate the deferred list
 112		 */
 113		mutex_unlock(&deferred_probe_mutex);
 114
 115		/*
 116		 * Force the device to the end of the dpm_list since
 117		 * the PM code assumes that the order we add things to
 118		 * the list is a good order for suspend but deferred
 119		 * probe makes that very unsafe.
 120		 */
 121		device_pm_move_to_tail(dev);
 122
 123		dev_dbg(dev, "Retrying from deferred list\n");
 124		bus_probe_device(dev);
 125		mutex_lock(&deferred_probe_mutex);
 126
 127		put_device(dev);
 128	}
 129	mutex_unlock(&deferred_probe_mutex);
 130}
 131static DECLARE_WORK(deferred_probe_work, deferred_probe_work_func);
 132
 133void driver_deferred_probe_add(struct device *dev)
 134{
 135	if (!dev->can_match)
 136		return;
 137
 138	mutex_lock(&deferred_probe_mutex);
 139	if (list_empty(&dev->p->deferred_probe)) {
 140		dev_dbg(dev, "Added to deferred list\n");
 141		list_add_tail(&dev->p->deferred_probe, &deferred_probe_pending_list);
 142	}
 143	mutex_unlock(&deferred_probe_mutex);
 144}
 145
 146void driver_deferred_probe_del(struct device *dev)
 147{
 148	mutex_lock(&deferred_probe_mutex);
 149	if (!list_empty(&dev->p->deferred_probe)) {
 150		dev_dbg(dev, "Removed from deferred list\n");
 151		list_del_init(&dev->p->deferred_probe);
 152		__device_set_deferred_probe_reason(dev, NULL);
 153	}
 154	mutex_unlock(&deferred_probe_mutex);
 155}
 156
 157static bool driver_deferred_probe_enable;
 158/**
 159 * driver_deferred_probe_trigger() - Kick off re-probing deferred devices
 160 *
 161 * This functions moves all devices from the pending list to the active
 162 * list and schedules the deferred probe workqueue to process them.  It
 163 * should be called anytime a driver is successfully bound to a device.
 164 *
 165 * Note, there is a race condition in multi-threaded probe. In the case where
 166 * more than one device is probing at the same time, it is possible for one
 167 * probe to complete successfully while another is about to defer. If the second
 168 * depends on the first, then it will get put on the pending list after the
 169 * trigger event has already occurred and will be stuck there.
 170 *
 171 * The atomic 'deferred_trigger_count' is used to determine if a successful
 172 * trigger has occurred in the midst of probing a driver. If the trigger count
 173 * changes in the midst of a probe, then deferred processing should be triggered
 174 * again.
 175 */
 176void driver_deferred_probe_trigger(void)
 177{
 178	if (!driver_deferred_probe_enable)
 179		return;
 180
 181	/*
 182	 * A successful probe means that all the devices in the pending list
 183	 * should be triggered to be reprobed.  Move all the deferred devices
 184	 * into the active list so they can be retried by the workqueue
 185	 */
 186	mutex_lock(&deferred_probe_mutex);
 187	atomic_inc(&deferred_trigger_count);
 188	list_splice_tail_init(&deferred_probe_pending_list,
 189			      &deferred_probe_active_list);
 190	mutex_unlock(&deferred_probe_mutex);
 191
 192	/*
 193	 * Kick the re-probe thread.  It may already be scheduled, but it is
 194	 * safe to kick it again.
 195	 */
 196	queue_work(system_dfl_wq, &deferred_probe_work);
 197}
 198
 199/**
 200 * device_block_probing() - Block/defer device's probes
 201 *
 202 *	It will disable probing of devices and defer their probes instead.
 203 */
 204void device_block_probing(void)
 205{
 206	defer_all_probes = true;
 207	/* sync with probes to avoid races. */
 208	wait_for_device_probe();
 209}
 210
 211/**
 212 * device_unblock_probing() - Unblock/enable device's probes
 213 *
 214 *	It will restore normal behavior and trigger re-probing of deferred
 215 * devices.
 216 */
 217void device_unblock_probing(void)
 218{
 219	defer_all_probes = false;
 220	driver_deferred_probe_trigger();
 221}
 222
 223/**
 224 * device_set_deferred_probe_reason() - Set defer probe reason message for device
 225 * @dev: the pointer to the struct device
 226 * @vaf: the pointer to va_format structure with message
 227 */
 228void device_set_deferred_probe_reason(const struct device *dev, struct va_format *vaf)
 229{
 230	const char *drv = dev_driver_string(dev);
 231	char *reason;
 232
 233	mutex_lock(&deferred_probe_mutex);
 234
 235	reason = kasprintf(GFP_KERNEL, "%s: %pV", drv, vaf);
 236	__device_set_deferred_probe_reason(dev, reason);
 237
 238	mutex_unlock(&deferred_probe_mutex);
 239}
 240
 241/*
 242 * deferred_devs_show() - Show the devices in the deferred probe pending list.
 243 */
 244static int deferred_devs_show(struct seq_file *s, void *data)
 245{
 246	struct device_private *curr;
 247
 248	mutex_lock(&deferred_probe_mutex);
 249
 250	list_for_each_entry(curr, &deferred_probe_pending_list, deferred_probe)
 251		seq_printf(s, "%s\t%s", dev_name(curr->device),
 252			   curr->deferred_probe_reason ?: "\n");
 253
 254	mutex_unlock(&deferred_probe_mutex);
 255
 256	return 0;
 257}
 258DEFINE_SHOW_ATTRIBUTE(deferred_devs);
 259
 260#ifdef CONFIG_MODULES
 261static int driver_deferred_probe_timeout = 10;
 262#else
 263static int driver_deferred_probe_timeout;
 264#endif
 265
 266static int __init deferred_probe_timeout_setup(char *str)
 267{
 268	int timeout;
 269
 270	if (!kstrtoint(str, 10, &timeout))
 271		driver_deferred_probe_timeout = timeout;
 272	return 1;
 273}
 274__setup("deferred_probe_timeout=", deferred_probe_timeout_setup);
 275
 276/**
 277 * driver_deferred_probe_check_state() - Check deferred probe state
 278 * @dev: device to check
 279 *
 280 * Return:
 281 * * -ENODEV if initcalls have completed and modules are disabled.
 282 * * -ETIMEDOUT if the deferred probe timeout was set and has expired
 283 *   and modules are enabled.
 284 * * -EPROBE_DEFER in other cases.
 285 *
 286 * Drivers or subsystems can opt-in to calling this function instead of directly
 287 * returning -EPROBE_DEFER.
 288 */
 289int driver_deferred_probe_check_state(struct device *dev)
 290{
 291	if (!IS_ENABLED(CONFIG_MODULES) && initcalls_done) {
 292		dev_warn(dev, "ignoring dependency for device, assuming no driver\n");
 293		return -ENODEV;
 294	}
 295
 296	if (!driver_deferred_probe_timeout && initcalls_done) {
 297		dev_warn(dev, "deferred probe timeout, ignoring dependency\n");
 298		return -ETIMEDOUT;
 299	}
 300
 301	return -EPROBE_DEFER;
 302}
 303EXPORT_SYMBOL_GPL(driver_deferred_probe_check_state);
 304
 305static void deferred_probe_timeout_work_func(struct work_struct *work)
 306{
 307	struct device_private *p;
 308
 309	fw_devlink_drivers_done();
 310
 311	driver_deferred_probe_timeout = 0;
 312	driver_deferred_probe_trigger();
 313	flush_work(&deferred_probe_work);
 314
 315	mutex_lock(&deferred_probe_mutex);
 316	list_for_each_entry(p, &deferred_probe_pending_list, deferred_probe)
 317		dev_warn(p->device, "deferred probe pending: %s", p->deferred_probe_reason ?: "(reason unknown)\n");
 318	mutex_unlock(&deferred_probe_mutex);
 319
 320	fw_devlink_probing_done();
 321}
 322static DECLARE_DELAYED_WORK(deferred_probe_timeout_work, deferred_probe_timeout_work_func);
 323
 324void deferred_probe_extend_timeout(void)
 325{
 326	/*
 327	 * If the work hasn't been queued yet or if the work expired, don't
 328	 * start a new one.
 329	 */
 330	if (cancel_delayed_work(&deferred_probe_timeout_work)) {
 331		schedule_delayed_work(&deferred_probe_timeout_work,
 332				driver_deferred_probe_timeout * HZ);
 333		pr_debug("Extended deferred probe timeout by %d secs\n",
 334					driver_deferred_probe_timeout);
 335	}
 336}
 337
 338/**
 339 * deferred_probe_initcall() - Enable probing of deferred devices
 340 *
 341 * We don't want to get in the way when the bulk of drivers are getting probed.
 342 * Instead, this initcall makes sure that deferred probing is delayed until
 343 * late_initcall time.
 344 */
 345static int deferred_probe_initcall(void)
 346{
 347	debugfs_create_file("devices_deferred", 0444, NULL, NULL,
 348			    &deferred_devs_fops);
 349
 350	driver_deferred_probe_enable = true;
 351	driver_deferred_probe_trigger();
 352	/* Sort as many dependencies as possible before exiting initcalls */
 353	flush_work(&deferred_probe_work);
 354	initcalls_done = true;
 355
 356	if (!IS_ENABLED(CONFIG_MODULES))
 357		fw_devlink_drivers_done();
 358
 359	/*
 360	 * Trigger deferred probe again, this time we won't defer anything
 361	 * that is optional
 362	 */
 363	driver_deferred_probe_trigger();
 364	flush_work(&deferred_probe_work);
 365
 366	if (driver_deferred_probe_timeout > 0) {
 367		schedule_delayed_work(&deferred_probe_timeout_work,
 368			driver_deferred_probe_timeout * HZ);
 369	}
 370
 371	if (!IS_ENABLED(CONFIG_MODULES))
 372		fw_devlink_probing_done();
 373
 374	return 0;
 375}
 376late_initcall(deferred_probe_initcall);
 377
 378static void __exit deferred_probe_exit(void)
 379{
 380	debugfs_lookup_and_remove("devices_deferred", NULL);
 381}
 382__exitcall(deferred_probe_exit);
 383
 384int __device_set_driver_override(struct device *dev, const char *s, size_t len)
 385{
 386	const char *new, *old;
 387	char *cp;
 388
 389	if (!s)
 390		return -EINVAL;
 391
 392	/*
 393	 * The stored value will be used in sysfs show callback (sysfs_emit()),
 394	 * which has a length limit of PAGE_SIZE and adds a trailing newline.
 395	 * Thus we can store one character less to avoid truncation during sysfs
 396	 * show.
 397	 */
 398	if (len >= (PAGE_SIZE - 1))
 399		return -EINVAL;
 400
 401	/*
 402	 * Compute the real length of the string in case userspace sends us a
 403	 * bunch of \0 characters like python likes to do.
 404	 */
 405	len = strlen(s);
 406
 407	if (!len) {
 408		/* Empty string passed - clear override */
 409		spin_lock(&dev->driver_override.lock);
 410		old = dev->driver_override.name;
 411		dev->driver_override.name = NULL;
 412		spin_unlock(&dev->driver_override.lock);
 413		kfree(old);
 414
 415		return 0;
 416	}
 417
 418	cp = strnchr(s, len, '\n');
 419	if (cp)
 420		len = cp - s;
 421
 422	new = kstrndup(s, len, GFP_KERNEL);
 423	if (!new)
 424		return -ENOMEM;
 425
 426	spin_lock(&dev->driver_override.lock);
 427	old = dev->driver_override.name;
 428	if (cp != s) {
 429		dev->driver_override.name = new;
 430		spin_unlock(&dev->driver_override.lock);
 431	} else {
 432		/* "\n" passed - clear override */
 433		dev->driver_override.name = NULL;
 434		spin_unlock(&dev->driver_override.lock);
 435
 436		kfree(new);
 437	}
 438	kfree(old);
 439
 440	return 0;
 441}
 442EXPORT_SYMBOL_GPL(__device_set_driver_override);
 443
 444/**
 445 * device_is_bound() - Check if device is bound to a driver
 446 * @dev: device to check
 447 *
 448 * Returns true if passed device has already finished probing successfully
 449 * against a driver.
 450 *
 451 * This function must be called with the device lock held.
 452 */
 453bool device_is_bound(struct device *dev)
 454{
 455	return dev->p && klist_node_attached(&dev->p->knode_driver);
 456}
 457EXPORT_SYMBOL_GPL(device_is_bound);
 458
 459static void driver_bound(struct device *dev)
 460{
 461	if (device_is_bound(dev)) {
 462		dev_warn(dev, "%s: device already bound\n", __func__);
 463		return;
 464	}
 465
 466	dev_dbg(dev, "driver: '%s': %s: bound to device\n", dev->driver->name,
 467		__func__);
 468
 469	klist_add_tail(&dev->p->knode_driver, &dev->driver->p->klist_devices);
 470	device_links_driver_bound(dev);
 471
 472	device_pm_check_callbacks(dev);
 473
 474	/*
 475	 * Make sure the device is no longer in one of the deferred lists and
 476	 * kick off retrying all pending devices
 477	 */
 478	driver_deferred_probe_del(dev);
 479	driver_deferred_probe_trigger();
 480
 481	bus_notify(dev, BUS_NOTIFY_BOUND_DRIVER);
 482	kobject_uevent(&dev->kobj, KOBJ_BIND);
 483}
 484
 485static ssize_t coredump_store(struct device *dev, struct device_attribute *attr,
 486			    const char *buf, size_t count)
 487{
 488	device_lock(dev);
 489	dev->driver->coredump(dev);
 490	device_unlock(dev);
 491
 492	return count;
 493}
 494static DEVICE_ATTR_WO(coredump);
 495
 496static int driver_sysfs_add(struct device *dev)
 497{
 498	int ret;
 499
 500	bus_notify(dev, BUS_NOTIFY_BIND_DRIVER);
 501
 502	ret = sysfs_create_link(&dev->driver->p->kobj, &dev->kobj,
 503				kobject_name(&dev->kobj));
 504	if (ret)
 505		goto fail;
 506
 507	ret = sysfs_create_link(&dev->kobj, &dev->driver->p->kobj,
 508				"driver");
 509	if (ret)
 510		goto rm_dev;
 511
 512	if (!IS_ENABLED(CONFIG_DEV_COREDUMP) || !dev->driver->coredump)
 513		return 0;
 514
 515	ret = device_create_file(dev, &dev_attr_coredump);
 516	if (!ret)
 517		return 0;
 518
 519	sysfs_remove_link(&dev->kobj, "driver");
 520
 521rm_dev:
 522	sysfs_remove_link(&dev->driver->p->kobj,
 523			  kobject_name(&dev->kobj));
 524
 525fail:
 526	return ret;
 527}
 528
 529static void driver_sysfs_remove(struct device *dev)
 530{
 531	struct device_driver *drv = dev->driver;
 532
 533	if (drv) {
 534		if (drv->coredump)
 535			device_remove_file(dev, &dev_attr_coredump);
 536		sysfs_remove_link(&drv->p->kobj, kobject_name(&dev->kobj));
 537		sysfs_remove_link(&dev->kobj, "driver");
 538	}
 539}
 540
 541/**
 542 * device_bind_driver - bind a driver to one device.
 543 * @dev: device.
 544 *
 545 * Allow manual attachment of a driver to a device.
 546 * Caller must have already set @dev->driver.
 547 *
 548 * Note that this does not modify the bus reference count.
 549 * Please verify that is accounted for before calling this.
 550 * (It is ok to call with no other effort from a driver's probe() method.)
 551 *
 552 * This function must be called with the device lock held.
 553 *
 554 * Callers should prefer to use device_driver_attach() instead.
 555 */
 556int device_bind_driver(struct device *dev)
 557{
 558	int ret;
 559
 560	ret = driver_sysfs_add(dev);
 561	if (!ret) {
 562		device_links_force_bind(dev);
 563		driver_bound(dev);
 564	}
 565	else
 566		bus_notify(dev, BUS_NOTIFY_DRIVER_NOT_BOUND);
 567	return ret;
 568}
 569EXPORT_SYMBOL_GPL(device_bind_driver);
 570
 571static atomic_t probe_count = ATOMIC_INIT(0);
 572static DECLARE_WAIT_QUEUE_HEAD(probe_waitqueue);
 573
 574static ssize_t state_synced_store(struct device *dev,
 575				  struct device_attribute *attr,
 576				  const char *buf, size_t count)
 577{
 578	int ret = 0;
 579
 580	if (strcmp("1", buf))
 581		return -EINVAL;
 582
 583	device_lock(dev);
 584	if (!dev->state_synced) {
 585		dev->state_synced = true;
 586		dev_sync_state(dev);
 587	} else {
 588		ret = -EINVAL;
 589	}
 590	device_unlock(dev);
 591
 592	return ret ? ret : count;
 593}
 594
 595static ssize_t state_synced_show(struct device *dev,
 596				 struct device_attribute *attr, char *buf)
 597{
 598	bool val;
 599
 600	device_lock(dev);
 601	val = dev->state_synced;
 602	device_unlock(dev);
 603
 604	return sysfs_emit(buf, "%u\n", val);
 605}
 606static DEVICE_ATTR_RW(state_synced);
 607
 608static void device_unbind_cleanup(struct device *dev)
 609{
 610	devres_release_all(dev);
 611	if (dev->driver->p_cb.post_unbind_rust)
 612		dev->driver->p_cb.post_unbind_rust(dev);
 613	arch_teardown_dma_ops(dev);
 614	kfree(dev->dma_range_map);
 615	dev->dma_range_map = NULL;
 616	device_set_driver(dev, NULL);
 617	dev_set_drvdata(dev, NULL);
 618	dev_pm_domain_detach(dev, dev->power.detach_power_off);
 619	if (dev->pm_domain && dev->pm_domain->dismiss)
 620		dev->pm_domain->dismiss(dev);
 621	pm_runtime_reinit(dev);
 622	dev_pm_set_driver_flags(dev, 0);
 623}
 624
 625static void device_remove(struct device *dev)
 626{
 627	device_remove_file(dev, &dev_attr_state_synced);
 628	device_remove_groups(dev, dev->driver->dev_groups);
 629
 630	if (dev->bus && dev->bus->remove)
 631		dev->bus->remove(dev);
 632	else if (dev->driver->remove)
 633		dev->driver->remove(dev);
 634}
 635
 636static int call_driver_probe(struct device *dev, const struct device_driver *drv)
 637{
 638	int ret = 0;
 639
 640	if (dev->bus->probe)
 641		ret = dev->bus->probe(dev);
 642	else if (drv->probe)
 643		ret = drv->probe(dev);
 644
 645	switch (ret) {
 646	case 0:
 647		break;
 648	case -EPROBE_DEFER:
 649		/* Driver requested deferred probing */
 650		dev_dbg(dev, "Driver %s requests probe deferral\n", drv->name);
 651		break;
 652	case -ENODEV:
 653	case -ENXIO:
 654		dev_dbg(dev, "probe with driver %s rejects match %d\n",
 655			drv->name, ret);
 656		break;
 657	default:
 658		/* driver matched but the probe failed */
 659		dev_err(dev, "probe with driver %s failed with error %d\n",
 660			drv->name, ret);
 661		break;
 662	}
 663
 664	return ret;
 665}
 666
 667static int really_probe(struct device *dev, const struct device_driver *drv)
 668{
 669	bool test_remove = IS_ENABLED(CONFIG_DEBUG_TEST_DRIVER_REMOVE) &&
 670			   !drv->suppress_bind_attrs;
 671	int ret, link_ret;
 672
 673	if (defer_all_probes) {
 674		/*
 675		 * Value of defer_all_probes can be set only by
 676		 * device_block_probing() which, in turn, will call
 677		 * wait_for_device_probe() right after that to avoid any races.
 678		 */
 679		dev_dbg(dev, "Driver %s force probe deferral\n", drv->name);
 680		return -EPROBE_DEFER;
 681	}
 682
 683	link_ret = device_links_check_suppliers(dev);
 684	if (link_ret == -EPROBE_DEFER)
 685		return link_ret;
 686
 687	dev_dbg(dev, "bus: '%s': %s: probing driver %s with device\n",
 688		drv->bus->name, __func__, drv->name);
 689	if (!list_empty(&dev->devres_head)) {
 690		dev_crit(dev, "Resources present before probing\n");
 691		ret = -EBUSY;
 692		goto done;
 693	}
 694
 695re_probe:
 696	device_set_driver(dev, drv);
 697
 698	/* If using pinctrl, bind pins now before probing */
 699	ret = pinctrl_bind_pins(dev);
 700	if (ret)
 701		goto pinctrl_bind_failed;
 702
 703	if (dev->bus->dma_configure) {
 704		ret = dev->bus->dma_configure(dev);
 705		if (ret)
 706			goto pinctrl_bind_failed;
 707	}
 708
 709	ret = driver_sysfs_add(dev);
 710	if (ret) {
 711		dev_err(dev, "%s: driver_sysfs_add failed\n", __func__);
 712		goto sysfs_failed;
 713	}
 714
 715	if (dev->pm_domain && dev->pm_domain->activate) {
 716		ret = dev->pm_domain->activate(dev);
 717		if (ret)
 718			goto probe_failed;
 719	}
 720
 721	ret = call_driver_probe(dev, drv);
 722	if (ret) {
 723		/*
 724		 * If fw_devlink_best_effort is active (denoted by -EAGAIN), the
 725		 * device might actually probe properly once some of its missing
 726		 * suppliers have probed. So, treat this as if the driver
 727		 * returned -EPROBE_DEFER.
 728		 */
 729		if (link_ret == -EAGAIN)
 730			ret = -EPROBE_DEFER;
 731
 732		/*
 733		 * Return probe errors as positive values so that the callers
 734		 * can distinguish them from other errors.
 735		 */
 736		ret = -ret;
 737		goto probe_failed;
 738	}
 739
 740	ret = device_add_groups(dev, drv->dev_groups);
 741	if (ret) {
 742		dev_err(dev, "device_add_groups() failed\n");
 743		goto dev_groups_failed;
 744	}
 745
 746	if (dev_has_sync_state(dev)) {
 747		ret = device_create_file(dev, &dev_attr_state_synced);
 748		if (ret) {
 749			dev_err(dev, "state_synced sysfs add failed\n");
 750			goto dev_sysfs_state_synced_failed;
 751		}
 752	}
 753
 754	if (test_remove) {
 755		test_remove = false;
 756
 757		device_remove(dev);
 758		driver_sysfs_remove(dev);
 759		if (dev->bus && dev->bus->dma_cleanup)
 760			dev->bus->dma_cleanup(dev);
 761		device_unbind_cleanup(dev);
 762
 763		goto re_probe;
 764	}
 765
 766	pinctrl_init_done(dev);
 767
 768	if (dev->pm_domain && dev->pm_domain->sync)
 769		dev->pm_domain->sync(dev);
 770
 771	driver_bound(dev);
 772	dev_dbg(dev, "bus: '%s': %s: bound device to driver %s\n",
 773		drv->bus->name, __func__, drv->name);
 774	goto done;
 775
 776dev_sysfs_state_synced_failed:
 777dev_groups_failed:
 778	device_remove(dev);
 779probe_failed:
 780	driver_sysfs_remove(dev);
 781sysfs_failed:
 782	bus_notify(dev, BUS_NOTIFY_DRIVER_NOT_BOUND);
 783	if (dev->bus && dev->bus->dma_cleanup)
 784		dev->bus->dma_cleanup(dev);
 785pinctrl_bind_failed:
 786	device_links_no_driver(dev);
 787	device_unbind_cleanup(dev);
 788done:
 789	return ret;
 790}
 791
 792/*
 793 * For initcall_debug, show the driver probe time.
 794 */
 795static int really_probe_debug(struct device *dev, const struct device_driver *drv)
 796{
 797	ktime_t calltime, rettime;
 798	int ret;
 799
 800	calltime = ktime_get();
 801	ret = really_probe(dev, drv);
 802	rettime = ktime_get();
 803	/*
 804	 * Don't change this to pr_debug() because that requires
 805	 * CONFIG_DYNAMIC_DEBUG and we want a simple 'initcall_debug' on the
 806	 * kernel commandline to print this all the time at the debug level.
 807	 */
 808	printk(KERN_DEBUG "probe of %s returned %d after %lld usecs\n",
 809		 dev_name(dev), ret, ktime_us_delta(rettime, calltime));
 810	return ret;
 811}
 812
 813/**
 814 * driver_probe_done
 815 * Determine if the probe sequence is finished or not.
 816 *
 817 * Should somehow figure out how to use a semaphore, not an atomic variable...
 818 */
 819bool __init driver_probe_done(void)
 820{
 821	int local_probe_count = atomic_read(&probe_count);
 822
 823	pr_debug("%s: probe_count = %d\n", __func__, local_probe_count);
 824	return !local_probe_count;
 825}
 826
 827/**
 828 * wait_for_device_probe
 829 * Wait for device probing to be completed.
 830 */
 831void wait_for_device_probe(void)
 832{
 833	/* wait for the deferred probe workqueue to finish */
 834	flush_work(&deferred_probe_work);
 835
 836	/* wait for the known devices to complete their probing */
 837	wait_event(probe_waitqueue, atomic_read(&probe_count) == 0);
 838	async_synchronize_full();
 839}
 840EXPORT_SYMBOL_GPL(wait_for_device_probe);
 841
 842static int __driver_probe_device(const struct device_driver *drv, struct device *dev)
 843{
 844	int ret = 0;
 845
 846	if (dev->p->dead || !device_is_registered(dev))
 847		return -ENODEV;
 848	if (dev->driver)
 849		return -EBUSY;
 850
 851	dev->can_match = true;
 852	dev_dbg(dev, "bus: '%s': %s: matched device with driver %s\n",
 853		drv->bus->name, __func__, drv->name);
 854
 855	pm_runtime_get_suppliers(dev);
 856	if (dev->parent)
 857		pm_runtime_get_sync(dev->parent);
 858
 859	pm_runtime_barrier(dev);
 860	if (initcall_debug)
 861		ret = really_probe_debug(dev, drv);
 862	else
 863		ret = really_probe(dev, drv);
 864	pm_request_idle(dev);
 865
 866	if (dev->parent)
 867		pm_runtime_put(dev->parent);
 868
 869	pm_runtime_put_suppliers(dev);
 870	return ret;
 871}
 872
 873/**
 874 * driver_probe_device - attempt to bind device & driver together
 875 * @drv: driver to bind a device to
 876 * @dev: device to try to bind to the driver
 877 *
 878 * This function returns -ENODEV if the device is not registered, -EBUSY if it
 879 * already has a driver, 0 if the device is bound successfully and a positive
 880 * (inverted) error code for failures from the ->probe method.
 881 *
 882 * This function must be called with @dev lock held.  When called for a
 883 * USB interface, @dev->parent lock must be held as well.
 884 *
 885 * If the device has a parent, runtime-resume the parent before driver probing.
 886 */
 887static int driver_probe_device(const struct device_driver *drv, struct device *dev)
 888{
 889	int trigger_count = atomic_read(&deferred_trigger_count);
 890	int ret;
 891
 892	atomic_inc(&probe_count);
 893	ret = __driver_probe_device(drv, dev);
 894	if (ret == -EPROBE_DEFER || ret == EPROBE_DEFER) {
 895		driver_deferred_probe_add(dev);
 896
 897		/*
 898		 * Did a trigger occur while probing? Need to re-trigger if yes
 899		 */
 900		if (trigger_count != atomic_read(&deferred_trigger_count) &&
 901		    !defer_all_probes)
 902			driver_deferred_probe_trigger();
 903	}
 904	atomic_dec(&probe_count);
 905	wake_up_all(&probe_waitqueue);
 906	return ret;
 907}
 908
 909static inline bool cmdline_requested_async_probing(const char *drv_name)
 910{
 911	bool async_drv;
 912
 913	async_drv = parse_option_str(async_probe_drv_names, drv_name);
 914
 915	return (async_probe_default != async_drv);
 916}
 917
 918/* The option format is "driver_async_probe=drv_name1,drv_name2,..." */
 919static int __init save_async_options(char *buf)
 920{
 921	if (strlen(buf) >= ASYNC_DRV_NAMES_MAX_LEN)
 922		pr_warn("Too long list of driver names for 'driver_async_probe'!\n");
 923
 924	strscpy(async_probe_drv_names, buf, ASYNC_DRV_NAMES_MAX_LEN);
 925	async_probe_default = parse_option_str(async_probe_drv_names, "*");
 926
 927	return 1;
 928}
 929__setup("driver_async_probe=", save_async_options);
 930
 931static bool driver_allows_async_probing(const struct device_driver *drv)
 932{
 933	switch (drv->probe_type) {
 934	case PROBE_PREFER_ASYNCHRONOUS:
 935		return true;
 936
 937	case PROBE_FORCE_SYNCHRONOUS:
 938		return false;
 939
 940	default:
 941		if (cmdline_requested_async_probing(drv->name))
 942			return true;
 943
 944		if (module_requested_async_probing(drv->owner))
 945			return true;
 946
 947		return false;
 948	}
 949}
 950
 951struct device_attach_data {
 952	struct device *dev;
 953
 954	/*
 955	 * Indicates whether we are considering asynchronous probing or
 956	 * not. Only initial binding after device or driver registration
 957	 * (including deferral processing) may be done asynchronously, the
 958	 * rest is always synchronous, as we expect it is being done by
 959	 * request from userspace.
 960	 */
 961	bool check_async;
 962
 963	/*
 964	 * Indicates if we are binding synchronous or asynchronous drivers.
 965	 * When asynchronous probing is enabled we'll execute 2 passes
 966	 * over drivers: first pass doing synchronous probing and second
 967	 * doing asynchronous probing (if synchronous did not succeed -
 968	 * most likely because there was no driver requiring synchronous
 969	 * probing - and we found asynchronous driver during first pass).
 970	 * The 2 passes are done because we can't shoot asynchronous
 971	 * probe for given device and driver from bus_for_each_drv() since
 972	 * driver pointer is not guaranteed to stay valid once
 973	 * bus_for_each_drv() iterates to the next driver on the bus.
 974	 */
 975	bool want_async;
 976
 977	/*
 978	 * We'll set have_async to 'true' if, while scanning for matching
 979	 * driver, we'll encounter one that requests asynchronous probing.
 980	 */
 981	bool have_async;
 982};
 983
 984static int __device_attach_driver(struct device_driver *drv, void *_data)
 985{
 986	struct device_attach_data *data = _data;
 987	struct device *dev = data->dev;
 988	bool async_allowed;
 989	int ret;
 990
 991	ret = driver_match_device(drv, dev);
 992	if (ret == 0) {
 993		/* no match */
 994		return 0;
 995	} else if (ret == -EPROBE_DEFER) {
 996		dev_dbg(dev, "Device match requests probe deferral\n");
 997		dev->can_match = true;
 998		driver_deferred_probe_add(dev);
 999		/*
1000		 * Device can't match with a driver right now, so don't attempt
1001		 * to match or bind with other drivers on the bus.
1002		 */
1003		return ret;
1004	} else if (ret < 0) {
1005		dev_dbg(dev, "Bus failed to match device: %d\n", ret);
1006		return ret;
1007	} /* ret > 0 means positive match */
1008
1009	async_allowed = driver_allows_async_probing(drv);
1010
1011	if (async_allowed)
1012		data->have_async = true;
1013
1014	if (data->check_async && async_allowed != data->want_async)
1015		return 0;
1016
1017	/*
1018	 * Ignore errors returned by ->probe so that the next driver can try
1019	 * its luck.
1020	 */
1021	ret = driver_probe_device(drv, dev);
1022	if (ret < 0)
1023		return ret;
1024	return ret == 0;
1025}
1026
1027static void __device_attach_async_helper(void *_dev, async_cookie_t cookie)
1028{
1029	struct device *dev = _dev;
1030	struct device_attach_data data = {
1031		.dev		= dev,
1032		.check_async	= true,
1033		.want_async	= true,
1034	};
1035
1036	device_lock(dev);
1037
1038	/*
1039	 * Check if device has already been removed or claimed. This may
1040	 * happen with driver loading, device discovery/registration,
1041	 * and deferred probe processing happens all at once with
1042	 * multiple threads.
1043	 */
1044	if (dev->p->dead || dev->driver)
1045		goto out_unlock;
1046
1047	if (dev->parent)
1048		pm_runtime_get_sync(dev->parent);
1049
1050	bus_for_each_drv(dev->bus, NULL, &data, __device_attach_driver);
1051	dev_dbg(dev, "async probe completed\n");
1052
1053	pm_request_idle(dev);
1054
1055	if (dev->parent)
1056		pm_runtime_put(dev->parent);
1057out_unlock:
1058	device_unlock(dev);
1059
1060	put_device(dev);
1061}
1062
1063static int __device_attach(struct device *dev, bool allow_async)
1064{
1065	int ret = 0;
1066	bool async = false;
1067
1068	device_lock(dev);
1069	if (dev->p->dead) {
1070		goto out_unlock;
1071	} else if (dev->driver) {
1072		if (device_is_bound(dev)) {
1073			ret = 1;
1074			goto out_unlock;
1075		}
1076		ret = device_bind_driver(dev);
1077		if (ret == 0)
1078			ret = 1;
1079		else {
1080			device_set_driver(dev, NULL);
1081			ret = 0;
1082		}
1083	} else {
1084		struct device_attach_data data = {
1085			.dev = dev,
1086			.check_async = allow_async,
1087			.want_async = false,
1088		};
1089
1090		if (dev->parent)
1091			pm_runtime_get_sync(dev->parent);
1092
1093		ret = bus_for_each_drv(dev->bus, NULL, &data,
1094					__device_attach_driver);
1095		if (!ret && allow_async && data.have_async) {
1096			/*
1097			 * If we could not find appropriate driver
1098			 * synchronously and we are allowed to do
1099			 * async probes and there are drivers that
1100			 * want to probe asynchronously, we'll
1101			 * try them.
1102			 */
1103			dev_dbg(dev, "scheduling asynchronous probe\n");
1104			get_device(dev);
1105			async = true;
1106		} else {
1107			pm_request_idle(dev);
1108		}
1109
1110		if (dev->parent)
1111			pm_runtime_put(dev->parent);
1112	}
1113out_unlock:
1114	device_unlock(dev);
1115	if (async)
1116		async_schedule_dev(__device_attach_async_helper, dev);
1117	return ret;
1118}
1119
1120/**
1121 * device_attach - try to attach device to a driver.
1122 * @dev: device.
1123 *
1124 * Walk the list of drivers that the bus has and call
1125 * driver_probe_device() for each pair. If a compatible
1126 * pair is found, break out and return.
1127 *
1128 * Returns 1 if the device was bound to a driver;
1129 * 0 if no matching driver was found;
1130 * -ENODEV if the device is not registered.
1131 *
1132 * When called for a USB interface, @dev->parent lock must be held.
1133 */
1134int device_attach(struct device *dev)
1135{
1136	return __device_attach(dev, false);
1137}
1138EXPORT_SYMBOL_GPL(device_attach);
1139
1140void device_initial_probe(struct device *dev)
1141{
1142	struct subsys_private *sp = bus_to_subsys(dev->bus);
1143
1144	if (!sp)
1145		return;
1146
1147	if (sp->drivers_autoprobe)
1148		__device_attach(dev, true);
1149
1150	subsys_put(sp);
1151}
1152
1153/*
1154 * __device_driver_lock - acquire locks needed to manipulate dev->drv
1155 * @dev: Device we will update driver info for
1156 * @parent: Parent device. Needed if the bus requires parent lock
1157 *
1158 * This function will take the required locks for manipulating dev->drv.
1159 * Normally this will just be the @dev lock, but when called for a USB
1160 * interface, @parent lock will be held as well.
1161 */
1162static void __device_driver_lock(struct device *dev, struct device *parent)
1163{
1164	if (parent && dev->bus->need_parent_lock)
1165		device_lock(parent);
1166	device_lock(dev);
1167}
1168
1169/*
1170 * __device_driver_unlock - release locks needed to manipulate dev->drv
1171 * @dev: Device we will update driver info for
1172 * @parent: Parent device. Needed if the bus requires parent lock
1173 *
1174 * This function will release the required locks for manipulating dev->drv.
1175 * Normally this will just be the @dev lock, but when called for a
1176 * USB interface, @parent lock will be released as well.
1177 */
1178static void __device_driver_unlock(struct device *dev, struct device *parent)
1179{
1180	device_unlock(dev);
1181	if (parent && dev->bus->need_parent_lock)
1182		device_unlock(parent);
1183}
1184
1185/**
1186 * device_driver_attach - attach a specific driver to a specific device
1187 * @drv: Driver to attach
1188 * @dev: Device to attach it to
1189 *
1190 * Manually attach driver to a device. Will acquire both @dev lock and
1191 * @dev->parent lock if needed. Returns 0 on success, -ERR on failure.
1192 */
1193int device_driver_attach(const struct device_driver *drv, struct device *dev)
1194{
1195	int ret;
1196
1197	__device_driver_lock(dev, dev->parent);
1198	ret = __driver_probe_device(drv, dev);
1199	__device_driver_unlock(dev, dev->parent);
1200
1201	/* also return probe errors as normal negative errnos */
1202	if (ret > 0)
1203		ret = -ret;
1204	if (ret == -EPROBE_DEFER)
1205		return -EAGAIN;
1206	return ret;
1207}
1208EXPORT_SYMBOL_GPL(device_driver_attach);
1209
1210static void __driver_attach_async_helper(void *_dev, async_cookie_t cookie)
1211{
1212	struct device *dev = _dev;
1213	const struct device_driver *drv;
1214	int ret;
1215
1216	__device_driver_lock(dev, dev->parent);
1217	drv = dev->p->async_driver;
1218	dev->p->async_driver = NULL;
1219	ret = driver_probe_device(drv, dev);
1220	__device_driver_unlock(dev, dev->parent);
1221
1222	dev_dbg(dev, "driver %s async attach completed: %d\n", drv->name, ret);
1223
1224	put_device(dev);
1225}
1226
1227static int __driver_attach(struct device *dev, void *data)
1228{
1229	const struct device_driver *drv = data;
1230	bool async = false;
1231	int ret;
1232
1233	/*
1234	 * Lock device and try to bind to it. We drop the error
1235	 * here and always return 0, because we need to keep trying
1236	 * to bind to devices and some drivers will return an error
1237	 * simply if it didn't support the device.
1238	 *
1239	 * driver_probe_device() will spit a warning if there
1240	 * is an error.
1241	 */
1242
1243	ret = driver_match_device(drv, dev);
1244	if (ret == 0) {
1245		/* no match */
1246		return 0;
1247	} else if (ret == -EPROBE_DEFER) {
1248		dev_dbg(dev, "Device match requests probe deferral\n");
1249		dev->can_match = true;
1250		driver_deferred_probe_add(dev);
1251		/*
1252		 * Driver could not match with device, but may match with
1253		 * another device on the bus.
1254		 */
1255		return 0;
1256	} else if (ret < 0) {
1257		dev_dbg(dev, "Bus failed to match device: %d\n", ret);
1258		/*
1259		 * Driver could not match with device, but may match with
1260		 * another device on the bus.
1261		 */
1262		return 0;
1263	} /* ret > 0 means positive match */
1264
1265	if (driver_allows_async_probing(drv)) {
1266		/*
1267		 * Instead of probing the device synchronously we will
1268		 * probe it asynchronously to allow for more parallelism.
1269		 *
1270		 * We only take the device lock here in order to guarantee
1271		 * that the dev->driver and async_driver fields are protected
1272		 */
1273		dev_dbg(dev, "probing driver %s asynchronously\n", drv->name);
1274		device_lock(dev);
1275		if (!dev->driver && !dev->p->async_driver) {
1276			get_device(dev);
1277			dev->p->async_driver = drv;
1278			async = true;
1279		}
1280		device_unlock(dev);
1281		if (async)
1282			async_schedule_dev(__driver_attach_async_helper, dev);
1283		return 0;
1284	}
1285
1286	__device_driver_lock(dev, dev->parent);
1287	driver_probe_device(drv, dev);
1288	__device_driver_unlock(dev, dev->parent);
1289
1290	return 0;
1291}
1292
1293/**
1294 * driver_attach - try to bind driver to devices.
1295 * @drv: driver.
1296 *
1297 * Walk the list of devices that the bus has on it and try to
1298 * match the driver with each one.  If driver_probe_device()
1299 * returns 0 and the @dev->driver is set, we've found a
1300 * compatible pair.
1301 */
1302int driver_attach(const struct device_driver *drv)
1303{
1304	/* The (void *) will be put back to const * in __driver_attach() */
1305	return bus_for_each_dev(drv->bus, NULL, (void *)drv, __driver_attach);
1306}
1307EXPORT_SYMBOL_GPL(driver_attach);
1308
1309/*
1310 * __device_release_driver() must be called with @dev lock held.
1311 * When called for a USB interface, @dev->parent lock must be held as well.
1312 */
1313static void __device_release_driver(struct device *dev, struct device *parent)
1314{
1315	struct device_driver *drv;
1316
1317	drv = dev->driver;
1318	if (drv) {
1319		pm_runtime_get_sync(dev);
1320
1321		while (device_links_busy(dev)) {
1322			__device_driver_unlock(dev, parent);
1323
1324			device_links_unbind_consumers(dev);
1325
1326			__device_driver_lock(dev, parent);
1327			/*
1328			 * A concurrent invocation of the same function might
1329			 * have released the driver successfully while this one
1330			 * was waiting, so check for that.
1331			 */
1332			if (dev->driver != drv) {
1333				pm_runtime_put(dev);
1334				return;
1335			}
1336		}
1337
1338		driver_sysfs_remove(dev);
1339
1340		bus_notify(dev, BUS_NOTIFY_UNBIND_DRIVER);
1341
1342		pm_runtime_put_sync(dev);
1343
1344		device_remove(dev);
1345
1346		if (dev->bus && dev->bus->dma_cleanup)
1347			dev->bus->dma_cleanup(dev);
1348
1349		device_unbind_cleanup(dev);
1350		device_links_driver_cleanup(dev);
1351
1352		klist_remove(&dev->p->knode_driver);
1353		device_pm_check_callbacks(dev);
1354
1355		bus_notify(dev, BUS_NOTIFY_UNBOUND_DRIVER);
1356		kobject_uevent(&dev->kobj, KOBJ_UNBIND);
1357	}
1358}
1359
1360void device_release_driver_internal(struct device *dev,
1361				    const struct device_driver *drv,
1362				    struct device *parent)
1363{
1364	__device_driver_lock(dev, parent);
1365
1366	if (!drv || drv == dev->driver)
1367		__device_release_driver(dev, parent);
1368
1369	__device_driver_unlock(dev, parent);
1370}
1371
1372/**
1373 * device_release_driver - manually detach device from driver.
1374 * @dev: device.
1375 *
1376 * Manually detach device from driver.
1377 * When called for a USB interface, @dev->parent lock must be held.
1378 *
1379 * If this function is to be called with @dev->parent lock held, ensure that
1380 * the device's consumers are unbound in advance or that their locks can be
1381 * acquired under the @dev->parent lock.
1382 */
1383void device_release_driver(struct device *dev)
1384{
1385	/*
1386	 * If anyone calls device_release_driver() recursively from
1387	 * within their ->remove callback for the same device, they
1388	 * will deadlock right here.
1389	 */
1390	device_release_driver_internal(dev, NULL, NULL);
1391}
1392EXPORT_SYMBOL_GPL(device_release_driver);
1393
1394/**
1395 * device_driver_detach - detach driver from a specific device
1396 * @dev: device to detach driver from
1397 *
1398 * Detach driver from device. Will acquire both @dev lock and @dev->parent
1399 * lock if needed.
1400 */
1401void device_driver_detach(struct device *dev)
1402{
1403	device_release_driver_internal(dev, NULL, dev->parent);
1404}
1405
1406/**
1407 * driver_detach - detach driver from all devices it controls.
1408 * @drv: driver.
1409 */
1410void driver_detach(const struct device_driver *drv)
1411{
1412	struct device_private *dev_prv;
1413	struct device *dev;
1414
1415	if (driver_allows_async_probing(drv))
1416		async_synchronize_full();
1417
1418	for (;;) {
1419		spin_lock(&drv->p->klist_devices.k_lock);
1420		if (list_empty(&drv->p->klist_devices.k_list)) {
1421			spin_unlock(&drv->p->klist_devices.k_lock);
1422			break;
1423		}
1424		dev_prv = list_last_entry(&drv->p->klist_devices.k_list,
1425				     struct device_private,
1426				     knode_driver.n_node);
1427		dev = dev_prv->device;
1428		get_device(dev);
1429		spin_unlock(&drv->p->klist_devices.k_lock);
1430		device_release_driver_internal(dev, drv, dev->parent);
1431		put_device(dev);
1432	}
1433}