remoteproc_core.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Remote Processor Framework
   4 *
   5 * Copyright (C) 2011 Texas Instruments, Inc.
   6 * Copyright (C) 2011 Google, Inc.
   7 *
   8 * Ohad Ben-Cohen <ohad@wizery.com>
   9 * Brian Swetland <swetland@google.com>
  10 * Mark Grosen <mgrosen@ti.com>
  11 * Fernando Guzman Lugo <fernando.lugo@ti.com>
  12 * Suman Anna <s-anna@ti.com>
  13 * Robert Tivy <rtivy@ti.com>
  14 * Armando Uribe De Leon <x0095078@ti.com>
  15 */
  16
  17#define pr_fmt(fmt)    "%s: " fmt, __func__
  18
  19#include <asm/byteorder.h>
  20#include <linux/delay.h>
  21#include <linux/device.h>
  22#include <linux/dma-mapping.h>
  23#include <linux/elf.h>
  24#include <linux/firmware.h>
  25#include <linux/idr.h>
  26#include <linux/iommu.h>
  27#include <linux/kernel.h>
  28#include <linux/module.h>
  29#include <linux/mutex.h>
  30#include <linux/of_platform.h>
  31#include <linux/panic_notifier.h>
  32#include <linux/platform_device.h>
  33#include <linux/rculist.h>
  34#include <linux/remoteproc.h>
  35#include <linux/slab.h>
  36#include <linux/string.h>
  37#include <linux/virtio_ring.h>
  38
  39#include "remoteproc_internal.h"
  40
  41#define HIGH_BITS_MASK 0xFFFFFFFF00000000ULL
  42
  43static DEFINE_MUTEX(rproc_list_mutex);
  44static LIST_HEAD(rproc_list);
  45static struct notifier_block rproc_panic_nb;
  46
  47typedef int (*rproc_handle_resource_t)(struct rproc *rproc,
  48				 void *, int offset, int avail);
  49
  50static int rproc_alloc_carveout(struct rproc *rproc,
  51				struct rproc_mem_entry *mem);
  52static int rproc_release_carveout(struct rproc *rproc,
  53				  struct rproc_mem_entry *mem);
  54
  55/* Unique indices for remoteproc devices */
  56static DEFINE_IDA(rproc_dev_index);
  57static struct workqueue_struct *rproc_recovery_wq;
  58
  59static const char * const rproc_crash_names[] = {
  60	[RPROC_MMUFAULT]	= "mmufault",
  61	[RPROC_WATCHDOG]	= "watchdog",
  62	[RPROC_FATAL_ERROR]	= "fatal error",
  63};
  64
  65/* translate rproc_crash_type to string */
  66static const char *rproc_crash_to_string(enum rproc_crash_type type)
  67{
  68	if (type < ARRAY_SIZE(rproc_crash_names))
  69		return rproc_crash_names[type];
  70	return "unknown";
  71}
  72
  73/*
  74 * This is the IOMMU fault handler we register with the IOMMU API
  75 * (when relevant; not all remote processors access memory through
  76 * an IOMMU).
  77 *
  78 * IOMMU core will invoke this handler whenever the remote processor
  79 * will try to access an unmapped device address.
  80 */
  81static int rproc_iommu_fault(struct iommu_domain *domain, struct device *dev,
  82			     unsigned long iova, int flags, void *token)
  83{
  84	struct rproc *rproc = token;
  85
  86	dev_err(dev, "iommu fault: da 0x%lx flags 0x%x\n", iova, flags);
  87
  88	rproc_report_crash(rproc, RPROC_MMUFAULT);
  89
  90	/*
  91	 * Let the iommu core know we're not really handling this fault;
  92	 * we just used it as a recovery trigger.
  93	 */
  94	return -ENOSYS;
  95}
  96
  97static int rproc_enable_iommu(struct rproc *rproc)
  98{
  99	struct iommu_domain *domain;
 100	struct device *dev = rproc->dev.parent;
 101	int ret;
 102
 103	if (!rproc->has_iommu) {
 104		dev_dbg(dev, "iommu not present\n");
 105		return 0;
 106	}
 107
 108	domain = iommu_paging_domain_alloc(dev);
 109	if (IS_ERR(domain)) {
 110		dev_err(dev, "can't alloc iommu domain\n");
 111		return PTR_ERR(domain);
 112	}
 113
 114	iommu_set_fault_handler(domain, rproc_iommu_fault, rproc);
 115
 116	ret = iommu_attach_device(domain, dev);
 117	if (ret) {
 118		dev_err(dev, "can't attach iommu device: %d\n", ret);
 119		goto free_domain;
 120	}
 121
 122	rproc->domain = domain;
 123
 124	return 0;
 125
 126free_domain:
 127	iommu_domain_free(domain);
 128	return ret;
 129}
 130
 131static void rproc_disable_iommu(struct rproc *rproc)
 132{
 133	struct iommu_domain *domain = rproc->domain;
 134	struct device *dev = rproc->dev.parent;
 135
 136	if (!domain)
 137		return;
 138
 139	iommu_detach_device(domain, dev);
 140	iommu_domain_free(domain);
 141}
 142
 143phys_addr_t rproc_va_to_pa(void *cpu_addr)
 144{
 145	/*
 146	 * Return physical address according to virtual address location
 147	 * - in vmalloc: if region ioremapped or defined as dma_alloc_coherent
 148	 * - in kernel: if region allocated in generic dma memory pool
 149	 */
 150	if (is_vmalloc_addr(cpu_addr)) {
 151		return page_to_phys(vmalloc_to_page(cpu_addr)) +
 152				    offset_in_page(cpu_addr);
 153	}
 154
 155	WARN_ON(!virt_addr_valid(cpu_addr));
 156	return virt_to_phys(cpu_addr);
 157}
 158
 159/**
 160 * rproc_da_to_va() - lookup the kernel virtual address for a remoteproc address
 161 * @rproc: handle of a remote processor
 162 * @da: remoteproc device address to translate
 163 * @len: length of the memory region @da is pointing to
 164 * @is_iomem: optional pointer filled in to indicate if @da is iomapped memory
 165 *
 166 * Some remote processors will ask us to allocate them physically contiguous
 167 * memory regions (which we call "carveouts"), and map them to specific
 168 * device addresses (which are hardcoded in the firmware). They may also have
 169 * dedicated memory regions internal to the processors, and use them either
 170 * exclusively or alongside carveouts.
 171 *
 172 * They may then ask us to copy objects into specific device addresses (e.g.
 173 * code/data sections) or expose us certain symbols in other device address
 174 * (e.g. their trace buffer).
 175 *
 176 * This function is a helper function with which we can go over the allocated
 177 * carveouts and translate specific device addresses to kernel virtual addresses
 178 * so we can access the referenced memory. This function also allows to perform
 179 * translations on the internal remoteproc memory regions through a platform
 180 * implementation specific da_to_va ops, if present.
 181 *
 182 * Note: phys_to_virt(iommu_iova_to_phys(rproc->domain, da)) will work too,
 183 * but only on kernel direct mapped RAM memory. Instead, we're just using
 184 * here the output of the DMA API for the carveouts, which should be more
 185 * correct.
 186 *
 187 * Return: a valid kernel address on success or NULL on failure
 188 */
 189void *rproc_da_to_va(struct rproc *rproc, u64 da, size_t len, bool *is_iomem)
 190{
 191	struct rproc_mem_entry *carveout;
 192	void *ptr = NULL;
 193
 194	if (rproc->ops->da_to_va) {
 195		ptr = rproc->ops->da_to_va(rproc, da, len, is_iomem);
 196		if (ptr)
 197			goto out;
 198	}
 199
 200	list_for_each_entry(carveout, &rproc->carveouts, node) {
 201		int offset = da - carveout->da;
 202
 203		/*  Verify that carveout is allocated */
 204		if (!carveout->va)
 205			continue;
 206
 207		/* try next carveout if da is too small */
 208		if (offset < 0)
 209			continue;
 210
 211		/* try next carveout if da is too large */
 212		if (offset + len > carveout->len)
 213			continue;
 214
 215		ptr = carveout->va + offset;
 216
 217		if (is_iomem)
 218			*is_iomem = carveout->is_iomem;
 219
 220		break;
 221	}
 222
 223out:
 224	return ptr;
 225}
 226EXPORT_SYMBOL(rproc_da_to_va);
 227
 228/**
 229 * rproc_find_carveout_by_name() - lookup the carveout region by a name
 230 * @rproc: handle of a remote processor
 231 * @name: carveout name to find (format string)
 232 * @...: optional parameters matching @name string
 233 *
 234 * Platform driver has the capability to register some pre-allacoted carveout
 235 * (physically contiguous memory regions) before rproc firmware loading and
 236 * associated resource table analysis. These regions may be dedicated memory
 237 * regions internal to the coprocessor or specified DDR region with specific
 238 * attributes
 239 *
 240 * This function is a helper function with which we can go over the
 241 * allocated carveouts and return associated region characteristics like
 242 * coprocessor address, length or processor virtual address.
 243 *
 244 * Return: a valid pointer on carveout entry on success or NULL on failure.
 245 */
 246__printf(2, 3)
 247struct rproc_mem_entry *
 248rproc_find_carveout_by_name(struct rproc *rproc, const char *name, ...)
 249{
 250	va_list args;
 251	char _name[32];
 252	struct rproc_mem_entry *carveout, *mem = NULL;
 253
 254	if (!name)
 255		return NULL;
 256
 257	va_start(args, name);
 258	vsnprintf(_name, sizeof(_name), name, args);
 259	va_end(args);
 260
 261	list_for_each_entry(carveout, &rproc->carveouts, node) {
 262		/* Compare carveout and requested names */
 263		if (!strcmp(carveout->name, _name)) {
 264			mem = carveout;
 265			break;
 266		}
 267	}
 268
 269	return mem;
 270}
 271
 272/**
 273 * rproc_check_carveout_da() - Check specified carveout da configuration
 274 * @rproc: handle of a remote processor
 275 * @mem: pointer on carveout to check
 276 * @da: area device address
 277 * @len: associated area size
 278 *
 279 * This function is a helper function to verify requested device area (couple
 280 * da, len) is part of specified carveout.
 281 * If da is not set (defined as FW_RSC_ADDR_ANY), only requested length is
 282 * checked.
 283 *
 284 * Return: 0 if carveout matches request else error
 285 */
 286static int rproc_check_carveout_da(struct rproc *rproc,
 287				   struct rproc_mem_entry *mem, u32 da, u32 len)
 288{
 289	struct device *dev = &rproc->dev;
 290	int delta;
 291
 292	/* Check requested resource length */
 293	if (len > mem->len) {
 294		dev_err(dev, "Registered carveout doesn't fit len request\n");
 295		return -EINVAL;
 296	}
 297
 298	if (da != FW_RSC_ADDR_ANY && mem->da == FW_RSC_ADDR_ANY) {
 299		/* Address doesn't match registered carveout configuration */
 300		return -EINVAL;
 301	} else if (da != FW_RSC_ADDR_ANY && mem->da != FW_RSC_ADDR_ANY) {
 302		delta = da - mem->da;
 303
 304		/* Check requested resource belongs to registered carveout */
 305		if (delta < 0) {
 306			dev_err(dev,
 307				"Registered carveout doesn't fit da request\n");
 308			return -EINVAL;
 309		}
 310
 311		if (delta + len > mem->len) {
 312			dev_err(dev,
 313				"Registered carveout doesn't fit len request\n");
 314			return -EINVAL;
 315		}
 316	}
 317
 318	return 0;
 319}
 320
 321int rproc_alloc_vring(struct rproc_vdev *rvdev, int i)
 322{
 323	struct rproc *rproc = rvdev->rproc;
 324	struct device *dev = &rproc->dev;
 325	struct rproc_vring *rvring = &rvdev->vring[i];
 326	struct fw_rsc_vdev *rsc;
 327	int ret, notifyid;
 328	struct rproc_mem_entry *mem;
 329	size_t size;
 330
 331	/* actual size of vring (in bytes) */
 332	size = PAGE_ALIGN(vring_size(rvring->num, rvring->align));
 333
 334	rsc = (void *)rproc->table_ptr + rvdev->rsc_offset;
 335
 336	/* Search for pre-registered carveout */
 337	mem = rproc_find_carveout_by_name(rproc, "vdev%dvring%d", rvdev->index,
 338					  i);
 339	if (mem) {
 340		if (rproc_check_carveout_da(rproc, mem, rsc->vring[i].da, size))
 341			return -ENOMEM;
 342	} else {
 343		/* Register carveout in list */
 344		mem = rproc_mem_entry_init(dev, NULL, 0,
 345					   size, rsc->vring[i].da,
 346					   rproc_alloc_carveout,
 347					   rproc_release_carveout,
 348					   "vdev%dvring%d",
 349					   rvdev->index, i);
 350		if (!mem) {
 351			dev_err(dev, "Can't allocate memory entry structure\n");
 352			return -ENOMEM;
 353		}
 354
 355		rproc_add_carveout(rproc, mem);
 356	}
 357
 358	/*
 359	 * Assign an rproc-wide unique index for this vring
 360	 * TODO: assign a notifyid for rvdev updates as well
 361	 * TODO: support predefined notifyids (via resource table)
 362	 */
 363	ret = idr_alloc(&rproc->notifyids, rvring, 0, 0, GFP_KERNEL);
 364	if (ret < 0) {
 365		dev_err(dev, "idr_alloc failed: %d\n", ret);
 366		return ret;
 367	}
 368	notifyid = ret;
 369
 370	/* Potentially bump max_notifyid */
 371	if (notifyid > rproc->max_notifyid)
 372		rproc->max_notifyid = notifyid;
 373
 374	rvring->notifyid = notifyid;
 375
 376	/* Let the rproc know the notifyid of this vring.*/
 377	rsc->vring[i].notifyid = notifyid;
 378	return 0;
 379}
 380
 381int
 382rproc_parse_vring(struct rproc_vdev *rvdev, struct fw_rsc_vdev *rsc, int i)
 383{
 384	struct rproc *rproc = rvdev->rproc;
 385	struct device *dev = &rproc->dev;
 386	struct fw_rsc_vdev_vring *vring = &rsc->vring[i];
 387	struct rproc_vring *rvring = &rvdev->vring[i];
 388
 389	dev_dbg(dev, "vdev rsc: vring%d: da 0x%x, qsz %d, align %d\n",
 390		i, vring->da, vring->num, vring->align);
 391
 392	/* verify queue size and vring alignment are sane */
 393	if (!vring->num || !vring->align) {
 394		dev_err(dev, "invalid qsz (%d) or alignment (%d)\n",
 395			vring->num, vring->align);
 396		return -EINVAL;
 397	}
 398
 399	rvring->num = vring->num;
 400	rvring->align = vring->align;
 401	rvring->rvdev = rvdev;
 402
 403	return 0;
 404}
 405
 406void rproc_free_vring(struct rproc_vring *rvring)
 407{
 408	struct rproc *rproc = rvring->rvdev->rproc;
 409	int idx = rvring - rvring->rvdev->vring;
 410	struct fw_rsc_vdev *rsc;
 411
 412	idr_remove(&rproc->notifyids, rvring->notifyid);
 413
 414	/*
 415	 * At this point rproc_stop() has been called and the installed resource
 416	 * table in the remote processor memory may no longer be accessible. As
 417	 * such and as per rproc_stop(), rproc->table_ptr points to the cached
 418	 * resource table (rproc->cached_table).  The cached resource table is
 419	 * only available when a remote processor has been booted by the
 420	 * remoteproc core, otherwise it is NULL.
 421	 *
 422	 * Based on the above, reset the virtio device section in the cached
 423	 * resource table only if there is one to work with.
 424	 */
 425	if (rproc->table_ptr) {
 426		rsc = (void *)rproc->table_ptr + rvring->rvdev->rsc_offset;
 427		rsc->vring[idx].da = 0;
 428		rsc->vring[idx].notifyid = -1;
 429	}
 430}
 431
 432void rproc_add_rvdev(struct rproc *rproc, struct rproc_vdev *rvdev)
 433{
 434	if (rvdev && rproc)
 435		list_add_tail(&rvdev->node, &rproc->rvdevs);
 436}
 437
 438void rproc_remove_rvdev(struct rproc_vdev *rvdev)
 439{
 440	if (rvdev)
 441		list_del(&rvdev->node);
 442}
 443/**
 444 * rproc_handle_vdev() - handle a vdev fw resource
 445 * @rproc: the remote processor
 446 * @ptr: the vring resource descriptor
 447 * @offset: offset of the resource entry
 448 * @avail: size of available data (for sanity checking the image)
 449 *
 450 * This resource entry requests the host to statically register a virtio
 451 * device (vdev), and setup everything needed to support it. It contains
 452 * everything needed to make it possible: the virtio device id, virtio
 453 * device features, vrings information, virtio config space, etc...
 454 *
 455 * Before registering the vdev, the vrings are allocated from non-cacheable
 456 * physically contiguous memory. Currently we only support two vrings per
 457 * remote processor (temporary limitation). We might also want to consider
 458 * doing the vring allocation only later when ->find_vqs() is invoked, and
 459 * then release them upon ->del_vqs().
 460 *
 461 * Note: @da is currently not really handled correctly: we dynamically
 462 * allocate it using the DMA API, ignoring requested hard coded addresses,
 463 * and we don't take care of any required IOMMU programming. This is all
 464 * going to be taken care of when the generic iommu-based DMA API will be
 465 * merged. Meanwhile, statically-addressed iommu-based firmware images should
 466 * use RSC_DEVMEM resource entries to map their required @da to the physical
 467 * address of their base CMA region (ouch, hacky!).
 468 *
 469 * Return: 0 on success, or an appropriate error code otherwise
 470 */
 471static int rproc_handle_vdev(struct rproc *rproc, void *ptr,
 472			     int offset, int avail)
 473{
 474	struct fw_rsc_vdev *rsc = ptr;
 475	struct device *dev = &rproc->dev;
 476	struct rproc_vdev *rvdev;
 477	size_t rsc_size;
 478	struct rproc_vdev_data rvdev_data;
 479	struct platform_device *pdev;
 480
 481	/* make sure resource isn't truncated */
 482	rsc_size = struct_size(rsc, vring, rsc->num_of_vrings);
 483	if (size_add(rsc_size, rsc->config_len) > avail) {
 484		dev_err(dev, "vdev rsc is truncated\n");
 485		return -EINVAL;
 486	}
 487
 488	/* make sure reserved bytes are zeroes */
 489	if (rsc->reserved[0] || rsc->reserved[1]) {
 490		dev_err(dev, "vdev rsc has non zero reserved bytes\n");
 491		return -EINVAL;
 492	}
 493
 494	dev_dbg(dev, "vdev rsc: id %d, dfeatures 0x%x, cfg len %d, %d vrings\n",
 495		rsc->id, rsc->dfeatures, rsc->config_len, rsc->num_of_vrings);
 496
 497	/* we currently support only two vrings per rvdev */
 498	if (rsc->num_of_vrings > ARRAY_SIZE(rvdev->vring)) {
 499		dev_err(dev, "too many vrings: %d\n", rsc->num_of_vrings);
 500		return -EINVAL;
 501	}
 502
 503	rvdev_data.id = rsc->id;
 504	rvdev_data.index = rproc->nb_vdev++;
 505	rvdev_data.rsc_offset = offset;
 506	rvdev_data.rsc = rsc;
 507
 508	/*
 509	 * When there is more than one remote processor, rproc->nb_vdev number is
 510	 * same for each separate instances of "rproc". If rvdev_data.index is used
 511	 * as device id, then we get duplication in sysfs, so need to use
 512	 * PLATFORM_DEVID_AUTO to auto select device id.
 513	 */
 514	pdev = platform_device_register_data(dev, "rproc-virtio", PLATFORM_DEVID_AUTO, &rvdev_data,
 515					     sizeof(rvdev_data));
 516	if (IS_ERR(pdev)) {
 517		dev_err(dev, "failed to create rproc-virtio device\n");
 518		return PTR_ERR(pdev);
 519	}
 520
 521	return 0;
 522}
 523
 524/**
 525 * rproc_handle_trace() - handle a shared trace buffer resource
 526 * @rproc: the remote processor
 527 * @ptr: the trace resource descriptor
 528 * @offset: offset of the resource entry
 529 * @avail: size of available data (for sanity checking the image)
 530 *
 531 * In case the remote processor dumps trace logs into memory,
 532 * export it via debugfs.
 533 *
 534 * Currently, the 'da' member of @rsc should contain the device address
 535 * where the remote processor is dumping the traces. Later we could also
 536 * support dynamically allocating this address using the generic
 537 * DMA API (but currently there isn't a use case for that).
 538 *
 539 * Return: 0 on success, or an appropriate error code otherwise
 540 */
 541static int rproc_handle_trace(struct rproc *rproc, void *ptr,
 542			      int offset, int avail)
 543{
 544	struct fw_rsc_trace *rsc = ptr;
 545	struct rproc_debug_trace *trace;
 546	struct device *dev = &rproc->dev;
 547	char name[15];
 548
 549	if (sizeof(*rsc) > avail) {
 550		dev_err(dev, "trace rsc is truncated\n");
 551		return -EINVAL;
 552	}
 553
 554	/* make sure reserved bytes are zeroes */
 555	if (rsc->reserved) {
 556		dev_err(dev, "trace rsc has non zero reserved bytes\n");
 557		return -EINVAL;
 558	}
 559
 560	trace = kzalloc(sizeof(*trace), GFP_KERNEL);
 561	if (!trace)
 562		return -ENOMEM;
 563
 564	/* set the trace buffer dma properties */
 565	trace->trace_mem.len = rsc->len;
 566	trace->trace_mem.da = rsc->da;
 567
 568	/* set pointer on rproc device */
 569	trace->rproc = rproc;
 570
 571	/* make sure snprintf always null terminates, even if truncating */
 572	snprintf(name, sizeof(name), "trace%d", rproc->num_traces);
 573
 574	/* create the debugfs entry */
 575	trace->tfile = rproc_create_trace_file(name, rproc, trace);
 576
 577	list_add_tail(&trace->node, &rproc->traces);
 578
 579	rproc->num_traces++;
 580
 581	dev_dbg(dev, "%s added: da 0x%x, len 0x%x\n",
 582		name, rsc->da, rsc->len);
 583
 584	return 0;
 585}
 586
 587/**
 588 * rproc_handle_devmem() - handle devmem resource entry
 589 * @rproc: remote processor handle
 590 * @ptr: the devmem resource entry
 591 * @offset: offset of the resource entry
 592 * @avail: size of available data (for sanity checking the image)
 593 *
 594 * Remote processors commonly need to access certain on-chip peripherals.
 595 *
 596 * Some of these remote processors access memory via an iommu device,
 597 * and might require us to configure their iommu before they can access
 598 * the on-chip peripherals they need.
 599 *
 600 * This resource entry is a request to map such a peripheral device.
 601 *
 602 * These devmem entries will contain the physical address of the device in
 603 * the 'pa' member. If a specific device address is expected, then 'da' will
 604 * contain it (currently this is the only use case supported). 'len' will
 605 * contain the size of the physical region we need to map.
 606 *
 607 * Currently we just "trust" those devmem entries to contain valid physical
 608 * addresses, but this is going to change: we want the implementations to
 609 * tell us ranges of physical addresses the firmware is allowed to request,
 610 * and not allow firmwares to request access to physical addresses that
 611 * are outside those ranges.
 612 *
 613 * Return: 0 on success, or an appropriate error code otherwise
 614 */
 615static int rproc_handle_devmem(struct rproc *rproc, void *ptr,
 616			       int offset, int avail)
 617{
 618	struct fw_rsc_devmem *rsc = ptr;
 619	struct rproc_mem_entry *mapping;
 620	struct device *dev = &rproc->dev;
 621	int ret;
 622
 623	/* no point in handling this resource without a valid iommu domain */
 624	if (!rproc->domain)
 625		return -EINVAL;
 626
 627	if (sizeof(*rsc) > avail) {
 628		dev_err(dev, "devmem rsc is truncated\n");
 629		return -EINVAL;
 630	}
 631
 632	/* make sure reserved bytes are zeroes */
 633	if (rsc->reserved) {
 634		dev_err(dev, "devmem rsc has non zero reserved bytes\n");
 635		return -EINVAL;
 636	}
 637
 638	mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
 639	if (!mapping)
 640		return -ENOMEM;
 641
 642	ret = iommu_map(rproc->domain, rsc->da, rsc->pa, rsc->len, rsc->flags,
 643			GFP_KERNEL);
 644	if (ret) {
 645		dev_err(dev, "failed to map devmem: %d\n", ret);
 646		goto out;
 647	}
 648
 649	/*
 650	 * We'll need this info later when we'll want to unmap everything
 651	 * (e.g. on shutdown).
 652	 *
 653	 * We can't trust the remote processor not to change the resource
 654	 * table, so we must maintain this info independently.
 655	 */
 656	mapping->da = rsc->da;
 657	mapping->len = rsc->len;
 658	list_add_tail(&mapping->node, &rproc->mappings);
 659
 660	dev_dbg(dev, "mapped devmem pa 0x%x, da 0x%x, len 0x%x\n",
 661		rsc->pa, rsc->da, rsc->len);
 662
 663	return 0;
 664
 665out:
 666	kfree(mapping);
 667	return ret;
 668}
 669
 670/**
 671 * rproc_alloc_carveout() - allocated specified carveout
 672 * @rproc: rproc handle
 673 * @mem: the memory entry to allocate
 674 *
 675 * This function allocate specified memory entry @mem using
 676 * dma_alloc_coherent() as default allocator
 677 *
 678 * Return: 0 on success, or an appropriate error code otherwise
 679 */
 680static int rproc_alloc_carveout(struct rproc *rproc,
 681				struct rproc_mem_entry *mem)
 682{
 683	struct rproc_mem_entry *mapping = NULL;
 684	struct device *dev = &rproc->dev;
 685	dma_addr_t dma;
 686	void *va;
 687	int ret;
 688
 689	va = dma_alloc_coherent(dev->parent, mem->len, &dma, GFP_KERNEL);
 690	if (!va) {
 691		dev_err(dev->parent,
 692			"failed to allocate dma memory: len 0x%zx\n",
 693			mem->len);
 694		return -ENOMEM;
 695	}
 696
 697	dev_dbg(dev, "carveout va %p, dma %pad, len 0x%zx\n",
 698		va, &dma, mem->len);
 699
 700	if (mem->da != FW_RSC_ADDR_ANY && !rproc->domain) {
 701		/*
 702		 * Check requested da is equal to dma address
 703		 * and print a warn message in case of missalignment.
 704		 * Don't stop rproc_start sequence as coprocessor may
 705		 * build pa to da translation on its side.
 706		 */
 707		if (mem->da != (u32)dma)
 708			dev_warn(dev->parent,
 709				 "Allocated carveout doesn't fit device address request\n");
 710	}
 711
 712	/*
 713	 * Ok, this is non-standard.
 714	 *
 715	 * Sometimes we can't rely on the generic iommu-based DMA API
 716	 * to dynamically allocate the device address and then set the IOMMU
 717	 * tables accordingly, because some remote processors might
 718	 * _require_ us to use hard coded device addresses that their
 719	 * firmware was compiled with.
 720	 *
 721	 * In this case, we must use the IOMMU API directly and map
 722	 * the memory to the device address as expected by the remote
 723	 * processor.
 724	 *
 725	 * Obviously such remote processor devices should not be configured
 726	 * to use the iommu-based DMA API: we expect 'dma' to contain the
 727	 * physical address in this case.
 728	 */
 729	if (mem->da != FW_RSC_ADDR_ANY && rproc->domain) {
 730		mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
 731		if (!mapping) {
 732			ret = -ENOMEM;
 733			goto dma_free;
 734		}
 735
 736		ret = iommu_map(rproc->domain, mem->da, dma, mem->len,
 737				mem->flags, GFP_KERNEL);
 738		if (ret) {
 739			dev_err(dev, "iommu_map failed: %d\n", ret);
 740			goto free_mapping;
 741		}
 742
 743		/*
 744		 * We'll need this info later when we'll want to unmap
 745		 * everything (e.g. on shutdown).
 746		 *
 747		 * We can't trust the remote processor not to change the
 748		 * resource table, so we must maintain this info independently.
 749		 */
 750		mapping->da = mem->da;
 751		mapping->len = mem->len;
 752		list_add_tail(&mapping->node, &rproc->mappings);
 753
 754		dev_dbg(dev, "carveout mapped 0x%x to %pad\n",
 755			mem->da, &dma);
 756	}
 757
 758	if (mem->da == FW_RSC_ADDR_ANY) {
 759		/* Update device address as undefined by requester */
 760		if ((u64)dma & HIGH_BITS_MASK)
 761			dev_warn(dev, "DMA address cast in 32bit to fit resource table format\n");
 762
 763		mem->da = (u32)dma;
 764	}
 765
 766	mem->dma = dma;
 767	mem->va = va;
 768
 769	return 0;
 770
 771free_mapping:
 772	kfree(mapping);
 773dma_free:
 774	dma_free_coherent(dev->parent, mem->len, va, dma);
 775	return ret;
 776}
 777
 778/**
 779 * rproc_release_carveout() - release acquired carveout
 780 * @rproc: rproc handle
 781 * @mem: the memory entry to release
 782 *
 783 * This function releases specified memory entry @mem allocated via
 784 * rproc_alloc_carveout() function by @rproc.
 785 *
 786 * Return: 0 on success, or an appropriate error code otherwise
 787 */
 788static int rproc_release_carveout(struct rproc *rproc,
 789				  struct rproc_mem_entry *mem)
 790{
 791	struct device *dev = &rproc->dev;
 792
 793	/* clean up carveout allocations */
 794	dma_free_coherent(dev->parent, mem->len, mem->va, mem->dma);
 795	return 0;
 796}
 797
 798/**
 799 * rproc_handle_carveout() - handle phys contig memory allocation requests
 800 * @rproc: rproc handle
 801 * @ptr: the resource entry
 802 * @offset: offset of the resource entry
 803 * @avail: size of available data (for image validation)
 804 *
 805 * This function will handle firmware requests for allocation of physically
 806 * contiguous memory regions.
 807 *
 808 * These request entries should come first in the firmware's resource table,
 809 * as other firmware entries might request placing other data objects inside
 810 * these memory regions (e.g. data/code segments, trace resource entries, ...).
 811 *
 812 * Allocating memory this way helps utilizing the reserved physical memory
 813 * (e.g. CMA) more efficiently, and also minimizes the number of TLB entries
 814 * needed to map it (in case @rproc is using an IOMMU). Reducing the TLB
 815 * pressure is important; it may have a substantial impact on performance.
 816 *
 817 * Return: 0 on success, or an appropriate error code otherwise
 818 */
 819static int rproc_handle_carveout(struct rproc *rproc,
 820				 void *ptr, int offset, int avail)
 821{
 822	struct fw_rsc_carveout *rsc = ptr;
 823	struct rproc_mem_entry *carveout;
 824	struct device *dev = &rproc->dev;
 825
 826	if (sizeof(*rsc) > avail) {
 827		dev_err(dev, "carveout rsc is truncated\n");
 828		return -EINVAL;
 829	}
 830
 831	/* make sure reserved bytes are zeroes */
 832	if (rsc->reserved) {
 833		dev_err(dev, "carveout rsc has non zero reserved bytes\n");
 834		return -EINVAL;
 835	}
 836
 837	dev_dbg(dev, "carveout rsc: name: %s, da 0x%x, pa 0x%x, len 0x%x, flags 0x%x\n",
 838		rsc->name, rsc->da, rsc->pa, rsc->len, rsc->flags);
 839
 840	/*
 841	 * Check carveout rsc already part of a registered carveout,
 842	 * Search by name, then check the da and length
 843	 */
 844	carveout = rproc_find_carveout_by_name(rproc, rsc->name);
 845
 846	if (carveout) {
 847		if (carveout->rsc_offset != FW_RSC_ADDR_ANY) {
 848			dev_err(dev,
 849				"Carveout already associated to resource table\n");
 850			return -ENOMEM;
 851		}
 852
 853		if (rproc_check_carveout_da(rproc, carveout, rsc->da, rsc->len))
 854			return -ENOMEM;
 855
 856		/* Update memory carveout with resource table info */
 857		carveout->rsc_offset = offset;
 858		carveout->flags = rsc->flags;
 859
 860		return 0;
 861	}
 862
 863	/* Register carveout in list */
 864	carveout = rproc_mem_entry_init(dev, NULL, 0, rsc->len, rsc->da,
 865					rproc_alloc_carveout,
 866					rproc_release_carveout, rsc->name);
 867	if (!carveout) {
 868		dev_err(dev, "Can't allocate memory entry structure\n");
 869		return -ENOMEM;
 870	}
 871
 872	carveout->flags = rsc->flags;
 873	carveout->rsc_offset = offset;
 874	rproc_add_carveout(rproc, carveout);
 875
 876	return 0;
 877}
 878
 879/**
 880 * rproc_add_carveout() - register an allocated carveout region
 881 * @rproc: rproc handle
 882 * @mem: memory entry to register
 883 *
 884 * This function registers specified memory entry in @rproc carveouts list.
 885 * Specified carveout should have been allocated before registering.
 886 */
 887void rproc_add_carveout(struct rproc *rproc, struct rproc_mem_entry *mem)
 888{
 889	list_add_tail(&mem->node, &rproc->carveouts);
 890}
 891EXPORT_SYMBOL(rproc_add_carveout);
 892
 893/**
 894 * rproc_mem_entry_init() - allocate and initialize rproc_mem_entry struct
 895 * @dev: pointer on device struct
 896 * @va: virtual address
 897 * @dma: dma address
 898 * @len: memory carveout length
 899 * @da: device address
 900 * @alloc: memory carveout allocation function
 901 * @release: memory carveout release function
 902 * @name: carveout name
 903 *
 904 * This function allocates a rproc_mem_entry struct and fill it with parameters
 905 * provided by client.
 906 *
 907 * Return: a valid pointer on success, or NULL on failure
 908 */
 909__printf(8, 9)
 910struct rproc_mem_entry *
 911rproc_mem_entry_init(struct device *dev,
 912		     void *va, dma_addr_t dma, size_t len, u32 da,
 913		     int (*alloc)(struct rproc *, struct rproc_mem_entry *),
 914		     int (*release)(struct rproc *, struct rproc_mem_entry *),
 915		     const char *name, ...)
 916{
 917	struct rproc_mem_entry *mem;
 918	va_list args;
 919
 920	mem = kzalloc(sizeof(*mem), GFP_KERNEL);
 921	if (!mem)
 922		return mem;
 923
 924	mem->va = va;
 925	mem->dma = dma;
 926	mem->da = da;
 927	mem->len = len;
 928	mem->alloc = alloc;
 929	mem->release = release;
 930	mem->rsc_offset = FW_RSC_ADDR_ANY;
 931	mem->of_resm_idx = -1;
 932
 933	va_start(args, name);
 934	vsnprintf(mem->name, sizeof(mem->name), name, args);
 935	va_end(args);
 936
 937	return mem;
 938}
 939EXPORT_SYMBOL(rproc_mem_entry_init);
 940
 941/**
 942 * rproc_of_resm_mem_entry_init() - allocate and initialize rproc_mem_entry struct
 943 * from a reserved memory phandle
 944 * @dev: pointer on device struct
 945 * @of_resm_idx: reserved memory phandle index in "memory-region"
 946 * @len: memory carveout length
 947 * @da: device address
 948 * @name: carveout name
 949 *
 950 * This function allocates a rproc_mem_entry struct and fill it with parameters
 951 * provided by client.
 952 *
 953 * Return: a valid pointer on success, or NULL on failure
 954 */
 955__printf(5, 6)
 956struct rproc_mem_entry *
 957rproc_of_resm_mem_entry_init(struct device *dev, u32 of_resm_idx, size_t len,
 958			     u32 da, const char *name, ...)
 959{
 960	struct rproc_mem_entry *mem;
 961	va_list args;
 962
 963	mem = kzalloc(sizeof(*mem), GFP_KERNEL);
 964	if (!mem)
 965		return mem;
 966
 967	mem->da = da;
 968	mem->len = len;
 969	mem->rsc_offset = FW_RSC_ADDR_ANY;
 970	mem->of_resm_idx = of_resm_idx;
 971
 972	va_start(args, name);
 973	vsnprintf(mem->name, sizeof(mem->name), name, args);
 974	va_end(args);
 975
 976	return mem;
 977}
 978EXPORT_SYMBOL(rproc_of_resm_mem_entry_init);
 979
 980/**
 981 * rproc_of_parse_firmware() - parse and return the firmware-name
 982 * @dev: pointer on device struct representing a rproc
 983 * @index: index to use for the firmware-name retrieval
 984 * @fw_name: pointer to a character string, in which the firmware
 985 *           name is returned on success and unmodified otherwise.
 986 *
 987 * This is an OF helper function that parses a device's DT node for
 988 * the "firmware-name" property and returns the firmware name pointer
 989 * in @fw_name on success.
 990 *
 991 * Return: 0 on success, or an appropriate failure.
 992 */
 993int rproc_of_parse_firmware(struct device *dev, int index, const char **fw_name)
 994{
 995	int ret;
 996
 997	ret = of_property_read_string_index(dev->of_node, "firmware-name",
 998					    index, fw_name);
 999	return ret ? ret : 0;
1000}
1001EXPORT_SYMBOL(rproc_of_parse_firmware);
1002
1003/*
1004 * A lookup table for resource handlers. The indices are defined in
1005 * enum fw_resource_type.
1006 */
1007static rproc_handle_resource_t rproc_loading_handlers[RSC_LAST] = {
1008	[RSC_CARVEOUT] = rproc_handle_carveout,
1009	[RSC_DEVMEM] = rproc_handle_devmem,
1010	[RSC_TRACE] = rproc_handle_trace,
1011	[RSC_VDEV] = rproc_handle_vdev,
1012};
1013
1014/* handle firmware resource entries before booting the remote processor */
1015static int rproc_handle_resources(struct rproc *rproc,
1016				  rproc_handle_resource_t handlers[RSC_LAST])
1017{
1018	struct device *dev = &rproc->dev;
1019	rproc_handle_resource_t handler;
1020	int ret = 0, i;
1021
1022	if (!rproc->table_ptr)
1023		return 0;
1024
1025	for (i = 0; i < rproc->table_ptr->num; i++) {
1026		int offset = rproc->table_ptr->offset[i];
1027		struct fw_rsc_hdr *hdr = (void *)rproc->table_ptr + offset;
1028		int avail = rproc->table_sz - offset - sizeof(*hdr);
1029		void *rsc = (void *)hdr + sizeof(*hdr);
1030
1031		/* make sure table isn't truncated */
1032		if (avail < 0) {
1033			dev_err(dev, "rsc table is truncated\n");
1034			return -EINVAL;
1035		}
1036
1037		dev_dbg(dev, "rsc: type %d\n", hdr->type);
1038
1039		if (hdr->type >= RSC_VENDOR_START &&
1040		    hdr->type <= RSC_VENDOR_END) {
1041			ret = rproc_handle_rsc(rproc, hdr->type, rsc,
1042					       offset + sizeof(*hdr), avail);
1043			if (ret == RSC_HANDLED)
1044				continue;
1045			else if (ret < 0)
1046				break;
1047
1048			dev_warn(dev, "unsupported vendor resource %d\n",
1049				 hdr->type);
1050			continue;
1051		}
1052
1053		if (hdr->type >= RSC_LAST) {
1054			dev_warn(dev, "unsupported resource %d\n", hdr->type);
1055			continue;
1056		}
1057
1058		handler = handlers[hdr->type];
1059		if (!handler)
1060			continue;
1061
1062		ret = handler(rproc, rsc, offset + sizeof(*hdr), avail);
1063		if (ret)
1064			break;
1065	}
1066
1067	return ret;
1068}
1069
1070static int rproc_prepare_subdevices(struct rproc *rproc)
1071{
1072	struct rproc_subdev *subdev;
1073	int ret;
1074
1075	list_for_each_entry(subdev, &rproc->subdevs, node) {
1076		if (subdev->prepare) {
1077			ret = subdev->prepare(subdev);
1078			if (ret)
1079				goto unroll_preparation;
1080		}
1081	}
1082
1083	return 0;
1084
1085unroll_preparation:
1086	list_for_each_entry_continue_reverse(subdev, &rproc->subdevs, node) {
1087		if (subdev->unprepare)
1088			subdev->unprepare(subdev);
1089	}
1090
1091	return ret;
1092}
1093
1094static int rproc_start_subdevices(struct rproc *rproc)
1095{
1096	struct rproc_subdev *subdev;
1097	int ret;
1098
1099	list_for_each_entry(subdev, &rproc->subdevs, node) {
1100		if (subdev->start) {
1101			ret = subdev->start(subdev);
1102			if (ret)
1103				goto unroll_registration;
1104		}
1105	}
1106
1107	return 0;
1108
1109unroll_registration:
1110	list_for_each_entry_continue_reverse(subdev, &rproc->subdevs, node) {
1111		if (subdev->stop)
1112			subdev->stop(subdev, true);
1113	}
1114
1115	return ret;
1116}
1117
1118static void rproc_stop_subdevices(struct rproc *rproc, bool crashed)
1119{
1120	struct rproc_subdev *subdev;
1121
1122	list_for_each_entry_reverse(subdev, &rproc->subdevs, node) {
1123		if (subdev->stop)
1124			subdev->stop(subdev, crashed);
1125	}
1126}
1127
1128static void rproc_unprepare_subdevices(struct rproc *rproc)
1129{
1130	struct rproc_subdev *subdev;
1131
1132	list_for_each_entry_reverse(subdev, &rproc->subdevs, node) {
1133		if (subdev->unprepare)
1134			subdev->unprepare(subdev);
1135	}
1136}
1137
1138/**
1139 * rproc_alloc_registered_carveouts() - allocate all carveouts registered
1140 * in the list
1141 * @rproc: the remote processor handle
1142 *
1143 * This function parses registered carveout list, performs allocation
1144 * if alloc() ops registered and updates resource table information
1145 * if rsc_offset set.
1146 *
1147 * Return: 0 on success
1148 */
1149static int rproc_alloc_registered_carveouts(struct rproc *rproc)
1150{
1151	struct rproc_mem_entry *entry, *tmp;
1152	struct fw_rsc_carveout *rsc;
1153	struct device *dev = &rproc->dev;
1154	u64 pa;
1155	int ret;
1156
1157	list_for_each_entry_safe(entry, tmp, &rproc->carveouts, node) {
1158		if (entry->alloc) {
1159			ret = entry->alloc(rproc, entry);
1160			if (ret) {
1161				dev_err(dev, "Unable to allocate carveout %s: %d\n",
1162					entry->name, ret);
1163				return -ENOMEM;
1164			}
1165		}
1166
1167		if (entry->rsc_offset != FW_RSC_ADDR_ANY) {
1168			/* update resource table */
1169			rsc = (void *)rproc->table_ptr + entry->rsc_offset;
1170
1171			/*
1172			 * Some remote processors might need to know the pa
1173			 * even though they are behind an IOMMU. E.g., OMAP4's
1174			 * remote M3 processor needs this so it can control
1175			 * on-chip hardware accelerators that are not behind
1176			 * the IOMMU, and therefor must know the pa.
1177			 *
1178			 * Generally we don't want to expose physical addresses
1179			 * if we don't have to (remote processors are generally
1180			 * _not_ trusted), so we might want to do this only for
1181			 * remote processor that _must_ have this (e.g. OMAP4's
1182			 * dual M3 subsystem).
1183			 *
1184			 * Non-IOMMU processors might also want to have this info.
1185			 * In this case, the device address and the physical address
1186			 * are the same.
1187			 */
1188
1189			/* Use va if defined else dma to generate pa */
1190			if (entry->va)
1191				pa = (u64)rproc_va_to_pa(entry->va);
1192			else
1193				pa = (u64)entry->dma;
1194
1195			if (((u64)pa) & HIGH_BITS_MASK)
1196				dev_warn(dev,
1197					 "Physical address cast in 32bit to fit resource table format\n");
1198
1199			rsc->pa = (u32)pa;
1200			rsc->da = entry->da;
1201			rsc->len = entry->len;
1202		}
1203	}
1204
1205	return 0;
1206}
1207
1208
1209/**
1210 * rproc_resource_cleanup() - clean up and free all acquired resources
1211 * @rproc: rproc handle
1212 *
1213 * This function will free all resources acquired for @rproc, and it
1214 * is called whenever @rproc either shuts down or fails to boot.
1215 */
1216void rproc_resource_cleanup(struct rproc *rproc)
1217{
1218	struct rproc_mem_entry *entry, *tmp;
1219	struct rproc_debug_trace *trace, *ttmp;
1220	struct rproc_vdev *rvdev, *rvtmp;
1221	struct device *dev = &rproc->dev;
1222
1223	/* clean up debugfs trace entries */
1224	list_for_each_entry_safe(trace, ttmp, &rproc->traces, node) {
1225		rproc_remove_trace_file(trace->tfile);
1226		rproc->num_traces--;
1227		list_del(&trace->node);
1228		kfree(trace);
1229	}
1230
1231	/* clean up iommu mapping entries */
1232	list_for_each_entry_safe(entry, tmp, &rproc->mappings, node) {
1233		size_t unmapped;
1234
1235		unmapped = iommu_unmap(rproc->domain, entry->da, entry->len);
1236		if (unmapped != entry->len) {
1237			/* nothing much to do besides complaining */
1238			dev_err(dev, "failed to unmap %zx/%zu\n", entry->len,
1239				unmapped);
1240		}
1241
1242		list_del(&entry->node);
1243		kfree(entry);
1244	}
1245
1246	/* clean up carveout allocations */
1247	list_for_each_entry_safe(entry, tmp, &rproc->carveouts, node) {
1248		if (entry->release)
1249			entry->release(rproc, entry);
1250		list_del(&entry->node);
1251		kfree(entry);
1252	}
1253
1254	/* clean up remote vdev entries */
1255	list_for_each_entry_safe(rvdev, rvtmp, &rproc->rvdevs, node)
1256		platform_device_unregister(rvdev->pdev);
1257
1258	rproc_coredump_cleanup(rproc);
1259}
1260EXPORT_SYMBOL(rproc_resource_cleanup);
1261
1262static int rproc_start(struct rproc *rproc, const struct firmware *fw)
1263{
1264	struct resource_table *loaded_table;
1265	struct device *dev = &rproc->dev;
1266	int ret;
1267
1268	/* load the ELF segments to memory */
1269	ret = rproc_load_segments(rproc, fw);
1270	if (ret) {
1271		dev_err(dev, "Failed to load program segments: %d\n", ret);
1272		return ret;
1273	}
1274
1275	/*
1276	 * The starting device has been given the rproc->cached_table as the
1277	 * resource table. The address of the vring along with the other
1278	 * allocated resources (carveouts etc) is stored in cached_table.
1279	 * In order to pass this information to the remote device we must copy
1280	 * this information to device memory. We also update the table_ptr so
1281	 * that any subsequent changes will be applied to the loaded version.
1282	 */
1283	loaded_table = rproc_find_loaded_rsc_table(rproc, fw);
1284	if (loaded_table) {
1285		memcpy(loaded_table, rproc->cached_table, rproc->table_sz);
1286		rproc->table_ptr = loaded_table;
1287	}
1288
1289	ret = rproc_prepare_subdevices(rproc);
1290	if (ret) {
1291		dev_err(dev, "failed to prepare subdevices for %s: %d\n",
1292			rproc->name, ret);
1293		goto reset_table_ptr;
1294	}
1295
1296	/* power up the remote processor */
1297	ret = rproc->ops->start(rproc);
1298	if (ret) {
1299		dev_err(dev, "can't start rproc %s: %d\n", rproc->name, ret);
1300		goto unprepare_subdevices;
1301	}
1302
1303	/* Start any subdevices for the remote processor */
1304	ret = rproc_start_subdevices(rproc);
1305	if (ret) {
1306		dev_err(dev, "failed to probe subdevices for %s: %d\n",
1307			rproc->name, ret);
1308		goto stop_rproc;
1309	}
1310
1311	rproc->state = RPROC_RUNNING;
1312
1313	dev_info(dev, "remote processor %s is now up\n", rproc->name);
1314
1315	return 0;
1316
1317stop_rproc:
1318	rproc->ops->stop(rproc);
1319unprepare_subdevices:
1320	rproc_unprepare_subdevices(rproc);
1321reset_table_ptr:
1322	rproc->table_ptr = rproc->cached_table;
1323
1324	return ret;
1325}
1326
1327static int __rproc_attach(struct rproc *rproc)
1328{
1329	struct device *dev = &rproc->dev;
1330	int ret;
1331
1332	ret = rproc_prepare_subdevices(rproc);
1333	if (ret) {
1334		dev_err(dev, "failed to prepare subdevices for %s: %d\n",
1335			rproc->name, ret);
1336		goto out;
1337	}
1338
1339	/* Attach to the remote processor */
1340	ret = rproc_attach_device(rproc);
1341	if (ret) {
1342		dev_err(dev, "can't attach to rproc %s: %d\n",
1343			rproc->name, ret);
1344		goto unprepare_subdevices;
1345	}
1346
1347	/* Start any subdevices for the remote processor */
1348	ret = rproc_start_subdevices(rproc);
1349	if (ret) {
1350		dev_err(dev, "failed to probe subdevices for %s: %d\n",
1351			rproc->name, ret);
1352		goto stop_rproc;
1353	}
1354
1355	rproc->state = RPROC_ATTACHED;
1356
1357	dev_info(dev, "remote processor %s is now attached\n", rproc->name);
1358
1359	return 0;
1360
1361stop_rproc:
1362	rproc->ops->stop(rproc);
1363unprepare_subdevices:
1364	rproc_unprepare_subdevices(rproc);
1365out:
1366	return ret;
1367}
1368
1369/*
1370 * take a firmware and boot a remote processor with it.
1371 */
1372static int rproc_fw_boot(struct rproc *rproc, const struct firmware *fw)
1373{
1374	struct device *dev = &rproc->dev;
1375	const char *name = rproc->firmware;
1376	int ret;
1377
1378	ret = rproc_fw_sanity_check(rproc, fw);
1379	if (ret)
1380		return ret;
1381
1382	dev_info(dev, "Booting fw image %s, size %zd\n", name, fw->size);
1383
1384	/*
1385	 * if enabling an IOMMU isn't relevant for this rproc, this is
1386	 * just a nop
1387	 */
1388	ret = rproc_enable_iommu(rproc);
1389	if (ret) {
1390		dev_err(dev, "can't enable iommu: %d\n", ret);
1391		return ret;
1392	}
1393
1394	/* Prepare rproc for firmware loading if needed */
1395	ret = rproc_prepare_device(rproc);
1396	if (ret) {
1397		dev_err(dev, "can't prepare rproc %s: %d\n", rproc->name, ret);
1398		goto disable_iommu;
1399	}
1400
1401	rproc->bootaddr = rproc_get_boot_addr(rproc, fw);
1402
1403	/* Load resource table, core dump segment list etc from the firmware */
1404	ret = rproc_parse_fw(rproc, fw);
1405	if (ret)
1406		goto unprepare_rproc;
1407
1408	/* reset max_notifyid */
1409	rproc->max_notifyid = -1;
1410
1411	/* reset handled vdev */
1412	rproc->nb_vdev = 0;
1413
1414	/* handle fw resources which are required to boot rproc */
1415	ret = rproc_handle_resources(rproc, rproc_loading_handlers);
1416	if (ret) {
1417		dev_err(dev, "Failed to process resources: %d\n", ret);
1418		goto clean_up_resources;
1419	}
1420
1421	/* Allocate carveout resources associated to rproc */
1422	ret = rproc_alloc_registered_carveouts(rproc);
1423	if (ret) {
1424		dev_err(dev, "Failed to allocate associated carveouts: %d\n",
1425			ret);
1426		goto clean_up_resources;
1427	}
1428
1429	ret = rproc_start(rproc, fw);
1430	if (ret)
1431		goto clean_up_resources;
1432
1433	return 0;
1434
1435clean_up_resources:
1436	rproc_resource_cleanup(rproc);
1437	kfree(rproc->cached_table);
1438	rproc->cached_table = NULL;
1439	rproc->table_ptr = NULL;
1440unprepare_rproc:
1441	/* release HW resources if needed */
1442	rproc_unprepare_device(rproc);
1443disable_iommu:
1444	rproc_disable_iommu(rproc);
1445	return ret;
1446}
1447
1448static int rproc_set_rsc_table(struct rproc *rproc)
1449{
1450	struct resource_table *table_ptr;
1451	struct device *dev = &rproc->dev;
1452	size_t table_sz;
1453	int ret;
1454
1455	table_ptr = rproc_get_loaded_rsc_table(rproc, &table_sz);
1456	if (!table_ptr) {
1457		/* Not having a resource table is acceptable */
1458		return 0;
1459	}
1460
1461	if (IS_ERR(table_ptr)) {
1462		ret = PTR_ERR(table_ptr);
1463		dev_err(dev, "can't load resource table: %d\n", ret);
1464		return ret;
1465	}
1466
1467	/*
1468	 * If it is possible to detach the remote processor, keep an untouched
1469	 * copy of the resource table.  That way we can start fresh again when
1470	 * the remote processor is re-attached, that is:
1471	 *
1472	 *      DETACHED -> ATTACHED -> DETACHED -> ATTACHED
1473	 *
1474	 * Free'd in rproc_reset_rsc_table_on_detach() and
1475	 * rproc_reset_rsc_table_on_stop().
1476	 */
1477	if (rproc->ops->detach) {
1478		rproc->clean_table = kmemdup(table_ptr, table_sz, GFP_KERNEL);
1479		if (!rproc->clean_table)
1480			return -ENOMEM;
1481	} else {
1482		rproc->clean_table = NULL;
1483	}
1484
1485	rproc->cached_table = NULL;
1486	rproc->table_ptr = table_ptr;
1487	rproc->table_sz = table_sz;
1488
1489	return 0;
1490}
1491
1492static int rproc_reset_rsc_table_on_detach(struct rproc *rproc)
1493{
1494	struct resource_table *table_ptr;
1495
1496	/* A resource table was never retrieved, nothing to do here */
1497	if (!rproc->table_ptr)
1498		return 0;
1499
1500	/*
1501	 * If we made it to this point a clean_table _must_ have been
1502	 * allocated in rproc_set_rsc_table().  If one isn't present
1503	 * something went really wrong and we must complain.
1504	 */
1505	if (WARN_ON(!rproc->clean_table))
1506		return -EINVAL;
1507
1508	/* Remember where the external entity installed the resource table */
1509	table_ptr = rproc->table_ptr;
1510
1511	/*
1512	 * If we made it here the remote processor was started by another
1513	 * entity and a cache table doesn't exist.  As such make a copy of
1514	 * the resource table currently used by the remote processor and
1515	 * use that for the rest of the shutdown process.  The memory
1516	 * allocated here is free'd in rproc_detach().
1517	 */
1518	rproc->cached_table = kmemdup(rproc->table_ptr,
1519				      rproc->table_sz, GFP_KERNEL);
1520	if (!rproc->cached_table)
1521		return -ENOMEM;
1522
1523	/*
1524	 * Use a copy of the resource table for the remainder of the
1525	 * shutdown process.
1526	 */
1527	rproc->table_ptr = rproc->cached_table;
1528
1529	/*
1530	 * Reset the memory area where the firmware loaded the resource table
1531	 * to its original value.  That way when we re-attach the remote
1532	 * processor the resource table is clean and ready to be used again.
1533	 */
1534	memcpy(table_ptr, rproc->clean_table, rproc->table_sz);
1535
1536	/*
1537	 * The clean resource table is no longer needed.  Allocated in
1538	 * rproc_set_rsc_table().
1539	 */
1540	kfree(rproc->clean_table);
1541
1542	return 0;
1543}
1544
1545static int rproc_reset_rsc_table_on_stop(struct rproc *rproc)
1546{
1547	/* A resource table was never retrieved, nothing to do here */
1548	if (!rproc->table_ptr)
1549		return 0;
1550
1551	/*
1552	 * If a cache table exists the remote processor was started by
1553	 * the remoteproc core.  That cache table should be used for
1554	 * the rest of the shutdown process.
1555	 */
1556	if (rproc->cached_table)
1557		goto out;
1558
1559	/*
1560	 * If we made it here the remote processor was started by another
1561	 * entity and a cache table doesn't exist.  As such make a copy of
1562	 * the resource table currently used by the remote processor and
1563	 * use that for the rest of the shutdown process.  The memory
1564	 * allocated here is free'd in rproc_shutdown().
1565	 */
1566	rproc->cached_table = kmemdup(rproc->table_ptr,
1567				      rproc->table_sz, GFP_KERNEL);
1568	if (!rproc->cached_table)
1569		return -ENOMEM;
1570
1571	/*
1572	 * Since the remote processor is being switched off the clean table
1573	 * won't be needed.  Allocated in rproc_set_rsc_table().
1574	 */
1575	kfree(rproc->clean_table);
1576
1577out:
1578	/*
1579	 * Use a copy of the resource table for the remainder of the
1580	 * shutdown process.
1581	 */
1582	rproc->table_ptr = rproc->cached_table;
1583	return 0;
1584}
1585
1586/*
1587 * Attach to remote processor - similar to rproc_fw_boot() but without
1588 * the steps that deal with the firmware image.
1589 */
1590static int rproc_attach(struct rproc *rproc)
1591{
1592	struct device *dev = &rproc->dev;
1593	int ret;
1594
1595	/*
1596	 * if enabling an IOMMU isn't relevant for this rproc, this is
1597	 * just a nop
1598	 */
1599	ret = rproc_enable_iommu(rproc);
1600	if (ret) {
1601		dev_err(dev, "can't enable iommu: %d\n", ret);
1602		return ret;
1603	}
1604
1605	/* Do anything that is needed to boot the remote processor */
1606	ret = rproc_prepare_device(rproc);
1607	if (ret) {
1608		dev_err(dev, "can't prepare rproc %s: %d\n", rproc->name, ret);
1609		goto disable_iommu;
1610	}
1611
1612	ret = rproc_set_rsc_table(rproc);
1613	if (ret) {
1614		dev_err(dev, "can't load resource table: %d\n", ret);
1615		goto clean_up_resources;
1616	}
1617
1618	/* reset max_notifyid */
1619	rproc->max_notifyid = -1;
1620
1621	/* reset handled vdev */
1622	rproc->nb_vdev = 0;
1623
1624	/*
1625	 * Handle firmware resources required to attach to a remote processor.
1626	 * Because we are attaching rather than booting the remote processor,
1627	 * we expect the platform driver to properly set rproc->table_ptr.
1628	 */
1629	ret = rproc_handle_resources(rproc, rproc_loading_handlers);
1630	if (ret) {
1631		dev_err(dev, "Failed to process resources: %d\n", ret);
1632		goto clean_up_resources;
1633	}
1634
1635	/* Allocate carveout resources associated to rproc */
1636	ret = rproc_alloc_registered_carveouts(rproc);
1637	if (ret) {
1638		dev_err(dev, "Failed to allocate associated carveouts: %d\n",
1639			ret);
1640		goto clean_up_resources;
1641	}
1642
1643	ret = __rproc_attach(rproc);
1644	if (ret)
1645		goto clean_up_resources;
1646
1647	return 0;
1648
1649clean_up_resources:
1650	rproc_resource_cleanup(rproc);
1651	/* release HW resources if needed */
1652	rproc_unprepare_device(rproc);
1653	kfree(rproc->clean_table);
1654disable_iommu:
1655	rproc_disable_iommu(rproc);
1656	return ret;
1657}
1658
1659/*
1660 * take a firmware and boot it up.
1661 *
1662 * Note: this function is called asynchronously upon registration of the
1663 * remote processor (so we must wait until it completes before we try
1664 * to unregister the device. one other option is just to use kref here,
1665 * that might be cleaner).
1666 */
1667static void rproc_auto_boot_callback(const struct firmware *fw, void *context)
1668{
1669	struct rproc *rproc = context;
1670
1671	rproc_boot(rproc);
1672
1673	release_firmware(fw);
1674}
1675
1676static int rproc_trigger_auto_boot(struct rproc *rproc)
1677{
1678	int ret;
1679
1680	/*
1681	 * Since the remote processor is in a detached state, it has already
1682	 * been booted by another entity.  As such there is no point in waiting
1683	 * for a firmware image to be loaded, we can simply initiate the process
1684	 * of attaching to it immediately.
1685	 */
1686	if (rproc->state == RPROC_DETACHED)
1687		return rproc_boot(rproc);
1688
1689	/*
1690	 * We're initiating an asynchronous firmware loading, so we can
1691	 * be built-in kernel code, without hanging the boot process.
1692	 */
1693	ret = request_firmware_nowait(THIS_MODULE, FW_ACTION_UEVENT,
1694				      rproc->firmware, &rproc->dev, GFP_KERNEL,
1695				      rproc, rproc_auto_boot_callback);
1696	if (ret < 0)
1697		dev_err(&rproc->dev, "request_firmware_nowait err: %d\n", ret);
1698
1699	return ret;
1700}
1701
1702static int rproc_stop(struct rproc *rproc, bool crashed)
1703{
1704	struct device *dev = &rproc->dev;
1705	int ret;
1706
1707	/* No need to continue if a stop() operation has not been provided */
1708	if (!rproc->ops->stop)
1709		return -EINVAL;
1710
1711	/* Stop any subdevices for the remote processor */
1712	rproc_stop_subdevices(rproc, crashed);
1713
1714	/* the installed resource table is no longer accessible */
1715	ret = rproc_reset_rsc_table_on_stop(rproc);
1716	if (ret) {
1717		dev_err(dev, "can't reset resource table: %d\n", ret);
1718		return ret;
1719	}
1720
1721
1722	/* power off the remote processor */
1723	ret = rproc->ops->stop(rproc);
1724	if (ret) {
1725		dev_err(dev, "can't stop rproc: %d\n", ret);
1726		return ret;
1727	}
1728
1729	rproc_unprepare_subdevices(rproc);
1730
1731	rproc->state = RPROC_OFFLINE;
1732
1733	dev_info(dev, "stopped remote processor %s\n", rproc->name);
1734
1735	return 0;
1736}
1737
1738/*
1739 * __rproc_detach(): Does the opposite of __rproc_attach()
1740 */
1741static int __rproc_detach(struct rproc *rproc)
1742{
1743	struct device *dev = &rproc->dev;
1744	int ret;
1745
1746	/* No need to continue if a detach() operation has not been provided */
1747	if (!rproc->ops->detach)
1748		return -EINVAL;
1749
1750	/* Stop any subdevices for the remote processor */
1751	rproc_stop_subdevices(rproc, false);
1752
1753	/* the installed resource table is no longer accessible */
1754	ret = rproc_reset_rsc_table_on_detach(rproc);
1755	if (ret) {
1756		dev_err(dev, "can't reset resource table: %d\n", ret);
1757		return ret;
1758	}
1759
1760	/* Tell the remote processor the core isn't available anymore */
1761	ret = rproc->ops->detach(rproc);
1762	if (ret) {
1763		dev_err(dev, "can't detach from rproc: %d\n", ret);
1764		return ret;
1765	}
1766
1767	rproc_unprepare_subdevices(rproc);
1768
1769	rproc->state = RPROC_DETACHED;
1770
1771	dev_info(dev, "detached remote processor %s\n", rproc->name);
1772
1773	return 0;
1774}
1775
1776static int rproc_attach_recovery(struct rproc *rproc)
1777{
1778	int ret;
1779
1780	ret = __rproc_detach(rproc);
1781	if (ret)
1782		return ret;
1783
1784	return __rproc_attach(rproc);
1785}
1786
1787static int rproc_boot_recovery(struct rproc *rproc)
1788{
1789	const struct firmware *firmware_p;
1790	struct device *dev = &rproc->dev;
1791	int ret;
1792
1793	ret = rproc_stop(rproc, true);
1794	if (ret)
1795		return ret;
1796
1797	/* generate coredump */
1798	rproc->ops->coredump(rproc);
1799
1800	/* load firmware */
1801	ret = request_firmware(&firmware_p, rproc->firmware, dev);
1802	if (ret < 0) {
1803		dev_err(dev, "request_firmware failed: %d\n", ret);
1804		return ret;
1805	}
1806
1807	/* boot the remote processor up again */
1808	ret = rproc_start(rproc, firmware_p);
1809
1810	release_firmware(firmware_p);
1811
1812	return ret;
1813}
1814
1815/**
1816 * rproc_trigger_recovery() - recover a remoteproc
1817 * @rproc: the remote processor
1818 *
1819 * The recovery is done by resetting all the virtio devices, that way all the
1820 * rpmsg drivers will be reseted along with the remote processor making the
1821 * remoteproc functional again.
1822 *
1823 * This function can sleep, so it cannot be called from atomic context.
1824 *
1825 * Return: 0 on success or a negative value upon failure
1826 */
1827int rproc_trigger_recovery(struct rproc *rproc)
1828{
1829	struct device *dev = &rproc->dev;
1830	int ret;
1831
1832	ret = mutex_lock_interruptible(&rproc->lock);
1833	if (ret)
1834		return ret;
1835
1836	/* State could have changed before we got the mutex */
1837	if (rproc->state != RPROC_CRASHED)
1838		goto unlock_mutex;
1839
1840	dev_err(dev, "recovering %s\n", rproc->name);
1841
1842	if (rproc_has_feature(rproc, RPROC_FEAT_ATTACH_ON_RECOVERY))
1843		ret = rproc_attach_recovery(rproc);
1844	else
1845		ret = rproc_boot_recovery(rproc);
1846
1847unlock_mutex:
1848	mutex_unlock(&rproc->lock);
1849	return ret;
1850}
1851
1852/**
1853 * rproc_crash_handler_work() - handle a crash
1854 * @work: work treating the crash
1855 *
1856 * This function needs to handle everything related to a crash, like cpu
1857 * registers and stack dump, information to help to debug the fatal error, etc.
1858 */
1859static void rproc_crash_handler_work(struct work_struct *work)
1860{
1861	struct rproc *rproc = container_of(work, struct rproc, crash_handler);
1862	struct device *dev = &rproc->dev;
1863
1864	dev_dbg(dev, "enter %s\n", __func__);
1865
1866	mutex_lock(&rproc->lock);
1867
1868	if (rproc->state == RPROC_CRASHED) {
1869		/* handle only the first crash detected */
1870		mutex_unlock(&rproc->lock);
1871		return;
1872	}
1873
1874	if (rproc->state == RPROC_OFFLINE) {
1875		/* Don't recover if the remote processor was stopped */
1876		mutex_unlock(&rproc->lock);
1877		goto out;
1878	}
1879
1880	rproc->state = RPROC_CRASHED;
1881	dev_err(dev, "handling crash #%u in %s\n", ++rproc->crash_cnt,
1882		rproc->name);
1883
1884	mutex_unlock(&rproc->lock);
1885
1886	if (!rproc->recovery_disabled)
1887		rproc_trigger_recovery(rproc);
1888
1889out:
1890	pm_relax(rproc->dev.parent);
1891}
1892
1893/**
1894 * rproc_boot() - boot a remote processor
1895 * @rproc: handle of a remote processor
1896 *
1897 * Boot a remote processor (i.e. load its firmware, power it on, ...).
1898 *
1899 * If the remote processor is already powered on, this function immediately
1900 * returns (successfully).
1901 *
1902 * Return: 0 on success, and an appropriate error value otherwise
1903 */
1904int rproc_boot(struct rproc *rproc)
1905{
1906	const struct firmware *firmware_p;
1907	struct device *dev;
1908	int ret;
1909
1910	if (!rproc) {
1911		pr_err("invalid rproc handle\n");
1912		return -EINVAL;
1913	}
1914
1915	dev = &rproc->dev;
1916
1917	ret = mutex_lock_interruptible(&rproc->lock);
1918	if (ret) {
1919		dev_err(dev, "can't lock rproc %s: %d\n", rproc->name, ret);
1920		return ret;
1921	}
1922
1923	if (rproc->state == RPROC_DELETED) {
1924		ret = -ENODEV;
1925		dev_err(dev, "can't boot deleted rproc %s\n", rproc->name);
1926		goto unlock_mutex;
1927	}
1928
1929	/* skip the boot or attach process if rproc is already powered up */
1930	if (atomic_inc_return(&rproc->power) > 1) {
1931		ret = 0;
1932		goto unlock_mutex;
1933	}
1934
1935	if (rproc->state == RPROC_DETACHED) {
1936		dev_info(dev, "attaching to %s\n", rproc->name);
1937
1938		ret = rproc_attach(rproc);
1939	} else {
1940		dev_info(dev, "powering up %s\n", rproc->name);
1941
1942		/* load firmware */
1943		ret = request_firmware(&firmware_p, rproc->firmware, dev);
1944		if (ret < 0) {
1945			dev_err(dev, "request_firmware failed: %d\n", ret);
1946			goto downref_rproc;
1947		}
1948
1949		ret = rproc_fw_boot(rproc, firmware_p);
1950
1951		release_firmware(firmware_p);
1952	}
1953
1954downref_rproc:
1955	if (ret)
1956		atomic_dec(&rproc->power);
1957unlock_mutex:
1958	mutex_unlock(&rproc->lock);
1959	return ret;
1960}
1961EXPORT_SYMBOL(rproc_boot);
1962
1963/**
1964 * rproc_shutdown() - power off the remote processor
1965 * @rproc: the remote processor
1966 *
1967 * Power off a remote processor (previously booted with rproc_boot()).
1968 *
1969 * In case @rproc is still being used by an additional user(s), then
1970 * this function will just decrement the power refcount and exit,
1971 * without really powering off the device.
1972 *
1973 * Every call to rproc_boot() must (eventually) be accompanied by a call
1974 * to rproc_shutdown(). Calling rproc_shutdown() redundantly is a bug.
1975 *
1976 * Notes:
1977 * - we're not decrementing the rproc's refcount, only the power refcount.
1978 *   which means that the @rproc handle stays valid even after rproc_shutdown()
1979 *   returns, and users can still use it with a subsequent rproc_boot(), if
1980 *   needed.
1981 *
1982 * Return: 0 on success, and an appropriate error value otherwise
1983 */
1984int rproc_shutdown(struct rproc *rproc)
1985{
1986	struct device *dev = &rproc->dev;
1987	int ret;
1988
1989	ret = mutex_lock_interruptible(&rproc->lock);
1990	if (ret) {
1991		dev_err(dev, "can't lock rproc %s: %d\n", rproc->name, ret);
1992		return ret;
1993	}
1994
1995	if (rproc->state != RPROC_RUNNING &&
1996	    rproc->state != RPROC_ATTACHED) {
1997		ret = -EINVAL;
1998		goto out;
1999	}
2000
2001	/* if the remote proc is still needed, bail out */
2002	if (!atomic_dec_and_test(&rproc->power))
2003		goto out;
2004
2005	ret = rproc_stop(rproc, false);
2006	if (ret) {
2007		atomic_inc(&rproc->power);
2008		goto out;
2009	}
2010
2011	/* clean up all acquired resources */
2012	rproc_resource_cleanup(rproc);
2013
2014	/* release HW resources if needed */
2015	rproc_unprepare_device(rproc);
2016
2017	rproc_disable_iommu(rproc);
2018
2019	/* Free the copy of the resource table */
2020	kfree(rproc->cached_table);
2021	rproc->cached_table = NULL;
2022	rproc->table_ptr = NULL;
2023out:
2024	mutex_unlock(&rproc->lock);
2025	return ret;
2026}
2027EXPORT_SYMBOL(rproc_shutdown);
2028
2029/**
2030 * rproc_detach() - Detach the remote processor from the
2031 * remoteproc core
2032 *
2033 * @rproc: the remote processor
2034 *
2035 * Detach a remote processor (previously attached to with rproc_attach()).
2036 *
2037 * In case @rproc is still being used by an additional user(s), then
2038 * this function will just decrement the power refcount and exit,
2039 * without disconnecting the device.
2040 *
2041 * Function rproc_detach() calls __rproc_detach() in order to let a remote
2042 * processor know that services provided by the application processor are
2043 * no longer available.  From there it should be possible to remove the
2044 * platform driver and even power cycle the application processor (if the HW
2045 * supports it) without needing to switch off the remote processor.
2046 *
2047 * Return: 0 on success, and an appropriate error value otherwise
2048 */
2049int rproc_detach(struct rproc *rproc)
2050{
2051	struct device *dev = &rproc->dev;
2052	int ret;
2053
2054	ret = mutex_lock_interruptible(&rproc->lock);
2055	if (ret) {
2056		dev_err(dev, "can't lock rproc %s: %d\n", rproc->name, ret);
2057		return ret;
2058	}
2059
2060	if (rproc->state != RPROC_ATTACHED) {
2061		ret = -EINVAL;
2062		goto out;
2063	}
2064
2065	/* if the remote proc is still needed, bail out */
2066	if (!atomic_dec_and_test(&rproc->power)) {
2067		ret = 0;
2068		goto out;
2069	}
2070
2071	ret = __rproc_detach(rproc);
2072	if (ret) {
2073		atomic_inc(&rproc->power);
2074		goto out;
2075	}
2076
2077	/* clean up all acquired resources */
2078	rproc_resource_cleanup(rproc);
2079
2080	/* release HW resources if needed */
2081	rproc_unprepare_device(rproc);
2082
2083	rproc_disable_iommu(rproc);
2084
2085	/* Free the copy of the resource table */
2086	kfree(rproc->cached_table);
2087	rproc->cached_table = NULL;
2088	rproc->table_ptr = NULL;
2089out:
2090	mutex_unlock(&rproc->lock);
2091	return ret;
2092}
2093EXPORT_SYMBOL(rproc_detach);
2094
2095/**
2096 * rproc_get_by_phandle() - find a remote processor by phandle
2097 * @phandle: phandle to the rproc
2098 *
2099 * Finds an rproc handle using the remote processor's phandle, and then
2100 * return a handle to the rproc.
2101 *
2102 * This function increments the remote processor's refcount, so always
2103 * use rproc_put() to decrement it back once rproc isn't needed anymore.
2104 *
2105 * Return: rproc handle on success, and NULL on failure
2106 */
2107#ifdef CONFIG_OF
2108struct rproc *rproc_get_by_phandle(phandle phandle)
2109{
2110	struct rproc *rproc = NULL, *r;
2111	struct device_driver *driver;
2112	struct device_node *np;
2113
2114	np = of_find_node_by_phandle(phandle);
2115	if (!np)
2116		return NULL;
2117
2118	rcu_read_lock();
2119	list_for_each_entry_rcu(r, &rproc_list, node) {
2120		if (r->dev.parent && device_match_of_node(r->dev.parent, np)) {
2121			/* prevent underlying implementation from being removed */
2122
2123			/*
2124			 * If the remoteproc's parent has a driver, the
2125			 * remoteproc is not part of a cluster and we can use
2126			 * that driver.
2127			 */
2128			driver = r->dev.parent->driver;
2129
2130			/*
2131			 * If the remoteproc's parent does not have a driver,
2132			 * look for the driver associated with the cluster.
2133			 */
2134			if (!driver) {
2135				if (r->dev.parent->parent)
2136					driver = r->dev.parent->parent->driver;
2137				if (!driver)
2138					break;
2139			}
2140
2141			if (!try_module_get(driver->owner)) {
2142				dev_err(&r->dev, "can't get owner\n");
2143				break;
2144			}
2145
2146			rproc = r;
2147			get_device(&rproc->dev);
2148			break;
2149		}
2150	}
2151	rcu_read_unlock();
2152
2153	of_node_put(np);
2154
2155	return rproc;
2156}
2157#else
2158struct rproc *rproc_get_by_phandle(phandle phandle)
2159{
2160	return NULL;
2161}
2162#endif
2163EXPORT_SYMBOL(rproc_get_by_phandle);
2164
2165/**
2166 * rproc_set_firmware() - assign a new firmware
2167 * @rproc: rproc handle to which the new firmware is being assigned
2168 * @fw_name: new firmware name to be assigned
2169 *
2170 * This function allows remoteproc drivers or clients to configure a custom
2171 * firmware name that is different from the default name used during remoteproc
2172 * registration. The function does not trigger a remote processor boot,
2173 * only sets the firmware name used for a subsequent boot. This function
2174 * should also be called only when the remote processor is offline.
2175 *
2176 * This allows either the userspace to configure a different name through
2177 * sysfs or a kernel-level remoteproc or a remoteproc client driver to set
2178 * a specific firmware when it is controlling the boot and shutdown of the
2179 * remote processor.
2180 *
2181 * Return: 0 on success or a negative value upon failure
2182 */
2183int rproc_set_firmware(struct rproc *rproc, const char *fw_name)
2184{
2185	struct device *dev;
2186	int ret, len;
2187	char *p;
2188
2189	if (!rproc || !fw_name)
2190		return -EINVAL;
2191
2192	dev = rproc->dev.parent;
2193
2194	ret = mutex_lock_interruptible(&rproc->lock);
2195	if (ret) {
2196		dev_err(dev, "can't lock rproc %s: %d\n", rproc->name, ret);
2197		return -EINVAL;
2198	}
2199
2200	if (rproc->state != RPROC_OFFLINE) {
2201		dev_err(dev, "can't change firmware while running\n");
2202		ret = -EBUSY;
2203		goto out;
2204	}
2205
2206	len = strcspn(fw_name, "\n");
2207	if (!len) {
2208		dev_err(dev, "can't provide empty string for firmware name\n");
2209		ret = -EINVAL;
2210		goto out;
2211	}
2212
2213	p = kstrndup(fw_name, len, GFP_KERNEL);
2214	if (!p) {
2215		ret = -ENOMEM;
2216		goto out;
2217	}
2218
2219	kfree_const(rproc->firmware);
2220	rproc->firmware = p;
2221
2222out:
2223	mutex_unlock(&rproc->lock);
2224	return ret;
2225}
2226EXPORT_SYMBOL(rproc_set_firmware);
2227
2228static int rproc_validate(struct rproc *rproc)
2229{
2230	switch (rproc->state) {
2231	case RPROC_OFFLINE:
2232		/*
2233		 * An offline processor without a start()
2234		 * function makes no sense.
2235		 */
2236		if (!rproc->ops->start)
2237			return -EINVAL;
2238		break;
2239	case RPROC_DETACHED:
2240		/*
2241		 * A remote processor in a detached state without an
2242		 * attach() function makes not sense.
2243		 */
2244		if (!rproc->ops->attach)
2245			return -EINVAL;
2246		/*
2247		 * When attaching to a remote processor the device memory
2248		 * is already available and as such there is no need to have a
2249		 * cached table.
2250		 */
2251		if (rproc->cached_table)
2252			return -EINVAL;
2253		break;
2254	default:
2255		/*
2256		 * When adding a remote processor, the state of the device
2257		 * can be offline or detached, nothing else.
2258		 */
2259		return -EINVAL;
2260	}
2261
2262	return 0;
2263}
2264
2265/**
2266 * rproc_add() - register a remote processor
2267 * @rproc: the remote processor handle to register
2268 *
2269 * Registers @rproc with the remoteproc framework, after it has been
2270 * allocated with rproc_alloc().
2271 *
2272 * This is called by the platform-specific rproc implementation, whenever
2273 * a new remote processor device is probed.
2274 *
2275 * Note: this function initiates an asynchronous firmware loading
2276 * context, which will look for virtio devices supported by the rproc's
2277 * firmware.
2278 *
2279 * If found, those virtio devices will be created and added, so as a result
2280 * of registering this remote processor, additional virtio drivers might be
2281 * probed.
2282 *
2283 * Return: 0 on success and an appropriate error code otherwise
2284 */
2285int rproc_add(struct rproc *rproc)
2286{
2287	struct device *dev = &rproc->dev;
2288	int ret;
2289
2290	ret = rproc_validate(rproc);
2291	if (ret < 0)
2292		return ret;
2293
2294	/* add char device for this remoteproc */
2295	ret = rproc_char_device_add(rproc);
2296	if (ret < 0)
2297		return ret;
2298
2299	ret = device_add(dev);
2300	if (ret < 0) {
2301		put_device(dev);
2302		goto rproc_remove_cdev;
2303	}
2304
2305	dev_info(dev, "%s is available\n", rproc->name);
2306
2307	/* create debugfs entries */
2308	rproc_create_debug_dir(rproc);
2309
2310	/* if rproc is marked always-on, request it to boot */
2311	if (rproc->auto_boot) {
2312		ret = rproc_trigger_auto_boot(rproc);
2313		if (ret < 0)
2314			goto rproc_remove_dev;
2315	}
2316
2317	/* expose to rproc_get_by_phandle users */
2318	mutex_lock(&rproc_list_mutex);
2319	list_add_rcu(&rproc->node, &rproc_list);
2320	mutex_unlock(&rproc_list_mutex);
2321
2322	return 0;
2323
2324rproc_remove_dev:
2325	rproc_delete_debug_dir(rproc);
2326	device_del(dev);
2327rproc_remove_cdev:
2328	rproc_char_device_remove(rproc);
2329	return ret;
2330}
2331EXPORT_SYMBOL(rproc_add);
2332
2333static void devm_rproc_remove(void *rproc)
2334{
2335	rproc_del(rproc);
2336}
2337
2338/**
2339 * devm_rproc_add() - resource managed rproc_add()
2340 * @dev: the underlying device
2341 * @rproc: the remote processor handle to register
2342 *
2343 * This function performs like rproc_add() but the registered rproc device will
2344 * automatically be removed on driver detach.
2345 *
2346 * Return: 0 on success, negative errno on failure
2347 */
2348int devm_rproc_add(struct device *dev, struct rproc *rproc)
2349{
2350	int err;
2351
2352	err = rproc_add(rproc);
2353	if (err)
2354		return err;
2355
2356	return devm_add_action_or_reset(dev, devm_rproc_remove, rproc);
2357}
2358EXPORT_SYMBOL(devm_rproc_add);
2359
2360/**
2361 * rproc_type_release() - release a remote processor instance
2362 * @dev: the rproc's device
2363 *
2364 * This function should _never_ be called directly.
2365 *
2366 * It will be called by the driver core when no one holds a valid pointer
2367 * to @dev anymore.
2368 */
2369static void rproc_type_release(struct device *dev)
2370{
2371	struct rproc *rproc = container_of(dev, struct rproc, dev);
2372
2373	dev_info(&rproc->dev, "releasing %s\n", rproc->name);
2374
2375	idr_destroy(&rproc->notifyids);
2376
2377	if (rproc->index >= 0)
2378		ida_free(&rproc_dev_index, rproc->index);
2379
2380	kfree_const(rproc->firmware);
2381	kfree_const(rproc->name);
2382	kfree(rproc->ops);
2383	kfree(rproc);
2384}
2385
2386static const struct device_type rproc_type = {
2387	.name		= "remoteproc",
2388	.release	= rproc_type_release,
2389};
2390
2391static int rproc_alloc_firmware(struct rproc *rproc,
2392				const char *name, const char *firmware)
2393{
2394	const char *p;
2395
2396	/*
2397	 * Allocate a firmware name if the caller gave us one to work
2398	 * with.  Otherwise construct a new one using a default pattern.
2399	 */
2400	if (firmware)
2401		p = kstrdup_const(firmware, GFP_KERNEL);
2402	else
2403		p = kasprintf(GFP_KERNEL, "rproc-%s-fw", name);
2404
2405	if (!p)
2406		return -ENOMEM;
2407
2408	rproc->firmware = p;
2409
2410	return 0;
2411}
2412
2413static int rproc_alloc_ops(struct rproc *rproc, const struct rproc_ops *ops)
2414{
2415	rproc->ops = kmemdup(ops, sizeof(*ops), GFP_KERNEL);
2416	if (!rproc->ops)
2417		return -ENOMEM;
2418
2419	/* Default to rproc_coredump if no coredump function is specified */
2420	if (!rproc->ops->coredump)
2421		rproc->ops->coredump = rproc_coredump;
2422
2423	if (rproc->ops->load)
2424		return 0;
2425
2426	/* Default to ELF loader if no load function is specified */
2427	rproc->ops->load = rproc_elf_load_segments;
2428	rproc->ops->parse_fw = rproc_elf_load_rsc_table;
2429	rproc->ops->find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table;
2430	rproc->ops->sanity_check = rproc_elf_sanity_check;
2431	rproc->ops->get_boot_addr = rproc_elf_get_boot_addr;
2432
2433	return 0;
2434}
2435
2436/**
2437 * rproc_alloc() - allocate a remote processor handle
2438 * @dev: the underlying device
2439 * @name: name of this remote processor
2440 * @ops: platform-specific handlers (mainly start/stop)
2441 * @firmware: name of firmware file to load, can be NULL
2442 * @len: length of private data needed by the rproc driver (in bytes)
2443 *
2444 * Allocates a new remote processor handle, but does not register
2445 * it yet. if @firmware is NULL, a default name is used.
2446 *
2447 * This function should be used by rproc implementations during initialization
2448 * of the remote processor.
2449 *
2450 * After creating an rproc handle using this function, and when ready,
2451 * implementations should then call rproc_add() to complete
2452 * the registration of the remote processor.
2453 *
2454 * Note: _never_ directly deallocate @rproc, even if it was not registered
2455 * yet. Instead, when you need to unroll rproc_alloc(), use rproc_free().
2456 *
2457 * Return: new rproc pointer on success, and NULL on failure
2458 */
2459struct rproc *rproc_alloc(struct device *dev, const char *name,
2460			  const struct rproc_ops *ops,
2461			  const char *firmware, int len)
2462{
2463	struct rproc *rproc;
2464
2465	if (!dev || !name || !ops)
2466		return NULL;
2467
2468	rproc = kzalloc(sizeof(struct rproc) + len, GFP_KERNEL);
2469	if (!rproc)
2470		return NULL;
2471
2472	rproc->priv = &rproc[1];
2473	rproc->auto_boot = true;
2474	rproc->elf_class = ELFCLASSNONE;
2475	rproc->elf_machine = EM_NONE;
2476
2477	device_initialize(&rproc->dev);
2478	rproc->dev.parent = dev;
2479	rproc->dev.type = &rproc_type;
2480	rproc->dev.class = &rproc_class;
2481	rproc->dev.driver_data = rproc;
2482	idr_init(&rproc->notifyids);
2483
2484	/* Assign a unique device index and name */
2485	rproc->index = ida_alloc(&rproc_dev_index, GFP_KERNEL);
2486	if (rproc->index < 0) {
2487		dev_err(dev, "ida_alloc failed: %d\n", rproc->index);
2488		goto put_device;
2489	}
2490
2491	rproc->name = kstrdup_const(name, GFP_KERNEL);
2492	if (!rproc->name)
2493		goto put_device;
2494
2495	if (rproc_alloc_firmware(rproc, name, firmware))
2496		goto put_device;
2497
2498	if (rproc_alloc_ops(rproc, ops))
2499		goto put_device;
2500
2501	dev_set_name(&rproc->dev, "remoteproc%d", rproc->index);
2502
2503	atomic_set(&rproc->power, 0);
2504
2505	mutex_init(&rproc->lock);
2506
2507	INIT_LIST_HEAD(&rproc->carveouts);
2508	INIT_LIST_HEAD(&rproc->mappings);
2509	INIT_LIST_HEAD(&rproc->traces);
2510	INIT_LIST_HEAD(&rproc->rvdevs);
2511	INIT_LIST_HEAD(&rproc->subdevs);
2512	INIT_LIST_HEAD(&rproc->dump_segments);
2513
2514	INIT_WORK(&rproc->crash_handler, rproc_crash_handler_work);
2515
2516	rproc->state = RPROC_OFFLINE;
2517
2518	return rproc;
2519
2520put_device:
2521	put_device(&rproc->dev);
2522	return NULL;
2523}
2524EXPORT_SYMBOL(rproc_alloc);
2525
2526/**
2527 * rproc_free() - unroll rproc_alloc()
2528 * @rproc: the remote processor handle
2529 *
2530 * This function decrements the rproc dev refcount.
2531 *
2532 * If no one holds any reference to rproc anymore, then its refcount would
2533 * now drop to zero, and it would be freed.
2534 */
2535void rproc_free(struct rproc *rproc)
2536{
2537	put_device(&rproc->dev);
2538}
2539EXPORT_SYMBOL(rproc_free);
2540
2541/**
2542 * rproc_put() - release rproc reference
2543 * @rproc: the remote processor handle
2544 *
2545 * This function decrements the rproc dev refcount.
2546 *
2547 * If no one holds any reference to rproc anymore, then its refcount would
2548 * now drop to zero, and it would be freed.
2549 */
2550void rproc_put(struct rproc *rproc)
2551{
2552	if (rproc->dev.parent->driver)
2553		module_put(rproc->dev.parent->driver->owner);
2554	else
2555		module_put(rproc->dev.parent->parent->driver->owner);
2556
2557	put_device(&rproc->dev);
2558}
2559EXPORT_SYMBOL(rproc_put);
2560
2561/**
2562 * rproc_del() - unregister a remote processor
2563 * @rproc: rproc handle to unregister
2564 *
2565 * This function should be called when the platform specific rproc
2566 * implementation decides to remove the rproc device. it should
2567 * _only_ be called if a previous invocation of rproc_add()
2568 * has completed successfully.
2569 *
2570 * After rproc_del() returns, @rproc isn't freed yet, because
2571 * of the outstanding reference created by rproc_alloc. To decrement that
2572 * one last refcount, one still needs to call rproc_free().
2573 *
2574 * Return: 0 on success and -EINVAL if @rproc isn't valid
2575 */
2576int rproc_del(struct rproc *rproc)
2577{
2578	if (!rproc)
2579		return -EINVAL;
2580
2581	/* TODO: make sure this works with rproc->power > 1 */
2582	rproc_shutdown(rproc);
2583
2584	mutex_lock(&rproc->lock);
2585	rproc->state = RPROC_DELETED;
2586	mutex_unlock(&rproc->lock);
2587
2588	rproc_delete_debug_dir(rproc);
2589
2590	/* the rproc is downref'ed as soon as it's removed from the klist */
2591	mutex_lock(&rproc_list_mutex);
2592	list_del_rcu(&rproc->node);
2593	mutex_unlock(&rproc_list_mutex);
2594
2595	/* Ensure that no readers of rproc_list are still active */
2596	synchronize_rcu();
2597
2598	device_del(&rproc->dev);
2599	rproc_char_device_remove(rproc);
2600
2601	return 0;
2602}
2603EXPORT_SYMBOL(rproc_del);
2604
2605static void devm_rproc_free(struct device *dev, void *res)
2606{
2607	rproc_free(*(struct rproc **)res);
2608}
2609
2610/**
2611 * devm_rproc_alloc() - resource managed rproc_alloc()
2612 * @dev: the underlying device
2613 * @name: name of this remote processor
2614 * @ops: platform-specific handlers (mainly start/stop)
2615 * @firmware: name of firmware file to load, can be NULL
2616 * @len: length of private data needed by the rproc driver (in bytes)
2617 *
2618 * This function performs like rproc_alloc() but the acquired rproc device will
2619 * automatically be released on driver detach.
2620 *
2621 * Return: new rproc instance, or NULL on failure
2622 */
2623struct rproc *devm_rproc_alloc(struct device *dev, const char *name,
2624			       const struct rproc_ops *ops,
2625			       const char *firmware, int len)
2626{
2627	struct rproc **ptr, *rproc;
2628
2629	ptr = devres_alloc(devm_rproc_free, sizeof(*ptr), GFP_KERNEL);
2630	if (!ptr)
2631		return NULL;
2632
2633	rproc = rproc_alloc(dev, name, ops, firmware, len);
2634	if (rproc) {
2635		*ptr = rproc;
2636		devres_add(dev, ptr);
2637	} else {
2638		devres_free(ptr);
2639	}
2640
2641	return rproc;
2642}
2643EXPORT_SYMBOL(devm_rproc_alloc);
2644
2645/**
2646 * rproc_add_subdev() - add a subdevice to a remoteproc
2647 * @rproc: rproc handle to add the subdevice to
2648 * @subdev: subdev handle to register
2649 *
2650 * Caller is responsible for populating optional subdevice function pointers.
2651 */
2652void rproc_add_subdev(struct rproc *rproc, struct rproc_subdev *subdev)
2653{
2654	list_add_tail(&subdev->node, &rproc->subdevs);
2655}
2656EXPORT_SYMBOL(rproc_add_subdev);
2657
2658/**
2659 * rproc_remove_subdev() - remove a subdevice from a remoteproc
2660 * @rproc: rproc handle to remove the subdevice from
2661 * @subdev: subdev handle, previously registered with rproc_add_subdev()
2662 */
2663void rproc_remove_subdev(struct rproc *rproc, struct rproc_subdev *subdev)
2664{
2665	list_del(&subdev->node);
2666}
2667EXPORT_SYMBOL(rproc_remove_subdev);
2668
2669/**
2670 * rproc_get_by_child() - acquire rproc handle of @dev's ancestor
2671 * @dev:	child device to find ancestor of
2672 *
2673 * Return: the ancestor rproc instance, or NULL if not found
2674 */
2675struct rproc *rproc_get_by_child(struct device *dev)
2676{
2677	for (dev = dev->parent; dev; dev = dev->parent) {
2678		if (dev->type == &rproc_type)
2679			return dev->driver_data;
2680	}
2681
2682	return NULL;
2683}
2684EXPORT_SYMBOL(rproc_get_by_child);
2685
2686/**
2687 * rproc_report_crash() - rproc crash reporter function
2688 * @rproc: remote processor
2689 * @type: crash type
2690 *
2691 * This function must be called every time a crash is detected by the low-level
2692 * drivers implementing a specific remoteproc. This should not be called from a
2693 * non-remoteproc driver.
2694 *
2695 * This function can be called from atomic/interrupt context.
2696 */
2697void rproc_report_crash(struct rproc *rproc, enum rproc_crash_type type)
2698{
2699	if (!rproc) {
2700		pr_err("NULL rproc pointer\n");
2701		return;
2702	}
2703
2704	/* Prevent suspend while the remoteproc is being recovered */
2705	pm_stay_awake(rproc->dev.parent);
2706
2707	dev_err(&rproc->dev, "crash detected in %s: type %s\n",
2708		rproc->name, rproc_crash_to_string(type));
2709
2710	queue_work(rproc_recovery_wq, &rproc->crash_handler);
2711}
2712EXPORT_SYMBOL(rproc_report_crash);
2713
2714static int rproc_panic_handler(struct notifier_block *nb, unsigned long event,
2715			       void *ptr)
2716{
2717	unsigned int longest = 0;
2718	struct rproc *rproc;
2719	unsigned int d;
2720
2721	rcu_read_lock();
2722	list_for_each_entry_rcu(rproc, &rproc_list, node) {
2723		if (!rproc->ops->panic)
2724			continue;
2725
2726		if (rproc->state != RPROC_RUNNING &&
2727		    rproc->state != RPROC_ATTACHED)
2728			continue;
2729
2730		d = rproc->ops->panic(rproc);
2731		longest = max(longest, d);
2732	}
2733	rcu_read_unlock();
2734
2735	/*
2736	 * Delay for the longest requested duration before returning. This can
2737	 * be used by the remoteproc drivers to give the remote processor time
2738	 * to perform any requested operations (such as flush caches), when
2739	 * it's not possible to signal the Linux side due to the panic.
2740	 */
2741	mdelay(longest);
2742
2743	return NOTIFY_DONE;
2744}
2745
2746static void __init rproc_init_panic(void)
2747{
2748	rproc_panic_nb.notifier_call = rproc_panic_handler;
2749	atomic_notifier_chain_register(&panic_notifier_list, &rproc_panic_nb);
2750}
2751
2752static void __exit rproc_exit_panic(void)
2753{
2754	atomic_notifier_chain_unregister(&panic_notifier_list, &rproc_panic_nb);
2755}
2756
2757static int __init remoteproc_init(void)
2758{
2759	rproc_recovery_wq = alloc_workqueue("rproc_recovery_wq",
2760						WQ_UNBOUND | WQ_FREEZABLE, 0);
2761	if (!rproc_recovery_wq) {
2762		pr_err("remoteproc: creation of rproc_recovery_wq failed\n");
2763		return -ENOMEM;
2764	}
2765
2766	rproc_init_sysfs();
2767	rproc_init_debugfs();
2768	rproc_init_cdev();
2769	rproc_init_panic();
2770
2771	return 0;
2772}
2773subsys_initcall(remoteproc_init);
2774
2775static void __exit remoteproc_exit(void)
2776{
2777	ida_destroy(&rproc_dev_index);
2778
2779	if (!rproc_recovery_wq)
2780		return;
2781
2782	rproc_exit_panic();
2783	rproc_exit_debugfs();
2784	rproc_exit_sysfs();
2785	destroy_workqueue(rproc_recovery_wq);
2786}
2787module_exit(remoteproc_exit);
2788
2789MODULE_DESCRIPTION("Generic Remote Processor Framework");