coresight-tmc-etr.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright(C) 2016 Linaro Limited. All rights reserved.
   4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   5 */
   6
   7#include <linux/atomic.h>
   8#include <linux/coresight.h>
   9#include <linux/dma-mapping.h>
  10#include <linux/iommu.h>
  11#include <linux/idr.h>
  12#include <linux/mutex.h>
  13#include <linux/refcount.h>
  14#include <linux/slab.h>
  15#include <linux/types.h>
  16#include <linux/vmalloc.h>
  17#include "coresight-catu.h"
  18#include "coresight-etm-perf.h"
  19#include "coresight-priv.h"
  20#include "coresight-tmc.h"
  21
  22struct etr_flat_buf {
  23	struct device	*dev;
  24	dma_addr_t	daddr;
  25	void		*vaddr;
  26	size_t		size;
  27};
  28
  29struct etr_buf_hw {
  30	bool	has_iommu;
  31	bool	has_etr_sg;
  32	bool	has_catu;
  33	bool	has_resrv;
  34};
  35
  36/*
  37 * etr_perf_buffer - Perf buffer used for ETR
  38 * @drvdata		- The ETR drvdaga this buffer has been allocated for.
  39 * @etr_buf		- Actual buffer used by the ETR
  40 * @pid			- The PID of the session owner that etr_perf_buffer
  41 *			  belongs to.
  42 * @snaphost		- Perf session mode
  43 * @nr_pages		- Number of pages in the ring buffer.
  44 * @pages		- Array of Pages in the ring buffer.
  45 */
  46struct etr_perf_buffer {
  47	struct tmc_drvdata	*drvdata;
  48	struct etr_buf		*etr_buf;
  49	pid_t			pid;
  50	bool			snapshot;
  51	int			nr_pages;
  52	void			**pages;
  53};
  54
  55/* Convert the perf index to an offset within the ETR buffer */
  56#define PERF_IDX2OFF(idx, buf)		\
  57		((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT))
  58
  59/* Lower limit for ETR hardware buffer */
  60#define TMC_ETR_PERF_MIN_BUF_SIZE	SZ_1M
  61
  62/*
  63 * The TMC ETR SG has a page size of 4K. The SG table contains pointers
  64 * to 4KB buffers. However, the OS may use a PAGE_SIZE different from
  65 * 4K (i.e, 16KB or 64KB). This implies that a single OS page could
  66 * contain more than one SG buffer and tables.
  67 *
  68 * A table entry has the following format:
  69 *
  70 * ---Bit31------------Bit4-------Bit1-----Bit0--
  71 * |     Address[39:12]    | SBZ |  Entry Type  |
  72 * ----------------------------------------------
  73 *
  74 * Address: Bits [39:12] of a physical page address. Bits [11:0] are
  75 *	    always zero.
  76 *
  77 * Entry type:
  78 *	b00 - Reserved.
  79 *	b01 - Last entry in the tables, points to 4K page buffer.
  80 *	b10 - Normal entry, points to 4K page buffer.
  81 *	b11 - Link. The address points to the base of next table.
  82 */
  83
  84typedef u32 sgte_t;
  85
  86#define ETR_SG_PAGE_SHIFT		12
  87#define ETR_SG_PAGE_SIZE		(1UL << ETR_SG_PAGE_SHIFT)
  88#define ETR_SG_PAGES_PER_SYSPAGE	(PAGE_SIZE / ETR_SG_PAGE_SIZE)
  89#define ETR_SG_PTRS_PER_PAGE		(ETR_SG_PAGE_SIZE / sizeof(sgte_t))
  90#define ETR_SG_PTRS_PER_SYSPAGE		(PAGE_SIZE / sizeof(sgte_t))
  91
  92#define ETR_SG_ET_MASK			0x3
  93#define ETR_SG_ET_LAST			0x1
  94#define ETR_SG_ET_NORMAL		0x2
  95#define ETR_SG_ET_LINK			0x3
  96
  97#define ETR_SG_ADDR_SHIFT		4
  98
  99#define ETR_SG_ENTRY(addr, type) \
 100	(sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
 101		 (type & ETR_SG_ET_MASK))
 102
 103#define ETR_SG_ADDR(entry) \
 104	(((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
 105#define ETR_SG_ET(entry)		((entry) & ETR_SG_ET_MASK)
 106
 107/*
 108 * struct etr_sg_table : ETR SG Table
 109 * @sg_table:		Generic SG Table holding the data/table pages.
 110 * @hwaddr:		hwaddress used by the TMC, which is the base
 111 *			address of the table.
 112 */
 113struct etr_sg_table {
 114	struct tmc_sg_table	*sg_table;
 115	dma_addr_t		hwaddr;
 116};
 117
 118/*
 119 * tmc_etr_sg_table_entries: Total number of table entries required to map
 120 * @nr_pages system pages.
 121 *
 122 * We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages.
 123 * Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers,
 124 * with the last entry pointing to another page of table entries.
 125 * If we spill over to a new page for mapping 1 entry, we could as
 126 * well replace the link entry of the previous page with the last entry.
 127 */
 128static unsigned long __attribute_const__
 129tmc_etr_sg_table_entries(int nr_pages)
 130{
 131	unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
 132	unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
 133	/*
 134	 * If we spill over to a new page for 1 entry, we could as well
 135	 * make it the LAST entry in the previous page, skipping the Link
 136	 * address.
 137	 */
 138	if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
 139		nr_sglinks--;
 140	return nr_sgpages + nr_sglinks;
 141}
 142
 143/*
 144 * tmc_pages_get_offset:  Go through all the pages in the tmc_pages
 145 * and map the device address @addr to an offset within the virtual
 146 * contiguous buffer.
 147 */
 148static long
 149tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
 150{
 151	int i;
 152	dma_addr_t page_start;
 153
 154	for (i = 0; i < tmc_pages->nr_pages; i++) {
 155		page_start = tmc_pages->daddrs[i];
 156		if (addr >= page_start && addr < (page_start + PAGE_SIZE))
 157			return i * PAGE_SIZE + (addr - page_start);
 158	}
 159
 160	return -EINVAL;
 161}
 162
 163/*
 164 * tmc_pages_free : Unmap and free the pages used by tmc_pages.
 165 * If the pages were not allocated in tmc_pages_alloc(), we would
 166 * simply drop the refcount.
 167 */
 168static void tmc_pages_free(struct tmc_pages *tmc_pages,
 169			   struct device *dev, enum dma_data_direction dir)
 170{
 171	int i;
 172	struct device *real_dev = dev->parent;
 173
 174	for (i = 0; i < tmc_pages->nr_pages; i++) {
 175		if (tmc_pages->daddrs && tmc_pages->daddrs[i])
 176			dma_unmap_page(real_dev, tmc_pages->daddrs[i],
 177					 PAGE_SIZE, dir);
 178		if (tmc_pages->pages && tmc_pages->pages[i])
 179			__free_page(tmc_pages->pages[i]);
 180	}
 181
 182	kfree(tmc_pages->pages);
 183	kfree(tmc_pages->daddrs);
 184	tmc_pages->pages = NULL;
 185	tmc_pages->daddrs = NULL;
 186	tmc_pages->nr_pages = 0;
 187}
 188
 189/*
 190 * tmc_pages_alloc : Allocate and map pages for a given @tmc_pages.
 191 * If @pages is not NULL, the list of page virtual addresses are
 192 * used as the data pages. The pages are then dma_map'ed for @dev
 193 * with dma_direction @dir.
 194 *
 195 * Returns 0 upon success, else the error number.
 196 */
 197static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
 198			   struct device *dev, int node,
 199			   enum dma_data_direction dir, void **pages)
 200{
 201	int i, nr_pages;
 202	dma_addr_t paddr;
 203	struct page *page;
 204	struct device *real_dev = dev->parent;
 205
 206	nr_pages = tmc_pages->nr_pages;
 207	tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs),
 208					 GFP_KERNEL);
 209	if (!tmc_pages->daddrs)
 210		return -ENOMEM;
 211	tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages),
 212					 GFP_KERNEL);
 213	if (!tmc_pages->pages) {
 214		kfree(tmc_pages->daddrs);
 215		tmc_pages->daddrs = NULL;
 216		return -ENOMEM;
 217	}
 218
 219	for (i = 0; i < nr_pages; i++) {
 220		if (pages && pages[i]) {
 221			page = virt_to_page(pages[i]);
 222			/* Hold a refcount on the page */
 223			get_page(page);
 224		} else {
 225			page = alloc_pages_node(node,
 226						GFP_KERNEL | __GFP_ZERO, 0);
 227			if (!page)
 228				goto err;
 229		}
 230		paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir);
 231		if (dma_mapping_error(real_dev, paddr))
 232			goto err;
 233		tmc_pages->daddrs[i] = paddr;
 234		tmc_pages->pages[i] = page;
 235	}
 236	return 0;
 237err:
 238	tmc_pages_free(tmc_pages, dev, dir);
 239	return -ENOMEM;
 240}
 241
 242static long
 243tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
 244{
 245	return tmc_pages_get_offset(&sg_table->data_pages, addr);
 246}
 247
 248static void tmc_free_table_pages(struct tmc_sg_table *sg_table)
 249{
 250	if (sg_table->table_vaddr)
 251		vunmap(sg_table->table_vaddr);
 252	tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE);
 253}
 254
 255static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
 256{
 257	if (sg_table->data_vaddr)
 258		vunmap(sg_table->data_vaddr);
 259	tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE);
 260}
 261
 262void tmc_free_sg_table(struct tmc_sg_table *sg_table)
 263{
 264	tmc_free_table_pages(sg_table);
 265	tmc_free_data_pages(sg_table);
 266	kfree(sg_table);
 267}
 268EXPORT_SYMBOL_GPL(tmc_free_sg_table);
 269
 270/*
 271 * Alloc pages for the table. Since this will be used by the device,
 272 * allocate the pages closer to the device (i.e, dev_to_node(dev)
 273 * rather than the CPU node).
 274 */
 275static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
 276{
 277	int rc;
 278	struct tmc_pages *table_pages = &sg_table->table_pages;
 279
 280	rc = tmc_pages_alloc(table_pages, sg_table->dev,
 281			     dev_to_node(sg_table->dev),
 282			     DMA_TO_DEVICE, NULL);
 283	if (rc)
 284		return rc;
 285	sg_table->table_vaddr = vmap(table_pages->pages,
 286				     table_pages->nr_pages,
 287				     VM_MAP,
 288				     PAGE_KERNEL);
 289	if (!sg_table->table_vaddr)
 290		rc = -ENOMEM;
 291	else
 292		sg_table->table_daddr = table_pages->daddrs[0];
 293	return rc;
 294}
 295
 296static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
 297{
 298	int rc;
 299
 300	/* Allocate data pages on the node requested by the caller */
 301	rc = tmc_pages_alloc(&sg_table->data_pages,
 302			     sg_table->dev, sg_table->node,
 303			     DMA_FROM_DEVICE, pages);
 304	if (!rc) {
 305		sg_table->data_vaddr = vmap(sg_table->data_pages.pages,
 306					    sg_table->data_pages.nr_pages,
 307					    VM_MAP,
 308					    PAGE_KERNEL);
 309		if (!sg_table->data_vaddr)
 310			rc = -ENOMEM;
 311	}
 312	return rc;
 313}
 314
 315/*
 316 * tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table
 317 * and data buffers. TMC writes to the data buffers and reads from the SG
 318 * Table pages.
 319 *
 320 * @dev		- Coresight device to which page should be DMA mapped.
 321 * @node	- Numa node for mem allocations
 322 * @nr_tpages	- Number of pages for the table entries.
 323 * @nr_dpages	- Number of pages for Data buffer.
 324 * @pages	- Optional list of virtual address of pages.
 325 */
 326struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
 327					int node,
 328					int nr_tpages,
 329					int nr_dpages,
 330					void **pages)
 331{
 332	long rc;
 333	struct tmc_sg_table *sg_table;
 334
 335	sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL);
 336	if (!sg_table)
 337		return ERR_PTR(-ENOMEM);
 338	sg_table->data_pages.nr_pages = nr_dpages;
 339	sg_table->table_pages.nr_pages = nr_tpages;
 340	sg_table->node = node;
 341	sg_table->dev = dev;
 342
 343	rc  = tmc_alloc_data_pages(sg_table, pages);
 344	if (!rc)
 345		rc = tmc_alloc_table_pages(sg_table);
 346	if (rc) {
 347		tmc_free_sg_table(sg_table);
 348		return ERR_PTR(rc);
 349	}
 350
 351	return sg_table;
 352}
 353EXPORT_SYMBOL_GPL(tmc_alloc_sg_table);
 354
 355/*
 356 * tmc_sg_table_sync_data_range: Sync the data buffer written
 357 * by the device from @offset upto a @size bytes.
 358 */
 359void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
 360				  u64 offset, u64 size)
 361{
 362	int i, index, start;
 363	int npages = DIV_ROUND_UP(size, PAGE_SIZE);
 364	struct device *real_dev = table->dev->parent;
 365	struct tmc_pages *data = &table->data_pages;
 366
 367	start = offset >> PAGE_SHIFT;
 368	for (i = start; i < (start + npages); i++) {
 369		index = i % data->nr_pages;
 370		dma_sync_single_for_cpu(real_dev, data->daddrs[index],
 371					PAGE_SIZE, DMA_FROM_DEVICE);
 372	}
 373}
 374EXPORT_SYMBOL_GPL(tmc_sg_table_sync_data_range);
 375
 376/* tmc_sg_sync_table: Sync the page table */
 377void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
 378{
 379	int i;
 380	struct device *real_dev = sg_table->dev->parent;
 381	struct tmc_pages *table_pages = &sg_table->table_pages;
 382
 383	for (i = 0; i < table_pages->nr_pages; i++)
 384		dma_sync_single_for_device(real_dev, table_pages->daddrs[i],
 385					   PAGE_SIZE, DMA_TO_DEVICE);
 386}
 387EXPORT_SYMBOL_GPL(tmc_sg_table_sync_table);
 388
 389/*
 390 * tmc_sg_table_get_data: Get the buffer pointer for data @offset
 391 * in the SG buffer. The @bufpp is updated to point to the buffer.
 392 * Returns :
 393 *	the length of linear data available at @offset.
 394 *	or
 395 *	<= 0 if no data is available.
 396 */
 397ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
 398			      u64 offset, size_t len, char **bufpp)
 399{
 400	size_t size;
 401	int pg_idx = offset >> PAGE_SHIFT;
 402	int pg_offset = offset & (PAGE_SIZE - 1);
 403	struct tmc_pages *data_pages = &sg_table->data_pages;
 404
 405	size = tmc_sg_table_buf_size(sg_table);
 406	if (offset >= size)
 407		return -EINVAL;
 408
 409	/* Make sure we don't go beyond the end */
 410	len = (len < (size - offset)) ? len : size - offset;
 411	/* Respect the page boundaries */
 412	len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
 413	if (len > 0)
 414		*bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
 415	return len;
 416}
 417EXPORT_SYMBOL_GPL(tmc_sg_table_get_data);
 418
 419#ifdef ETR_SG_DEBUG
 420/* Map a dma address to virtual address */
 421static unsigned long
 422tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
 423		      dma_addr_t addr, bool table)
 424{
 425	long offset;
 426	unsigned long base;
 427	struct tmc_pages *tmc_pages;
 428
 429	if (table) {
 430		tmc_pages = &sg_table->table_pages;
 431		base = (unsigned long)sg_table->table_vaddr;
 432	} else {
 433		tmc_pages = &sg_table->data_pages;
 434		base = (unsigned long)sg_table->data_vaddr;
 435	}
 436
 437	offset = tmc_pages_get_offset(tmc_pages, addr);
 438	if (offset < 0)
 439		return 0;
 440	return base + offset;
 441}
 442
 443/* Dump the given sg_table */
 444static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
 445{
 446	sgte_t *ptr;
 447	int i = 0;
 448	dma_addr_t addr;
 449	struct tmc_sg_table *sg_table = etr_table->sg_table;
 450
 451	ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
 452					      etr_table->hwaddr, true);
 453	while (ptr) {
 454		addr = ETR_SG_ADDR(*ptr);
 455		switch (ETR_SG_ET(*ptr)) {
 456		case ETR_SG_ET_NORMAL:
 457			dev_dbg(sg_table->dev,
 458				"%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
 459			ptr++;
 460			break;
 461		case ETR_SG_ET_LINK:
 462			dev_dbg(sg_table->dev,
 463				"%05d: *** %p\t:{L} 0x%llx ***\n",
 464				 i, ptr, addr);
 465			ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
 466							      addr, true);
 467			break;
 468		case ETR_SG_ET_LAST:
 469			dev_dbg(sg_table->dev,
 470				"%05d: ### %p\t:[L] 0x%llx ###\n",
 471				 i, ptr, addr);
 472			return;
 473		default:
 474			dev_dbg(sg_table->dev,
 475				"%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
 476				 i, ptr, addr);
 477			return;
 478		}
 479		i++;
 480	}
 481	dev_dbg(sg_table->dev, "******* End of Table *****\n");
 482}
 483#else
 484static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
 485#endif
 486
 487/*
 488 * Populate the SG Table page table entries from table/data
 489 * pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages.
 490 * So does a Table page. So we keep track of indices of the tables
 491 * in each system page and move the pointers accordingly.
 492 */
 493#define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
 494static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
 495{
 496	dma_addr_t paddr;
 497	int i, type, nr_entries;
 498	int tpidx = 0; /* index to the current system table_page */
 499	int sgtidx = 0;	/* index to the sg_table within the current syspage */
 500	int sgtentry = 0; /* the entry within the sg_table */
 501	int dpidx = 0; /* index to the current system data_page */
 502	int spidx = 0; /* index to the SG page within the current data page */
 503	sgte_t *ptr; /* pointer to the table entry to fill */
 504	struct tmc_sg_table *sg_table = etr_table->sg_table;
 505	dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
 506	dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
 507
 508	nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages);
 509	/*
 510	 * Use the contiguous virtual address of the table to update entries.
 511	 */
 512	ptr = sg_table->table_vaddr;
 513	/*
 514	 * Fill all the entries, except the last entry to avoid special
 515	 * checks within the loop.
 516	 */
 517	for (i = 0; i < nr_entries - 1; i++) {
 518		if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
 519			/*
 520			 * Last entry in a sg_table page is a link address to
 521			 * the next table page. If this sg_table is the last
 522			 * one in the system page, it links to the first
 523			 * sg_table in the next system page. Otherwise, it
 524			 * links to the next sg_table page within the system
 525			 * page.
 526			 */
 527			if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
 528				paddr = table_daddrs[tpidx + 1];
 529			} else {
 530				paddr = table_daddrs[tpidx] +
 531					(ETR_SG_PAGE_SIZE * (sgtidx + 1));
 532			}
 533			type = ETR_SG_ET_LINK;
 534		} else {
 535			/*
 536			 * Update the indices to the data_pages to point to the
 537			 * next sg_page in the data buffer.
 538			 */
 539			type = ETR_SG_ET_NORMAL;
 540			paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
 541			if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
 542				dpidx++;
 543		}
 544		*ptr++ = ETR_SG_ENTRY(paddr, type);
 545		/*
 546		 * Move to the next table pointer, moving the table page index
 547		 * if necessary
 548		 */
 549		if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
 550			if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
 551				tpidx++;
 552		}
 553	}
 554
 555	/* Set up the last entry, which is always a data pointer */
 556	paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
 557	*ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
 558}
 559
 560/*
 561 * tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and
 562 * populate the table.
 563 *
 564 * @dev		- Device pointer for the TMC
 565 * @node	- NUMA node where the memory should be allocated
 566 * @size	- Total size of the data buffer
 567 * @pages	- Optional list of page virtual address
 568 */
 569static struct etr_sg_table *
 570tmc_init_etr_sg_table(struct device *dev, int node,
 571		      unsigned long size, void **pages)
 572{
 573	int nr_entries, nr_tpages;
 574	int nr_dpages = size >> PAGE_SHIFT;
 575	struct tmc_sg_table *sg_table;
 576	struct etr_sg_table *etr_table;
 577
 578	etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL);
 579	if (!etr_table)
 580		return ERR_PTR(-ENOMEM);
 581	nr_entries = tmc_etr_sg_table_entries(nr_dpages);
 582	nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
 583
 584	sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
 585	if (IS_ERR(sg_table)) {
 586		kfree(etr_table);
 587		return ERR_CAST(sg_table);
 588	}
 589
 590	etr_table->sg_table = sg_table;
 591	/* TMC should use table base address for DBA */
 592	etr_table->hwaddr = sg_table->table_daddr;
 593	tmc_etr_sg_table_populate(etr_table);
 594	/* Sync the table pages for the HW */
 595	tmc_sg_table_sync_table(sg_table);
 596	tmc_etr_sg_table_dump(etr_table);
 597
 598	return etr_table;
 599}
 600
 601/*
 602 * tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer.
 603 */
 604static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
 605				  struct etr_buf *etr_buf, int node,
 606				  void **pages)
 607{
 608	struct etr_flat_buf *flat_buf;
 609	struct device *real_dev = drvdata->csdev->dev.parent;
 610
 611	/* We cannot reuse existing pages for flat buf */
 612	if (pages)
 613		return -EINVAL;
 614
 615	flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL);
 616	if (!flat_buf)
 617		return -ENOMEM;
 618
 619	flat_buf->vaddr = dma_alloc_noncoherent(real_dev, etr_buf->size,
 620						&flat_buf->daddr,
 621						DMA_FROM_DEVICE,
 622						GFP_KERNEL | __GFP_NOWARN);
 623	if (!flat_buf->vaddr) {
 624		kfree(flat_buf);
 625		return -ENOMEM;
 626	}
 627
 628	flat_buf->size = etr_buf->size;
 629	flat_buf->dev = &drvdata->csdev->dev;
 630	etr_buf->hwaddr = flat_buf->daddr;
 631	etr_buf->mode = ETR_MODE_FLAT;
 632	etr_buf->private = flat_buf;
 633	return 0;
 634}
 635
 636static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
 637{
 638	struct etr_flat_buf *flat_buf = etr_buf->private;
 639
 640	if (flat_buf && flat_buf->daddr) {
 641		struct device *real_dev = flat_buf->dev->parent;
 642
 643		dma_free_noncoherent(real_dev, etr_buf->size,
 644				     flat_buf->vaddr, flat_buf->daddr,
 645				     DMA_FROM_DEVICE);
 646	}
 647	kfree(flat_buf);
 648}
 649
 650static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
 651{
 652	struct etr_flat_buf *flat_buf = etr_buf->private;
 653	struct device *real_dev = flat_buf->dev->parent;
 654
 655	/*
 656	 * Adjust the buffer to point to the beginning of the trace data
 657	 * and update the available trace data.
 658	 */
 659	etr_buf->offset = rrp - etr_buf->hwaddr;
 660	if (etr_buf->full)
 661		etr_buf->len = etr_buf->size;
 662	else
 663		etr_buf->len = rwp - rrp;
 664
 665	/*
 666	 * The driver always starts tracing at the beginning of the buffer,
 667	 * the only reason why we would get a wrap around is when the buffer
 668	 * is full.  Sync the entire buffer in one go for this case.
 669	 */
 670	if (etr_buf->offset + etr_buf->len > etr_buf->size)
 671		dma_sync_single_for_cpu(real_dev, flat_buf->daddr,
 672					etr_buf->size, DMA_FROM_DEVICE);
 673	else
 674		dma_sync_single_for_cpu(real_dev,
 675					flat_buf->daddr + etr_buf->offset,
 676					etr_buf->len, DMA_FROM_DEVICE);
 677}
 678
 679static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
 680					 u64 offset, size_t len, char **bufpp)
 681{
 682	struct etr_flat_buf *flat_buf = etr_buf->private;
 683
 684	*bufpp = (char *)flat_buf->vaddr + offset;
 685	/*
 686	 * tmc_etr_buf_get_data already adjusts the length to handle
 687	 * buffer wrapping around.
 688	 */
 689	return len;
 690}
 691
 692static const struct etr_buf_operations etr_flat_buf_ops = {
 693	.alloc = tmc_etr_alloc_flat_buf,
 694	.free = tmc_etr_free_flat_buf,
 695	.sync = tmc_etr_sync_flat_buf,
 696	.get_data = tmc_etr_get_data_flat_buf,
 697};
 698
 699/*
 700 * tmc_etr_alloc_resrv_buf: Allocate a contiguous DMA buffer from reserved region.
 701 */
 702static int tmc_etr_alloc_resrv_buf(struct tmc_drvdata *drvdata,
 703				  struct etr_buf *etr_buf, int node,
 704				  void **pages)
 705{
 706	struct etr_flat_buf *resrv_buf;
 707	struct device *real_dev = drvdata->csdev->dev.parent;
 708
 709	/* We cannot reuse existing pages for resrv buf */
 710	if (pages)
 711		return -EINVAL;
 712
 713	resrv_buf = kzalloc(sizeof(*resrv_buf), GFP_KERNEL);
 714	if (!resrv_buf)
 715		return -ENOMEM;
 716
 717	resrv_buf->daddr = dma_map_resource(real_dev, drvdata->resrv_buf.paddr,
 718					   drvdata->resrv_buf.size,
 719					   DMA_FROM_DEVICE, 0);
 720	if (dma_mapping_error(real_dev, resrv_buf->daddr)) {
 721		dev_err(real_dev, "failed to map source buffer address\n");
 722		kfree(resrv_buf);
 723		return -ENOMEM;
 724	}
 725
 726	resrv_buf->vaddr = drvdata->resrv_buf.vaddr;
 727	resrv_buf->size = etr_buf->size = drvdata->resrv_buf.size;
 728	resrv_buf->dev = &drvdata->csdev->dev;
 729	etr_buf->hwaddr = resrv_buf->daddr;
 730	etr_buf->mode = ETR_MODE_RESRV;
 731	etr_buf->private = resrv_buf;
 732	return 0;
 733}
 734
 735static void tmc_etr_free_resrv_buf(struct etr_buf *etr_buf)
 736{
 737	struct etr_flat_buf *resrv_buf = etr_buf->private;
 738
 739	if (resrv_buf && resrv_buf->daddr) {
 740		struct device *real_dev = resrv_buf->dev->parent;
 741
 742		dma_unmap_resource(real_dev, resrv_buf->daddr,
 743				resrv_buf->size, DMA_FROM_DEVICE, 0);
 744	}
 745	kfree(resrv_buf);
 746}
 747
 748static void tmc_etr_sync_resrv_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
 749{
 750	/*
 751	 * Adjust the buffer to point to the beginning of the trace data
 752	 * and update the available trace data.
 753	 */
 754	etr_buf->offset = rrp - etr_buf->hwaddr;
 755	if (etr_buf->full)
 756		etr_buf->len = etr_buf->size;
 757	else
 758		etr_buf->len = rwp - rrp;
 759}
 760
 761static const struct etr_buf_operations etr_resrv_buf_ops = {
 762	.alloc = tmc_etr_alloc_resrv_buf,
 763	.free = tmc_etr_free_resrv_buf,
 764	.sync = tmc_etr_sync_resrv_buf,
 765	.get_data = tmc_etr_get_data_flat_buf,
 766};
 767
 768/*
 769 * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters
 770 * appropriately.
 771 */
 772static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
 773				struct etr_buf *etr_buf, int node,
 774				void **pages)
 775{
 776	struct etr_sg_table *etr_table;
 777	struct device *dev = &drvdata->csdev->dev;
 778
 779	etr_table = tmc_init_etr_sg_table(dev, node,
 780					  etr_buf->size, pages);
 781	if (IS_ERR(etr_table))
 782		return -ENOMEM;
 783	etr_buf->hwaddr = etr_table->hwaddr;
 784	etr_buf->mode = ETR_MODE_ETR_SG;
 785	etr_buf->private = etr_table;
 786	return 0;
 787}
 788
 789static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
 790{
 791	struct etr_sg_table *etr_table = etr_buf->private;
 792
 793	if (etr_table) {
 794		tmc_free_sg_table(etr_table->sg_table);
 795		kfree(etr_table);
 796	}
 797}
 798
 799static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
 800				       size_t len, char **bufpp)
 801{
 802	struct etr_sg_table *etr_table = etr_buf->private;
 803
 804	return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
 805}
 806
 807static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
 808{
 809	long r_offset, w_offset;
 810	struct etr_sg_table *etr_table = etr_buf->private;
 811	struct tmc_sg_table *table = etr_table->sg_table;
 812
 813	/* Convert hw address to offset in the buffer */
 814	r_offset = tmc_sg_get_data_page_offset(table, rrp);
 815	if (r_offset < 0) {
 816		dev_warn(table->dev,
 817			 "Unable to map RRP %llx to offset\n", rrp);
 818		etr_buf->len = 0;
 819		return;
 820	}
 821
 822	w_offset = tmc_sg_get_data_page_offset(table, rwp);
 823	if (w_offset < 0) {
 824		dev_warn(table->dev,
 825			 "Unable to map RWP %llx to offset\n", rwp);
 826		etr_buf->len = 0;
 827		return;
 828	}
 829
 830	etr_buf->offset = r_offset;
 831	if (etr_buf->full)
 832		etr_buf->len = etr_buf->size;
 833	else
 834		etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
 835				w_offset - r_offset;
 836	tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
 837}
 838
 839static const struct etr_buf_operations etr_sg_buf_ops = {
 840	.alloc = tmc_etr_alloc_sg_buf,
 841	.free = tmc_etr_free_sg_buf,
 842	.sync = tmc_etr_sync_sg_buf,
 843	.get_data = tmc_etr_get_data_sg_buf,
 844};
 845
 846/*
 847 * TMC ETR could be connected to a CATU device, which can provide address
 848 * translation service. This is represented by the Output port of the TMC
 849 * (ETR) connected to the input port of the CATU.
 850 *
 851 * Returns	: coresight_device ptr for the CATU device if a CATU is found.
 852 *		: NULL otherwise.
 853 */
 854struct coresight_device *
 855tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
 856{
 857	struct coresight_device *etr = drvdata->csdev;
 858	union coresight_dev_subtype catu_subtype = {
 859		.helper_subtype = CORESIGHT_DEV_SUBTYPE_HELPER_CATU
 860	};
 861
 862	if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
 863		return NULL;
 864
 865	return coresight_find_output_type(etr->pdata, CORESIGHT_DEV_TYPE_HELPER,
 866					  catu_subtype);
 867}
 868EXPORT_SYMBOL_GPL(tmc_etr_get_catu_device);
 869
 870static const struct etr_buf_operations *etr_buf_ops[] = {
 871	[ETR_MODE_FLAT] = &etr_flat_buf_ops,
 872	[ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
 873	[ETR_MODE_CATU] = NULL,
 874	[ETR_MODE_RESRV] = &etr_resrv_buf_ops
 875};
 876
 877void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu)
 878{
 879	etr_buf_ops[ETR_MODE_CATU] = catu;
 880}
 881EXPORT_SYMBOL_GPL(tmc_etr_set_catu_ops);
 882
 883void tmc_etr_remove_catu_ops(void)
 884{
 885	etr_buf_ops[ETR_MODE_CATU] = NULL;
 886}
 887EXPORT_SYMBOL_GPL(tmc_etr_remove_catu_ops);
 888
 889static int tmc_etr_mode_alloc_buf(int mode, struct tmc_drvdata *drvdata, struct etr_buf *etr_buf,
 890				  int node, void **pages)
 891{
 892	int rc = -EINVAL;
 893
 894	switch (mode) {
 895	case ETR_MODE_FLAT:
 896	case ETR_MODE_ETR_SG:
 897	case ETR_MODE_CATU:
 898	case ETR_MODE_RESRV:
 899		if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
 900			rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
 901						      node, pages);
 902		if (!rc)
 903			etr_buf->ops = etr_buf_ops[mode];
 904		return rc;
 905	default:
 906		return -EINVAL;
 907	}
 908}
 909
 910static void get_etr_buf_hw(struct device *dev, struct etr_buf_hw *buf_hw)
 911{
 912	struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
 913
 914	buf_hw->has_iommu = iommu_get_domain_for_dev(dev->parent);
 915	buf_hw->has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
 916	buf_hw->has_catu = !!tmc_etr_get_catu_device(drvdata);
 917	buf_hw->has_resrv = tmc_has_reserved_buffer(drvdata);
 918}
 919
 920static bool etr_can_use_flat_mode(struct etr_buf_hw *buf_hw, ssize_t etr_buf_size)
 921{
 922	bool has_sg = buf_hw->has_catu || buf_hw->has_etr_sg;
 923
 924	return !has_sg || buf_hw->has_iommu || etr_buf_size < SZ_1M;
 925}
 926
 927/*
 928 * tmc_alloc_etr_buf: Allocate a buffer use by ETR.
 929 * @drvdata	: ETR device details.
 930 * @size	: size of the requested buffer.
 931 * @flags	: Required properties for the buffer.
 932 * @node	: Node for memory allocations.
 933 * @pages	: An optional list of pages.
 934 */
 935static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
 936					 ssize_t size, int flags,
 937					 int node, void **pages)
 938{
 939	int rc = -ENOMEM;
 940	struct etr_buf *etr_buf;
 941	struct etr_buf_hw buf_hw;
 942	struct device *dev = &drvdata->csdev->dev;
 943
 944	get_etr_buf_hw(dev, &buf_hw);
 945	etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
 946	if (!etr_buf)
 947		return ERR_PTR(-ENOMEM);
 948
 949	etr_buf->size = size;
 950
 951	/* If there is user directive for buffer mode, try that first */
 952	if (drvdata->etr_mode != ETR_MODE_AUTO)
 953		rc = tmc_etr_mode_alloc_buf(drvdata->etr_mode, drvdata,
 954					    etr_buf, node, pages);
 955
 956	/*
 957	 * If we have to use an existing list of pages, we cannot reliably
 958	 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
 959	 * we use the contiguous DMA memory if at least one of the following
 960	 * conditions is true:
 961	 *  a) The ETR cannot use Scatter-Gather.
 962	 *  b) we have a backing IOMMU
 963	 *  c) The requested memory size is smaller (< 1M).
 964	 *
 965	 * Fallback to available mechanisms.
 966	 *
 967	 */
 968	if (rc && !pages && etr_can_use_flat_mode(&buf_hw, size))
 969		rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
 970					    etr_buf, node, pages);
 971	if (rc && buf_hw.has_etr_sg)
 972		rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
 973					    etr_buf, node, pages);
 974	if (rc && buf_hw.has_catu)
 975		rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
 976					    etr_buf, node, pages);
 977	if (rc) {
 978		kfree(etr_buf);
 979		return ERR_PTR(rc);
 980	}
 981
 982	refcount_set(&etr_buf->refcount, 1);
 983	dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",
 984		(unsigned long)size >> 10, etr_buf->mode);
 985	return etr_buf;
 986}
 987
 988static void tmc_free_etr_buf(struct etr_buf *etr_buf)
 989{
 990	WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
 991	etr_buf->ops->free(etr_buf);
 992	kfree(etr_buf);
 993}
 994
 995/*
 996 * tmc_etr_buf_get_data: Get the pointer the trace data at @offset
 997 * with a maximum of @len bytes.
 998 * Returns: The size of the linear data available @pos, with *bufpp
 999 * updated to point to the buffer.
1000 */
1001static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
1002				    u64 offset, size_t len, char **bufpp)
1003{
1004	/* Adjust the length to limit this transaction to end of buffer */
1005	len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
1006
1007	return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
1008}
1009
1010static s64
1011tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
1012{
1013	ssize_t len;
1014	char *bufp;
1015
1016	len = tmc_etr_buf_get_data(etr_buf, offset,
1017				   CORESIGHT_BARRIER_PKT_SIZE, &bufp);
1018	if (WARN_ON(len < 0 || len < CORESIGHT_BARRIER_PKT_SIZE))
1019		return -EINVAL;
1020	coresight_insert_barrier_packet(bufp);
1021	return offset + CORESIGHT_BARRIER_PKT_SIZE;
1022}
1023
1024/*
1025 * tmc_sync_etr_buf: Sync the trace buffer availability with drvdata.
1026 * Makes sure the trace data is synced to the memory for consumption.
1027 * @etr_buf->offset will hold the offset to the beginning of the trace data
1028 * within the buffer, with @etr_buf->len bytes to consume.
1029 */
1030static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
1031{
1032	struct etr_buf *etr_buf = drvdata->etr_buf;
1033	u64 rrp, rwp;
1034	u32 status;
1035
1036	rrp = tmc_read_rrp(drvdata);
1037	rwp = tmc_read_rwp(drvdata);
1038	status = readl_relaxed(drvdata->base + TMC_STS);
1039
1040	/*
1041	 * If there were memory errors in the session, truncate the
1042	 * buffer.
1043	 */
1044	if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) {
1045		dev_dbg(&drvdata->csdev->dev,
1046			"tmc memory error detected, truncating buffer\n");
1047		etr_buf->len = 0;
1048		etr_buf->full = false;
1049		return;
1050	}
1051
1052	etr_buf->full = !!(status & TMC_STS_FULL);
1053
1054	WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
1055
1056	etr_buf->ops->sync(etr_buf, rrp, rwp);
1057}
1058
1059static int __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
1060{
1061	u32 axictl, sts, ffcr;
1062	struct etr_buf *etr_buf = drvdata->etr_buf;
1063	int rc = 0;
1064
1065	CS_UNLOCK(drvdata->base);
1066
1067	/* Wait for TMCSReady bit to be set */
1068	rc = tmc_wait_for_tmcready(drvdata);
1069	if (rc) {
1070		dev_err(&drvdata->csdev->dev,
1071			"Failed to enable : TMC not ready\n");
1072		CS_LOCK(drvdata->base);
1073		return rc;
1074	}
1075
1076	writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
1077	writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
1078
1079	axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
1080	axictl &= ~TMC_AXICTL_CLEAR_MASK;
1081	axictl |= TMC_AXICTL_PROT_CTL_B1;
1082	axictl |= TMC_AXICTL_WR_BURST(drvdata->max_burst_size);
1083	axictl |= TMC_AXICTL_AXCACHE_OS;
1084
1085	if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
1086		axictl &= ~TMC_AXICTL_ARCACHE_MASK;
1087		axictl |= TMC_AXICTL_ARCACHE_OS;
1088	}
1089
1090	if (etr_buf->mode == ETR_MODE_ETR_SG)
1091		axictl |= TMC_AXICTL_SCT_GAT_MODE;
1092
1093	writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
1094	tmc_write_dba(drvdata, etr_buf->hwaddr);
1095	/*
1096	 * If the TMC pointers must be programmed before the session,
1097	 * we have to set it properly (i.e, RRP/RWP to base address and
1098	 * STS to "not full").
1099	 */
1100	if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
1101		tmc_write_rrp(drvdata, etr_buf->hwaddr);
1102		tmc_write_rwp(drvdata, etr_buf->hwaddr);
1103		sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
1104		writel_relaxed(sts, drvdata->base + TMC_STS);
1105	}
1106
1107	ffcr = TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI | TMC_FFCR_FON_FLIN |
1108		TMC_FFCR_FON_TRIG_EVT | TMC_FFCR_TRIGON_TRIGIN;
1109	if (drvdata->stop_on_flush)
1110		ffcr |= TMC_FFCR_STOP_ON_FLUSH;
1111	writel_relaxed(ffcr, drvdata->base + TMC_FFCR);
1112
1113	writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
1114	tmc_enable_hw(drvdata);
1115
1116	CS_LOCK(drvdata->base);
1117	return rc;
1118}
1119
1120static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,
1121			     struct etr_buf *etr_buf)
1122{
1123	int rc;
1124
1125	/* Callers should provide an appropriate buffer for use */
1126	if (WARN_ON(!etr_buf))
1127		return -EINVAL;
1128
1129	if ((etr_buf->mode == ETR_MODE_ETR_SG) &&
1130	    WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
1131		return -EINVAL;
1132
1133	if (WARN_ON(drvdata->etr_buf))
1134		return -EBUSY;
1135
1136	rc = coresight_claim_device(drvdata->csdev);
1137	if (!rc) {
1138		drvdata->etr_buf = etr_buf;
1139		rc = __tmc_etr_enable_hw(drvdata);
1140		if (rc) {
1141			drvdata->etr_buf = NULL;
1142			coresight_disclaim_device(drvdata->csdev);
1143		}
1144	}
1145
1146	return rc;
1147}
1148
1149/*
1150 * Return the available trace data in the buffer (starts at etr_buf->offset,
1151 * limited by etr_buf->len) from @pos, with a maximum limit of @len,
1152 * also updating the @bufpp on where to find it. Since the trace data
1153 * starts at anywhere in the buffer, depending on the RRP, we adjust the
1154 * @len returned to handle buffer wrapping around.
1155 *
1156 * We are protected here by drvdata->reading != 0, which ensures the
1157 * sysfs_buf stays alive.
1158 */
1159ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
1160				loff_t pos, size_t len, char **bufpp)
1161{
1162	s64 offset;
1163	ssize_t actual = len;
1164	struct etr_buf *etr_buf = drvdata->sysfs_buf;
1165
1166	if (pos + actual > etr_buf->len)
1167		actual = etr_buf->len - pos;
1168	if (actual <= 0)
1169		return actual;
1170
1171	/* Compute the offset from which we read the data */
1172	offset = etr_buf->offset + pos;
1173	if (offset >= etr_buf->size)
1174		offset -= etr_buf->size;
1175	return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp);
1176}
1177
1178static struct etr_buf *
1179tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
1180{
1181	return tmc_alloc_etr_buf(drvdata, drvdata->size,
1182				 0, cpu_to_node(0), NULL);
1183}
1184
1185static void
1186tmc_etr_free_sysfs_buf(struct etr_buf *buf)
1187{
1188	if (buf)
1189		tmc_free_etr_buf(buf);
1190}
1191
1192static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
1193{
1194	struct etr_buf *etr_buf = drvdata->etr_buf;
1195
1196	if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {
1197		tmc_etr_free_sysfs_buf(drvdata->sysfs_buf);
1198		drvdata->sysfs_buf = NULL;
1199	} else {
1200		tmc_sync_etr_buf(drvdata);
1201		/*
1202		 * Insert barrier packets at the beginning, if there was
1203		 * an overflow.
1204		 */
1205		if (etr_buf->full)
1206			tmc_etr_buf_insert_barrier_packet(etr_buf,
1207							  etr_buf->offset);
1208	}
1209}
1210
1211static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1212{
1213	CS_UNLOCK(drvdata->base);
1214
1215	tmc_flush_and_stop(drvdata);
1216	/*
1217	 * When operating in sysFS mode the content of the buffer needs to be
1218	 * read before the TMC is disabled.
1219	 */
1220	if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS)
1221		tmc_etr_sync_sysfs_buf(drvdata);
1222
1223	tmc_disable_hw(drvdata);
1224
1225	CS_LOCK(drvdata->base);
1226
1227}
1228
1229void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1230{
1231	__tmc_etr_disable_hw(drvdata);
1232	coresight_disclaim_device(drvdata->csdev);
1233	/* Reset the ETR buf used by hardware */
1234	drvdata->etr_buf = NULL;
1235}
1236
1237static struct etr_buf *tmc_etr_get_sysfs_buffer(struct coresight_device *csdev)
1238{
1239	int ret = 0;
1240	unsigned long flags;
1241	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1242	struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;
1243
1244	/*
1245	 * If we are enabling the ETR from disabled state, we need to make
1246	 * sure we have a buffer with the right size. The etr_buf is not reset
1247	 * immediately after we stop the tracing in SYSFS mode as we wait for
1248	 * the user to collect the data. We may be able to reuse the existing
1249	 * buffer, provided the size matches. Any allocation has to be done
1250	 * with the lock released.
1251	 */
1252	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1253
1254	/*
1255	 * If the ETR is already enabled, continue with the existing buffer.
1256	 */
1257	if (coresight_get_mode(csdev) == CS_MODE_SYSFS)
1258		goto out;
1259
1260	sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1261	if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1262		raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1263
1264		/* Allocate memory with the locks released */
1265		free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1266		if (IS_ERR(new_buf))
1267			return new_buf;
1268
1269		/* Let's try again */
1270		raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1271	}
1272
1273	if (drvdata->reading || coresight_get_mode(csdev) == CS_MODE_PERF) {
1274		ret = -EBUSY;
1275		goto out;
1276	}
1277
1278	/*
1279	 * If we don't have a buffer or it doesn't match the requested size,
1280	 * use the buffer allocated above. Otherwise reuse the existing buffer.
1281	 */
1282	sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1283	if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {
1284		free_buf = sysfs_buf;
1285		drvdata->sysfs_buf = new_buf;
1286	}
1287
1288out:
1289	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1290
1291	/* Free memory outside the spinlock if need be */
1292	if (free_buf)
1293		tmc_etr_free_sysfs_buf(free_buf);
1294	return ret ? ERR_PTR(ret) : drvdata->sysfs_buf;
1295}
1296
1297static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1298{
1299	int ret = 0;
1300	unsigned long flags;
1301	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1302	struct etr_buf *sysfs_buf = tmc_etr_get_sysfs_buffer(csdev);
1303
1304	if (IS_ERR(sysfs_buf))
1305		return PTR_ERR(sysfs_buf);
1306
1307	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1308
1309	/*
1310	 * Since the sysfs buffer allocation and the hardware enablement is not
1311	 * in the same critical region, it's possible to race with the perf.
1312	 */
1313	if (coresight_get_mode(csdev) == CS_MODE_PERF) {
1314		drvdata->sysfs_buf = NULL;
1315		raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1316
1317		/* Free allocated memory out side of the spinlock */
1318		tmc_etr_free_sysfs_buf(sysfs_buf);
1319		return -EBUSY;
1320	}
1321
1322	/*
1323	 * In sysFS mode we can have multiple writers per sink.  Since this
1324	 * sink is already enabled no memory is needed and the HW need not be
1325	 * touched, even if the buffer size has changed.
1326	 */
1327	if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
1328		csdev->refcnt++;
1329		goto out;
1330	}
1331
1332	ret = tmc_etr_enable_hw(drvdata, sysfs_buf);
1333	if (!ret) {
1334		coresight_set_mode(csdev, CS_MODE_SYSFS);
1335		csdev->refcnt++;
1336	}
1337
1338out:
1339	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1340
1341	if (!ret)
1342		dev_dbg(&csdev->dev, "TMC-ETR enabled\n");
1343
1344	return ret;
1345}
1346
1347struct etr_buf *tmc_etr_get_buffer(struct coresight_device *csdev,
1348				   enum cs_mode mode,
1349				   struct coresight_path *path)
1350{
1351	struct perf_output_handle *handle = path->handle;
1352	struct etr_perf_buffer *etr_perf;
1353
1354	switch (mode) {
1355	case CS_MODE_SYSFS:
1356		return tmc_etr_get_sysfs_buffer(csdev);
1357	case CS_MODE_PERF:
1358		etr_perf = etm_perf_sink_config(handle);
1359		if (WARN_ON(!etr_perf || !etr_perf->etr_buf))
1360			return ERR_PTR(-EINVAL);
1361		return etr_perf->etr_buf;
1362	default:
1363		return ERR_PTR(-EINVAL);
1364	}
1365}
1366EXPORT_SYMBOL_GPL(tmc_etr_get_buffer);
1367
1368/*
1369 * alloc_etr_buf: Allocate ETR buffer for use by perf.
1370 * The size of the hardware buffer is dependent on the size configured
1371 * via sysfs and the perf ring buffer size. We prefer to allocate the
1372 * largest possible size, scaling down the size by half until it
1373 * reaches a minimum limit (1M), beyond which we give up.
1374 */
1375static struct etr_buf *
1376alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1377	      int nr_pages, void **pages, bool snapshot)
1378{
1379	int node;
1380	struct etr_buf *etr_buf;
1381	unsigned long size;
1382
1383	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1384	/*
1385	 * Try to match the perf ring buffer size if it is larger
1386	 * than the size requested via sysfs.
1387	 */
1388	if ((nr_pages << PAGE_SHIFT) > drvdata->size) {
1389		etr_buf = tmc_alloc_etr_buf(drvdata, ((ssize_t)nr_pages << PAGE_SHIFT),
1390					    0, node, NULL);
1391		if (!IS_ERR(etr_buf))
1392			goto done;
1393	}
1394
1395	/*
1396	 * Else switch to configured size for this ETR
1397	 * and scale down until we hit the minimum limit.
1398	 */
1399	size = drvdata->size;
1400	do {
1401		etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL);
1402		if (!IS_ERR(etr_buf))
1403			goto done;
1404		size /= 2;
1405	} while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);
1406
1407	return ERR_PTR(-ENOMEM);
1408
1409done:
1410	return etr_buf;
1411}
1412
1413static struct etr_buf *
1414get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,
1415			  struct perf_event *event, int nr_pages,
1416			  void **pages, bool snapshot)
1417{
1418	int ret;
1419	pid_t pid = task_pid_nr(event->owner);
1420	struct etr_buf *etr_buf;
1421
1422retry:
1423	/*
1424	 * An etr_perf_buffer is associated with an event and holds a reference
1425	 * to the AUX ring buffer that was created for that event.  In CPU-wide
1426	 * N:1 mode multiple events (one per CPU), each with its own AUX ring
1427	 * buffer, share a sink.  As such an etr_perf_buffer is created for each
1428	 * event but a single etr_buf associated with the ETR is shared between
1429	 * them.  The last event in a trace session will copy the content of the
1430	 * etr_buf to its AUX ring buffer.  Ring buffer associated to other
1431	 * events are simply not used an freed as events are destoyed.  We still
1432	 * need to allocate a ring buffer for each event since we don't know
1433	 * which event will be last.
1434	 */
1435
1436	/*
1437	 * The first thing to do here is check if an etr_buf has already been
1438	 * allocated for this session.  If so it is shared with this event,
1439	 * otherwise it is created.
1440	 */
1441	mutex_lock(&drvdata->idr_mutex);
1442	etr_buf = idr_find(&drvdata->idr, pid);
1443	if (etr_buf) {
1444		refcount_inc(&etr_buf->refcount);
1445		mutex_unlock(&drvdata->idr_mutex);
1446		return etr_buf;
1447	}
1448
1449	/* If we made it here no buffer has been allocated, do so now. */
1450	mutex_unlock(&drvdata->idr_mutex);
1451
1452	etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1453	if (IS_ERR(etr_buf))
1454		return etr_buf;
1455
1456	/* Now that we have a buffer, add it to the IDR. */
1457	mutex_lock(&drvdata->idr_mutex);
1458	ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL);
1459	mutex_unlock(&drvdata->idr_mutex);
1460
1461	/* Another event with this session ID has allocated this buffer. */
1462	if (ret == -ENOSPC) {
1463		tmc_free_etr_buf(etr_buf);
1464		goto retry;
1465	}
1466
1467	/* The IDR can't allocate room for a new session, abandon ship. */
1468	if (ret == -ENOMEM) {
1469		tmc_free_etr_buf(etr_buf);
1470		return ERR_PTR(ret);
1471	}
1472
1473
1474	return etr_buf;
1475}
1476
1477static struct etr_buf *
1478get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
1479			    struct perf_event *event, int nr_pages,
1480			    void **pages, bool snapshot)
1481{
1482	/*
1483	 * In per-thread mode the etr_buf isn't shared, so just go ahead
1484	 * with memory allocation.
1485	 */
1486	return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1487}
1488
1489static struct etr_buf *
1490get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1491		 int nr_pages, void **pages, bool snapshot)
1492{
1493	if (event->cpu == -1)
1494		return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,
1495						   pages, snapshot);
1496
1497	return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,
1498					 pages, snapshot);
1499}
1500
1501static struct etr_perf_buffer *
1502tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1503		       int nr_pages, void **pages, bool snapshot)
1504{
1505	int node;
1506	struct etr_buf *etr_buf;
1507	struct etr_perf_buffer *etr_perf;
1508
1509	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
1510
1511	etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node);
1512	if (!etr_perf)
1513		return ERR_PTR(-ENOMEM);
1514
1515	etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1516	if (!IS_ERR(etr_buf))
1517		goto done;
1518
1519	kfree(etr_perf);
1520	return ERR_PTR(-ENOMEM);
1521
1522done:
1523	/*
1524	 * Keep a reference to the ETR this buffer has been allocated for
1525	 * in order to have access to the IDR in tmc_free_etr_buffer().
1526	 */
1527	etr_perf->drvdata = drvdata;
1528	etr_perf->etr_buf = etr_buf;
1529
1530	return etr_perf;
1531}
1532
1533
1534static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,
1535				  struct perf_event *event, void **pages,
1536				  int nr_pages, bool snapshot)
1537{
1538	struct etr_perf_buffer *etr_perf;
1539	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1540
1541	etr_perf = tmc_etr_setup_perf_buf(drvdata, event,
1542					  nr_pages, pages, snapshot);
1543	if (IS_ERR(etr_perf)) {
1544		dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n");
1545		return NULL;
1546	}
1547
1548	etr_perf->pid = task_pid_nr(event->owner);
1549	etr_perf->snapshot = snapshot;
1550	etr_perf->nr_pages = nr_pages;
1551	etr_perf->pages = pages;
1552
1553	return etr_perf;
1554}
1555
1556static void tmc_free_etr_buffer(void *config)
1557{
1558	struct etr_perf_buffer *etr_perf = config;
1559	struct tmc_drvdata *drvdata = etr_perf->drvdata;
1560	struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;
1561
1562	if (!etr_buf)
1563		goto free_etr_perf_buffer;
1564
1565	mutex_lock(&drvdata->idr_mutex);
1566	/* If we are not the last one to use the buffer, don't touch it. */
1567	if (!refcount_dec_and_test(&etr_buf->refcount)) {
1568		mutex_unlock(&drvdata->idr_mutex);
1569		goto free_etr_perf_buffer;
1570	}
1571
1572	/* We are the last one, remove from the IDR and free the buffer. */
1573	buf = idr_remove(&drvdata->idr, etr_perf->pid);
1574	mutex_unlock(&drvdata->idr_mutex);
1575
1576	/*
1577	 * Something went very wrong if the buffer associated with this ID
1578	 * is not the same in the IDR.  Leak to avoid use after free.
1579	 */
1580	if (buf && WARN_ON(buf != etr_buf))
1581		goto free_etr_perf_buffer;
1582
1583	tmc_free_etr_buf(etr_perf->etr_buf);
1584
1585free_etr_perf_buffer:
1586	kfree(etr_perf);
1587}
1588
1589/*
1590 * tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware
1591 * buffer to the perf ring buffer.
1592 */
1593static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf,
1594				     unsigned long head,
1595				     unsigned long src_offset,
1596				     unsigned long to_copy)
1597{
1598	long bytes;
1599	long pg_idx, pg_offset;
1600	char **dst_pages, *src_buf;
1601	struct etr_buf *etr_buf = etr_perf->etr_buf;
1602
1603	head = PERF_IDX2OFF(head, etr_perf);
1604	pg_idx = head >> PAGE_SHIFT;
1605	pg_offset = head & (PAGE_SIZE - 1);
1606	dst_pages = (char **)etr_perf->pages;
1607
1608	while (to_copy > 0) {
1609		/*
1610		 * In one iteration, we can copy minimum of :
1611		 *  1) what is available in the source buffer,
1612		 *  2) what is available in the source buffer, before it
1613		 *     wraps around.
1614		 *  3) what is available in the destination page.
1615		 * in one iteration.
1616		 */
1617		if (src_offset >= etr_buf->size)
1618			src_offset -= etr_buf->size;
1619		bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy,
1620					     &src_buf);
1621		if (WARN_ON_ONCE(bytes <= 0))
1622			break;
1623		bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));
1624
1625		memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);
1626
1627		to_copy -= bytes;
1628
1629		/* Move destination pointers */
1630		pg_offset += bytes;
1631		if (pg_offset == PAGE_SIZE) {
1632			pg_offset = 0;
1633			if (++pg_idx == etr_perf->nr_pages)
1634				pg_idx = 0;
1635		}
1636
1637		/* Move source pointers */
1638		src_offset += bytes;
1639	}
1640}
1641
1642/*
1643 * tmc_update_etr_buffer : Update the perf ring buffer with the
1644 * available trace data. We use software double buffering at the moment.
1645 *
1646 * TODO: Add support for reusing the perf ring buffer.
1647 */
1648static unsigned long
1649tmc_update_etr_buffer(struct coresight_device *csdev,
1650		      struct perf_output_handle *handle,
1651		      void *config)
1652{
1653	bool lost = false;
1654	unsigned long flags, offset, size = 0;
1655	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1656	struct etr_perf_buffer *etr_perf = config;
1657	struct etr_buf *etr_buf = etr_perf->etr_buf;
1658	struct perf_event *event = handle->event;
1659
1660	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1661
1662	/* Don't do anything if another tracer is using this sink */
1663	if (csdev->refcnt != 1) {
1664		raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1665		goto out;
1666	}
1667
1668	if (WARN_ON(drvdata->perf_buf != etr_buf)) {
1669		lost = true;
1670		raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1671		goto out;
1672	}
1673
1674	CS_UNLOCK(drvdata->base);
1675
1676	tmc_flush_and_stop(drvdata);
1677	tmc_sync_etr_buf(drvdata);
1678
1679	CS_LOCK(drvdata->base);
1680	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1681
1682	lost = etr_buf->full;
1683	offset = etr_buf->offset;
1684	size = etr_buf->len;
1685
1686	/*
1687	 * The ETR buffer may be bigger than the space available in the
1688	 * perf ring buffer (handle->size).  If so advance the offset so that we
1689	 * get the latest trace data.  In snapshot mode none of that matters
1690	 * since we are expected to clobber stale data in favour of the latest
1691	 * traces.
1692	 */
1693	if (!etr_perf->snapshot && size > handle->size) {
1694		u32 mask = tmc_get_memwidth_mask(drvdata);
1695
1696		/*
1697		 * Make sure the new size is aligned in accordance with the
1698		 * requirement explained in function tmc_get_memwidth_mask().
1699		 */
1700		size = handle->size & mask;
1701		offset = etr_buf->offset + etr_buf->len - size;
1702
1703		if (offset >= etr_buf->size)
1704			offset -= etr_buf->size;
1705		lost = true;
1706	}
1707
1708	/* Insert barrier packets at the beginning, if there was an overflow */
1709	if (lost)
1710		tmc_etr_buf_insert_barrier_packet(etr_buf, offset);
1711	tmc_etr_sync_perf_buffer(etr_perf, handle->head, offset, size);
1712
1713	/*
1714	 * In snapshot mode we simply increment the head by the number of byte
1715	 * that were written.  User space will figure out how many bytes to get
1716	 * from the AUX buffer based on the position of the head.
1717	 */
1718	if (etr_perf->snapshot)
1719		handle->head += size;
1720
1721	/*
1722	 * Ensure that the AUX trace data is visible before the aux_head
1723	 * is updated via perf_aux_output_end(), as expected by the
1724	 * perf ring buffer.
1725	 */
1726	smp_wmb();
1727
1728	/*
1729	 * If the event is active, it is triggered during an AUX pause.
1730	 * Re-enable the sink so that it is ready when AUX resume is invoked.
1731	 */
1732	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1733	if (csdev->refcnt && !event->hw.state)
1734		__tmc_etr_enable_hw(drvdata);
1735	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1736
1737out:
1738	/*
1739	 * Don't set the TRUNCATED flag in snapshot mode because 1) the
1740	 * captured buffer is expected to be truncated and 2) a full buffer
1741	 * prevents the event from being re-enabled by the perf core,
1742	 * resulting in stale data being send to user space.
1743	 */
1744	if (!etr_perf->snapshot && lost)
1745		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
1746	return size;
1747}
1748
1749static int tmc_enable_etr_sink_perf(struct coresight_device *csdev,
1750				    struct coresight_path *path)
1751{
1752	int rc = 0;
1753	pid_t pid;
1754	unsigned long flags;
1755	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1756	struct perf_output_handle *handle = path->handle;
1757	struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1758
1759	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1760	 /* Don't use this sink if it is already claimed by sysFS */
1761	if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
1762		rc = -EBUSY;
1763		goto unlock_out;
1764	}
1765
1766	if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {
1767		rc = -EINVAL;
1768		goto unlock_out;
1769	}
1770
1771	/* Get a handle on the pid of the session owner */
1772	pid = etr_perf->pid;
1773
1774	/* Do not proceed if this device is associated with another session */
1775	if (drvdata->pid != -1 && drvdata->pid != pid) {
1776		rc = -EBUSY;
1777		goto unlock_out;
1778	}
1779
1780	/*
1781	 * No HW configuration is needed if the sink is already in
1782	 * use for this session.
1783	 */
1784	if (drvdata->pid == pid) {
1785		csdev->refcnt++;
1786		goto unlock_out;
1787	}
1788
1789	rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf);
1790	if (!rc) {
1791		/* Associate with monitored process. */
1792		drvdata->pid = pid;
1793		coresight_set_mode(csdev, CS_MODE_PERF);
1794		drvdata->perf_buf = etr_perf->etr_buf;
1795		csdev->refcnt++;
1796	}
1797
1798unlock_out:
1799	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1800	return rc;
1801}
1802
1803static int tmc_enable_etr_sink(struct coresight_device *csdev,
1804			       enum cs_mode mode,
1805			       struct coresight_path *path)
1806{
1807	switch (mode) {
1808	case CS_MODE_SYSFS:
1809		return tmc_enable_etr_sink_sysfs(csdev);
1810	case CS_MODE_PERF:
1811		return tmc_enable_etr_sink_perf(csdev, path);
1812	default:
1813		return -EINVAL;
1814	}
1815}
1816
1817static int tmc_disable_etr_sink(struct coresight_device *csdev)
1818{
1819	unsigned long flags;
1820	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1821
1822	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1823
1824	if (drvdata->reading) {
1825		raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1826		return -EBUSY;
1827	}
1828
1829	csdev->refcnt--;
1830	if (csdev->refcnt) {
1831		raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1832		return -EBUSY;
1833	}
1834
1835	/* Complain if we (somehow) got out of sync */
1836	WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED);
1837	tmc_etr_disable_hw(drvdata);
1838	/* Dissociate from monitored process. */
1839	drvdata->pid = -1;
1840	coresight_set_mode(csdev, CS_MODE_DISABLED);
1841	/* Reset perf specific data */
1842	drvdata->perf_buf = NULL;
1843
1844	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1845
1846	dev_dbg(&csdev->dev, "TMC-ETR disabled\n");
1847	return 0;
1848}
1849
1850static int tmc_panic_sync_etr(struct coresight_device *csdev)
1851{
1852	u32 val;
1853	struct tmc_crash_metadata *mdata;
1854	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1855
1856	mdata = (struct tmc_crash_metadata *)drvdata->crash_mdata.vaddr;
1857
1858	if (!drvdata->etr_buf)
1859		return 0;
1860
1861	/* Being in RESRV mode implies valid reserved memory as well */
1862	if (drvdata->etr_buf->mode != ETR_MODE_RESRV)
1863		return 0;
1864
1865	if (!tmc_has_crash_mdata_buffer(drvdata))
1866		return 0;
1867
1868	CS_UNLOCK(drvdata->base);
1869
1870	/* Proceed only if ETR is enabled */
1871	val = readl(drvdata->base + TMC_CTL);
1872	if (!(val & TMC_CTL_CAPT_EN))
1873		goto out;
1874
1875	val = readl(drvdata->base + TMC_FFSR);
1876	/* Do manual flush and stop only if its not auto-stopped */
1877	if (!(val & TMC_FFSR_FT_STOPPED)) {
1878		dev_dbg(&csdev->dev,
1879			 "%s: Triggering manual flush\n", __func__);
1880		tmc_flush_and_stop(drvdata);
1881	} else
1882		tmc_wait_for_tmcready(drvdata);
1883
1884	/* Sync registers from hardware to metadata region */
1885	mdata->tmc_ram_size = readl(drvdata->base + TMC_RSZ);
1886	mdata->tmc_sts = readl(drvdata->base + TMC_STS);
1887	mdata->tmc_mode = readl(drvdata->base + TMC_MODE);
1888	mdata->tmc_ffcr = readl(drvdata->base + TMC_FFCR);
1889	mdata->tmc_ffsr = readl(drvdata->base + TMC_FFSR);
1890	mdata->tmc_rrp = tmc_read_rrp(drvdata);
1891	mdata->tmc_rwp = tmc_read_rwp(drvdata);
1892	mdata->tmc_dba = tmc_read_dba(drvdata);
1893	mdata->trace_paddr = drvdata->resrv_buf.paddr;
1894	mdata->version = CS_CRASHDATA_VERSION;
1895
1896	/*
1897	 * Make sure all previous writes are ordered,
1898	 * before we mark valid
1899	 */
1900	dmb(sy);
1901	mdata->valid = true;
1902	/*
1903	 * Below order need to maintained, since crc of metadata
1904	 * is dependent on first
1905	 */
1906	mdata->crc32_tdata = find_crash_tracedata_crc(drvdata, mdata);
1907	mdata->crc32_mdata = find_crash_metadata_crc(mdata);
1908
1909	tmc_disable_hw(drvdata);
1910
1911	dev_dbg(&csdev->dev, "%s: success\n", __func__);
1912out:
1913	CS_UNLOCK(drvdata->base);
1914
1915	return 0;
1916}
1917
1918static const struct coresight_ops_sink tmc_etr_sink_ops = {
1919	.enable		= tmc_enable_etr_sink,
1920	.disable	= tmc_disable_etr_sink,
1921	.alloc_buffer	= tmc_alloc_etr_buffer,
1922	.update_buffer	= tmc_update_etr_buffer,
1923	.free_buffer	= tmc_free_etr_buffer,
1924};
1925
1926static const struct coresight_ops_panic tmc_etr_sync_ops = {
1927	.sync		= tmc_panic_sync_etr,
1928};
1929
1930const struct coresight_ops tmc_etr_cs_ops = {
1931	.sink_ops	= &tmc_etr_sink_ops,
1932	.panic_ops	= &tmc_etr_sync_ops,
1933};
1934
1935int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1936{
1937	int ret = 0;
1938	unsigned long flags;
1939
1940	/* config types are set a boot time and never change */
1941	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1942		return -EINVAL;
1943
1944	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1945	if (drvdata->reading) {
1946		ret = -EBUSY;
1947		goto out;
1948	}
1949
1950	/*
1951	 * We can safely allow reads even if the ETR is operating in PERF mode,
1952	 * since the sysfs session is captured in mode specific data.
1953	 * If drvdata::sysfs_data is NULL the trace data has been read already.
1954	 */
1955	if (!drvdata->sysfs_buf) {
1956		ret = -EINVAL;
1957		goto out;
1958	}
1959
1960	/* Disable the TMC if we are trying to read from a running session. */
1961	if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS)
1962		__tmc_etr_disable_hw(drvdata);
1963
1964	drvdata->reading = true;
1965out:
1966	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1967
1968	return ret;
1969}
1970
1971int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1972{
1973	unsigned long flags;
1974	struct etr_buf *sysfs_buf = NULL;
1975
1976	/* config types are set a boot time and never change */
1977	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1978		return -EINVAL;
1979
1980	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1981
1982	/* RE-enable the TMC if need be */
1983	if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) {
1984		/*
1985		 * The trace run will continue with the same allocated trace
1986		 * buffer. Since the tracer is still enabled drvdata::buf can't
1987		 * be NULL.
1988		 */
1989		__tmc_etr_enable_hw(drvdata);
1990	} else {
1991		/*
1992		 * The ETR is not tracing and the buffer was just read.
1993		 * As such prepare to free the trace buffer.
1994		 */
1995		sysfs_buf = drvdata->sysfs_buf;
1996		drvdata->sysfs_buf = NULL;
1997	}
1998
1999	drvdata->reading = false;
2000	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
2001
2002	/* Free allocated memory out side of the spinlock */
2003	if (sysfs_buf)
2004		tmc_etr_free_sysfs_buf(sysfs_buf);
2005
2006	return 0;
2007}
2008
2009static const char *const buf_modes_str[] = {
2010	[ETR_MODE_FLAT]		= "flat",
2011	[ETR_MODE_ETR_SG]	= "tmc-sg",
2012	[ETR_MODE_CATU]		= "catu",
2013	[ETR_MODE_RESRV]	= "resrv",
2014	[ETR_MODE_AUTO]		= "auto",
2015};
2016
2017static ssize_t buf_modes_available_show(struct device *dev,
2018					    struct device_attribute *attr, char *buf)
2019{
2020	struct etr_buf_hw buf_hw;
2021	ssize_t size = 0;
2022
2023	get_etr_buf_hw(dev, &buf_hw);
2024	size += sysfs_emit(buf, "%s ", buf_modes_str[ETR_MODE_AUTO]);
2025	size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_FLAT]);
2026	if (buf_hw.has_etr_sg)
2027		size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_ETR_SG]);
2028
2029	if (buf_hw.has_catu)
2030		size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_CATU]);
2031
2032	if (buf_hw.has_resrv)
2033		size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_RESRV]);
2034
2035	size += sysfs_emit_at(buf, size, "\n");
2036	return size;
2037}
2038static DEVICE_ATTR_RO(buf_modes_available);
2039
2040static ssize_t buf_mode_preferred_show(struct device *dev,
2041					 struct device_attribute *attr, char *buf)
2042{
2043	struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
2044
2045	return sysfs_emit(buf, "%s\n", buf_modes_str[drvdata->etr_mode]);
2046}
2047
2048static int buf_mode_set_resrv(struct tmc_drvdata *drvdata)
2049{
2050	int err = -EBUSY;
2051	unsigned long flags;
2052	struct tmc_resrv_buf *rbuf;
2053
2054	rbuf = &drvdata->resrv_buf;
2055
2056	/* Ensure there are no active crashdata read sessions */
2057	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
2058	if (!rbuf->reading) {
2059		tmc_crashdata_set_invalid(drvdata);
2060		rbuf->len = 0;
2061		drvdata->etr_mode = ETR_MODE_RESRV;
2062		err = 0;
2063	}
2064	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
2065	return err;
2066}
2067
2068static ssize_t buf_mode_preferred_store(struct device *dev,
2069					  struct device_attribute *attr,
2070					  const char *buf, size_t size)
2071{
2072	struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
2073	struct etr_buf_hw buf_hw;
2074
2075	get_etr_buf_hw(dev, &buf_hw);
2076	if (sysfs_streq(buf, buf_modes_str[ETR_MODE_FLAT]))
2077		drvdata->etr_mode = ETR_MODE_FLAT;
2078	else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_ETR_SG]) && buf_hw.has_etr_sg)
2079		drvdata->etr_mode = ETR_MODE_ETR_SG;
2080	else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_CATU]) && buf_hw.has_catu)
2081		drvdata->etr_mode = ETR_MODE_CATU;
2082	else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_RESRV]) && buf_hw.has_resrv)
2083		return buf_mode_set_resrv(drvdata) ? : size;
2084	else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_AUTO]))
2085		drvdata->etr_mode = ETR_MODE_AUTO;
2086	else
2087		return -EINVAL;
2088	return size;
2089}
2090static DEVICE_ATTR_RW(buf_mode_preferred);
2091
2092static struct attribute *coresight_etr_attrs[] = {
2093	&dev_attr_buf_modes_available.attr,
2094	&dev_attr_buf_mode_preferred.attr,
2095	NULL,
2096};
2097
2098const struct attribute_group coresight_etr_group = {
2099	.attrs = coresight_etr_attrs,
2100};