coresight-tmc-etf.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright(C) 2016 Linaro Limited. All rights reserved.
  4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
  5 */
  6
  7#include <linux/atomic.h>
  8#include <linux/circ_buf.h>
  9#include <linux/coresight.h>
 10#include <linux/perf_event.h>
 11#include <linux/slab.h>
 12#include "coresight-priv.h"
 13#include "coresight-tmc.h"
 14#include "coresight-etm-perf.h"
 15
 16static int tmc_set_etf_buffer(struct coresight_device *csdev,
 17			      struct perf_output_handle *handle);
 18
 19static int __tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
 20{
 21	int rc = 0;
 22	u32 ffcr;
 23
 24	CS_UNLOCK(drvdata->base);
 25
 26	/* Wait for TMCSReady bit to be set */
 27	rc = tmc_wait_for_tmcready(drvdata);
 28	if (rc) {
 29		dev_err(&drvdata->csdev->dev,
 30			"Failed to enable: TMC not ready\n");
 31		CS_LOCK(drvdata->base);
 32		return rc;
 33	}
 34
 35	writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
 36
 37	ffcr = TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI | TMC_FFCR_FON_FLIN |
 38		TMC_FFCR_FON_TRIG_EVT | TMC_FFCR_TRIGON_TRIGIN;
 39	if (drvdata->stop_on_flush)
 40		ffcr |= TMC_FFCR_STOP_ON_FLUSH;
 41	writel_relaxed(ffcr, drvdata->base + TMC_FFCR);
 42
 43	writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
 44	tmc_enable_hw(drvdata);
 45
 46	CS_LOCK(drvdata->base);
 47	return rc;
 48}
 49
 50static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
 51{
 52	int rc = coresight_claim_device(drvdata->csdev);
 53
 54	if (rc)
 55		return rc;
 56
 57	rc = __tmc_etb_enable_hw(drvdata);
 58	if (rc)
 59		coresight_disclaim_device(drvdata->csdev);
 60	return rc;
 61}
 62
 63static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
 64{
 65	char *bufp;
 66	u32 read_data, lost;
 67
 68	/* Check if the buffer wrapped around. */
 69	lost = readl_relaxed(drvdata->base + TMC_STS) & TMC_STS_FULL;
 70	bufp = drvdata->buf;
 71	drvdata->len = 0;
 72	while (1) {
 73		read_data = readl_relaxed(drvdata->base + TMC_RRD);
 74		if (read_data == 0xFFFFFFFF)
 75			break;
 76		memcpy(bufp, &read_data, 4);
 77		bufp += 4;
 78		drvdata->len += 4;
 79	}
 80
 81	if (lost)
 82		coresight_insert_barrier_packet(drvdata->buf);
 83	return;
 84}
 85
 86static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
 87{
 88	CS_UNLOCK(drvdata->base);
 89
 90	tmc_flush_and_stop(drvdata);
 91	/*
 92	 * When operating in sysFS mode the content of the buffer needs to be
 93	 * read before the TMC is disabled.
 94	 */
 95	if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS)
 96		tmc_etb_dump_hw(drvdata);
 97	tmc_disable_hw(drvdata);
 98
 99	CS_LOCK(drvdata->base);
100}
101
102static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
103{
104	__tmc_etb_disable_hw(drvdata);
105	coresight_disclaim_device(drvdata->csdev);
106}
107
108static int __tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
109{
110	int rc = 0;
111
112	CS_UNLOCK(drvdata->base);
113
114	/* Wait for TMCSReady bit to be set */
115	rc = tmc_wait_for_tmcready(drvdata);
116	if (rc) {
117		dev_err(&drvdata->csdev->dev,
118			"Failed to enable : TMC is not ready\n");
119		CS_LOCK(drvdata->base);
120		return rc;
121	}
122
123	writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE);
124	writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI,
125		       drvdata->base + TMC_FFCR);
126	writel_relaxed(0x0, drvdata->base + TMC_BUFWM);
127	tmc_enable_hw(drvdata);
128
129	CS_LOCK(drvdata->base);
130	return rc;
131}
132
133static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
134{
135	int rc = coresight_claim_device(drvdata->csdev);
136
137	if (rc)
138		return rc;
139
140	rc = __tmc_etf_enable_hw(drvdata);
141	if (rc)
142		coresight_disclaim_device(drvdata->csdev);
143	return rc;
144}
145
146static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata)
147{
148	struct coresight_device *csdev = drvdata->csdev;
149
150	CS_UNLOCK(drvdata->base);
151
152	tmc_flush_and_stop(drvdata);
153	tmc_disable_hw(drvdata);
154	coresight_disclaim_device_unlocked(csdev);
155	CS_LOCK(drvdata->base);
156}
157
158/*
159 * Return the available trace data in the buffer from @pos, with
160 * a maximum limit of @len, updating the @bufpp on where to
161 * find it.
162 */
163ssize_t tmc_etb_get_sysfs_trace(struct tmc_drvdata *drvdata,
164				loff_t pos, size_t len, char **bufpp)
165{
166	ssize_t actual = len;
167
168	/* Adjust the len to available size @pos */
169	if (pos + actual > drvdata->len)
170		actual = drvdata->len - pos;
171	if (actual > 0)
172		*bufpp = drvdata->buf + pos;
173	return actual;
174}
175
176static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev)
177{
178	int ret = 0;
179	bool used = false;
180	char *buf = NULL;
181	unsigned long flags;
182	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
183
184	/*
185	 * If we don't have a buffer release the lock and allocate memory.
186	 * Otherwise keep the lock and move along.
187	 */
188	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
189	if (!drvdata->buf) {
190		raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
191
192		/* Allocating the memory here while outside of the spinlock */
193		buf = kzalloc(drvdata->size, GFP_KERNEL);
194		if (!buf)
195			return -ENOMEM;
196
197		/* Let's try again */
198		raw_spin_lock_irqsave(&drvdata->spinlock, flags);
199	}
200
201	if (drvdata->reading) {
202		ret = -EBUSY;
203		goto out;
204	}
205
206	/*
207	 * In sysFS mode we can have multiple writers per sink.  Since this
208	 * sink is already enabled no memory is needed and the HW need not be
209	 * touched.
210	 */
211	if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
212		csdev->refcnt++;
213		goto out;
214	}
215
216	/*
217	 * If drvdata::buf isn't NULL, memory was allocated for a previous
218	 * trace run but wasn't read.  If so simply zero-out the memory.
219	 * Otherwise use the memory allocated above.
220	 *
221	 * The memory is freed when users read the buffer using the
222	 * /dev/xyz.{etf|etb} interface.  See tmc_read_unprepare_etf() for
223	 * details.
224	 */
225	if (drvdata->buf) {
226		memset(drvdata->buf, 0, drvdata->size);
227	} else {
228		used = true;
229		drvdata->buf = buf;
230	}
231	ret = tmc_etb_enable_hw(drvdata);
232	if (!ret) {
233		coresight_set_mode(csdev, CS_MODE_SYSFS);
234		csdev->refcnt++;
235	} else {
236		/* Free up the buffer if we failed to enable */
237		used = false;
238	}
239out:
240	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
241
242	/* Free memory outside the spinlock if need be */
243	if (!used)
244		kfree(buf);
245
246	return ret;
247}
248
249static int tmc_enable_etf_sink_perf(struct coresight_device *csdev,
250				    struct coresight_path *path)
251{
252	int ret = 0;
253	pid_t pid;
254	unsigned long flags;
255	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
256	struct perf_output_handle *handle = path->handle;
257	struct cs_buffers *buf = etm_perf_sink_config(handle);
258
259	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
260	do {
261		ret = -EINVAL;
262		if (drvdata->reading)
263			break;
264		/*
265		 * No need to continue if the ETB/ETF is already operated
266		 * from sysFS.
267		 */
268		if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
269			ret = -EBUSY;
270			break;
271		}
272
273		/* Get a handle on the pid of the process to monitor */
274		pid = buf->pid;
275
276		if (drvdata->pid != -1 && drvdata->pid != pid) {
277			ret = -EBUSY;
278			break;
279		}
280
281		ret = tmc_set_etf_buffer(csdev, handle);
282		if (ret)
283			break;
284
285		/*
286		 * No HW configuration is needed if the sink is already in
287		 * use for this session.
288		 */
289		if (drvdata->pid == pid) {
290			csdev->refcnt++;
291			break;
292		}
293
294		ret  = tmc_etb_enable_hw(drvdata);
295		if (!ret) {
296			/* Associate with monitored process. */
297			drvdata->pid = pid;
298			coresight_set_mode(csdev, CS_MODE_PERF);
299			csdev->refcnt++;
300		}
301	} while (0);
302	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
303
304	return ret;
305}
306
307static int tmc_enable_etf_sink(struct coresight_device *csdev,
308			       enum cs_mode mode,
309			       struct coresight_path *path)
310{
311	int ret;
312
313	switch (mode) {
314	case CS_MODE_SYSFS:
315		ret = tmc_enable_etf_sink_sysfs(csdev);
316		break;
317	case CS_MODE_PERF:
318		ret = tmc_enable_etf_sink_perf(csdev, path);
319		break;
320	/* We shouldn't be here */
321	default:
322		ret = -EINVAL;
323		break;
324	}
325
326	if (ret)
327		return ret;
328
329	dev_dbg(&csdev->dev, "TMC-ETB/ETF enabled\n");
330	return 0;
331}
332
333static int tmc_disable_etf_sink(struct coresight_device *csdev)
334{
335	unsigned long flags;
336	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
337
338	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
339
340	if (drvdata->reading) {
341		raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
342		return -EBUSY;
343	}
344
345	csdev->refcnt--;
346	if (csdev->refcnt) {
347		raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
348		return -EBUSY;
349	}
350
351	/* Complain if we (somehow) got out of sync */
352	WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED);
353	tmc_etb_disable_hw(drvdata);
354	/* Dissociate from monitored process. */
355	drvdata->pid = -1;
356	coresight_set_mode(csdev, CS_MODE_DISABLED);
357
358	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
359
360	dev_dbg(&csdev->dev, "TMC-ETB/ETF disabled\n");
361	return 0;
362}
363
364static int tmc_enable_etf_link(struct coresight_device *csdev,
365			       struct coresight_connection *in,
366			       struct coresight_connection *out)
367{
368	int ret = 0;
369	unsigned long flags;
370	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
371	bool first_enable = false;
372
373	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
374	if (drvdata->reading) {
375		raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
376		return -EBUSY;
377	}
378
379	if (csdev->refcnt == 0) {
380		ret = tmc_etf_enable_hw(drvdata);
381		if (!ret) {
382			coresight_set_mode(csdev, CS_MODE_SYSFS);
383			first_enable = true;
384		}
385	}
386	if (!ret)
387		csdev->refcnt++;
388	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
389
390	if (first_enable)
391		dev_dbg(&csdev->dev, "TMC-ETF enabled\n");
392	return ret;
393}
394
395static void tmc_disable_etf_link(struct coresight_device *csdev,
396				 struct coresight_connection *in,
397				 struct coresight_connection *out)
398{
399	unsigned long flags;
400	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
401	bool last_disable = false;
402
403	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
404	if (drvdata->reading) {
405		raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
406		return;
407	}
408
409	csdev->refcnt--;
410	if (csdev->refcnt == 0) {
411		tmc_etf_disable_hw(drvdata);
412		coresight_set_mode(csdev, CS_MODE_DISABLED);
413		last_disable = true;
414	}
415	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
416
417	if (last_disable)
418		dev_dbg(&csdev->dev, "TMC-ETF disabled\n");
419}
420
421static void *tmc_alloc_etf_buffer(struct coresight_device *csdev,
422				  struct perf_event *event, void **pages,
423				  int nr_pages, bool overwrite)
424{
425	int node;
426	struct cs_buffers *buf;
427
428	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
429
430	/* Allocate memory structure for interaction with Perf */
431	buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node);
432	if (!buf)
433		return NULL;
434
435	buf->pid = task_pid_nr(event->owner);
436	buf->snapshot = overwrite;
437	buf->nr_pages = nr_pages;
438	buf->data_pages = pages;
439
440	return buf;
441}
442
443static void tmc_free_etf_buffer(void *config)
444{
445	struct cs_buffers *buf = config;
446
447	kfree(buf);
448}
449
450static int tmc_set_etf_buffer(struct coresight_device *csdev,
451			      struct perf_output_handle *handle)
452{
453	int ret = 0;
454	unsigned long head;
455	struct cs_buffers *buf = etm_perf_sink_config(handle);
456
457	if (!buf)
458		return -EINVAL;
459
460	/* wrap head around to the amount of space we have */
461	head = handle->head & (((unsigned long)buf->nr_pages << PAGE_SHIFT) - 1);
462
463	/* find the page to write to */
464	buf->cur = head / PAGE_SIZE;
465
466	/* and offset within that page */
467	buf->offset = head % PAGE_SIZE;
468
469	local_set(&buf->data_size, 0);
470
471	return ret;
472}
473
474static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev,
475				  struct perf_output_handle *handle,
476				  void *sink_config)
477{
478	bool lost = false;
479	int i, cur;
480	const u32 *barrier;
481	u32 *buf_ptr;
482	u64 read_ptr, write_ptr;
483	u32 status;
484	unsigned long offset, to_read = 0, flags;
485	struct cs_buffers *buf = sink_config;
486	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
487	struct perf_event *event = handle->event;
488
489	if (!buf)
490		return 0;
491
492	/* This shouldn't happen */
493	if (WARN_ON_ONCE(coresight_get_mode(csdev) != CS_MODE_PERF))
494		return 0;
495
496	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
497
498	/* Don't do anything if another tracer is using this sink */
499	if (csdev->refcnt != 1)
500		goto out;
501
502	CS_UNLOCK(drvdata->base);
503
504	tmc_flush_and_stop(drvdata);
505
506	read_ptr = tmc_read_rrp(drvdata);
507	write_ptr = tmc_read_rwp(drvdata);
508
509	/*
510	 * Get a hold of the status register and see if a wrap around
511	 * has occurred.  If so adjust things accordingly.
512	 */
513	status = readl_relaxed(drvdata->base + TMC_STS);
514	if (status & TMC_STS_FULL) {
515		lost = true;
516		to_read = drvdata->size;
517	} else {
518		to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size);
519	}
520
521	/*
522	 * The TMC RAM buffer may be bigger than the space available in the
523	 * perf ring buffer (handle->size).  If so advance the RRP so that we
524	 * get the latest trace data.  In snapshot mode none of that matters
525	 * since we are expected to clobber stale data in favour of the latest
526	 * traces.
527	 */
528	if (!buf->snapshot && to_read > handle->size) {
529		u32 mask = tmc_get_memwidth_mask(drvdata);
530
531		/*
532		 * Make sure the new size is aligned in accordance with the
533		 * requirement explained in function tmc_get_memwidth_mask().
534		 */
535		to_read = handle->size & mask;
536		/* Move the RAM read pointer up */
537		read_ptr = (write_ptr + drvdata->size) - to_read;
538		/* Make sure we are still within our limits */
539		if (read_ptr > (drvdata->size - 1))
540			read_ptr -= drvdata->size;
541		/* Tell the HW */
542		tmc_write_rrp(drvdata, read_ptr);
543		lost = true;
544	}
545
546	/*
547	 * Don't set the TRUNCATED flag in snapshot mode because 1) the
548	 * captured buffer is expected to be truncated and 2) a full buffer
549	 * prevents the event from being re-enabled by the perf core,
550	 * resulting in stale data being send to user space.
551	 */
552	if (!buf->snapshot && lost)
553		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
554
555	cur = buf->cur;
556	offset = buf->offset;
557	barrier = coresight_barrier_pkt;
558
559	/* for every byte to read */
560	for (i = 0; i < to_read; i += 4) {
561		buf_ptr = buf->data_pages[cur] + offset;
562		*buf_ptr = readl_relaxed(drvdata->base + TMC_RRD);
563
564		if (lost && i < CORESIGHT_BARRIER_PKT_SIZE) {
565			*buf_ptr = *barrier;
566			barrier++;
567		}
568
569		offset += 4;
570		if (offset >= PAGE_SIZE) {
571			offset = 0;
572			cur++;
573			/* wrap around at the end of the buffer */
574			cur &= buf->nr_pages - 1;
575		}
576	}
577
578	/*
579	 * In snapshot mode we simply increment the head by the number of byte
580	 * that were written.  User space will figure out how many bytes to get
581	 * from the AUX buffer based on the position of the head.
582	 */
583	if (buf->snapshot)
584		handle->head += to_read;
585
586	/*
587	 * CS_LOCK() contains mb() so it can ensure visibility of the AUX trace
588	 * data before the aux_head is updated via perf_aux_output_end(), which
589	 * is expected by the perf ring buffer.
590	 */
591	CS_LOCK(drvdata->base);
592
593	/*
594	 * If the event is active, it is triggered during an AUX pause.
595	 * Re-enable the sink so that it is ready when AUX resume is invoked.
596	 */
597	if (!event->hw.state)
598		__tmc_etb_enable_hw(drvdata);
599
600out:
601	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
602
603	return to_read;
604}
605
606static int tmc_panic_sync_etf(struct coresight_device *csdev)
607{
608	u32 val;
609	struct tmc_crash_metadata *mdata;
610	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
611
612	mdata = (struct tmc_crash_metadata *)drvdata->crash_mdata.vaddr;
613
614	/* Make sure we have valid reserved memory */
615	if (!tmc_has_reserved_buffer(drvdata) ||
616	    !tmc_has_crash_mdata_buffer(drvdata))
617		return 0;
618
619	tmc_crashdata_set_invalid(drvdata);
620
621	CS_UNLOCK(drvdata->base);
622
623	/* Proceed only if ETF is enabled or configured as sink */
624	val = readl(drvdata->base + TMC_CTL);
625	if (!(val & TMC_CTL_CAPT_EN))
626		goto out;
627	val = readl(drvdata->base + TMC_MODE);
628	if (val != TMC_MODE_CIRCULAR_BUFFER)
629		goto out;
630
631	val = readl(drvdata->base + TMC_FFSR);
632	/* Do manual flush and stop only if its not auto-stopped */
633	if (!(val & TMC_FFSR_FT_STOPPED)) {
634		dev_dbg(&csdev->dev,
635			 "%s: Triggering manual flush\n", __func__);
636		tmc_flush_and_stop(drvdata);
637	} else
638		tmc_wait_for_tmcready(drvdata);
639
640	/* Sync registers from hardware to metadata region */
641	mdata->tmc_sts = readl(drvdata->base + TMC_STS);
642	mdata->tmc_mode = readl(drvdata->base + TMC_MODE);
643	mdata->tmc_ffcr = readl(drvdata->base + TMC_FFCR);
644	mdata->tmc_ffsr = readl(drvdata->base + TMC_FFSR);
645
646	/* Sync Internal SRAM to reserved trace buffer region */
647	drvdata->buf = drvdata->resrv_buf.vaddr;
648	tmc_etb_dump_hw(drvdata);
649	/* Store as per RSZ register convention */
650	mdata->tmc_ram_size = drvdata->len >> 2;
651
652	/* Other fields for processing trace buffer reads */
653	mdata->tmc_rrp = 0;
654	mdata->tmc_dba = 0;
655	mdata->tmc_rwp = drvdata->len;
656	mdata->trace_paddr = drvdata->resrv_buf.paddr;
657
658	mdata->version = CS_CRASHDATA_VERSION;
659
660	/*
661	 * Make sure all previous writes are ordered,
662	 * before we mark valid
663	 */
664	dmb(sy);
665	mdata->valid = true;
666	/*
667	 * Below order need to maintained, since crc of metadata
668	 * is dependent on first
669	 */
670	mdata->crc32_tdata = find_crash_tracedata_crc(drvdata, mdata);
671	mdata->crc32_mdata = find_crash_metadata_crc(mdata);
672
673	tmc_disable_hw(drvdata);
674
675	dev_dbg(&csdev->dev, "%s: success\n", __func__);
676out:
677	CS_UNLOCK(drvdata->base);
678	return 0;
679}
680
681static const struct coresight_ops_sink tmc_etf_sink_ops = {
682	.enable		= tmc_enable_etf_sink,
683	.disable	= tmc_disable_etf_sink,
684	.alloc_buffer	= tmc_alloc_etf_buffer,
685	.free_buffer	= tmc_free_etf_buffer,
686	.update_buffer	= tmc_update_etf_buffer,
687};
688
689static const struct coresight_ops_link tmc_etf_link_ops = {
690	.enable		= tmc_enable_etf_link,
691	.disable	= tmc_disable_etf_link,
692};
693
694static const struct coresight_ops_panic tmc_etf_sync_ops = {
695	.sync		= tmc_panic_sync_etf,
696};
697
698const struct coresight_ops tmc_etb_cs_ops = {
699	.sink_ops	= &tmc_etf_sink_ops,
700};
701
702const struct coresight_ops tmc_etf_cs_ops = {
703	.sink_ops	= &tmc_etf_sink_ops,
704	.link_ops	= &tmc_etf_link_ops,
705	.panic_ops	= &tmc_etf_sync_ops,
706};
707
708int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
709{
710	enum tmc_mode mode;
711	int ret = 0;
712	unsigned long flags;
713
714	/* config types are set a boot time and never change */
715	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
716			 drvdata->config_type != TMC_CONFIG_TYPE_ETF))
717		return -EINVAL;
718
719	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
720
721	if (drvdata->reading) {
722		ret = -EBUSY;
723		goto out;
724	}
725
726	/* Don't interfere if operated from Perf */
727	if (coresight_get_mode(drvdata->csdev) == CS_MODE_PERF) {
728		ret = -EINVAL;
729		goto out;
730	}
731
732	/* If drvdata::buf is NULL the trace data has been read already */
733	if (drvdata->buf == NULL) {
734		ret = -EINVAL;
735		goto out;
736	}
737
738	/* Disable the TMC if need be */
739	if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) {
740		/* There is no point in reading a TMC in HW FIFO mode */
741		mode = readl_relaxed(drvdata->base + TMC_MODE);
742		if (mode != TMC_MODE_CIRCULAR_BUFFER) {
743			ret = -EINVAL;
744			goto out;
745		}
746		__tmc_etb_disable_hw(drvdata);
747	}
748
749	drvdata->reading = true;
750out:
751	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
752
753	return ret;
754}
755
756int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata)
757{
758	char *buf = NULL;
759	enum tmc_mode mode;
760	unsigned long flags;
761
762	/* config types are set a boot time and never change */
763	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
764			 drvdata->config_type != TMC_CONFIG_TYPE_ETF))
765		return -EINVAL;
766
767	raw_spin_lock_irqsave(&drvdata->spinlock, flags);
768
769	/* Re-enable the TMC if need be */
770	if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) {
771		/* There is no point in reading a TMC in HW FIFO mode */
772		mode = readl_relaxed(drvdata->base + TMC_MODE);
773		if (mode != TMC_MODE_CIRCULAR_BUFFER) {
774			raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
775			return -EINVAL;
776		}
777		/*
778		 * The trace run will continue with the same allocated trace
779		 * buffer. As such zero-out the buffer so that we don't end
780		 * up with stale data.
781		 *
782		 * Since the tracer is still enabled drvdata::buf
783		 * can't be NULL.
784		 */
785		memset(drvdata->buf, 0, drvdata->size);
786		/*
787		 * Ignore failures to enable the TMC to make sure, we don't
788		 * leave the TMC in a "reading" state.
789		 */
790		__tmc_etb_enable_hw(drvdata);
791	} else {
792		/*
793		 * The ETB/ETF is not tracing and the buffer was just read.
794		 * As such prepare to free the trace buffer.
795		 */
796		buf = drvdata->buf;
797		drvdata->buf = NULL;
798	}
799
800	drvdata->reading = false;
801	raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
802
803	/*
804	 * Free allocated memory outside of the spinlock.  There is no need
805	 * to assert the validity of 'buf' since calling kfree(NULL) is safe.
806	 */
807	kfree(buf);
808
809	return 0;
810}