dma.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#undef TRACE_SYSTEM
  3#define TRACE_SYSTEM dma
  4
  5#if !defined(_TRACE_DMA_H) || defined(TRACE_HEADER_MULTI_READ)
  6#define _TRACE_DMA_H
  7
  8#include <linux/tracepoint.h>
  9#include <linux/dma-direction.h>
 10#include <linux/dma-mapping.h>
 11#include <trace/events/mmflags.h>
 12
 13TRACE_DEFINE_ENUM(DMA_BIDIRECTIONAL);
 14TRACE_DEFINE_ENUM(DMA_TO_DEVICE);
 15TRACE_DEFINE_ENUM(DMA_FROM_DEVICE);
 16TRACE_DEFINE_ENUM(DMA_NONE);
 17
 18#define decode_dma_data_direction(dir) \
 19	__print_symbolic(dir, \
 20		{ DMA_BIDIRECTIONAL, "BIDIRECTIONAL" }, \
 21		{ DMA_TO_DEVICE, "TO_DEVICE" }, \
 22		{ DMA_FROM_DEVICE, "FROM_DEVICE" }, \
 23		{ DMA_NONE, "NONE" })
 24
 25#define decode_dma_attrs(attrs) \
 26	__print_flags(attrs, "|", \
 27		{ DMA_ATTR_WEAK_ORDERING, "WEAK_ORDERING" }, \
 28		{ DMA_ATTR_WRITE_COMBINE, "WRITE_COMBINE" }, \
 29		{ DMA_ATTR_NO_KERNEL_MAPPING, "NO_KERNEL_MAPPING" }, \
 30		{ DMA_ATTR_SKIP_CPU_SYNC, "SKIP_CPU_SYNC" }, \
 31		{ DMA_ATTR_FORCE_CONTIGUOUS, "FORCE_CONTIGUOUS" }, \
 32		{ DMA_ATTR_ALLOC_SINGLE_PAGES, "ALLOC_SINGLE_PAGES" }, \
 33		{ DMA_ATTR_NO_WARN, "NO_WARN" }, \
 34		{ DMA_ATTR_PRIVILEGED, "PRIVILEGED" }, \
 35		{ DMA_ATTR_MMIO, "MMIO" }, \
 36		{ DMA_ATTR_CPU_CACHE_CLEAN, "CACHE_CLEAN" })
 37
 38DECLARE_EVENT_CLASS(dma_map,
 39	TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr,
 40		 size_t size, enum dma_data_direction dir, unsigned long attrs),
 41	TP_ARGS(dev, phys_addr, dma_addr, size, dir, attrs),
 42
 43	TP_STRUCT__entry(
 44		__string(device, dev_name(dev))
 45		__field(u64, phys_addr)
 46		__field(u64, dma_addr)
 47		__field(size_t, size)
 48		__field(enum dma_data_direction, dir)
 49		__field(unsigned long, attrs)
 50	),
 51
 52	TP_fast_assign(
 53		__assign_str(device);
 54		__entry->phys_addr = phys_addr;
 55		__entry->dma_addr = dma_addr;
 56		__entry->size = size;
 57		__entry->dir = dir;
 58		__entry->attrs = attrs;
 59	),
 60
 61	TP_printk("%s dir=%s dma_addr=%llx size=%zu phys_addr=%llx attrs=%s",
 62		__get_str(device),
 63		decode_dma_data_direction(__entry->dir),
 64		__entry->dma_addr,
 65		__entry->size,
 66		__entry->phys_addr,
 67		decode_dma_attrs(__entry->attrs))
 68);
 69
 70#define DEFINE_MAP_EVENT(name) \
 71DEFINE_EVENT(dma_map, name, \
 72	TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr, \
 73		 size_t size, enum dma_data_direction dir, unsigned long attrs), \
 74	TP_ARGS(dev, phys_addr, dma_addr, size, dir, attrs))
 75
 76DEFINE_MAP_EVENT(dma_map_phys);
 77
 78DECLARE_EVENT_CLASS(dma_unmap,
 79	TP_PROTO(struct device *dev, dma_addr_t addr, size_t size,
 80		 enum dma_data_direction dir, unsigned long attrs),
 81	TP_ARGS(dev, addr, size, dir, attrs),
 82
 83	TP_STRUCT__entry(
 84		__string(device, dev_name(dev))
 85		__field(u64, addr)
 86		__field(size_t, size)
 87		__field(enum dma_data_direction, dir)
 88		__field(unsigned long, attrs)
 89	),
 90
 91	TP_fast_assign(
 92		__assign_str(device);
 93		__entry->addr = addr;
 94		__entry->size = size;
 95		__entry->dir = dir;
 96		__entry->attrs = attrs;
 97	),
 98
 99	TP_printk("%s dir=%s dma_addr=%llx size=%zu attrs=%s",
100		__get_str(device),
101		decode_dma_data_direction(__entry->dir),
102		__entry->addr,
103		__entry->size,
104		decode_dma_attrs(__entry->attrs))
105);
106
107#define DEFINE_UNMAP_EVENT(name) \
108DEFINE_EVENT(dma_unmap, name, \
109	TP_PROTO(struct device *dev, dma_addr_t addr, size_t size, \
110		 enum dma_data_direction dir, unsigned long attrs), \
111	TP_ARGS(dev, addr, size, dir, attrs))
112
113DEFINE_UNMAP_EVENT(dma_unmap_phys);
114
115DECLARE_EVENT_CLASS(dma_alloc_class,
116	TP_PROTO(struct device *dev, void *virt_addr, dma_addr_t dma_addr,
117		 size_t size, enum dma_data_direction dir, gfp_t flags,
118		 unsigned long attrs),
119	TP_ARGS(dev, virt_addr, dma_addr, size, dir, flags, attrs),
120
121	TP_STRUCT__entry(
122		__string(device, dev_name(dev))
123		__field(void *, virt_addr)
124		__field(u64, dma_addr)
125		__field(size_t, size)
126		__field(gfp_t, flags)
127		__field(enum dma_data_direction, dir)
128		__field(unsigned long, attrs)
129	),
130
131	TP_fast_assign(
132		__assign_str(device);
133		__entry->virt_addr = virt_addr;
134		__entry->dma_addr = dma_addr;
135		__entry->size = size;
136		__entry->flags = flags;
137		__entry->dir = dir;
138		__entry->attrs = attrs;
139	),
140
141	TP_printk("%s dir=%s dma_addr=%llx size=%zu virt_addr=%p flags=%s attrs=%s",
142		__get_str(device),
143		decode_dma_data_direction(__entry->dir),
144		__entry->dma_addr,
145		__entry->size,
146		__entry->virt_addr,
147		show_gfp_flags(__entry->flags),
148		decode_dma_attrs(__entry->attrs))
149);
150
151#define DEFINE_ALLOC_EVENT(name) \
152DEFINE_EVENT(dma_alloc_class, name, \
153	TP_PROTO(struct device *dev, void *virt_addr, dma_addr_t dma_addr, \
154		 size_t size, enum dma_data_direction dir, gfp_t flags, \
155		 unsigned long attrs), \
156	TP_ARGS(dev, virt_addr, dma_addr, size, dir, flags, attrs))
157
158DEFINE_ALLOC_EVENT(dma_alloc);
159DEFINE_ALLOC_EVENT(dma_alloc_pages);
160DEFINE_ALLOC_EVENT(dma_alloc_sgt_err);
161
162TRACE_EVENT(dma_alloc_sgt,
163	TP_PROTO(struct device *dev, struct sg_table *sgt, size_t size,
164		 enum dma_data_direction dir, gfp_t flags, unsigned long attrs),
165	TP_ARGS(dev, sgt, size, dir, flags, attrs),
166
167	TP_STRUCT__entry(
168		__string(device, dev_name(dev))
169		__dynamic_array(u64, phys_addrs, sgt->orig_nents)
170		__field(u64, dma_addr)
171		__field(size_t, size)
172		__field(enum dma_data_direction, dir)
173		__field(gfp_t, flags)
174		__field(unsigned long, attrs)
175	),
176
177	TP_fast_assign(
178		struct scatterlist *sg;
179		int i;
180
181		__assign_str(device);
182		for_each_sg(sgt->sgl, sg, sgt->orig_nents, i)
183			((u64 *)__get_dynamic_array(phys_addrs))[i] = sg_phys(sg);
184		__entry->dma_addr = sg_dma_address(sgt->sgl);
185		__entry->size = size;
186		__entry->dir = dir;
187		__entry->flags = flags;
188		__entry->attrs = attrs;
189	),
190
191	TP_printk("%s dir=%s dma_addr=%llx size=%zu phys_addrs=%s flags=%s attrs=%s",
192		__get_str(device),
193		decode_dma_data_direction(__entry->dir),
194		__entry->dma_addr,
195		__entry->size,
196		__print_array(__get_dynamic_array(phys_addrs),
197			      __get_dynamic_array_len(phys_addrs) /
198				sizeof(u64), sizeof(u64)),
199		show_gfp_flags(__entry->flags),
200		decode_dma_attrs(__entry->attrs))
201);
202
203DECLARE_EVENT_CLASS(dma_free_class,
204	TP_PROTO(struct device *dev, void *virt_addr, dma_addr_t dma_addr,
205		 size_t size, enum dma_data_direction dir, unsigned long attrs),
206	TP_ARGS(dev, virt_addr, dma_addr, size, dir, attrs),
207
208	TP_STRUCT__entry(
209		__string(device, dev_name(dev))
210		__field(void *, virt_addr)
211		__field(u64, dma_addr)
212		__field(size_t, size)
213		__field(enum dma_data_direction, dir)
214		__field(unsigned long, attrs)
215	),
216
217	TP_fast_assign(
218		__assign_str(device);
219		__entry->virt_addr = virt_addr;
220		__entry->dma_addr = dma_addr;
221		__entry->size = size;
222		__entry->dir = dir;
223		__entry->attrs = attrs;
224	),
225
226	TP_printk("%s dir=%s dma_addr=%llx size=%zu virt_addr=%p attrs=%s",
227		__get_str(device),
228		decode_dma_data_direction(__entry->dir),
229		__entry->dma_addr,
230		__entry->size,
231		__entry->virt_addr,
232		decode_dma_attrs(__entry->attrs))
233);
234
235#define DEFINE_FREE_EVENT(name) \
236DEFINE_EVENT(dma_free_class, name, \
237	TP_PROTO(struct device *dev, void *virt_addr, dma_addr_t dma_addr, \
238		 size_t size, enum dma_data_direction dir, unsigned long attrs), \
239	TP_ARGS(dev, virt_addr, dma_addr, size, dir, attrs))
240
241DEFINE_FREE_EVENT(dma_free);
242DEFINE_FREE_EVENT(dma_free_pages);
243
244TRACE_EVENT(dma_free_sgt,
245	TP_PROTO(struct device *dev, struct sg_table *sgt, size_t size,
246		 enum dma_data_direction dir),
247	TP_ARGS(dev, sgt, size, dir),
248
249	TP_STRUCT__entry(
250		__string(device, dev_name(dev))
251		__dynamic_array(u64, phys_addrs, sgt->orig_nents)
252		__field(u64, dma_addr)
253		__field(size_t, size)
254		__field(enum dma_data_direction, dir)
255	),
256
257	TP_fast_assign(
258		struct scatterlist *sg;
259		int i;
260
261		__assign_str(device);
262		for_each_sg(sgt->sgl, sg, sgt->orig_nents, i)
263			((u64 *)__get_dynamic_array(phys_addrs))[i] = sg_phys(sg);
264		__entry->dma_addr = sg_dma_address(sgt->sgl);
265		__entry->size = size;
266		__entry->dir = dir;
267	),
268
269	TP_printk("%s dir=%s dma_addr=%llx size=%zu phys_addrs=%s",
270		__get_str(device),
271		decode_dma_data_direction(__entry->dir),
272		__entry->dma_addr,
273		__entry->size,
274		__print_array(__get_dynamic_array(phys_addrs),
275			      __get_dynamic_array_len(phys_addrs) /
276				sizeof(u64), sizeof(u64)))
277);
278
279#define DMA_TRACE_MAX_ENTRIES 128
280
281TRACE_EVENT(dma_map_sg,
282	TP_PROTO(struct device *dev, struct scatterlist *sgl, int nents,
283		 int ents, enum dma_data_direction dir, unsigned long attrs),
284	TP_ARGS(dev, sgl, nents, ents, dir, attrs),
285
286	TP_STRUCT__entry(
287		__string(device, dev_name(dev))
288		__field(int, full_nents)
289		__field(int, full_ents)
290		__field(bool, truncated)
291		__dynamic_array(u64, phys_addrs,  min(nents, DMA_TRACE_MAX_ENTRIES))
292		__dynamic_array(u64, dma_addrs, min(ents, DMA_TRACE_MAX_ENTRIES))
293		__dynamic_array(unsigned int, lengths, min(ents, DMA_TRACE_MAX_ENTRIES))
294		__field(enum dma_data_direction, dir)
295		__field(unsigned long, attrs)
296	),
297
298	TP_fast_assign(
299		struct scatterlist *sg;
300		int i;
301		int traced_nents = min_t(int, nents, DMA_TRACE_MAX_ENTRIES);
302		int traced_ents = min_t(int, ents, DMA_TRACE_MAX_ENTRIES);
303
304		__assign_str(device);
305		__entry->full_nents = nents;
306		__entry->full_ents = ents;
307		__entry->truncated = (nents > DMA_TRACE_MAX_ENTRIES) || (ents > DMA_TRACE_MAX_ENTRIES);
308		for_each_sg(sgl, sg, traced_nents, i)
309			((u64 *)__get_dynamic_array(phys_addrs))[i] = sg_phys(sg);
310		for_each_sg(sgl, sg, traced_ents, i) {
311			((u64 *)__get_dynamic_array(dma_addrs))[i] =
312				sg_dma_address(sg);
313			((unsigned int *)__get_dynamic_array(lengths))[i] =
314				sg_dma_len(sg);
315		}
316		__entry->dir = dir;
317		__entry->attrs = attrs;
318	),
319
320	TP_printk("%s dir=%s nents=%d/%d ents=%d/%d%s dma_addrs=%s sizes=%s phys_addrs=%s attrs=%s",
321		__get_str(device),
322		decode_dma_data_direction(__entry->dir),
323		min_t(int, __entry->full_nents, DMA_TRACE_MAX_ENTRIES), __entry->full_nents,
324		min_t(int, __entry->full_ents, DMA_TRACE_MAX_ENTRIES), __entry->full_ents,
325		__entry->truncated ? " [TRUNCATED]" : "",
326		__print_array(__get_dynamic_array(dma_addrs),
327			      __get_dynamic_array_len(dma_addrs) /
328				sizeof(u64), sizeof(u64)),
329		__print_array(__get_dynamic_array(lengths),
330			      __get_dynamic_array_len(lengths) /
331				sizeof(unsigned int), sizeof(unsigned int)),
332		__print_array(__get_dynamic_array(phys_addrs),
333			      __get_dynamic_array_len(phys_addrs) /
334				sizeof(u64), sizeof(u64)),
335		decode_dma_attrs(__entry->attrs))
336);
337
338TRACE_EVENT(dma_map_sg_err,
339	TP_PROTO(struct device *dev, struct scatterlist *sgl, int nents,
340		 int err, enum dma_data_direction dir, unsigned long attrs),
341	TP_ARGS(dev, sgl, nents, err, dir, attrs),
342
343	TP_STRUCT__entry(
344		__string(device, dev_name(dev))
345		__dynamic_array(u64, phys_addrs, nents)
346		__field(int, err)
347		__field(enum dma_data_direction, dir)
348		__field(unsigned long, attrs)
349	),
350
351	TP_fast_assign(
352		struct scatterlist *sg;
353		int i;
354
355		__assign_str(device);
356		for_each_sg(sgl, sg, nents, i)
357			((u64 *)__get_dynamic_array(phys_addrs))[i] = sg_phys(sg);
358		__entry->err = err;
359		__entry->dir = dir;
360		__entry->attrs = attrs;
361	),
362
363	TP_printk("%s dir=%s dma_addrs=%s err=%d attrs=%s",
364		__get_str(device),
365		decode_dma_data_direction(__entry->dir),
366		__print_array(__get_dynamic_array(phys_addrs),
367			      __get_dynamic_array_len(phys_addrs) /
368				sizeof(u64), sizeof(u64)),
369		__entry->err,
370		decode_dma_attrs(__entry->attrs))
371);
372
373TRACE_EVENT(dma_unmap_sg,
374	TP_PROTO(struct device *dev, struct scatterlist *sgl, int nents,
375		 enum dma_data_direction dir, unsigned long attrs),
376	TP_ARGS(dev, sgl, nents, dir, attrs),
377
378	TP_STRUCT__entry(
379		__string(device, dev_name(dev))
380		__dynamic_array(u64, addrs, nents)
381		__field(enum dma_data_direction, dir)
382		__field(unsigned long, attrs)
383	),
384
385	TP_fast_assign(
386		struct scatterlist *sg;
387		int i;
388
389		__assign_str(device);
390		for_each_sg(sgl, sg, nents, i)
391			((u64 *)__get_dynamic_array(addrs))[i] = sg_phys(sg);
392		__entry->dir = dir;
393		__entry->attrs = attrs;
394	),
395
396	TP_printk("%s dir=%s phys_addrs=%s attrs=%s",
397		__get_str(device),
398		decode_dma_data_direction(__entry->dir),
399		__print_array(__get_dynamic_array(addrs),
400			      __get_dynamic_array_len(addrs) /
401				sizeof(u64), sizeof(u64)),
402		decode_dma_attrs(__entry->attrs))
403);
404
405DECLARE_EVENT_CLASS(dma_sync_single,
406	TP_PROTO(struct device *dev, dma_addr_t dma_addr, size_t size,
407		 enum dma_data_direction dir),
408	TP_ARGS(dev, dma_addr, size, dir),
409
410	TP_STRUCT__entry(
411		__string(device, dev_name(dev))
412		__field(u64, dma_addr)
413		__field(size_t, size)
414		__field(enum dma_data_direction, dir)
415	),
416
417	TP_fast_assign(
418		__assign_str(device);
419		__entry->dma_addr = dma_addr;
420		__entry->size = size;
421		__entry->dir = dir;
422	),
423
424	TP_printk("%s dir=%s dma_addr=%llx size=%zu",
425		__get_str(device),
426		decode_dma_data_direction(__entry->dir),
427		__entry->dma_addr,
428		__entry->size)
429);
430
431#define DEFINE_SYNC_SINGLE_EVENT(name) \
432DEFINE_EVENT(dma_sync_single, name, \
433	TP_PROTO(struct device *dev, dma_addr_t dma_addr, size_t size, \
434		 enum dma_data_direction dir), \
435	TP_ARGS(dev, dma_addr, size, dir))
436
437DEFINE_SYNC_SINGLE_EVENT(dma_sync_single_for_cpu);
438DEFINE_SYNC_SINGLE_EVENT(dma_sync_single_for_device);
439
440DECLARE_EVENT_CLASS(dma_sync_sg,
441	TP_PROTO(struct device *dev, struct scatterlist *sgl, int nents,
442		 enum dma_data_direction dir),
443	TP_ARGS(dev, sgl, nents, dir),
444
445	TP_STRUCT__entry(
446		__string(device, dev_name(dev))
447		__dynamic_array(u64, dma_addrs, nents)
448		__dynamic_array(unsigned int, lengths, nents)
449		__field(enum dma_data_direction, dir)
450	),
451
452	TP_fast_assign(
453		struct scatterlist *sg;
454		int i;
455
456		__assign_str(device);
457		for_each_sg(sgl, sg, nents, i) {
458			((u64 *)__get_dynamic_array(dma_addrs))[i] =
459				sg_dma_address(sg);
460			((unsigned int *)__get_dynamic_array(lengths))[i] =
461				sg_dma_len(sg);
462		}
463		__entry->dir = dir;
464	),
465
466	TP_printk("%s dir=%s dma_addrs=%s sizes=%s",
467		__get_str(device),
468		decode_dma_data_direction(__entry->dir),
469		__print_array(__get_dynamic_array(dma_addrs),
470			      __get_dynamic_array_len(dma_addrs) /
471				sizeof(u64), sizeof(u64)),
472		__print_array(__get_dynamic_array(lengths),
473			      __get_dynamic_array_len(lengths) /
474				sizeof(unsigned int), sizeof(unsigned int)))
475);
476
477#define DEFINE_SYNC_SG_EVENT(name) \
478DEFINE_EVENT(dma_sync_sg, name, \
479	TP_PROTO(struct device *dev, struct scatterlist *sg, int nents, \
480		 enum dma_data_direction dir), \
481	TP_ARGS(dev, sg, nents, dir))
482
483DEFINE_SYNC_SG_EVENT(dma_sync_sg_for_cpu);
484DEFINE_SYNC_SG_EVENT(dma_sync_sg_for_device);
485
486#endif /*  _TRACE_DMA_H */
487
488/* This part must be outside protection */
489#include <trace/define_trace.h>