pt_fmt_defaults.h

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES
  4 *
  5 * Default definitions for formats that don't define these functions.
  6 */
  7#ifndef __GENERIC_PT_PT_FMT_DEFAULTS_H
  8#define __GENERIC_PT_PT_FMT_DEFAULTS_H
  9
 10#include "pt_defs.h"
 11#include <linux/log2.h>
 12
 13/* Header self-compile default defines */
 14#ifndef pt_load_entry_raw
 15#include "fmt/amdv1.h"
 16#endif
 17
 18/*
 19 * The format must provide PT_GRANULE_LG2SZ, PT_TABLEMEM_LG2SZ, and
 20 * PT_ITEM_WORD_SIZE. They must be the same at every level excluding the top.
 21 */
 22#ifndef pt_table_item_lg2sz
 23static inline unsigned int pt_table_item_lg2sz(const struct pt_state *pts)
 24{
 25	return PT_GRANULE_LG2SZ +
 26	       (PT_TABLEMEM_LG2SZ - ilog2(PT_ITEM_WORD_SIZE)) * pts->level;
 27}
 28#endif
 29
 30#ifndef pt_pgsz_lg2_to_level
 31static inline unsigned int pt_pgsz_lg2_to_level(struct pt_common *common,
 32						unsigned int pgsize_lg2)
 33{
 34	return ((unsigned int)(pgsize_lg2 - PT_GRANULE_LG2SZ)) /
 35	       (PT_TABLEMEM_LG2SZ - ilog2(PT_ITEM_WORD_SIZE));
 36}
 37#endif
 38
 39/*
 40 * If not supplied by the format then contiguous pages are not supported.
 41 *
 42 * If contiguous pages are supported then the format must also provide
 43 * pt_contig_count_lg2() if it supports a single contiguous size per level,
 44 * or pt_possible_sizes() if it supports multiple sizes per level.
 45 */
 46#ifndef pt_entry_num_contig_lg2
 47static inline unsigned int pt_entry_num_contig_lg2(const struct pt_state *pts)
 48{
 49	return ilog2(1);
 50}
 51
 52/*
 53 * Return the number of contiguous OA items forming an entry at this table level
 54 */
 55static inline unsigned short pt_contig_count_lg2(const struct pt_state *pts)
 56{
 57	return ilog2(1);
 58}
 59#endif
 60
 61/* If not supplied by the format then dirty tracking is not supported */
 62#ifndef pt_entry_is_write_dirty
 63static inline bool pt_entry_is_write_dirty(const struct pt_state *pts)
 64{
 65	return false;
 66}
 67
 68static inline void pt_entry_make_write_clean(struct pt_state *pts)
 69{
 70}
 71
 72static inline bool pt_dirty_supported(struct pt_common *common)
 73{
 74	return false;
 75}
 76#else
 77/* If not supplied then dirty tracking is always enabled */
 78#ifndef pt_dirty_supported
 79static inline bool pt_dirty_supported(struct pt_common *common)
 80{
 81	return true;
 82}
 83#endif
 84#endif
 85
 86#ifndef pt_entry_make_write_dirty
 87static inline bool pt_entry_make_write_dirty(struct pt_state *pts)
 88{
 89	return false;
 90}
 91#endif
 92
 93/*
 94 * Format supplies either:
 95 *   pt_entry_oa - OA is at the start of a contiguous entry
 96 * or
 97 *   pt_item_oa  - OA is adjusted for every item in a contiguous entry
 98 *
 99 * Build the missing one
100 *
101 * The internal helper _pt_entry_oa_fast() allows generating
102 * an efficient pt_entry_oa_exact(), it doesn't care which
103 * option is selected.
104 */
105#ifdef pt_entry_oa
106static inline pt_oaddr_t pt_item_oa(const struct pt_state *pts)
107{
108	return pt_entry_oa(pts) |
109	       log2_mul(pts->index, pt_table_item_lg2sz(pts));
110}
111#define _pt_entry_oa_fast pt_entry_oa
112#endif
113
114#ifdef pt_item_oa
115static inline pt_oaddr_t pt_entry_oa(const struct pt_state *pts)
116{
117	return log2_set_mod(pt_item_oa(pts), 0,
118			    pt_entry_num_contig_lg2(pts) +
119				    pt_table_item_lg2sz(pts));
120}
121#define _pt_entry_oa_fast pt_item_oa
122#endif
123
124/*
125 * If not supplied by the format then use the constant
126 * PT_MAX_OUTPUT_ADDRESS_LG2.
127 */
128#ifndef pt_max_oa_lg2
129static inline unsigned int
130pt_max_oa_lg2(const struct pt_common *common)
131{
132	return PT_MAX_OUTPUT_ADDRESS_LG2;
133}
134#endif
135
136#ifndef pt_has_system_page_size
137static inline bool pt_has_system_page_size(const struct pt_common *common)
138{
139	return PT_GRANULE_LG2SZ == PAGE_SHIFT;
140}
141#endif
142
143/*
144 * If not supplied by the format then assume only one contiguous size determined
145 * by pt_contig_count_lg2()
146 */
147#ifndef pt_possible_sizes
148static inline unsigned short pt_contig_count_lg2(const struct pt_state *pts);
149
150/* Return a bitmap of possible leaf page sizes at this level */
151static inline pt_vaddr_t pt_possible_sizes(const struct pt_state *pts)
152{
153	unsigned int isz_lg2 = pt_table_item_lg2sz(pts);
154
155	if (!pt_can_have_leaf(pts))
156		return 0;
157	return log2_to_int(isz_lg2) |
158	       log2_to_int(pt_contig_count_lg2(pts) + isz_lg2);
159}
160#endif
161
162/* If not supplied by the format then use 0. */
163#ifndef pt_full_va_prefix
164static inline pt_vaddr_t pt_full_va_prefix(const struct pt_common *common)
165{
166	return 0;
167}
168#endif
169
170/* If not supplied by the format then zero fill using PT_ITEM_WORD_SIZE */
171#ifndef pt_clear_entries
172static inline void pt_clear_entries64(struct pt_state *pts,
173				      unsigned int num_contig_lg2)
174{
175	u64 *tablep = pt_cur_table(pts, u64) + pts->index;
176	u64 *end = tablep + log2_to_int(num_contig_lg2);
177
178	PT_WARN_ON(log2_mod(pts->index, num_contig_lg2));
179	for (; tablep != end; tablep++)
180		WRITE_ONCE(*tablep, 0);
181}
182
183static inline void pt_clear_entries32(struct pt_state *pts,
184				      unsigned int num_contig_lg2)
185{
186	u32 *tablep = pt_cur_table(pts, u32) + pts->index;
187	u32 *end = tablep + log2_to_int(num_contig_lg2);
188
189	PT_WARN_ON(log2_mod(pts->index, num_contig_lg2));
190	for (; tablep != end; tablep++)
191		WRITE_ONCE(*tablep, 0);
192}
193
194static inline void pt_clear_entries(struct pt_state *pts,
195				    unsigned int num_contig_lg2)
196{
197	if (PT_ITEM_WORD_SIZE == sizeof(u32))
198		pt_clear_entries32(pts, num_contig_lg2);
199	else
200		pt_clear_entries64(pts, num_contig_lg2);
201}
202#define pt_clear_entries pt_clear_entries
203#endif
204
205/* If not supplied then SW bits are not supported */
206#ifdef pt_sw_bit
207static inline bool pt_test_sw_bit_acquire(struct pt_state *pts,
208					  unsigned int bitnr)
209{
210	/* Acquire, pairs with pt_set_sw_bit_release() */
211	smp_mb();
212	/* For a contiguous entry the sw bit is only stored in the first item. */
213	return pts->entry & pt_sw_bit(bitnr);
214}
215#define pt_test_sw_bit_acquire pt_test_sw_bit_acquire
216
217static inline void pt_set_sw_bit_release(struct pt_state *pts,
218					 unsigned int bitnr)
219{
220#if !IS_ENABLED(CONFIG_GENERIC_ATOMIC64)
221	if (PT_ITEM_WORD_SIZE == sizeof(u64)) {
222		u64 *entryp = pt_cur_table(pts, u64) + pts->index;
223		u64 old_entry = pts->entry;
224		u64 new_entry;
225
226		do {
227			new_entry = old_entry | pt_sw_bit(bitnr);
228		} while (!try_cmpxchg64_release(entryp, &old_entry, new_entry));
229		pts->entry = new_entry;
230		return;
231	}
232#endif
233	if (PT_ITEM_WORD_SIZE == sizeof(u32)) {
234		u32 *entryp = pt_cur_table(pts, u32) + pts->index;
235		u32 old_entry = pts->entry;
236		u32 new_entry;
237
238		do {
239			new_entry = old_entry | pt_sw_bit(bitnr);
240		} while (!try_cmpxchg_release(entryp, &old_entry, new_entry));
241		pts->entry = new_entry;
242	} else
243		BUILD_BUG();
244}
245#define pt_set_sw_bit_release pt_set_sw_bit_release
246#else
247static inline unsigned int pt_max_sw_bit(struct pt_common *common)
248{
249	return 0;
250}
251
252extern void __pt_no_sw_bit(void);
253static inline bool pt_test_sw_bit_acquire(struct pt_state *pts,
254					  unsigned int bitnr)
255{
256	__pt_no_sw_bit();
257	return false;
258}
259
260static inline void pt_set_sw_bit_release(struct pt_state *pts,
261					 unsigned int bitnr)
262{
263	__pt_no_sw_bit();
264}
265#endif
266
267/*
268 * Format can call in the pt_install_leaf_entry() to check the arguments are all
269 * aligned correctly.
270 */
271static inline bool pt_check_install_leaf_args(struct pt_state *pts,
272					      pt_oaddr_t oa,
273					      unsigned int oasz_lg2)
274{
275	unsigned int isz_lg2 = pt_table_item_lg2sz(pts);
276
277	if (PT_WARN_ON(oalog2_mod(oa, oasz_lg2)))
278		return false;
279
280#ifdef pt_possible_sizes
281	if (PT_WARN_ON(isz_lg2 > oasz_lg2 ||
282		       oasz_lg2 > isz_lg2 + pt_num_items_lg2(pts)))
283		return false;
284#else
285	if (PT_WARN_ON(oasz_lg2 != isz_lg2 &&
286		       oasz_lg2 != isz_lg2 + pt_contig_count_lg2(pts)))
287		return false;
288#endif
289
290	if (PT_WARN_ON(oalog2_mod(pts->index, oasz_lg2 - isz_lg2)))
291		return false;
292	return true;
293}
294
295#endif