leafops.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * Describes operations that can be performed on software-defined page table
  4 * leaf entries. These are abstracted from the hardware page table entries
  5 * themselves by the softleaf_t type, see mm_types.h.
  6 */
  7#ifndef _LINUX_LEAFOPS_H
  8#define _LINUX_LEAFOPS_H
  9
 10#include <linux/mm_types.h>
 11#include <linux/swapops.h>
 12#include <linux/swap.h>
 13
 14#ifdef CONFIG_MMU
 15
 16/* Temporary until swp_entry_t eliminated. */
 17#define LEAF_TYPE_SHIFT SWP_TYPE_SHIFT
 18
 19enum softleaf_type {
 20	/* Fundamental types. */
 21	SOFTLEAF_NONE,
 22	SOFTLEAF_SWAP,
 23	/* Migration types. */
 24	SOFTLEAF_MIGRATION_READ,
 25	SOFTLEAF_MIGRATION_READ_EXCLUSIVE,
 26	SOFTLEAF_MIGRATION_WRITE,
 27	/* Device types. */
 28	SOFTLEAF_DEVICE_PRIVATE_READ,
 29	SOFTLEAF_DEVICE_PRIVATE_WRITE,
 30	SOFTLEAF_DEVICE_EXCLUSIVE,
 31	/* H/W posion types. */
 32	SOFTLEAF_HWPOISON,
 33	/* Marker types. */
 34	SOFTLEAF_MARKER,
 35};
 36
 37/**
 38 * softleaf_mk_none() - Create an empty ('none') leaf entry.
 39 * Returns: empty leaf entry.
 40 */
 41static inline softleaf_t softleaf_mk_none(void)
 42{
 43	return ((softleaf_t) { 0 });
 44}
 45
 46/**
 47 * softleaf_from_pte() - Obtain a leaf entry from a PTE entry.
 48 * @pte: PTE entry.
 49 *
 50 * If @pte is present (therefore not a leaf entry) the function returns an empty
 51 * leaf entry. Otherwise, it returns a leaf entry.
 52 *
 53 * Returns: Leaf entry.
 54 */
 55static inline softleaf_t softleaf_from_pte(pte_t pte)
 56{
 57	softleaf_t arch_entry;
 58
 59	if (pte_present(pte) || pte_none(pte))
 60		return softleaf_mk_none();
 61
 62	pte = pte_swp_clear_flags(pte);
 63	arch_entry = __pte_to_swp_entry(pte);
 64
 65	/* Temporary until swp_entry_t eliminated. */
 66	return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
 67}
 68
 69/**
 70 * softleaf_to_pte() - Obtain a PTE entry from a leaf entry.
 71 * @entry: Leaf entry.
 72 *
 73 * This generates an architecture-specific PTE entry that can be utilised to
 74 * encode the metadata the leaf entry encodes.
 75 *
 76 * Returns: Architecture-specific PTE entry encoding leaf entry.
 77 */
 78static inline pte_t softleaf_to_pte(softleaf_t entry)
 79{
 80	/* Temporary until swp_entry_t eliminated. */
 81	return swp_entry_to_pte(entry);
 82}
 83
 84#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
 85/**
 86 * softleaf_from_pmd() - Obtain a leaf entry from a PMD entry.
 87 * @pmd: PMD entry.
 88 *
 89 * If @pmd is present (therefore not a leaf entry) the function returns an empty
 90 * leaf entry. Otherwise, it returns a leaf entry.
 91 *
 92 * Returns: Leaf entry.
 93 */
 94static inline softleaf_t softleaf_from_pmd(pmd_t pmd)
 95{
 96	softleaf_t arch_entry;
 97
 98	if (pmd_present(pmd) || pmd_none(pmd))
 99		return softleaf_mk_none();
100
101	if (pmd_swp_soft_dirty(pmd))
102		pmd = pmd_swp_clear_soft_dirty(pmd);
103	if (pmd_swp_uffd_wp(pmd))
104		pmd = pmd_swp_clear_uffd_wp(pmd);
105	arch_entry = __pmd_to_swp_entry(pmd);
106
107	/* Temporary until swp_entry_t eliminated. */
108	return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
109}
110
111#else
112
113static inline softleaf_t softleaf_from_pmd(pmd_t pmd)
114{
115	return softleaf_mk_none();
116}
117
118#endif
119
120/**
121 * softleaf_is_none() - Is the leaf entry empty?
122 * @entry: Leaf entry.
123 *
124 * Empty entries are typically the result of a 'none' page table leaf entry
125 * being converted to a leaf entry.
126 *
127 * Returns: true if the entry is empty, false otherwise.
128 */
129static inline bool softleaf_is_none(softleaf_t entry)
130{
131	return entry.val == 0;
132}
133
134/**
135 * softleaf_type() - Identify the type of leaf entry.
136 * @entry: Leaf entry.
137 *
138 * Returns: the leaf entry type associated with @entry.
139 */
140static inline enum softleaf_type softleaf_type(softleaf_t entry)
141{
142	unsigned int type_num;
143
144	if (softleaf_is_none(entry))
145		return SOFTLEAF_NONE;
146
147	type_num = entry.val >> LEAF_TYPE_SHIFT;
148
149	if (type_num < MAX_SWAPFILES)
150		return SOFTLEAF_SWAP;
151
152	switch (type_num) {
153#ifdef CONFIG_MIGRATION
154	case SWP_MIGRATION_READ:
155		return SOFTLEAF_MIGRATION_READ;
156	case SWP_MIGRATION_READ_EXCLUSIVE:
157		return SOFTLEAF_MIGRATION_READ_EXCLUSIVE;
158	case SWP_MIGRATION_WRITE:
159		return SOFTLEAF_MIGRATION_WRITE;
160#endif
161#ifdef CONFIG_DEVICE_PRIVATE
162	case SWP_DEVICE_WRITE:
163		return SOFTLEAF_DEVICE_PRIVATE_WRITE;
164	case SWP_DEVICE_READ:
165		return SOFTLEAF_DEVICE_PRIVATE_READ;
166	case SWP_DEVICE_EXCLUSIVE:
167		return SOFTLEAF_DEVICE_EXCLUSIVE;
168#endif
169#ifdef CONFIG_MEMORY_FAILURE
170	case SWP_HWPOISON:
171		return SOFTLEAF_HWPOISON;
172#endif
173	case SWP_PTE_MARKER:
174		return SOFTLEAF_MARKER;
175	}
176
177	/* Unknown entry type. */
178	VM_WARN_ON_ONCE(1);
179	return SOFTLEAF_NONE;
180}
181
182/**
183 * softleaf_is_swap() - Is this leaf entry a swap entry?
184 * @entry: Leaf entry.
185 *
186 * Returns: true if the leaf entry is a swap entry, otherwise false.
187 */
188static inline bool softleaf_is_swap(softleaf_t entry)
189{
190	return softleaf_type(entry) == SOFTLEAF_SWAP;
191}
192
193/**
194 * softleaf_is_migration_write() - Is this leaf entry a writable migration entry?
195 * @entry: Leaf entry.
196 *
197 * Returns: true if the leaf entry is a writable migration entry, otherwise
198 * false.
199 */
200static inline bool softleaf_is_migration_write(softleaf_t entry)
201{
202	return softleaf_type(entry) == SOFTLEAF_MIGRATION_WRITE;
203}
204
205/**
206 * softleaf_is_migration_read() - Is this leaf entry a readable migration entry?
207 * @entry: Leaf entry.
208 *
209 * Returns: true if the leaf entry is a readable migration entry, otherwise
210 * false.
211 */
212static inline bool softleaf_is_migration_read(softleaf_t entry)
213{
214	return softleaf_type(entry) == SOFTLEAF_MIGRATION_READ;
215}
216
217/**
218 * softleaf_is_migration_read_exclusive() - Is this leaf entry an exclusive
219 * readable migration entry?
220 * @entry: Leaf entry.
221 *
222 * Returns: true if the leaf entry is an exclusive readable migration entry,
223 * otherwise false.
224 */
225static inline bool softleaf_is_migration_read_exclusive(softleaf_t entry)
226{
227	return softleaf_type(entry) == SOFTLEAF_MIGRATION_READ_EXCLUSIVE;
228}
229
230/**
231 * softleaf_is_migration() - Is this leaf entry a migration entry?
232 * @entry: Leaf entry.
233 *
234 * Returns: true if the leaf entry is a migration entry, otherwise false.
235 */
236static inline bool softleaf_is_migration(softleaf_t entry)
237{
238	switch (softleaf_type(entry)) {
239	case SOFTLEAF_MIGRATION_READ:
240	case SOFTLEAF_MIGRATION_READ_EXCLUSIVE:
241	case SOFTLEAF_MIGRATION_WRITE:
242		return true;
243	default:
244		return false;
245	}
246}
247
248/**
249 * softleaf_is_device_private_write() - Is this leaf entry a device private
250 * writable entry?
251 * @entry: Leaf entry.
252 *
253 * Returns: true if the leaf entry is a device private writable entry, otherwise
254 * false.
255 */
256static inline bool softleaf_is_device_private_write(softleaf_t entry)
257{
258	return softleaf_type(entry) == SOFTLEAF_DEVICE_PRIVATE_WRITE;
259}
260
261/**
262 * softleaf_is_device_private() - Is this leaf entry a device private entry?
263 * @entry: Leaf entry.
264 *
265 * Returns: true if the leaf entry is a device private entry, otherwise false.
266 */
267static inline bool softleaf_is_device_private(softleaf_t entry)
268{
269	switch (softleaf_type(entry)) {
270	case SOFTLEAF_DEVICE_PRIVATE_WRITE:
271	case SOFTLEAF_DEVICE_PRIVATE_READ:
272		return true;
273	default:
274		return false;
275	}
276}
277
278/**
279 * softleaf_is_device_exclusive() - Is this leaf entry a device-exclusive entry?
280 * @entry: Leaf entry.
281 *
282 * Returns: true if the leaf entry is a device-exclusive entry, otherwise false.
283 */
284static inline bool softleaf_is_device_exclusive(softleaf_t entry)
285{
286	return softleaf_type(entry) == SOFTLEAF_DEVICE_EXCLUSIVE;
287}
288
289/**
290 * softleaf_is_hwpoison() - Is this leaf entry a hardware poison entry?
291 * @entry: Leaf entry.
292 *
293 * Returns: true if the leaf entry is a hardware poison entry, otherwise false.
294 */
295static inline bool softleaf_is_hwpoison(softleaf_t entry)
296{
297	return softleaf_type(entry) == SOFTLEAF_HWPOISON;
298}
299
300/**
301 * softleaf_is_marker() - Is this leaf entry a marker?
302 * @entry: Leaf entry.
303 *
304 * Returns: true if the leaf entry is a marker entry, otherwise false.
305 */
306static inline bool softleaf_is_marker(softleaf_t entry)
307{
308	return softleaf_type(entry) == SOFTLEAF_MARKER;
309}
310
311/**
312 * softleaf_to_marker() - Obtain marker associated with leaf entry.
313 * @entry: Leaf entry, softleaf_is_marker(@entry) must return true.
314 *
315 * Returns: Marker associated with the leaf entry.
316 */
317static inline pte_marker softleaf_to_marker(softleaf_t entry)
318{
319	VM_WARN_ON_ONCE(!softleaf_is_marker(entry));
320
321	return swp_offset(entry) & PTE_MARKER_MASK;
322}
323
324/**
325 * softleaf_has_pfn() - Does this leaf entry encode a valid PFN number?
326 * @entry: Leaf entry.
327 *
328 * A pfn swap entry is a special type of swap entry that always has a pfn stored
329 * in the swap offset. They can either be used to represent unaddressable device
330 * memory, to restrict access to a page undergoing migration or to represent a
331 * pfn which has been hwpoisoned and unmapped.
332 *
333 * Returns: true if the leaf entry encodes a PFN, otherwise false.
334 */
335static inline bool softleaf_has_pfn(softleaf_t entry)
336{
337	/* Make sure the swp offset can always store the needed fields. */
338	BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS);
339
340	if (softleaf_is_migration(entry))
341		return true;
342	if (softleaf_is_device_private(entry))
343		return true;
344	if (softleaf_is_device_exclusive(entry))
345		return true;
346	if (softleaf_is_hwpoison(entry))
347		return true;
348
349	return false;
350}
351
352/**
353 * softleaf_to_pfn() - Obtain PFN encoded within leaf entry.
354 * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true.
355 *
356 * Returns: The PFN associated with the leaf entry.
357 */
358static inline unsigned long softleaf_to_pfn(softleaf_t entry)
359{
360	VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
361
362	/* Temporary until swp_entry_t eliminated. */
363	return swp_offset(entry) & SWP_PFN_MASK;
364}
365
366static inline void softleaf_migration_sync(softleaf_t entry,
367		struct folio *folio)
368{
369	/*
370	 * Ensure we do not race with split, which might alter tail pages into new
371	 * folios and thus result in observing an unlocked folio.
372	 * This matches the write barrier in __split_folio_to_order().
373	 */
374	smp_rmb();
375
376	/*
377	 * Any use of migration entries may only occur while the
378	 * corresponding page is locked
379	 */
380	VM_WARN_ON_ONCE(!folio_test_locked(folio));
381}
382
383/**
384 * softleaf_to_page() - Obtains struct page for PFN encoded within leaf entry.
385 * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true.
386 *
387 * Returns: Pointer to the struct page associated with the leaf entry's PFN.
388 */
389static inline struct page *softleaf_to_page(softleaf_t entry)
390{
391	struct page *page = pfn_to_page(softleaf_to_pfn(entry));
392
393	VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
394	if (softleaf_is_migration(entry))
395		softleaf_migration_sync(entry, page_folio(page));
396
397	return page;
398}
399
400/**
401 * softleaf_to_folio() - Obtains struct folio for PFN encoded within leaf entry.
402 * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true.
403 *
404 * Returns: Pointer to the struct folio associated with the leaf entry's PFN.
405 */
406static inline struct folio *softleaf_to_folio(softleaf_t entry)
407{
408	struct folio *folio = pfn_folio(softleaf_to_pfn(entry));
409
410	VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
411	if (softleaf_is_migration(entry))
412		softleaf_migration_sync(entry, folio);
413
414	return folio;
415}
416
417/**
418 * softleaf_is_poison_marker() - Is this leaf entry a poison marker?
419 * @entry: Leaf entry.
420 *
421 * The poison marker is set via UFFDIO_POISON. Userfaultfd-specific.
422 *
423 * Returns: true if the leaf entry is a poison marker, otherwise false.
424 */
425static inline bool softleaf_is_poison_marker(softleaf_t entry)
426{
427	if (!softleaf_is_marker(entry))
428		return false;
429
430	return softleaf_to_marker(entry) & PTE_MARKER_POISONED;
431}
432
433/**
434 * softleaf_is_guard_marker() - Is this leaf entry a guard region marker?
435 * @entry: Leaf entry.
436 *
437 * Returns: true if the leaf entry is a guard marker, otherwise false.
438 */
439static inline bool softleaf_is_guard_marker(softleaf_t entry)
440{
441	if (!softleaf_is_marker(entry))
442		return false;
443
444	return softleaf_to_marker(entry) & PTE_MARKER_GUARD;
445}
446
447/**
448 * softleaf_is_uffd_wp_marker() - Is this leaf entry a userfautlfd write protect
449 * marker?
450 * @entry: Leaf entry.
451 *
452 * Userfaultfd-specific.
453 *
454 * Returns: true if the leaf entry is a UFFD WP marker, otherwise false.
455 */
456static inline bool softleaf_is_uffd_wp_marker(softleaf_t entry)
457{
458	if (!softleaf_is_marker(entry))
459		return false;
460
461	return softleaf_to_marker(entry) & PTE_MARKER_UFFD_WP;
462}
463
464#ifdef CONFIG_MIGRATION
465
466/**
467 * softleaf_is_migration_young() - Does this migration entry contain an accessed
468 * bit?
469 * @entry: Leaf entry.
470 *
471 * If the architecture can support storing A/D bits in migration entries, this
472 * determines whether the accessed (or 'young') bit was set on the migrated page
473 * table entry.
474 *
475 * Returns: true if the entry contains an accessed bit, otherwise false.
476 */
477static inline bool softleaf_is_migration_young(softleaf_t entry)
478{
479	VM_WARN_ON_ONCE(!softleaf_is_migration(entry));
480
481	if (migration_entry_supports_ad())
482		return swp_offset(entry) & SWP_MIG_YOUNG;
483	/* Keep the old behavior of aging page after migration */
484	return false;
485}
486
487/**
488 * softleaf_is_migration_dirty() - Does this migration entry contain a dirty bit?
489 * @entry: Leaf entry.
490 *
491 * If the architecture can support storing A/D bits in migration entries, this
492 * determines whether the dirty bit was set on the migrated page table entry.
493 *
494 * Returns: true if the entry contains a dirty bit, otherwise false.
495 */
496static inline bool softleaf_is_migration_dirty(softleaf_t entry)
497{
498	VM_WARN_ON_ONCE(!softleaf_is_migration(entry));
499
500	if (migration_entry_supports_ad())
501		return swp_offset(entry) & SWP_MIG_DIRTY;
502	/* Keep the old behavior of clean page after migration */
503	return false;
504}
505
506#else /* CONFIG_MIGRATION */
507
508static inline bool softleaf_is_migration_young(softleaf_t entry)
509{
510	return false;
511}
512
513static inline bool softleaf_is_migration_dirty(softleaf_t entry)
514{
515	return false;
516}
517#endif /* CONFIG_MIGRATION */
518
519/**
520 * pte_is_marker() - Does the PTE entry encode a marker leaf entry?
521 * @pte: PTE entry.
522 *
523 * Returns: true if this PTE is a marker leaf entry, otherwise false.
524 */
525static inline bool pte_is_marker(pte_t pte)
526{
527	return softleaf_is_marker(softleaf_from_pte(pte));
528}
529
530/**
531 * pte_is_uffd_wp_marker() - Does this PTE entry encode a userfaultfd write
532 * protect marker leaf entry?
533 * @pte: PTE entry.
534 *
535 * Returns: true if this PTE is a UFFD WP marker leaf entry, otherwise false.
536 */
537static inline bool pte_is_uffd_wp_marker(pte_t pte)
538{
539	const softleaf_t entry = softleaf_from_pte(pte);
540
541	return softleaf_is_uffd_wp_marker(entry);
542}
543
544/**
545 * pte_is_uffd_marker() - Does this PTE entry encode a userfault-specific marker
546 * leaf entry?
547 * @pte: PTE entry.
548 *
549 * It's useful to be able to determine which leaf entries encode UFFD-specific
550 * markers so we can handle these correctly.
551 *
552 * Returns: true if this PTE entry is a UFFD-specific marker, otherwise false.
553 */
554static inline bool pte_is_uffd_marker(pte_t pte)
555{
556	const softleaf_t entry = softleaf_from_pte(pte);
557
558	if (!softleaf_is_marker(entry))
559		return false;
560
561	/* UFFD WP, poisoned swap entries are UFFD-handled. */
562	if (softleaf_is_uffd_wp_marker(entry))
563		return true;
564	if (softleaf_is_poison_marker(entry))
565		return true;
566
567	return false;
568}
569
570#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION)
571
572/**
573 * pmd_is_device_private_entry() - Check if PMD contains a device private swap
574 * entry.
575 * @pmd: The PMD to check.
576 *
577 * Returns true if the PMD contains a swap entry that represents a device private
578 * page mapping. This is used for zone device private pages that have been
579 * swapped out but still need special handling during various memory management
580 * operations.
581 *
582 * Return: true if PMD contains device private entry, false otherwise
583 */
584static inline bool pmd_is_device_private_entry(pmd_t pmd)
585{
586	return softleaf_is_device_private(softleaf_from_pmd(pmd));
587}
588
589#else  /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */
590
591static inline bool pmd_is_device_private_entry(pmd_t pmd)
592{
593	return false;
594}
595
596#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */
597
598/**
599 * pmd_is_migration_entry() - Does this PMD entry encode a migration entry?
600 * @pmd: PMD entry.
601 *
602 * Returns: true if the PMD encodes a migration entry, otherwise false.
603 */
604static inline bool pmd_is_migration_entry(pmd_t pmd)
605{
606	return softleaf_is_migration(softleaf_from_pmd(pmd));
607}
608
609/**
610 * pmd_is_valid_softleaf() - Is this PMD entry a valid leaf entry?
611 * @pmd: PMD entry.
612 *
613 * PMD leaf entries are valid only if they are device private or migration
614 * entries. This function asserts that a PMD leaf entry is valid in this
615 * respect.
616 *
617 * Returns: true if the PMD entry is a valid leaf entry, otherwise false.
618 */
619static inline bool pmd_is_valid_softleaf(pmd_t pmd)
620{
621	const softleaf_t entry = softleaf_from_pmd(pmd);
622
623	/* Only device private, migration entries valid for PMD. */
624	return softleaf_is_device_private(entry) ||
625		softleaf_is_migration(entry);
626}
627
628#endif  /* CONFIG_MMU */
629#endif  /* _LINUX_LEAFOPS_H */