copypage.c

 1// SPDX-License-Identifier: GPL-2.0-only
 2/*
 3 * Based on arch/arm/mm/copypage.c
 4 *
 5 * Copyright (C) 2002 Deep Blue Solutions Ltd, All Rights Reserved.
 6 * Copyright (C) 2012 ARM Ltd.
 7 */
 8
 9#include <linux/bitops.h>
10#include <linux/mm.h>
11
12#include <asm/page.h>
13#include <asm/cacheflush.h>
14#include <asm/cpufeature.h>
15#include <asm/mte.h>
16
17void copy_highpage(struct page *to, struct page *from)
18{
19	void *kto = page_address(to);
20	void *kfrom = page_address(from);
21	struct folio *src = page_folio(from);
22	struct folio *dst = page_folio(to);
23	unsigned int i, nr_pages;
24
25	copy_page(kto, kfrom);
26
27	if (kasan_hw_tags_enabled())
28		page_kasan_tag_reset(to);
29
30	if (!system_supports_mte())
31		return;
32
33	if (folio_test_hugetlb(src)) {
34		if (!folio_test_hugetlb_mte_tagged(src) ||
35		    from != folio_page(src, 0))
36			return;
37
38		folio_try_hugetlb_mte_tagging(dst);
39
40		/*
41		 * Populate tags for all subpages.
42		 *
43		 * Don't assume the first page is head page since
44		 * huge page copy may start from any subpage.
45		 */
46		nr_pages = folio_nr_pages(src);
47		for (i = 0; i < nr_pages; i++) {
48			kfrom = page_address(folio_page(src, i));
49			kto = page_address(folio_page(dst, i));
50			mte_copy_page_tags(kto, kfrom);
51		}
52		folio_set_hugetlb_mte_tagged(dst);
53	} else if (page_mte_tagged(from)) {
54		/*
55		 * Most of the time it's a new page that shouldn't have been
56		 * tagged yet. However, folio migration can end up reusing the
57		 * same page without untagging it. Ignore the warning if the
58		 * page is already tagged.
59		 */
60		try_page_mte_tagging(to);
61
62		mte_copy_page_tags(kto, kfrom);
63		set_page_mte_tagged(to);
64	}
65}
66EXPORT_SYMBOL(copy_highpage);
67
68void copy_user_highpage(struct page *to, struct page *from,
69			unsigned long vaddr, struct vm_area_struct *vma)
70{
71	copy_highpage(to, from);
72	flush_dcache_page(to);
73}
74EXPORT_SYMBOL_GPL(copy_user_highpage);