memtype.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Page Attribute Table (PAT) support: handle memory caching attributes in page tables.
   4 *
   5 * Authors: Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
   6 *          Suresh B Siddha <suresh.b.siddha@intel.com>
   7 *
   8 * Loosely based on earlier PAT patchset from Eric Biederman and Andi Kleen.
   9 *
  10 * Basic principles:
  11 *
  12 * PAT is a CPU feature supported by all modern x86 CPUs, to allow the firmware and
  13 * the kernel to set one of a handful of 'caching type' attributes for physical
  14 * memory ranges: uncached, write-combining, write-through, write-protected,
  15 * and the most commonly used and default attribute: write-back caching.
  16 *
  17 * PAT support supersedes and augments MTRR support in a compatible fashion: MTRR is
  18 * a hardware interface to enumerate a limited number of physical memory ranges
  19 * and set their caching attributes explicitly, programmed into the CPU via MSRs.
  20 * Even modern CPUs have MTRRs enabled - but these are typically not touched
  21 * by the kernel or by user-space (such as the X server), we rely on PAT for any
  22 * additional cache attribute logic.
  23 *
  24 * PAT doesn't work via explicit memory ranges, but uses page table entries to add
  25 * cache attribute information to the mapped memory range: there's 3 bits used,
  26 * (_PAGE_PWT, _PAGE_PCD, _PAGE_PAT), with the 8 possible values mapped by the
  27 * CPU to actual cache attributes via an MSR loaded into the CPU (MSR_IA32_CR_PAT).
  28 *
  29 * ( There's a metric ton of finer details, such as compatibility with CPU quirks
  30 *   that only support 4 types of PAT entries, and interaction with MTRRs, see
  31 *   below for details. )
  32 */
  33
  34#include <linux/seq_file.h>
  35#include <linux/memblock.h>
  36#include <linux/debugfs.h>
  37#include <linux/ioport.h>
  38#include <linux/kernel.h>
  39#include <linux/slab.h>
  40#include <linux/io.h>
  41#include <linux/mm.h>
  42#include <linux/highmem.h>
  43#include <linux/fs.h>
  44#include <linux/rbtree.h>
  45#include <linux/kvm_types.h>
  46
  47#include <asm/cpu_device_id.h>
  48#include <asm/cacheflush.h>
  49#include <asm/cacheinfo.h>
  50#include <asm/processor.h>
  51#include <asm/tlbflush.h>
  52#include <asm/x86_init.h>
  53#include <asm/fcntl.h>
  54#include <asm/e820/api.h>
  55#include <asm/mtrr.h>
  56#include <asm/page.h>
  57#include <asm/msr.h>
  58#include <asm/memtype.h>
  59#include <asm/io.h>
  60
  61#include "memtype.h"
  62#include "../mm_internal.h"
  63
  64#undef pr_fmt
  65#define pr_fmt(fmt) "" fmt
  66
  67static bool __read_mostly pat_disabled = !IS_ENABLED(CONFIG_X86_PAT);
  68static u64 __ro_after_init pat_msr_val;
  69
  70/*
  71 * PAT support is enabled by default, but can be disabled for
  72 * various user-requested or hardware-forced reasons:
  73 */
  74static void __init pat_disable(const char *msg_reason)
  75{
  76	if (pat_disabled)
  77		return;
  78
  79	pat_disabled = true;
  80	pr_info("x86/PAT: %s\n", msg_reason);
  81
  82	memory_caching_control &= ~CACHE_PAT;
  83}
  84
  85static int __init nopat(char *str)
  86{
  87	pat_disable("PAT support disabled via boot option.");
  88	return 0;
  89}
  90early_param("nopat", nopat);
  91
  92bool pat_enabled(void)
  93{
  94	return !pat_disabled;
  95}
  96EXPORT_SYMBOL_GPL(pat_enabled);
  97
  98int pat_debug_enable;
  99
 100static int __init pat_debug_setup(char *str)
 101{
 102	pat_debug_enable = 1;
 103	return 1;
 104}
 105__setup("debugpat", pat_debug_setup);
 106
 107#ifdef CONFIG_X86_PAT
 108/*
 109 * X86 PAT uses page flags arch_1 and arch_2 together to keep track of
 110 * memory type of pages that have backing page struct.
 111 *
 112 * X86 PAT supports 4 different memory types:
 113 *  - _PAGE_CACHE_MODE_WB
 114 *  - _PAGE_CACHE_MODE_WC
 115 *  - _PAGE_CACHE_MODE_UC_MINUS
 116 *  - _PAGE_CACHE_MODE_WT
 117 *
 118 * _PAGE_CACHE_MODE_WB is the default type.
 119 */
 120
 121#define _PGMT_WB		0
 122#define _PGMT_WC		(1UL << PG_arch_1)
 123#define _PGMT_UC_MINUS		(1UL << PG_arch_2)
 124#define _PGMT_WT		(1UL << PG_arch_2 | 1UL << PG_arch_1)
 125#define _PGMT_MASK		(1UL << PG_arch_2 | 1UL << PG_arch_1)
 126#define _PGMT_CLEAR_MASK	(~_PGMT_MASK)
 127
 128static inline enum page_cache_mode get_page_memtype(struct page *pg)
 129{
 130	unsigned long pg_flags = pg->flags.f & _PGMT_MASK;
 131
 132	if (pg_flags == _PGMT_WB)
 133		return _PAGE_CACHE_MODE_WB;
 134	else if (pg_flags == _PGMT_WC)
 135		return _PAGE_CACHE_MODE_WC;
 136	else if (pg_flags == _PGMT_UC_MINUS)
 137		return _PAGE_CACHE_MODE_UC_MINUS;
 138	else
 139		return _PAGE_CACHE_MODE_WT;
 140}
 141
 142static inline void set_page_memtype(struct page *pg,
 143				    enum page_cache_mode memtype)
 144{
 145	unsigned long memtype_flags;
 146	unsigned long old_flags;
 147	unsigned long new_flags;
 148
 149	switch (memtype) {
 150	case _PAGE_CACHE_MODE_WC:
 151		memtype_flags = _PGMT_WC;
 152		break;
 153	case _PAGE_CACHE_MODE_UC_MINUS:
 154		memtype_flags = _PGMT_UC_MINUS;
 155		break;
 156	case _PAGE_CACHE_MODE_WT:
 157		memtype_flags = _PGMT_WT;
 158		break;
 159	case _PAGE_CACHE_MODE_WB:
 160	default:
 161		memtype_flags = _PGMT_WB;
 162		break;
 163	}
 164
 165	old_flags = READ_ONCE(pg->flags.f);
 166	do {
 167		new_flags = (old_flags & _PGMT_CLEAR_MASK) | memtype_flags;
 168	} while (!try_cmpxchg(&pg->flags.f, &old_flags, new_flags));
 169}
 170#else
 171static inline enum page_cache_mode get_page_memtype(struct page *pg)
 172{
 173	return -1;
 174}
 175static inline void set_page_memtype(struct page *pg,
 176				    enum page_cache_mode memtype)
 177{
 178}
 179#endif
 180
 181#define CM(c) (_PAGE_CACHE_MODE_ ## c)
 182
 183static enum page_cache_mode __init pat_get_cache_mode(unsigned int pat_val,
 184						      char *msg)
 185{
 186	enum page_cache_mode cache;
 187	char *cache_mode;
 188
 189	switch (pat_val) {
 190	case X86_MEMTYPE_UC:       cache = CM(UC);       cache_mode = "UC  "; break;
 191	case X86_MEMTYPE_WC:       cache = CM(WC);       cache_mode = "WC  "; break;
 192	case X86_MEMTYPE_WT:       cache = CM(WT);       cache_mode = "WT  "; break;
 193	case X86_MEMTYPE_WP:       cache = CM(WP);       cache_mode = "WP  "; break;
 194	case X86_MEMTYPE_WB:       cache = CM(WB);       cache_mode = "WB  "; break;
 195	case X86_MEMTYPE_UC_MINUS: cache = CM(UC_MINUS); cache_mode = "UC- "; break;
 196	default:                   cache = CM(WB);       cache_mode = "WB  "; break;
 197	}
 198
 199	memcpy(msg, cache_mode, 4);
 200
 201	return cache;
 202}
 203
 204#undef CM
 205
 206/*
 207 * Update the cache mode to pgprot translation tables according to PAT
 208 * configuration.
 209 * Using lower indices is preferred, so we start with highest index.
 210 */
 211static void __init init_cache_modes(u64 pat)
 212{
 213	enum page_cache_mode cache;
 214	char pat_msg[33];
 215	int i;
 216
 217	pat_msg[32] = 0;
 218	for (i = 7; i >= 0; i--) {
 219		cache = pat_get_cache_mode((pat >> (i * 8)) & 7,
 220					   pat_msg + 4 * i);
 221		update_cache_mode_entry(i, cache);
 222	}
 223	pr_info("x86/PAT: Configuration [0-7]: %s\n", pat_msg);
 224}
 225
 226void pat_cpu_init(void)
 227{
 228	if (!boot_cpu_has(X86_FEATURE_PAT)) {
 229		/*
 230		 * If this happens we are on a secondary CPU, but switched to
 231		 * PAT on the boot CPU. We have no way to undo PAT.
 232		 */
 233		panic("x86/PAT: PAT enabled, but not supported by secondary CPU\n");
 234	}
 235
 236	wrmsrq(MSR_IA32_CR_PAT, pat_msr_val);
 237
 238	__flush_tlb_all();
 239}
 240
 241/**
 242 * pat_bp_init - Initialize the PAT MSR value and PAT table
 243 *
 244 * This function initializes PAT MSR value and PAT table with an OS-defined
 245 * value to enable additional cache attributes, WC, WT and WP.
 246 *
 247 * This function prepares the calls of pat_cpu_init() via cache_cpu_init()
 248 * on all CPUs.
 249 */
 250void __init pat_bp_init(void)
 251{
 252	struct cpuinfo_x86 *c = &boot_cpu_data;
 253
 254	if (!IS_ENABLED(CONFIG_X86_PAT))
 255		pr_info_once("x86/PAT: PAT support disabled because CONFIG_X86_PAT is disabled in the kernel.\n");
 256
 257	if (!cpu_feature_enabled(X86_FEATURE_PAT))
 258		pat_disable("PAT not supported by the CPU.");
 259	else
 260		rdmsrq(MSR_IA32_CR_PAT, pat_msr_val);
 261
 262	if (!pat_msr_val) {
 263		pat_disable("PAT support disabled by the firmware.");
 264
 265		/*
 266		 * No PAT. Emulate the PAT table that corresponds to the two
 267		 * cache bits, PWT (Write Through) and PCD (Cache Disable).
 268		 * This setup is also the same as the BIOS default setup.
 269		 *
 270		 * PTE encoding:
 271		 *
 272		 *       PCD
 273		 *       |PWT  PAT
 274		 *       ||    slot
 275		 *       00    0    WB : _PAGE_CACHE_MODE_WB
 276		 *       01    1    WT : _PAGE_CACHE_MODE_WT
 277		 *       10    2    UC-: _PAGE_CACHE_MODE_UC_MINUS
 278		 *       11    3    UC : _PAGE_CACHE_MODE_UC
 279		 *
 280		 * NOTE: When WC or WP is used, it is redirected to UC- per
 281		 * the default setup in __cachemode2pte_tbl[].
 282		 */
 283		pat_msr_val = PAT_VALUE(WB, WT, UC_MINUS, UC, WB, WT, UC_MINUS, UC);
 284	}
 285
 286	/*
 287	 * Xen PV doesn't allow to set PAT MSR, but all cache modes are
 288	 * supported.
 289	 */
 290	if (pat_disabled || cpu_feature_enabled(X86_FEATURE_XENPV)) {
 291		init_cache_modes(pat_msr_val);
 292		return;
 293	}
 294
 295	if ((c->x86_vfm >= INTEL_PENTIUM_PRO   && c->x86_vfm <= INTEL_PENTIUM_M_DOTHAN) ||
 296	    (c->x86_vfm >= INTEL_P4_WILLAMETTE && c->x86_vfm <= INTEL_P4_CEDARMILL)) {
 297		/*
 298		 * PAT support with the lower four entries. Intel Pentium 2,
 299		 * 3, M, and 4 are affected by PAT errata, which makes the
 300		 * upper four entries unusable. To be on the safe side, we don't
 301		 * use those.
 302		 *
 303		 *  PTE encoding:
 304		 *      PAT
 305		 *      |PCD
 306		 *      ||PWT  PAT
 307		 *      |||    slot
 308		 *      000    0    WB : _PAGE_CACHE_MODE_WB
 309		 *      001    1    WC : _PAGE_CACHE_MODE_WC
 310		 *      010    2    UC-: _PAGE_CACHE_MODE_UC_MINUS
 311		 *      011    3    UC : _PAGE_CACHE_MODE_UC
 312		 * PAT bit unused
 313		 *
 314		 * NOTE: When WT or WP is used, it is redirected to UC- per
 315		 * the default setup in __cachemode2pte_tbl[].
 316		 */
 317		pat_msr_val = PAT_VALUE(WB, WC, UC_MINUS, UC, WB, WC, UC_MINUS, UC);
 318	} else {
 319		/*
 320		 * Full PAT support.  We put WT in slot 7 to improve
 321		 * robustness in the presence of errata that might cause
 322		 * the high PAT bit to be ignored.  This way, a buggy slot 7
 323		 * access will hit slot 3, and slot 3 is UC, so at worst
 324		 * we lose performance without causing a correctness issue.
 325		 * Pentium 4 erratum N46 is an example for such an erratum,
 326		 * although we try not to use PAT at all on affected CPUs.
 327		 *
 328		 *  PTE encoding:
 329		 *      PAT
 330		 *      |PCD
 331		 *      ||PWT  PAT
 332		 *      |||    slot
 333		 *      000    0    WB : _PAGE_CACHE_MODE_WB
 334		 *      001    1    WC : _PAGE_CACHE_MODE_WC
 335		 *      010    2    UC-: _PAGE_CACHE_MODE_UC_MINUS
 336		 *      011    3    UC : _PAGE_CACHE_MODE_UC
 337		 *      100    4    WB : Reserved
 338		 *      101    5    WP : _PAGE_CACHE_MODE_WP
 339		 *      110    6    UC-: Reserved
 340		 *      111    7    WT : _PAGE_CACHE_MODE_WT
 341		 *
 342		 * The reserved slots are unused, but mapped to their
 343		 * corresponding types in the presence of PAT errata.
 344		 */
 345		pat_msr_val = PAT_VALUE(WB, WC, UC_MINUS, UC, WB, WP, UC_MINUS, WT);
 346	}
 347
 348	memory_caching_control |= CACHE_PAT;
 349
 350	init_cache_modes(pat_msr_val);
 351}
 352
 353static DEFINE_SPINLOCK(memtype_lock);	/* protects memtype accesses */
 354
 355/*
 356 * Does intersection of PAT memory type and MTRR memory type and returns
 357 * the resulting memory type as PAT understands it.
 358 * (Type in pat and mtrr will not have same value)
 359 * The intersection is based on "Effective Memory Type" tables in IA-32
 360 * SDM vol 3a
 361 */
 362static unsigned long pat_x_mtrr_type(u64 start, u64 end,
 363				     enum page_cache_mode req_type)
 364{
 365	/*
 366	 * Look for MTRR hint to get the effective type in case where PAT
 367	 * request is for WB.
 368	 */
 369	if (req_type == _PAGE_CACHE_MODE_WB) {
 370		u8 mtrr_type, uniform;
 371
 372		mtrr_type = mtrr_type_lookup(start, end, &uniform);
 373		if (mtrr_type != MTRR_TYPE_WRBACK)
 374			return _PAGE_CACHE_MODE_UC_MINUS;
 375
 376		return _PAGE_CACHE_MODE_WB;
 377	}
 378
 379	return req_type;
 380}
 381
 382struct pagerange_state {
 383	unsigned long		cur_pfn;
 384	int			ram;
 385	int			not_ram;
 386};
 387
 388static int
 389pagerange_is_ram_callback(unsigned long initial_pfn, unsigned long total_nr_pages, void *arg)
 390{
 391	struct pagerange_state *state = arg;
 392
 393	state->not_ram	|= initial_pfn > state->cur_pfn;
 394	state->ram	|= total_nr_pages > 0;
 395	state->cur_pfn	 = initial_pfn + total_nr_pages;
 396
 397	return state->ram && state->not_ram;
 398}
 399
 400static int pat_pagerange_is_ram(resource_size_t start, resource_size_t end)
 401{
 402	int ret = 0;
 403	unsigned long start_pfn = start >> PAGE_SHIFT;
 404	unsigned long end_pfn = (end + PAGE_SIZE - 1) >> PAGE_SHIFT;
 405	struct pagerange_state state = {start_pfn, 0, 0};
 406
 407	/*
 408	 * For legacy reasons, physical address range in the legacy ISA
 409	 * region is tracked as non-RAM. This will allow users of
 410	 * /dev/mem to map portions of legacy ISA region, even when
 411	 * some of those portions are listed(or not even listed) with
 412	 * different e820 types(RAM/reserved/..)
 413	 */
 414	if (start_pfn < ISA_END_ADDRESS >> PAGE_SHIFT)
 415		start_pfn = ISA_END_ADDRESS >> PAGE_SHIFT;
 416
 417	if (start_pfn < end_pfn) {
 418		ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn,
 419				&state, pagerange_is_ram_callback);
 420	}
 421
 422	return (ret > 0) ? -1 : (state.ram ? 1 : 0);
 423}
 424
 425/*
 426 * For RAM pages, we use page flags to mark the pages with appropriate type.
 427 * The page flags are limited to four types, WB (default), WC, WT and UC-.
 428 * WP request fails with -EINVAL, and UC gets redirected to UC-.  Setting
 429 * a new memory type is only allowed for a page mapped with the default WB
 430 * type.
 431 *
 432 * Here we do two passes:
 433 * - Find the memtype of all the pages in the range, look for any conflicts.
 434 * - In case of no conflicts, set the new memtype for pages in the range.
 435 */
 436static int reserve_ram_pages_type(u64 start, u64 end,
 437				  enum page_cache_mode req_type,
 438				  enum page_cache_mode *new_type)
 439{
 440	struct page *page;
 441	u64 pfn;
 442
 443	if (req_type == _PAGE_CACHE_MODE_WP) {
 444		if (new_type)
 445			*new_type = _PAGE_CACHE_MODE_UC_MINUS;
 446		return -EINVAL;
 447	}
 448
 449	if (req_type == _PAGE_CACHE_MODE_UC) {
 450		/* We do not support strong UC */
 451		WARN_ON_ONCE(1);
 452		req_type = _PAGE_CACHE_MODE_UC_MINUS;
 453	}
 454
 455	for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
 456		enum page_cache_mode type;
 457
 458		page = pfn_to_page(pfn);
 459		type = get_page_memtype(page);
 460		if (type != _PAGE_CACHE_MODE_WB) {
 461			pr_info("x86/PAT: reserve_ram_pages_type failed [mem %#010Lx-%#010Lx], track 0x%x, req 0x%x\n",
 462				start, end - 1, type, req_type);
 463			if (new_type)
 464				*new_type = type;
 465
 466			return -EBUSY;
 467		}
 468	}
 469
 470	if (new_type)
 471		*new_type = req_type;
 472
 473	for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
 474		page = pfn_to_page(pfn);
 475		set_page_memtype(page, req_type);
 476	}
 477	return 0;
 478}
 479
 480static int free_ram_pages_type(u64 start, u64 end)
 481{
 482	struct page *page;
 483	u64 pfn;
 484
 485	for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
 486		page = pfn_to_page(pfn);
 487		set_page_memtype(page, _PAGE_CACHE_MODE_WB);
 488	}
 489	return 0;
 490}
 491
 492static u64 sanitize_phys(u64 address)
 493{
 494	/*
 495	 * When changing the memtype for pages containing poison allow
 496	 * for a "decoy" virtual address (bit 63 clear) passed to
 497	 * set_memory_X(). __pa() on a "decoy" address results in a
 498	 * physical address with bit 63 set.
 499	 *
 500	 * Decoy addresses are not present for 32-bit builds, see
 501	 * set_mce_nospec().
 502	 */
 503	if (IS_ENABLED(CONFIG_X86_64))
 504		return address & __PHYSICAL_MASK;
 505	return address;
 506}
 507
 508/*
 509 * req_type typically has one of the:
 510 * - _PAGE_CACHE_MODE_WB
 511 * - _PAGE_CACHE_MODE_WC
 512 * - _PAGE_CACHE_MODE_UC_MINUS
 513 * - _PAGE_CACHE_MODE_UC
 514 * - _PAGE_CACHE_MODE_WT
 515 *
 516 * If new_type is NULL, function will return an error if it cannot reserve the
 517 * region with req_type. If new_type is non-NULL, function will return
 518 * available type in new_type in case of no error. In case of any error
 519 * it will return a negative return value.
 520 */
 521int memtype_reserve(u64 start, u64 end, enum page_cache_mode req_type,
 522		    enum page_cache_mode *new_type)
 523{
 524	struct memtype *entry_new;
 525	enum page_cache_mode actual_type;
 526	int is_range_ram;
 527	int err = 0;
 528
 529	start = sanitize_phys(start);
 530
 531	/*
 532	 * The end address passed into this function is exclusive, but
 533	 * sanitize_phys() expects an inclusive address.
 534	 */
 535	end = sanitize_phys(end - 1) + 1;
 536	if (start >= end) {
 537		WARN(1, "%s failed: [mem %#010Lx-%#010Lx], req %s\n", __func__,
 538				start, end - 1, cattr_name(req_type));
 539		return -EINVAL;
 540	}
 541
 542	if (!pat_enabled()) {
 543		/* This is identical to page table setting without PAT */
 544		if (new_type)
 545			*new_type = req_type;
 546		return 0;
 547	}
 548
 549	/* Low ISA region is always mapped WB in page table. No need to track */
 550	if (x86_platform.is_untracked_pat_range(start, end)) {
 551		if (new_type)
 552			*new_type = _PAGE_CACHE_MODE_WB;
 553		return 0;
 554	}
 555
 556	/*
 557	 * Call mtrr_lookup to get the type hint. This is an
 558	 * optimization for /dev/mem mmap'ers into WB memory (BIOS
 559	 * tools and ACPI tools). Use WB request for WB memory and use
 560	 * UC_MINUS otherwise.
 561	 */
 562	actual_type = pat_x_mtrr_type(start, end, req_type);
 563
 564	if (new_type)
 565		*new_type = actual_type;
 566
 567	is_range_ram = pat_pagerange_is_ram(start, end);
 568	if (is_range_ram == 1) {
 569
 570		err = reserve_ram_pages_type(start, end, req_type, new_type);
 571
 572		return err;
 573	} else if (is_range_ram < 0) {
 574		return -EINVAL;
 575	}
 576
 577	entry_new = kzalloc(sizeof(struct memtype), GFP_KERNEL);
 578	if (!entry_new)
 579		return -ENOMEM;
 580
 581	entry_new->start = start;
 582	entry_new->end	 = end;
 583	entry_new->type	 = actual_type;
 584
 585	spin_lock(&memtype_lock);
 586
 587	err = memtype_check_insert(entry_new, new_type);
 588	if (err) {
 589		pr_info("x86/PAT: memtype_reserve failed [mem %#010Lx-%#010Lx], track %s, req %s\n",
 590			start, end - 1,
 591			cattr_name(entry_new->type), cattr_name(req_type));
 592		kfree(entry_new);
 593		spin_unlock(&memtype_lock);
 594
 595		return err;
 596	}
 597
 598	spin_unlock(&memtype_lock);
 599
 600	dprintk("memtype_reserve added [mem %#010Lx-%#010Lx], track %s, req %s, ret %s\n",
 601		start, end - 1, cattr_name(entry_new->type), cattr_name(req_type),
 602		new_type ? cattr_name(*new_type) : "-");
 603
 604	return err;
 605}
 606
 607int memtype_free(u64 start, u64 end)
 608{
 609	int is_range_ram;
 610	struct memtype *entry_old;
 611
 612	if (!pat_enabled())
 613		return 0;
 614
 615	start = sanitize_phys(start);
 616	end = sanitize_phys(end);
 617
 618	/* Low ISA region is always mapped WB. No need to track */
 619	if (x86_platform.is_untracked_pat_range(start, end))
 620		return 0;
 621
 622	is_range_ram = pat_pagerange_is_ram(start, end);
 623	if (is_range_ram == 1)
 624		return free_ram_pages_type(start, end);
 625	if (is_range_ram < 0)
 626		return -EINVAL;
 627
 628	spin_lock(&memtype_lock);
 629	entry_old = memtype_erase(start, end);
 630	spin_unlock(&memtype_lock);
 631
 632	if (IS_ERR(entry_old)) {
 633		pr_info("x86/PAT: %s:%d freeing invalid memtype [mem %#010Lx-%#010Lx]\n",
 634			current->comm, current->pid, start, end - 1);
 635		return -EINVAL;
 636	}
 637
 638	kfree(entry_old);
 639
 640	dprintk("memtype_free request [mem %#010Lx-%#010Lx]\n", start, end - 1);
 641
 642	return 0;
 643}
 644
 645
 646/**
 647 * lookup_memtype - Looks up the memory type for a physical address
 648 * @paddr: physical address of which memory type needs to be looked up
 649 *
 650 * Only to be called when PAT is enabled
 651 *
 652 * Returns _PAGE_CACHE_MODE_WB, _PAGE_CACHE_MODE_WC, _PAGE_CACHE_MODE_UC_MINUS
 653 * or _PAGE_CACHE_MODE_WT.
 654 */
 655static enum page_cache_mode lookup_memtype(u64 paddr)
 656{
 657	enum page_cache_mode rettype = _PAGE_CACHE_MODE_WB;
 658	struct memtype *entry;
 659
 660	if (x86_platform.is_untracked_pat_range(paddr, paddr + PAGE_SIZE))
 661		return rettype;
 662
 663	if (pat_pagerange_is_ram(paddr, paddr + PAGE_SIZE)) {
 664		struct page *page;
 665
 666		page = pfn_to_page(paddr >> PAGE_SHIFT);
 667		return get_page_memtype(page);
 668	}
 669
 670	spin_lock(&memtype_lock);
 671
 672	entry = memtype_lookup(paddr);
 673	if (entry != NULL)
 674		rettype = entry->type;
 675	else
 676		rettype = _PAGE_CACHE_MODE_UC_MINUS;
 677
 678	spin_unlock(&memtype_lock);
 679
 680	return rettype;
 681}
 682
 683/**
 684 * pat_pfn_immune_to_uc_mtrr - Check whether the PAT memory type
 685 * of @pfn cannot be overridden by UC MTRR memory type.
 686 * @pfn: The page frame number to check.
 687 *
 688 * Only to be called when PAT is enabled.
 689 *
 690 * Returns true, if the PAT memory type of @pfn is UC, UC-, or WC.
 691 * Returns false in other cases.
 692 */
 693bool pat_pfn_immune_to_uc_mtrr(unsigned long pfn)
 694{
 695	enum page_cache_mode cm = lookup_memtype(PFN_PHYS(pfn));
 696
 697	return cm == _PAGE_CACHE_MODE_UC ||
 698	       cm == _PAGE_CACHE_MODE_UC_MINUS ||
 699	       cm == _PAGE_CACHE_MODE_WC;
 700}
 701EXPORT_SYMBOL_FOR_KVM(pat_pfn_immune_to_uc_mtrr);
 702
 703/**
 704 * memtype_reserve_io - Request a memory type mapping for a region of memory
 705 * @start: start (physical address) of the region
 706 * @end: end (physical address) of the region
 707 * @type: A pointer to memtype, with requested type. On success, requested
 708 * or any other compatible type that was available for the region is returned
 709 *
 710 * On success, returns 0
 711 * On failure, returns non-zero
 712 */
 713int memtype_reserve_io(resource_size_t start, resource_size_t end,
 714			enum page_cache_mode *type)
 715{
 716	resource_size_t size = end - start;
 717	enum page_cache_mode req_type = *type;
 718	enum page_cache_mode new_type;
 719	int ret;
 720
 721	WARN_ON_ONCE(iomem_map_sanity_check(start, size));
 722
 723	ret = memtype_reserve(start, end, req_type, &new_type);
 724	if (ret)
 725		goto out_err;
 726
 727	if (!is_new_memtype_allowed(start, size, req_type, new_type))
 728		goto out_free;
 729
 730	if (memtype_kernel_map_sync(start, size, new_type) < 0)
 731		goto out_free;
 732
 733	*type = new_type;
 734	return 0;
 735
 736out_free:
 737	memtype_free(start, end);
 738	ret = -EBUSY;
 739out_err:
 740	return ret;
 741}
 742
 743/**
 744 * memtype_free_io - Release a memory type mapping for a region of memory
 745 * @start: start (physical address) of the region
 746 * @end: end (physical address) of the region
 747 */
 748void memtype_free_io(resource_size_t start, resource_size_t end)
 749{
 750	memtype_free(start, end);
 751}
 752
 753#ifdef CONFIG_X86_PAT
 754int arch_io_reserve_memtype_wc(resource_size_t start, resource_size_t size)
 755{
 756	enum page_cache_mode type = _PAGE_CACHE_MODE_WC;
 757
 758	return memtype_reserve_io(start, start + size, &type);
 759}
 760EXPORT_SYMBOL(arch_io_reserve_memtype_wc);
 761
 762void arch_io_free_memtype_wc(resource_size_t start, resource_size_t size)
 763{
 764	memtype_free_io(start, start + size);
 765}
 766EXPORT_SYMBOL(arch_io_free_memtype_wc);
 767#endif
 768
 769pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
 770				unsigned long size, pgprot_t vma_prot)
 771{
 772	if (!phys_mem_access_encrypted(pfn << PAGE_SHIFT, size))
 773		vma_prot = pgprot_decrypted(vma_prot);
 774
 775	return vma_prot;
 776}
 777
 778static inline void pgprot_set_cachemode(pgprot_t *prot, enum page_cache_mode pcm)
 779{
 780	*prot = __pgprot((pgprot_val(*prot) & ~_PAGE_CACHE_MASK) |
 781			 cachemode2protval(pcm));
 782}
 783
 784int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
 785				unsigned long size, pgprot_t *vma_prot)
 786{
 787	enum page_cache_mode pcm = _PAGE_CACHE_MODE_WB;
 788
 789	if (!pat_enabled())
 790		return 1;
 791
 792	if (!range_is_allowed(pfn, size))
 793		return 0;
 794
 795	if (file->f_flags & O_DSYNC)
 796		pcm = _PAGE_CACHE_MODE_UC_MINUS;
 797
 798	pgprot_set_cachemode(vma_prot, pcm);
 799	return 1;
 800}
 801
 802/*
 803 * Change the memory type for the physical address range in kernel identity
 804 * mapping space if that range is a part of identity map.
 805 */
 806int memtype_kernel_map_sync(u64 base, unsigned long size,
 807			    enum page_cache_mode pcm)
 808{
 809	unsigned long id_sz;
 810
 811	if (base > __pa(high_memory-1))
 812		return 0;
 813
 814	/*
 815	 * Some areas in the middle of the kernel identity range
 816	 * are not mapped, for example the PCI space.
 817	 */
 818	if (!page_is_ram(base >> PAGE_SHIFT))
 819		return 0;
 820
 821	id_sz = (__pa(high_memory-1) <= base + size) ?
 822				__pa(high_memory) - base : size;
 823
 824	if (ioremap_change_attr((unsigned long)__va(base), id_sz, pcm) < 0) {
 825		pr_info("x86/PAT: %s:%d ioremap_change_attr failed %s for [mem %#010Lx-%#010Lx]\n",
 826			current->comm, current->pid,
 827			cattr_name(pcm),
 828			base, (unsigned long long)(base + size-1));
 829		return -EINVAL;
 830	}
 831	return 0;
 832}
 833
 834/*
 835 * Internal interface to reserve a range of physical memory with prot.
 836 * Reserved non RAM regions only and after successful memtype_reserve,
 837 * this func also keeps identity mapping (if any) in sync with this new prot.
 838 */
 839static int reserve_pfn_range(u64 paddr, unsigned long size, pgprot_t *vma_prot)
 840{
 841	int is_ram = 0;
 842	int ret;
 843	enum page_cache_mode want_pcm = pgprot2cachemode(*vma_prot);
 844	enum page_cache_mode pcm = want_pcm;
 845
 846	is_ram = pat_pagerange_is_ram(paddr, paddr + size);
 847
 848	/*
 849	 * reserve_pfn_range() for RAM pages. We do not refcount to keep
 850	 * track of number of mappings of RAM pages. We can assert that
 851	 * the type requested matches the type of first page in the range.
 852	 */
 853	if (is_ram) {
 854		if (!pat_enabled())
 855			return 0;
 856
 857		pcm = lookup_memtype(paddr);
 858		if (want_pcm != pcm) {
 859			pr_warn("x86/PAT: %s:%d map pfn RAM range req %s for [mem %#010Lx-%#010Lx], got %s\n",
 860				current->comm, current->pid,
 861				cattr_name(want_pcm),
 862				(unsigned long long)paddr,
 863				(unsigned long long)(paddr + size - 1),
 864				cattr_name(pcm));
 865			pgprot_set_cachemode(vma_prot, pcm);
 866		}
 867		return 0;
 868	}
 869
 870	ret = memtype_reserve(paddr, paddr + size, want_pcm, &pcm);
 871	if (ret)
 872		return ret;
 873
 874	if (pcm != want_pcm) {
 875		if (!is_new_memtype_allowed(paddr, size, want_pcm, pcm)) {
 876			memtype_free(paddr, paddr + size);
 877			pr_err("x86/PAT: %s:%d map pfn expected mapping type %s for [mem %#010Lx-%#010Lx], got %s\n",
 878			       current->comm, current->pid,
 879			       cattr_name(want_pcm),
 880			       (unsigned long long)paddr,
 881			       (unsigned long long)(paddr + size - 1),
 882			       cattr_name(pcm));
 883			return -EINVAL;
 884		}
 885		pgprot_set_cachemode(vma_prot, pcm);
 886	}
 887
 888	if (memtype_kernel_map_sync(paddr, size, pcm) < 0) {
 889		memtype_free(paddr, paddr + size);
 890		return -EINVAL;
 891	}
 892	return 0;
 893}
 894
 895/*
 896 * Internal interface to free a range of physical memory.
 897 * Frees non RAM regions only.
 898 */
 899static void free_pfn_range(u64 paddr, unsigned long size)
 900{
 901	int is_ram;
 902
 903	is_ram = pat_pagerange_is_ram(paddr, paddr + size);
 904	if (is_ram == 0)
 905		memtype_free(paddr, paddr + size);
 906}
 907
 908int pfnmap_setup_cachemode(unsigned long pfn, unsigned long size, pgprot_t *prot)
 909{
 910	resource_size_t paddr = (resource_size_t)pfn << PAGE_SHIFT;
 911	enum page_cache_mode pcm;
 912
 913	if (!pat_enabled())
 914		return 0;
 915
 916	pcm = lookup_memtype(paddr);
 917
 918	/* Check memtype for the remaining pages */
 919	while (size > PAGE_SIZE) {
 920		size -= PAGE_SIZE;
 921		paddr += PAGE_SIZE;
 922		if (pcm != lookup_memtype(paddr))
 923			return -EINVAL;
 924	}
 925
 926	pgprot_set_cachemode(prot, pcm);
 927	return 0;
 928}
 929
 930int pfnmap_track(unsigned long pfn, unsigned long size, pgprot_t *prot)
 931{
 932	const resource_size_t paddr = (resource_size_t)pfn << PAGE_SHIFT;
 933
 934	return reserve_pfn_range(paddr, size, prot);
 935}
 936
 937void pfnmap_untrack(unsigned long pfn, unsigned long size)
 938{
 939	const resource_size_t paddr = (resource_size_t)pfn << PAGE_SHIFT;
 940
 941	free_pfn_range(paddr, size);
 942}
 943
 944pgprot_t pgprot_writecombine(pgprot_t prot)
 945{
 946	pgprot_set_cachemode(&prot, _PAGE_CACHE_MODE_WC);
 947	return prot;
 948}
 949EXPORT_SYMBOL_GPL(pgprot_writecombine);
 950
 951pgprot_t pgprot_writethrough(pgprot_t prot)
 952{
 953	pgprot_set_cachemode(&prot, _PAGE_CACHE_MODE_WT);
 954	return prot;
 955}
 956EXPORT_SYMBOL_GPL(pgprot_writethrough);
 957
 958#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_X86_PAT)
 959
 960/*
 961 * We are allocating a temporary printout-entry to be passed
 962 * between seq_start()/next() and seq_show():
 963 */
 964static struct memtype *memtype_get_idx(loff_t pos)
 965{
 966	struct memtype *entry_print;
 967	int ret;
 968
 969	entry_print  = kzalloc(sizeof(struct memtype), GFP_KERNEL);
 970	if (!entry_print)
 971		return NULL;
 972
 973	spin_lock(&memtype_lock);
 974	ret = memtype_copy_nth_element(entry_print, pos);
 975	spin_unlock(&memtype_lock);
 976
 977	/* Free it on error: */
 978	if (ret) {
 979		kfree(entry_print);
 980		return NULL;
 981	}
 982
 983	return entry_print;
 984}
 985
 986static void *memtype_seq_start(struct seq_file *seq, loff_t *pos)
 987{
 988	if (*pos == 0) {
 989		++*pos;
 990		seq_puts(seq, "PAT memtype list:\n");
 991	}
 992
 993	return memtype_get_idx(*pos);
 994}
 995
 996static void *memtype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 997{
 998	kfree(v);
 999	++*pos;
1000	return memtype_get_idx(*pos);
1001}
1002
1003static void memtype_seq_stop(struct seq_file *seq, void *v)
1004{
1005	kfree(v);
1006}
1007
1008static int memtype_seq_show(struct seq_file *seq, void *v)
1009{
1010	struct memtype *entry_print = (struct memtype *)v;
1011
1012	seq_printf(seq, "PAT: [mem 0x%016Lx-0x%016Lx] %s\n",
1013			entry_print->start,
1014			entry_print->end,
1015			cattr_name(entry_print->type));
1016
1017	return 0;
1018}
1019
1020static const struct seq_operations memtype_seq_ops = {
1021	.start = memtype_seq_start,
1022	.next  = memtype_seq_next,
1023	.stop  = memtype_seq_stop,
1024	.show  = memtype_seq_show,
1025};
1026
1027static int memtype_seq_open(struct inode *inode, struct file *file)
1028{
1029	return seq_open(file, &memtype_seq_ops);
1030}
1031
1032static const struct file_operations memtype_fops = {
1033	.open    = memtype_seq_open,
1034	.read    = seq_read,
1035	.llseek  = seq_lseek,
1036	.release = seq_release,
1037};
1038
1039static int __init pat_memtype_list_init(void)
1040{
1041	if (pat_enabled()) {
1042		debugfs_create_file("pat_memtype_list", S_IRUSR,
1043				    arch_debugfs_dir, NULL, &memtype_fops);
1044	}
1045	return 0;
1046}
1047late_initcall(pat_memtype_list_init);
1048
1049#endif /* CONFIG_DEBUG_FS && CONFIG_X86_PAT */