cleanup.c

  1// SPDX-License-Identifier: LGPL-2.0+
  2/*
  3 * MTRR (Memory Type Range Register) cleanup
  4 *
  5 *  Copyright (C) 2009 Yinghai Lu
  6 */
  7#include <linux/init.h>
  8#include <linux/pci.h>
  9#include <linux/smp.h>
 10#include <linux/cpu.h>
 11#include <linux/mutex.h>
 12#include <linux/uaccess.h>
 13#include <linux/kvm_para.h>
 14#include <linux/range.h>
 15
 16#include <asm/processor.h>
 17#include <asm/e820/api.h>
 18#include <asm/mtrr.h>
 19#include <asm/msr.h>
 20
 21#include "mtrr.h"
 22
 23struct var_mtrr_range_state {
 24	unsigned long	base_pfn;
 25	unsigned long	size_pfn;
 26	mtrr_type	type;
 27};
 28
 29struct var_mtrr_state {
 30	unsigned long	range_startk;
 31	unsigned long	range_sizek;
 32	unsigned long	chunk_sizek;
 33	unsigned long	gran_sizek;
 34	unsigned int	reg;
 35};
 36
 37/* Should be related to MTRR_VAR_RANGES nums */
 38#define RANGE_NUM				256
 39
 40static struct range __initdata		range[RANGE_NUM];
 41static int __initdata				nr_range;
 42
 43static struct var_mtrr_range_state __initdata	range_state[RANGE_NUM];
 44
 45#define BIOS_BUG_MSG \
 46	"WARNING: BIOS bug: VAR MTRR %d contains strange UC entry under 1M, check with your system vendor!\n"
 47
 48static int __init
 49x86_get_mtrr_mem_range(struct range *range, int nr_range,
 50		       unsigned long extra_remove_base,
 51		       unsigned long extra_remove_size)
 52{
 53	unsigned long base, size;
 54	mtrr_type type;
 55	int i;
 56
 57	for (i = 0; i < num_var_ranges; i++) {
 58		type = range_state[i].type;
 59		if (type != MTRR_TYPE_WRBACK)
 60			continue;
 61		base = range_state[i].base_pfn;
 62		size = range_state[i].size_pfn;
 63		nr_range = add_range_with_merge(range, RANGE_NUM, nr_range,
 64						base, base + size);
 65	}
 66
 67	Dprintk("After WB checking\n");
 68	for (i = 0; i < nr_range; i++)
 69		Dprintk("MTRR MAP PFN: %016llx - %016llx\n",
 70			 range[i].start, range[i].end);
 71
 72	/* Take out UC ranges: */
 73	for (i = 0; i < num_var_ranges; i++) {
 74		type = range_state[i].type;
 75		if (type != MTRR_TYPE_UNCACHABLE &&
 76		    type != MTRR_TYPE_WRPROT)
 77			continue;
 78		size = range_state[i].size_pfn;
 79		if (!size)
 80			continue;
 81		base = range_state[i].base_pfn;
 82		if (base < (1<<(20-PAGE_SHIFT)) && mtrr_state.have_fixed &&
 83		    (mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED) &&
 84		    (mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED)) {
 85			/* Var MTRR contains UC entry below 1M? Skip it: */
 86			pr_warn(BIOS_BUG_MSG, i);
 87			if (base + size <= (1<<(20-PAGE_SHIFT)))
 88				continue;
 89			size -= (1<<(20-PAGE_SHIFT)) - base;
 90			base = 1<<(20-PAGE_SHIFT);
 91		}
 92		subtract_range(range, RANGE_NUM, base, base + size);
 93	}
 94	if (extra_remove_size)
 95		subtract_range(range, RANGE_NUM, extra_remove_base,
 96				 extra_remove_base + extra_remove_size);
 97
 98	Dprintk("After UC checking\n");
 99	for (i = 0; i < RANGE_NUM; i++) {
100		if (!range[i].end)
101			continue;
102
103		Dprintk("MTRR MAP PFN: %016llx - %016llx\n",
104			 range[i].start, range[i].end);
105	}
106
107	/* sort the ranges */
108	nr_range = clean_sort_range(range, RANGE_NUM);
109
110	Dprintk("After sorting\n");
111	for (i = 0; i < nr_range; i++)
112		Dprintk("MTRR MAP PFN: %016llx - %016llx\n",
113			range[i].start, range[i].end);
114
115	return nr_range;
116}
117
118#ifdef CONFIG_MTRR_SANITIZER
119
120static unsigned long __init sum_ranges(struct range *range, int nr_range)
121{
122	unsigned long sum = 0;
123	int i;
124
125	for (i = 0; i < nr_range; i++)
126		sum += range[i].end - range[i].start;
127
128	return sum;
129}
130
131static int enable_mtrr_cleanup __initdata =
132	CONFIG_MTRR_SANITIZER_ENABLE_DEFAULT;
133
134static int __init disable_mtrr_cleanup_setup(char *str)
135{
136	enable_mtrr_cleanup = 0;
137	return 0;
138}
139early_param("disable_mtrr_cleanup", disable_mtrr_cleanup_setup);
140
141static int __init enable_mtrr_cleanup_setup(char *str)
142{
143	enable_mtrr_cleanup = 1;
144	return 0;
145}
146early_param("enable_mtrr_cleanup", enable_mtrr_cleanup_setup);
147
148static void __init
149set_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek,
150	     unsigned char type)
151{
152	u32 base_lo, base_hi, mask_lo, mask_hi;
153	u64 base, mask;
154
155	if (!sizek) {
156		fill_mtrr_var_range(reg, 0, 0, 0, 0);
157		return;
158	}
159
160	mask = (1ULL << boot_cpu_data.x86_phys_bits) - 1;
161	mask &= ~((((u64)sizek) << 10) - 1);
162
163	base = ((u64)basek) << 10;
164
165	base |= type;
166	mask |= 0x800;
167
168	base_lo = base & ((1ULL<<32) - 1);
169	base_hi = base >> 32;
170
171	mask_lo = mask & ((1ULL<<32) - 1);
172	mask_hi = mask >> 32;
173
174	fill_mtrr_var_range(reg, base_lo, base_hi, mask_lo, mask_hi);
175}
176
177static void __init
178save_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek,
179	      unsigned char type)
180{
181	range_state[reg].base_pfn = basek >> (PAGE_SHIFT - 10);
182	range_state[reg].size_pfn = sizek >> (PAGE_SHIFT - 10);
183	range_state[reg].type = type;
184}
185
186static void __init set_var_mtrr_all(void)
187{
188	unsigned long basek, sizek;
189	unsigned char type;
190	unsigned int reg;
191
192	for (reg = 0; reg < num_var_ranges; reg++) {
193		basek = range_state[reg].base_pfn << (PAGE_SHIFT - 10);
194		sizek = range_state[reg].size_pfn << (PAGE_SHIFT - 10);
195		type = range_state[reg].type;
196
197		set_var_mtrr(reg, basek, sizek, type);
198	}
199}
200
201static unsigned long to_size_factor(unsigned long sizek, char *factorp)
202{
203	unsigned long base = sizek;
204	char factor;
205
206	if (base & ((1<<10) - 1)) {
207		/* Not MB-aligned: */
208		factor = 'K';
209	} else if (base & ((1<<20) - 1)) {
210		factor = 'M';
211		base >>= 10;
212	} else {
213		factor = 'G';
214		base >>= 20;
215	}
216
217	*factorp = factor;
218
219	return base;
220}
221
222static unsigned int __init
223range_to_mtrr(unsigned int reg, unsigned long range_startk,
224	      unsigned long range_sizek, unsigned char type)
225{
226	if (!range_sizek || (reg >= num_var_ranges))
227		return reg;
228
229	while (range_sizek) {
230		unsigned long max_align, align;
231		unsigned long sizek;
232
233		/* Compute the maximum size with which we can make a range: */
234		if (range_startk)
235			max_align = __ffs(range_startk);
236		else
237			max_align = BITS_PER_LONG - 1;
238
239		align = __fls(range_sizek);
240		if (align > max_align)
241			align = max_align;
242
243		sizek = 1UL << align;
244		if (mtrr_debug) {
245			char start_factor = 'K', size_factor = 'K';
246			unsigned long start_base, size_base;
247
248			start_base = to_size_factor(range_startk, &start_factor);
249			size_base = to_size_factor(sizek, &size_factor);
250
251			Dprintk("Setting variable MTRR %d, "
252				"base: %ld%cB, range: %ld%cB, type %s\n",
253				reg, start_base, start_factor,
254				size_base, size_factor,
255				(type == MTRR_TYPE_UNCACHABLE) ? "UC" :
256				   ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other")
257				);
258		}
259		save_var_mtrr(reg++, range_startk, sizek, type);
260		range_startk += sizek;
261		range_sizek -= sizek;
262		if (reg >= num_var_ranges)
263			break;
264	}
265	return reg;
266}
267
268static unsigned __init
269range_to_mtrr_with_hole(struct var_mtrr_state *state, unsigned long basek,
270			unsigned long sizek)
271{
272	unsigned long hole_basek, hole_sizek;
273	unsigned long second_sizek;
274	unsigned long range0_basek, range0_sizek;
275	unsigned long range_basek, range_sizek;
276	unsigned long chunk_sizek;
277	unsigned long gran_sizek;
278
279	hole_basek = 0;
280	hole_sizek = 0;
281	second_sizek = 0;
282	chunk_sizek = state->chunk_sizek;
283	gran_sizek = state->gran_sizek;
284
285	/* Align with gran size, prevent small block used up MTRRs: */
286	range_basek = ALIGN(state->range_startk, gran_sizek);
287	if ((range_basek > basek) && basek)
288		return second_sizek;
289
290	state->range_sizek -= (range_basek - state->range_startk);
291	range_sizek = ALIGN(state->range_sizek, gran_sizek);
292
293	while (range_sizek > state->range_sizek) {
294		range_sizek -= gran_sizek;
295		if (!range_sizek)
296			return 0;
297	}
298	state->range_sizek = range_sizek;
299
300	/* Try to append some small hole: */
301	range0_basek = state->range_startk;
302	range0_sizek = ALIGN(state->range_sizek, chunk_sizek);
303
304	/* No increase: */
305	if (range0_sizek == state->range_sizek) {
306		Dprintk("rangeX: %016lx - %016lx\n",
307			range0_basek<<10,
308			(range0_basek + state->range_sizek)<<10);
309		state->reg = range_to_mtrr(state->reg, range0_basek,
310				state->range_sizek, MTRR_TYPE_WRBACK);
311		return 0;
312	}
313
314	/* Only cut back when it is not the last: */
315	if (sizek) {
316		while (range0_basek + range0_sizek > (basek + sizek)) {
317			if (range0_sizek >= chunk_sizek)
318				range0_sizek -= chunk_sizek;
319			else
320				range0_sizek = 0;
321
322			if (!range0_sizek)
323				break;
324		}
325	}
326
327second_try:
328	range_basek = range0_basek + range0_sizek;
329
330	/* One hole in the middle: */
331	if (range_basek > basek && range_basek <= (basek + sizek))
332		second_sizek = range_basek - basek;
333
334	if (range0_sizek > state->range_sizek) {
335
336		/* One hole in middle or at the end: */
337		hole_sizek = range0_sizek - state->range_sizek - second_sizek;
338
339		/* Hole size should be less than half of range0 size: */
340		if (hole_sizek >= (range0_sizek >> 1) &&
341		    range0_sizek >= chunk_sizek) {
342			range0_sizek -= chunk_sizek;
343			second_sizek = 0;
344			hole_sizek = 0;
345
346			goto second_try;
347		}
348	}
349
350	if (range0_sizek) {
351		Dprintk("range0: %016lx - %016lx\n",
352			range0_basek<<10,
353			(range0_basek + range0_sizek)<<10);
354		state->reg = range_to_mtrr(state->reg, range0_basek,
355				range0_sizek, MTRR_TYPE_WRBACK);
356	}
357
358	if (range0_sizek < state->range_sizek) {
359		/* Need to handle left over range: */
360		range_sizek = state->range_sizek - range0_sizek;
361
362		Dprintk("range: %016lx - %016lx\n",
363			 range_basek<<10,
364			 (range_basek + range_sizek)<<10);
365
366		state->reg = range_to_mtrr(state->reg, range_basek,
367				 range_sizek, MTRR_TYPE_WRBACK);
368	}
369
370	if (hole_sizek) {
371		hole_basek = range_basek - hole_sizek - second_sizek;
372		Dprintk("hole: %016lx - %016lx\n",
373			 hole_basek<<10,
374			 (hole_basek + hole_sizek)<<10);
375		state->reg = range_to_mtrr(state->reg, hole_basek,
376				 hole_sizek, MTRR_TYPE_UNCACHABLE);
377	}
378
379	return second_sizek;
380}
381
382static void __init
383set_var_mtrr_range(struct var_mtrr_state *state, unsigned long base_pfn,
384		   unsigned long size_pfn)
385{
386	unsigned long basek, sizek;
387	unsigned long second_sizek = 0;
388
389	if (state->reg >= num_var_ranges)
390		return;
391
392	basek = base_pfn << (PAGE_SHIFT - 10);
393	sizek = size_pfn << (PAGE_SHIFT - 10);
394
395	/* See if I can merge with the last range: */
396	if ((basek <= 1024) ||
397	    (state->range_startk + state->range_sizek == basek)) {
398		unsigned long endk = basek + sizek;
399		state->range_sizek = endk - state->range_startk;
400		return;
401	}
402	/* Write the range mtrrs: */
403	if (state->range_sizek != 0)
404		second_sizek = range_to_mtrr_with_hole(state, basek, sizek);
405
406	/* Allocate an msr: */
407	state->range_startk = basek + second_sizek;
408	state->range_sizek  = sizek - second_sizek;
409}
410
411/* Minimum size of mtrr block that can take hole: */
412static u64 mtrr_chunk_size __initdata = (256ULL<<20);
413
414static int __init parse_mtrr_chunk_size_opt(char *p)
415{
416	if (!p)
417		return -EINVAL;
418	mtrr_chunk_size = memparse(p, &p);
419	return 0;
420}
421early_param("mtrr_chunk_size", parse_mtrr_chunk_size_opt);
422
423/* Granularity of mtrr of block: */
424static u64 mtrr_gran_size __initdata;
425
426static int __init parse_mtrr_gran_size_opt(char *p)
427{
428	if (!p)
429		return -EINVAL;
430	mtrr_gran_size = memparse(p, &p);
431	return 0;
432}
433early_param("mtrr_gran_size", parse_mtrr_gran_size_opt);
434
435static unsigned long nr_mtrr_spare_reg __initdata =
436				 CONFIG_MTRR_SANITIZER_SPARE_REG_NR_DEFAULT;
437
438static int __init parse_mtrr_spare_reg(char *arg)
439{
440	if (arg)
441		nr_mtrr_spare_reg = simple_strtoul(arg, NULL, 0);
442	return 0;
443}
444early_param("mtrr_spare_reg_nr", parse_mtrr_spare_reg);
445
446static int __init
447x86_setup_var_mtrrs(struct range *range, int nr_range,
448		    u64 chunk_size, u64 gran_size)
449{
450	struct var_mtrr_state var_state;
451	int num_reg;
452	int i;
453
454	var_state.range_startk	= 0;
455	var_state.range_sizek	= 0;
456	var_state.reg		= 0;
457	var_state.chunk_sizek	= chunk_size >> 10;
458	var_state.gran_sizek	= gran_size >> 10;
459
460	memset(range_state, 0, sizeof(range_state));
461
462	/* Write the range: */
463	for (i = 0; i < nr_range; i++) {
464		set_var_mtrr_range(&var_state, range[i].start,
465				   range[i].end - range[i].start);
466	}
467
468	/* Write the last range: */
469	if (var_state.range_sizek != 0)
470		range_to_mtrr_with_hole(&var_state, 0, 0);
471
472	num_reg = var_state.reg;
473	/* Clear out the extra MTRR's: */
474	while (var_state.reg < num_var_ranges) {
475		save_var_mtrr(var_state.reg, 0, 0, 0);
476		var_state.reg++;
477	}
478
479	return num_reg;
480}
481
482struct mtrr_cleanup_result {
483	unsigned long	gran_sizek;
484	unsigned long	chunk_sizek;
485	unsigned long	lose_cover_sizek;
486	unsigned int	num_reg;
487	int		bad;
488};
489
490/*
491 * gran_size: 64K, 128K, 256K, 512K, 1M, 2M, ..., 2G
492 * chunk size: gran_size, ..., 2G
493 * so we need (1+16)*8
494 */
495#define NUM_RESULT	136
496#define PSHIFT		(PAGE_SHIFT - 10)
497
498static struct mtrr_cleanup_result __initdata result[NUM_RESULT];
499static unsigned long __initdata min_loss_pfn[RANGE_NUM];
500
501static void __init print_out_mtrr_range_state(void)
502{
503	char start_factor = 'K', size_factor = 'K';
504	unsigned long start_base, size_base;
505	mtrr_type type;
506	int i;
507
508	for (i = 0; i < num_var_ranges; i++) {
509
510		size_base = range_state[i].size_pfn << (PAGE_SHIFT - 10);
511		if (!size_base)
512			continue;
513
514		size_base = to_size_factor(size_base, &size_factor);
515		start_base = range_state[i].base_pfn << (PAGE_SHIFT - 10);
516		start_base = to_size_factor(start_base, &start_factor);
517		type = range_state[i].type;
518
519		Dprintk("reg %d, base: %ld%cB, range: %ld%cB, type %s\n",
520			i, start_base, start_factor,
521			size_base, size_factor,
522			(type == MTRR_TYPE_UNCACHABLE) ? "UC" :
523			    ((type == MTRR_TYPE_WRPROT) ? "WP" :
524			     ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other"))
525			);
526	}
527}
528
529static int __init mtrr_need_cleanup(void)
530{
531	int i;
532	mtrr_type type;
533	unsigned long size;
534	/* Extra one for all 0: */
535	int num[MTRR_NUM_TYPES + 1];
536
537	/* Check entries number: */
538	memset(num, 0, sizeof(num));
539	for (i = 0; i < num_var_ranges; i++) {
540		type = range_state[i].type;
541		size = range_state[i].size_pfn;
542		if (type >= MTRR_NUM_TYPES)
543			continue;
544		if (!size)
545			type = MTRR_NUM_TYPES;
546		num[type]++;
547	}
548
549	/* Check if we got UC entries: */
550	if (!num[MTRR_TYPE_UNCACHABLE])
551		return 0;
552
553	/* Check if we only had WB and UC */
554	if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] !=
555	    num_var_ranges - num[MTRR_NUM_TYPES])
556		return 0;
557
558	return 1;
559}
560
561static unsigned long __initdata range_sums;
562
563static void __init
564mtrr_calc_range_state(u64 chunk_size, u64 gran_size,
565		      unsigned long x_remove_base,
566		      unsigned long x_remove_size, int i)
567{
568	/*
569	 * range_new should really be an automatic variable, but
570	 * putting 4096 bytes on the stack is frowned upon, to put it
571	 * mildly. It is safe to make it a static __initdata variable,
572	 * since mtrr_calc_range_state is only called during init and
573	 * there's no way it will call itself recursively.
574	 */
575	static struct range range_new[RANGE_NUM] __initdata;
576	unsigned long range_sums_new;
577	int nr_range_new;
578	int num_reg;
579
580	/* Convert ranges to var ranges state: */
581	num_reg = x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size);
582
583	/* We got new setting in range_state, check it: */
584	memset(range_new, 0, sizeof(range_new));
585	nr_range_new = x86_get_mtrr_mem_range(range_new, 0,
586				x_remove_base, x_remove_size);
587	range_sums_new = sum_ranges(range_new, nr_range_new);
588
589	result[i].chunk_sizek = chunk_size >> 10;
590	result[i].gran_sizek = gran_size >> 10;
591	result[i].num_reg = num_reg;
592
593	if (range_sums < range_sums_new) {
594		result[i].lose_cover_sizek = (range_sums_new - range_sums) << PSHIFT;
595		result[i].bad = 1;
596	} else {
597		result[i].lose_cover_sizek = (range_sums - range_sums_new) << PSHIFT;
598	}
599
600	/* Double check it: */
601	if (!result[i].bad && !result[i].lose_cover_sizek) {
602		if (nr_range_new != nr_range || memcmp(range, range_new, sizeof(range)))
603			result[i].bad = 1;
604	}
605
606	if (!result[i].bad && (range_sums - range_sums_new < min_loss_pfn[num_reg]))
607		min_loss_pfn[num_reg] = range_sums - range_sums_new;
608}
609
610static void __init mtrr_print_out_one_result(int i)
611{
612	unsigned long gran_base, chunk_base, lose_base;
613	char gran_factor, chunk_factor, lose_factor;
614
615	gran_base = to_size_factor(result[i].gran_sizek, &gran_factor);
616	chunk_base = to_size_factor(result[i].chunk_sizek, &chunk_factor);
617	lose_base = to_size_factor(result[i].lose_cover_sizek, &lose_factor);
618
619	pr_info("%sgran_size: %ld%c \tchunk_size: %ld%c \t",
620		result[i].bad ? "*BAD*" : " ",
621		gran_base, gran_factor, chunk_base, chunk_factor);
622	pr_cont("num_reg: %d  \tlose cover RAM: %s%ld%c\n",
623		result[i].num_reg, result[i].bad ? "-" : "",
624		lose_base, lose_factor);
625}
626
627static int __init mtrr_search_optimal_index(void)
628{
629	int num_reg_good;
630	int index_good;
631	int i;
632
633	if (nr_mtrr_spare_reg >= num_var_ranges)
634		nr_mtrr_spare_reg = num_var_ranges - 1;
635
636	num_reg_good = -1;
637	for (i = num_var_ranges - nr_mtrr_spare_reg; i > 0; i--) {
638		if (!min_loss_pfn[i])
639			num_reg_good = i;
640	}
641
642	index_good = -1;
643	if (num_reg_good != -1) {
644		for (i = 0; i < NUM_RESULT; i++) {
645			if (!result[i].bad &&
646			    result[i].num_reg == num_reg_good &&
647			    !result[i].lose_cover_sizek) {
648				index_good = i;
649				break;
650			}
651		}
652	}
653
654	return index_good;
655}
656
657int __init mtrr_cleanup(void)
658{
659	unsigned long x_remove_base, x_remove_size;
660	unsigned long base, size, def, dummy;
661	u64 chunk_size, gran_size;
662	mtrr_type type;
663	int index_good;
664	int i;
665
666	if (!mtrr_enabled())
667		return 0;
668
669	if (!cpu_feature_enabled(X86_FEATURE_MTRR) || enable_mtrr_cleanup < 1)
670		return 0;
671
672	rdmsr(MSR_MTRRdefType, def, dummy);
673	def &= 0xff;
674	if (def != MTRR_TYPE_UNCACHABLE)
675		return 0;
676
677	/* Get it and store it aside: */
678	memset(range_state, 0, sizeof(range_state));
679	for (i = 0; i < num_var_ranges; i++) {
680		mtrr_if->get(i, &base, &size, &type);
681		range_state[i].base_pfn = base;
682		range_state[i].size_pfn = size;
683		range_state[i].type = type;
684	}
685
686	/* Check if we need handle it and can handle it: */
687	if (!mtrr_need_cleanup())
688		return 0;
689
690	/* Print original var MTRRs at first, for debugging: */
691	Dprintk("original variable MTRRs\n");
692	print_out_mtrr_range_state();
693
694	memset(range, 0, sizeof(range));
695	x_remove_size = 0;
696	x_remove_base = 1 << (32 - PAGE_SHIFT);
697	if (mtrr_tom2)
698		x_remove_size = (mtrr_tom2 >> PAGE_SHIFT) - x_remove_base;
699
700	/*
701	 * [0, 1M) should always be covered by var mtrr with WB
702	 * and fixed mtrrs should take effect before var mtrr for it:
703	 */
704	nr_range = add_range_with_merge(range, RANGE_NUM, 0, 0,
705					1ULL<<(20 - PAGE_SHIFT));
706	/* add from var mtrr at last */
707	nr_range = x86_get_mtrr_mem_range(range, nr_range,
708					  x_remove_base, x_remove_size);
709
710	range_sums = sum_ranges(range, nr_range);
711	pr_info("total RAM covered: %ldM\n",
712	       range_sums >> (20 - PAGE_SHIFT));
713
714	if (mtrr_chunk_size && mtrr_gran_size) {
715		i = 0;
716		mtrr_calc_range_state(mtrr_chunk_size, mtrr_gran_size,
717				      x_remove_base, x_remove_size, i);
718
719		mtrr_print_out_one_result(i);
720
721		if (!result[i].bad) {
722			set_var_mtrr_all();
723			Dprintk("New variable MTRRs\n");
724			print_out_mtrr_range_state();
725			return 1;
726		}
727		pr_info("invalid mtrr_gran_size or mtrr_chunk_size, will find optimal one\n");
728	}
729
730	i = 0;
731	memset(min_loss_pfn, 0xff, sizeof(min_loss_pfn));
732	memset(result, 0, sizeof(result));
733	for (gran_size = (1ULL<<16); gran_size < (1ULL<<32); gran_size <<= 1) {
734
735		for (chunk_size = gran_size; chunk_size < (1ULL<<32);
736		     chunk_size <<= 1) {
737
738			if (i >= NUM_RESULT)
739				continue;
740
741			mtrr_calc_range_state(chunk_size, gran_size,
742				      x_remove_base, x_remove_size, i);
743			if (mtrr_debug) {
744				mtrr_print_out_one_result(i);
745				pr_info("\n");
746			}
747
748			i++;
749		}
750	}
751
752	/* Try to find the optimal index: */
753	index_good = mtrr_search_optimal_index();
754
755	if (index_good != -1) {
756		pr_info("Found optimal setting for mtrr clean up\n");
757		i = index_good;
758		mtrr_print_out_one_result(i);
759
760		/* Convert ranges to var ranges state: */
761		chunk_size = result[i].chunk_sizek;
762		chunk_size <<= 10;
763		gran_size = result[i].gran_sizek;
764		gran_size <<= 10;
765		x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size);
766		set_var_mtrr_all();
767		Dprintk("New variable MTRRs\n");
768		print_out_mtrr_range_state();
769		return 1;
770	} else {
771		/* print out all */
772		for (i = 0; i < NUM_RESULT; i++)
773			mtrr_print_out_one_result(i);
774	}
775
776	pr_info("mtrr_cleanup: can not find optimal value\n");
777	pr_info("please specify mtrr_gran_size/mtrr_chunk_size\n");
778
779	return 0;
780}
781#else
782int __init mtrr_cleanup(void)
783{
784	return 0;
785}
786#endif
787
788static int disable_mtrr_trim;
789
790static int __init disable_mtrr_trim_setup(char *str)
791{
792	disable_mtrr_trim = 1;
793	return 0;
794}
795early_param("disable_mtrr_trim", disable_mtrr_trim_setup);
796
797/*
798 * Newer AMD K8s and later CPUs have a special magic MSR way to force WB
799 * for memory >4GB. Check for that here.
800 * Note this won't check if the MTRRs < 4GB where the magic bit doesn't
801 * apply to are wrong, but so far we don't know of any such case in the wild.
802 */
803#define Tom2Enabled		(1U << 21)
804#define Tom2ForceMemTypeWB	(1U << 22)
805
806int __init amd_special_default_mtrr(void)
807{
808	u32 l, h;
809
810	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
811	    boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
812		return 0;
813	if (boot_cpu_data.x86 < 0xf)
814		return 0;
815	/* In case some hypervisor doesn't pass SYSCFG through: */
816	if (rdmsr_safe(MSR_AMD64_SYSCFG, &l, &h) < 0)
817		return 0;
818	/*
819	 * Memory between 4GB and top of mem is forced WB by this magic bit.
820	 * Reserved before K8RevF, but should be zero there.
821	 */
822	if ((l & (Tom2Enabled | Tom2ForceMemTypeWB)) ==
823		 (Tom2Enabled | Tom2ForceMemTypeWB))
824		return 1;
825	return 0;
826}
827
828static u64 __init
829real_trim_memory(unsigned long start_pfn, unsigned long limit_pfn)
830{
831	u64 trim_start, trim_size;
832
833	trim_start = start_pfn;
834	trim_start <<= PAGE_SHIFT;
835
836	trim_size = limit_pfn;
837	trim_size <<= PAGE_SHIFT;
838	trim_size -= trim_start;
839
840	return e820__range_update(trim_start, trim_size, E820_TYPE_RAM, E820_TYPE_RESERVED);
841}
842
843/**
844 * mtrr_trim_uncached_memory - trim RAM not covered by MTRRs
845 * @end_pfn: ending page frame number
846 *
847 * Some buggy BIOSes don't setup the MTRRs properly for systems with certain
848 * memory configurations.  This routine checks that the highest MTRR matches
849 * the end of memory, to make sure the MTRRs having a write back type cover
850 * all of the memory the kernel is intending to use.  If not, it'll trim any
851 * memory off the end by adjusting end_pfn, removing it from the kernel's
852 * allocation pools, warning the user with an obnoxious message.
853 */
854int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
855{
856	unsigned long i, base, size, highest_pfn = 0, def, dummy;
857	mtrr_type type;
858	u64 total_trim_size;
859	/* extra one for all 0 */
860	int num[MTRR_NUM_TYPES + 1];
861
862	if (!mtrr_enabled())
863		return 0;
864
865	/*
866	 * Make sure we only trim uncachable memory on machines that
867	 * support the Intel MTRR architecture:
868	 */
869	if (!cpu_feature_enabled(X86_FEATURE_MTRR) || disable_mtrr_trim)
870		return 0;
871
872	rdmsr(MSR_MTRRdefType, def, dummy);
873	def &= MTRR_DEF_TYPE_TYPE;
874	if (def != MTRR_TYPE_UNCACHABLE)
875		return 0;
876
877	/* Get it and store it aside: */
878	memset(range_state, 0, sizeof(range_state));
879	for (i = 0; i < num_var_ranges; i++) {
880		mtrr_if->get(i, &base, &size, &type);
881		range_state[i].base_pfn = base;
882		range_state[i].size_pfn = size;
883		range_state[i].type = type;
884	}
885
886	/* Find highest cached pfn: */
887	for (i = 0; i < num_var_ranges; i++) {
888		type = range_state[i].type;
889		if (type != MTRR_TYPE_WRBACK)
890			continue;
891		base = range_state[i].base_pfn;
892		size = range_state[i].size_pfn;
893		if (highest_pfn < base + size)
894			highest_pfn = base + size;
895	}
896
897	/* kvm/qemu doesn't have mtrr set right, don't trim them all: */
898	if (!highest_pfn) {
899		pr_info("CPU MTRRs all blank - virtualized system.\n");
900		return 0;
901	}
902
903	/* Check entries number: */
904	memset(num, 0, sizeof(num));
905	for (i = 0; i < num_var_ranges; i++) {
906		type = range_state[i].type;
907		if (type >= MTRR_NUM_TYPES)
908			continue;
909		size = range_state[i].size_pfn;
910		if (!size)
911			type = MTRR_NUM_TYPES;
912		num[type]++;
913	}
914
915	/* No entry for WB? */
916	if (!num[MTRR_TYPE_WRBACK])
917		return 0;
918
919	/* Check if we only had WB and UC: */
920	if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] !=
921		num_var_ranges - num[MTRR_NUM_TYPES])
922		return 0;
923
924	memset(range, 0, sizeof(range));
925	nr_range = 0;
926	if (mtrr_tom2) {
927		range[nr_range].start = (1ULL<<(32 - PAGE_SHIFT));
928		range[nr_range].end = mtrr_tom2 >> PAGE_SHIFT;
929		if (highest_pfn < range[nr_range].end)
930			highest_pfn = range[nr_range].end;
931		nr_range++;
932	}
933	nr_range = x86_get_mtrr_mem_range(range, nr_range, 0, 0);
934
935	/* Check the head: */
936	total_trim_size = 0;
937	if (range[0].start)
938		total_trim_size += real_trim_memory(0, range[0].start);
939
940	/* Check the holes: */
941	for (i = 0; i < nr_range - 1; i++) {
942		if (range[i].end < range[i+1].start)
943			total_trim_size += real_trim_memory(range[i].end,
944							    range[i+1].start);
945	}
946
947	/* Check the top: */
948	i = nr_range - 1;
949	if (range[i].end < end_pfn)
950		total_trim_size += real_trim_memory(range[i].end,
951							 end_pfn);
952
953	if (total_trim_size) {
954		pr_warn("WARNING: BIOS bug: CPU MTRRs don't cover all of memory, losing %lluMB of RAM.\n",
955			total_trim_size >> 20);
956
957		if (!changed_by_mtrr_cleanup)
958			WARN_ON(1);
959
960		pr_info("update e820 for mtrr\n");
961		e820__update_table_print();
962
963		return 1;
964	}
965
966	return 0;
967}