setup.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Machine specific setup for xen
   4 *
   5 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
   6 */
   7
   8#include <linux/init.h>
   9#include <linux/iscsi_ibft.h>
  10#include <linux/sched.h>
  11#include <linux/kstrtox.h>
  12#include <linux/mm.h>
  13#include <linux/pm.h>
  14#include <linux/memblock.h>
  15#include <linux/cpuidle.h>
  16#include <linux/cpufreq.h>
  17#include <linux/memory_hotplug.h>
  18#include <linux/acpi.h>
  19
  20#include <asm/elf.h>
  21#include <asm/vdso.h>
  22#include <asm/e820/api.h>
  23#include <asm/setup.h>
  24#include <asm/numa.h>
  25#include <asm/idtentry.h>
  26#include <asm/xen/hypervisor.h>
  27#include <asm/xen/hypercall.h>
  28
  29#include <xen/xen.h>
  30#include <xen/page.h>
  31#include <xen/interface/callback.h>
  32#include <xen/interface/memory.h>
  33#include <xen/interface/physdev.h>
  34#include <xen/features.h>
  35#include <xen/hvc-console.h>
  36#include "xen-ops.h"
  37
  38#define GB(x) ((uint64_t)(x) * 1024 * 1024 * 1024)
  39
  40/* Memory map would allow PCI passthrough. */
  41bool xen_pv_pci_possible;
  42
  43/* E820 map used during setting up memory. */
  44static struct e820_table xen_e820_table __initdata;
  45
  46/* Number of initially usable memory pages. */
  47static unsigned long ini_nr_pages __initdata;
  48
  49/*
  50 * Buffer used to remap identity mapped pages. We only need the virtual space.
  51 * The physical page behind this address is remapped as needed to different
  52 * buffer pages.
  53 */
  54#define REMAP_SIZE	(P2M_PER_PAGE - 3)
  55static struct {
  56	unsigned long	next_area_mfn;
  57	unsigned long	target_pfn;
  58	unsigned long	size;
  59	unsigned long	mfns[REMAP_SIZE];
  60} xen_remap_buf __initdata __aligned(PAGE_SIZE);
  61static unsigned long xen_remap_mfn __initdata = INVALID_P2M_ENTRY;
  62
  63static bool xen_512gb_limit __initdata = IS_ENABLED(CONFIG_XEN_512GB);
  64
  65static void __init xen_parse_512gb(void)
  66{
  67	bool val = false;
  68	char *arg;
  69
  70	arg = strstr(xen_start_info->cmd_line, "xen_512gb_limit");
  71	if (!arg)
  72		return;
  73
  74	arg = strstr(xen_start_info->cmd_line, "xen_512gb_limit=");
  75	if (!arg)
  76		val = true;
  77	else if (kstrtobool(arg + strlen("xen_512gb_limit="), &val))
  78		return;
  79
  80	xen_512gb_limit = val;
  81}
  82
  83static void __init xen_del_extra_mem(unsigned long start_pfn,
  84				     unsigned long n_pfns)
  85{
  86	int i;
  87	unsigned long start_r, size_r;
  88
  89	for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) {
  90		start_r = xen_extra_mem[i].start_pfn;
  91		size_r = xen_extra_mem[i].n_pfns;
  92
  93		/* Start of region. */
  94		if (start_r == start_pfn) {
  95			BUG_ON(n_pfns > size_r);
  96			xen_extra_mem[i].start_pfn += n_pfns;
  97			xen_extra_mem[i].n_pfns -= n_pfns;
  98			break;
  99		}
 100		/* End of region. */
 101		if (start_r + size_r == start_pfn + n_pfns) {
 102			BUG_ON(n_pfns > size_r);
 103			xen_extra_mem[i].n_pfns -= n_pfns;
 104			break;
 105		}
 106		/* Mid of region. */
 107		if (start_pfn > start_r && start_pfn < start_r + size_r) {
 108			BUG_ON(start_pfn + n_pfns > start_r + size_r);
 109			xen_extra_mem[i].n_pfns = start_pfn - start_r;
 110			/* Calling memblock_reserve() again is okay. */
 111			xen_add_extra_mem(start_pfn + n_pfns, start_r + size_r -
 112					  (start_pfn + n_pfns));
 113			break;
 114		}
 115	}
 116	memblock_phys_free(PFN_PHYS(start_pfn), PFN_PHYS(n_pfns));
 117}
 118
 119/*
 120 * Called during boot before the p2m list can take entries beyond the
 121 * hypervisor supplied p2m list. Entries in extra mem are to be regarded as
 122 * invalid.
 123 */
 124unsigned long __ref xen_chk_extra_mem(unsigned long pfn)
 125{
 126	int i;
 127
 128	for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) {
 129		if (pfn >= xen_extra_mem[i].start_pfn &&
 130		    pfn < xen_extra_mem[i].start_pfn + xen_extra_mem[i].n_pfns)
 131			return INVALID_P2M_ENTRY;
 132	}
 133
 134	return IDENTITY_FRAME(pfn);
 135}
 136
 137/*
 138 * Mark all pfns of extra mem as invalid in p2m list.
 139 */
 140void __init xen_inv_extra_mem(void)
 141{
 142	unsigned long pfn, pfn_s, pfn_e;
 143	int i;
 144
 145	for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) {
 146		if (!xen_extra_mem[i].n_pfns)
 147			continue;
 148		pfn_s = xen_extra_mem[i].start_pfn;
 149		pfn_e = pfn_s + xen_extra_mem[i].n_pfns;
 150		for (pfn = pfn_s; pfn < pfn_e; pfn++)
 151			set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
 152	}
 153}
 154
 155/*
 156 * Finds the next RAM pfn available in the E820 map after min_pfn.
 157 * This function updates min_pfn with the pfn found and returns
 158 * the size of that range or zero if not found.
 159 */
 160static unsigned long __init xen_find_pfn_range(unsigned long *min_pfn)
 161{
 162	const struct e820_entry *entry = xen_e820_table.entries;
 163	unsigned int i;
 164	unsigned long done = 0;
 165
 166	for (i = 0; i < xen_e820_table.nr_entries; i++, entry++) {
 167		unsigned long s_pfn;
 168		unsigned long e_pfn;
 169
 170		if (entry->type != E820_TYPE_RAM)
 171			continue;
 172
 173		e_pfn = PFN_DOWN(entry->addr + entry->size);
 174
 175		/* We only care about E820 after this */
 176		if (e_pfn <= *min_pfn)
 177			continue;
 178
 179		s_pfn = PFN_UP(entry->addr);
 180
 181		/* If min_pfn falls within the E820 entry, we want to start
 182		 * at the min_pfn PFN.
 183		 */
 184		if (s_pfn <= *min_pfn) {
 185			done = e_pfn - *min_pfn;
 186		} else {
 187			done = e_pfn - s_pfn;
 188			*min_pfn = s_pfn;
 189		}
 190		break;
 191	}
 192
 193	return done;
 194}
 195
 196static int __init xen_free_mfn(unsigned long mfn)
 197{
 198	struct xen_memory_reservation reservation = {
 199		.address_bits = 0,
 200		.extent_order = 0,
 201		.domid        = DOMID_SELF
 202	};
 203
 204	set_xen_guest_handle(reservation.extent_start, &mfn);
 205	reservation.nr_extents = 1;
 206
 207	return HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation);
 208}
 209
 210/*
 211 * This releases a chunk of memory and then does the identity map. It's used
 212 * as a fallback if the remapping fails.
 213 */
 214static void __init xen_set_identity_and_release_chunk(unsigned long start_pfn,
 215						      unsigned long end_pfn)
 216{
 217	unsigned long pfn, end;
 218	int ret;
 219
 220	WARN_ON(start_pfn > end_pfn);
 221
 222	/* Release pages first. */
 223	end = min(end_pfn, ini_nr_pages);
 224	for (pfn = start_pfn; pfn < end; pfn++) {
 225		unsigned long mfn = pfn_to_mfn(pfn);
 226
 227		/* Make sure pfn exists to start with */
 228		if (mfn == INVALID_P2M_ENTRY || mfn_to_pfn(mfn) != pfn)
 229			continue;
 230
 231		ret = xen_free_mfn(mfn);
 232		WARN(ret != 1, "Failed to release pfn %lx err=%d\n", pfn, ret);
 233
 234		if (ret == 1) {
 235			xen_released_pages++;
 236			if (!__set_phys_to_machine(pfn, INVALID_P2M_ENTRY))
 237				break;
 238		} else
 239			break;
 240	}
 241
 242	set_phys_range_identity(start_pfn, end_pfn);
 243}
 244
 245/*
 246 * Helper function to update the p2m and m2p tables and kernel mapping.
 247 */
 248static void __init xen_update_mem_tables(unsigned long pfn, unsigned long mfn)
 249{
 250	struct mmu_update update = {
 251		.ptr = ((uint64_t)mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE,
 252		.val = pfn
 253	};
 254
 255	/* Update p2m */
 256	if (!set_phys_to_machine(pfn, mfn)) {
 257		WARN(1, "Failed to set p2m mapping for pfn=%ld mfn=%ld\n",
 258		     pfn, mfn);
 259		BUG();
 260	}
 261
 262	/* Update m2p */
 263	if (HYPERVISOR_mmu_update(&update, 1, NULL, DOMID_SELF) < 0) {
 264		WARN(1, "Failed to set m2p mapping for mfn=%ld pfn=%ld\n",
 265		     mfn, pfn);
 266		BUG();
 267	}
 268
 269	if (HYPERVISOR_update_va_mapping((unsigned long)__va(pfn << PAGE_SHIFT),
 270					 mfn_pte(mfn, PAGE_KERNEL), 0)) {
 271		WARN(1, "Failed to update kernel mapping for mfn=%ld pfn=%ld\n",
 272		      mfn, pfn);
 273		BUG();
 274	}
 275}
 276
 277/*
 278 * This function updates the p2m and m2p tables with an identity map from
 279 * start_pfn to start_pfn+size and prepares remapping the underlying RAM of the
 280 * original allocation at remap_pfn. The information needed for remapping is
 281 * saved in the memory itself to avoid the need for allocating buffers. The
 282 * complete remap information is contained in a list of MFNs each containing
 283 * up to REMAP_SIZE MFNs and the start target PFN for doing the remap.
 284 * This enables us to preserve the original mfn sequence while doing the
 285 * remapping at a time when the memory management is capable of allocating
 286 * virtual and physical memory in arbitrary amounts, see 'xen_remap_memory' and
 287 * its callers.
 288 */
 289static void __init xen_do_set_identity_and_remap_chunk(
 290        unsigned long start_pfn, unsigned long size, unsigned long remap_pfn)
 291{
 292	unsigned long buf = (unsigned long)&xen_remap_buf;
 293	unsigned long mfn_save, mfn;
 294	unsigned long ident_pfn_iter, remap_pfn_iter;
 295	unsigned long ident_end_pfn = start_pfn + size;
 296	unsigned long left = size;
 297	unsigned int i, chunk;
 298
 299	WARN_ON(size == 0);
 300
 301	mfn_save = virt_to_mfn((void *)buf);
 302
 303	for (ident_pfn_iter = start_pfn, remap_pfn_iter = remap_pfn;
 304	     ident_pfn_iter < ident_end_pfn;
 305	     ident_pfn_iter += REMAP_SIZE, remap_pfn_iter += REMAP_SIZE) {
 306		chunk = (left < REMAP_SIZE) ? left : REMAP_SIZE;
 307
 308		/* Map first pfn to xen_remap_buf */
 309		mfn = pfn_to_mfn(ident_pfn_iter);
 310		set_pte_mfn(buf, mfn, PAGE_KERNEL);
 311
 312		/* Save mapping information in page */
 313		xen_remap_buf.next_area_mfn = xen_remap_mfn;
 314		xen_remap_buf.target_pfn = remap_pfn_iter;
 315		xen_remap_buf.size = chunk;
 316		for (i = 0; i < chunk; i++)
 317			xen_remap_buf.mfns[i] = pfn_to_mfn(ident_pfn_iter + i);
 318
 319		/* Put remap buf into list. */
 320		xen_remap_mfn = mfn;
 321
 322		/* Set identity map */
 323		set_phys_range_identity(ident_pfn_iter, ident_pfn_iter + chunk);
 324
 325		left -= chunk;
 326	}
 327
 328	/* Restore old xen_remap_buf mapping */
 329	set_pte_mfn(buf, mfn_save, PAGE_KERNEL);
 330}
 331
 332/*
 333 * This function takes a contiguous pfn range that needs to be identity mapped
 334 * and:
 335 *
 336 *  1) Finds a new range of pfns to use to remap based on E820 and remap_pfn.
 337 *  2) Calls the do_ function to actually do the mapping/remapping work.
 338 *
 339 * The goal is to not allocate additional memory but to remap the existing
 340 * pages. In the case of an error the underlying memory is simply released back
 341 * to Xen and not remapped.
 342 */
 343static unsigned long __init xen_set_identity_and_remap_chunk(
 344	unsigned long start_pfn, unsigned long end_pfn, unsigned long remap_pfn)
 345{
 346	unsigned long pfn;
 347	unsigned long i = 0;
 348	unsigned long n = end_pfn - start_pfn;
 349
 350	if (remap_pfn == 0)
 351		remap_pfn = ini_nr_pages;
 352
 353	while (i < n) {
 354		unsigned long cur_pfn = start_pfn + i;
 355		unsigned long left = n - i;
 356		unsigned long size = left;
 357		unsigned long remap_range_size;
 358
 359		/* Do not remap pages beyond the current allocation */
 360		if (cur_pfn >= ini_nr_pages) {
 361			/* Identity map remaining pages */
 362			set_phys_range_identity(cur_pfn, cur_pfn + size);
 363			break;
 364		}
 365		if (cur_pfn + size > ini_nr_pages)
 366			size = ini_nr_pages - cur_pfn;
 367
 368		remap_range_size = xen_find_pfn_range(&remap_pfn);
 369		if (!remap_range_size) {
 370			pr_warn("Unable to find available pfn range, not remapping identity pages\n");
 371			xen_set_identity_and_release_chunk(cur_pfn,
 372							   cur_pfn + left);
 373			break;
 374		}
 375		/* Adjust size to fit in current e820 RAM region */
 376		if (size > remap_range_size)
 377			size = remap_range_size;
 378
 379		xen_do_set_identity_and_remap_chunk(cur_pfn, size, remap_pfn);
 380
 381		/* Update variables to reflect new mappings. */
 382		i += size;
 383		remap_pfn += size;
 384	}
 385
 386	/*
 387	 * If the PFNs are currently mapped, their VA mappings need to be
 388	 * zapped.
 389	 */
 390	for (pfn = start_pfn; pfn <= max_pfn_mapped && pfn < end_pfn; pfn++)
 391		(void)HYPERVISOR_update_va_mapping(
 392			(unsigned long)__va(pfn << PAGE_SHIFT),
 393			native_make_pte(0), 0);
 394
 395	return remap_pfn;
 396}
 397
 398static unsigned long __init xen_count_remap_pages(
 399	unsigned long start_pfn, unsigned long end_pfn,
 400	unsigned long remap_pages)
 401{
 402	if (start_pfn >= ini_nr_pages)
 403		return remap_pages;
 404
 405	return remap_pages + min(end_pfn, ini_nr_pages) - start_pfn;
 406}
 407
 408static unsigned long __init xen_foreach_remap_area(
 409	unsigned long (*func)(unsigned long start_pfn, unsigned long end_pfn,
 410			      unsigned long last_val))
 411{
 412	phys_addr_t start = 0;
 413	unsigned long ret_val = 0;
 414	const struct e820_entry *entry = xen_e820_table.entries;
 415	int i;
 416
 417	/*
 418	 * Combine non-RAM regions and gaps until a RAM region (or the
 419	 * end of the map) is reached, then call the provided function
 420	 * to perform its duty on the non-RAM region.
 421	 *
 422	 * The combined non-RAM regions are rounded to a whole number
 423	 * of pages so any partial pages are accessible via the 1:1
 424	 * mapping.  This is needed for some BIOSes that put (for
 425	 * example) the DMI tables in a reserved region that begins on
 426	 * a non-page boundary.
 427	 */
 428	for (i = 0; i < xen_e820_table.nr_entries; i++, entry++) {
 429		phys_addr_t end = entry->addr + entry->size;
 430		if (entry->type == E820_TYPE_RAM || i == xen_e820_table.nr_entries - 1) {
 431			unsigned long start_pfn = PFN_DOWN(start);
 432			unsigned long end_pfn = PFN_UP(end);
 433
 434			if (entry->type == E820_TYPE_RAM)
 435				end_pfn = PFN_UP(entry->addr);
 436
 437			if (start_pfn < end_pfn)
 438				ret_val = func(start_pfn, end_pfn, ret_val);
 439			start = end;
 440		}
 441	}
 442
 443	return ret_val;
 444}
 445
 446/*
 447 * Remap the memory prepared in xen_do_set_identity_and_remap_chunk().
 448 * The remap information (which mfn remap to which pfn) is contained in the
 449 * to be remapped memory itself in a linked list anchored at xen_remap_mfn.
 450 * This scheme allows to remap the different chunks in arbitrary order while
 451 * the resulting mapping will be independent from the order.
 452 */
 453void __init xen_remap_memory(void)
 454{
 455	unsigned long buf = (unsigned long)&xen_remap_buf;
 456	unsigned long mfn_save, pfn;
 457	unsigned long remapped = 0;
 458	unsigned int i;
 459	unsigned long pfn_s = ~0UL;
 460	unsigned long len = 0;
 461
 462	mfn_save = virt_to_mfn((void *)buf);
 463
 464	while (xen_remap_mfn != INVALID_P2M_ENTRY) {
 465		/* Map the remap information */
 466		set_pte_mfn(buf, xen_remap_mfn, PAGE_KERNEL);
 467
 468		BUG_ON(xen_remap_mfn != xen_remap_buf.mfns[0]);
 469
 470		pfn = xen_remap_buf.target_pfn;
 471		for (i = 0; i < xen_remap_buf.size; i++) {
 472			xen_update_mem_tables(pfn, xen_remap_buf.mfns[i]);
 473			remapped++;
 474			pfn++;
 475		}
 476		if (pfn_s == ~0UL || pfn == pfn_s) {
 477			pfn_s = xen_remap_buf.target_pfn;
 478			len += xen_remap_buf.size;
 479		} else if (pfn_s + len == xen_remap_buf.target_pfn) {
 480			len += xen_remap_buf.size;
 481		} else {
 482			xen_del_extra_mem(pfn_s, len);
 483			pfn_s = xen_remap_buf.target_pfn;
 484			len = xen_remap_buf.size;
 485		}
 486		xen_remap_mfn = xen_remap_buf.next_area_mfn;
 487	}
 488
 489	if (pfn_s != ~0UL && len)
 490		xen_del_extra_mem(pfn_s, len);
 491
 492	set_pte_mfn(buf, mfn_save, PAGE_KERNEL);
 493
 494	pr_info("Remapped %ld page(s)\n", remapped);
 495
 496	xen_do_remap_nonram();
 497}
 498
 499static unsigned long __init xen_get_pages_limit(void)
 500{
 501	unsigned long limit;
 502
 503	limit = MAXMEM / PAGE_SIZE;
 504	if (!xen_initial_domain() && xen_512gb_limit)
 505		limit = GB(512) / PAGE_SIZE;
 506
 507	return limit;
 508}
 509
 510static unsigned long __init xen_get_max_pages(void)
 511{
 512	unsigned long max_pages, limit;
 513	domid_t domid = DOMID_SELF;
 514	long ret;
 515
 516	limit = xen_get_pages_limit();
 517	max_pages = limit;
 518
 519	/*
 520	 * For the initial domain we use the maximum reservation as
 521	 * the maximum page.
 522	 *
 523	 * For guest domains the current maximum reservation reflects
 524	 * the current maximum rather than the static maximum. In this
 525	 * case the e820 map provided to us will cover the static
 526	 * maximum region.
 527	 */
 528	if (xen_initial_domain()) {
 529		ret = HYPERVISOR_memory_op(XENMEM_maximum_reservation, &domid);
 530		if (ret > 0)
 531			max_pages = ret;
 532	}
 533
 534	return min(max_pages, limit);
 535}
 536
 537static void __init xen_align_and_add_e820_region(phys_addr_t start,
 538						 phys_addr_t size, int type)
 539{
 540	phys_addr_t end = start + size;
 541
 542	/* Align RAM regions to page boundaries. */
 543	if (type == E820_TYPE_RAM) {
 544		start = PAGE_ALIGN(start);
 545		end &= ~((phys_addr_t)PAGE_SIZE - 1);
 546#ifdef CONFIG_MEMORY_HOTPLUG
 547		/*
 548		 * Don't allow adding memory not in E820 map while booting the
 549		 * system. Once the balloon driver is up it will remove that
 550		 * restriction again.
 551		 */
 552		max_mem_size = end;
 553#endif
 554	}
 555
 556	e820__range_add(start, end - start, type);
 557}
 558
 559static void __init xen_ignore_unusable(void)
 560{
 561	struct e820_entry *entry = xen_e820_table.entries;
 562	unsigned int i;
 563
 564	for (i = 0; i < xen_e820_table.nr_entries; i++, entry++) {
 565		if (entry->type == E820_TYPE_UNUSABLE)
 566			entry->type = E820_TYPE_RAM;
 567	}
 568}
 569
 570static bool __init xen_is_e820_reserved(phys_addr_t start, phys_addr_t size)
 571{
 572	struct e820_entry *entry;
 573	unsigned mapcnt;
 574	phys_addr_t end;
 575
 576	if (!size)
 577		return false;
 578
 579	end = start + size;
 580	entry = xen_e820_table.entries;
 581
 582	for (mapcnt = 0; mapcnt < xen_e820_table.nr_entries; mapcnt++) {
 583		if (entry->type == E820_TYPE_RAM && entry->addr <= start &&
 584		    (entry->addr + entry->size) >= end)
 585			return false;
 586
 587		entry++;
 588	}
 589
 590	return true;
 591}
 592
 593/*
 594 * Find a free area in physical memory not yet reserved and compliant with
 595 * E820 map.
 596 * Used to relocate pre-allocated areas like initrd or p2m list which are in
 597 * conflict with the to be used E820 map.
 598 * In case no area is found, return 0. Otherwise return the physical address
 599 * of the area which is already reserved for convenience.
 600 */
 601phys_addr_t __init xen_find_free_area(phys_addr_t size)
 602{
 603	unsigned mapcnt;
 604	phys_addr_t addr, start;
 605	struct e820_entry *entry = xen_e820_table.entries;
 606
 607	for (mapcnt = 0; mapcnt < xen_e820_table.nr_entries; mapcnt++, entry++) {
 608		if (entry->type != E820_TYPE_RAM || entry->size < size)
 609			continue;
 610		start = entry->addr;
 611		for (addr = start; addr < start + size; addr += PAGE_SIZE) {
 612			if (!memblock_is_reserved(addr))
 613				continue;
 614			start = addr + PAGE_SIZE;
 615			if (start + size > entry->addr + entry->size)
 616				break;
 617		}
 618		if (addr >= start + size) {
 619			memblock_reserve(start, size);
 620			return start;
 621		}
 622	}
 623
 624	return 0;
 625}
 626
 627/*
 628 * Swap a non-RAM E820 map entry with RAM above ini_nr_pages.
 629 * Note that the E820 map is modified accordingly, but the P2M map isn't yet.
 630 * The adaption of the P2M must be deferred until page allocation is possible.
 631 */
 632static void __init xen_e820_swap_entry_with_ram(struct e820_entry *swap_entry)
 633{
 634	struct e820_entry *entry;
 635	unsigned int mapcnt;
 636	phys_addr_t mem_end = PFN_PHYS(ini_nr_pages);
 637	phys_addr_t swap_addr, swap_size, entry_end;
 638
 639	swap_addr = PAGE_ALIGN_DOWN(swap_entry->addr);
 640	swap_size = PAGE_ALIGN(swap_entry->addr - swap_addr + swap_entry->size);
 641	entry = xen_e820_table.entries;
 642
 643	for (mapcnt = 0; mapcnt < xen_e820_table.nr_entries; mapcnt++) {
 644		entry_end = entry->addr + entry->size;
 645		if (entry->type == E820_TYPE_RAM && entry->size >= swap_size &&
 646		    entry_end - swap_size >= mem_end) {
 647			/* Reduce RAM entry by needed space (whole pages). */
 648			entry->size -= swap_size;
 649
 650			/* Add new entry at the end of E820 map. */
 651			entry = xen_e820_table.entries +
 652				xen_e820_table.nr_entries;
 653			xen_e820_table.nr_entries++;
 654
 655			/* Fill new entry (keep size and page offset). */
 656			entry->type = swap_entry->type;
 657			entry->addr = entry_end - swap_size +
 658				      swap_addr - swap_entry->addr;
 659			entry->size = swap_entry->size;
 660
 661			/* Convert old entry to RAM, align to pages. */
 662			swap_entry->type = E820_TYPE_RAM;
 663			swap_entry->addr = swap_addr;
 664			swap_entry->size = swap_size;
 665
 666			/* Remember PFN<->MFN relation for P2M update. */
 667			xen_add_remap_nonram(swap_addr, entry_end - swap_size,
 668					     swap_size);
 669
 670			/* Order E820 table and merge entries. */
 671			e820__update_table(&xen_e820_table);
 672
 673			return;
 674		}
 675
 676		entry++;
 677	}
 678
 679	xen_raw_console_write("No suitable area found for required E820 entry remapping action\n");
 680	BUG();
 681}
 682
 683/*
 684 * Look for non-RAM memory types in a specific guest physical area and move
 685 * those away if possible (ACPI NVS only for now).
 686 */
 687static void __init xen_e820_resolve_conflicts(phys_addr_t start,
 688					      phys_addr_t size)
 689{
 690	struct e820_entry *entry;
 691	unsigned int mapcnt;
 692	phys_addr_t end;
 693
 694	if (!size)
 695		return;
 696
 697	end = start + size;
 698	entry = xen_e820_table.entries;
 699
 700	for (mapcnt = 0; mapcnt < xen_e820_table.nr_entries; mapcnt++) {
 701		if (entry->addr >= end)
 702			return;
 703
 704		if (entry->addr + entry->size > start &&
 705		    entry->type == E820_TYPE_NVS)
 706			xen_e820_swap_entry_with_ram(entry);
 707
 708		entry++;
 709	}
 710}
 711
 712/*
 713 * Check for an area in physical memory to be usable for non-movable purposes.
 714 * An area is considered to usable if the used E820 map lists it to be RAM or
 715 * some other type which can be moved to higher PFNs while keeping the MFNs.
 716 * In case the area is not usable, crash the system with an error message.
 717 */
 718void __init xen_chk_is_e820_usable(phys_addr_t start, phys_addr_t size,
 719				   const char *component)
 720{
 721	xen_e820_resolve_conflicts(start, size);
 722
 723	if (!xen_is_e820_reserved(start, size))
 724		return;
 725
 726	xen_raw_console_write("Xen hypervisor allocated ");
 727	xen_raw_console_write(component);
 728	xen_raw_console_write(" memory conflicts with E820 map\n");
 729	BUG();
 730}
 731
 732/*
 733 * Like memcpy, but with physical addresses for dest and src.
 734 */
 735static void __init xen_phys_memcpy(phys_addr_t dest, phys_addr_t src,
 736				   phys_addr_t n)
 737{
 738	phys_addr_t dest_off, src_off, dest_len, src_len, len;
 739	void *from, *to;
 740
 741	while (n) {
 742		dest_off = dest & ~PAGE_MASK;
 743		src_off = src & ~PAGE_MASK;
 744		dest_len = n;
 745		if (dest_len > (NR_FIX_BTMAPS << PAGE_SHIFT) - dest_off)
 746			dest_len = (NR_FIX_BTMAPS << PAGE_SHIFT) - dest_off;
 747		src_len = n;
 748		if (src_len > (NR_FIX_BTMAPS << PAGE_SHIFT) - src_off)
 749			src_len = (NR_FIX_BTMAPS << PAGE_SHIFT) - src_off;
 750		len = min(dest_len, src_len);
 751		to = early_memremap(dest - dest_off, dest_len + dest_off);
 752		from = early_memremap(src - src_off, src_len + src_off);
 753		memcpy(to, from, len);
 754		early_memunmap(to, dest_len + dest_off);
 755		early_memunmap(from, src_len + src_off);
 756		n -= len;
 757		dest += len;
 758		src += len;
 759	}
 760}
 761
 762/*
 763 * Reserve Xen mfn_list.
 764 */
 765static void __init xen_reserve_xen_mfnlist(void)
 766{
 767	phys_addr_t start, size;
 768
 769	if (xen_start_info->mfn_list >= __START_KERNEL_map) {
 770		start = __pa(xen_start_info->mfn_list);
 771		size = PFN_ALIGN(xen_start_info->nr_pages *
 772				 sizeof(unsigned long));
 773	} else {
 774		start = PFN_PHYS(xen_start_info->first_p2m_pfn);
 775		size = PFN_PHYS(xen_start_info->nr_p2m_frames);
 776	}
 777
 778	memblock_reserve(start, size);
 779	if (!xen_is_e820_reserved(start, size))
 780		return;
 781
 782	xen_relocate_p2m();
 783	memblock_phys_free(start, size);
 784}
 785
 786/**
 787 * xen_memory_setup - Hook for machine specific memory setup.
 788 **/
 789char * __init xen_memory_setup(void)
 790{
 791	unsigned long pfn_s, n_pfns;
 792	phys_addr_t mem_end, addr, size, chunk_size;
 793	u32 type;
 794	int rc;
 795	struct xen_memory_map memmap;
 796	unsigned long max_pages;
 797	unsigned long extra_pages = 0;
 798	unsigned long maxmem_pages;
 799	int i;
 800	int op;
 801
 802	xen_parse_512gb();
 803	ini_nr_pages = min(xen_get_pages_limit(), xen_start_info->nr_pages);
 804	mem_end = PFN_PHYS(ini_nr_pages);
 805
 806	memmap.nr_entries = ARRAY_SIZE(xen_e820_table.entries);
 807	set_xen_guest_handle(memmap.buffer, xen_e820_table.entries);
 808
 809#if defined(CONFIG_MEMORY_HOTPLUG) && defined(CONFIG_XEN_BALLOON)
 810	xen_saved_max_mem_size = max_mem_size;
 811#endif
 812
 813	op = xen_initial_domain() ?
 814		XENMEM_machine_memory_map :
 815		XENMEM_memory_map;
 816	rc = HYPERVISOR_memory_op(op, &memmap);
 817	if (rc == -ENOSYS) {
 818		BUG_ON(xen_initial_domain());
 819		memmap.nr_entries = 1;
 820		xen_e820_table.entries[0].addr = 0ULL;
 821		xen_e820_table.entries[0].size = mem_end;
 822		/* 8MB slack (to balance backend allocations). */
 823		xen_e820_table.entries[0].size += 8ULL << 20;
 824		xen_e820_table.entries[0].type = E820_TYPE_RAM;
 825		rc = 0;
 826	}
 827	BUG_ON(rc);
 828	BUG_ON(memmap.nr_entries == 0);
 829	xen_e820_table.nr_entries = memmap.nr_entries;
 830
 831	if (xen_initial_domain()) {
 832		/*
 833		 * Xen won't allow a 1:1 mapping to be created to UNUSABLE
 834		 * regions, so if we're using the machine memory map leave the
 835		 * region as RAM as it is in the pseudo-physical map.
 836		 *
 837		 * UNUSABLE regions in domUs are not handled and will need
 838		 * a patch in the future.
 839		 */
 840		xen_ignore_unusable();
 841
 842#ifdef CONFIG_ISCSI_IBFT_FIND
 843		/* Reserve 0.5 MiB to 1 MiB region so iBFT can be found */
 844		xen_e820_table.entries[xen_e820_table.nr_entries].addr = IBFT_START;
 845		xen_e820_table.entries[xen_e820_table.nr_entries].size = IBFT_END - IBFT_START;
 846		xen_e820_table.entries[xen_e820_table.nr_entries].type = E820_TYPE_RESERVED;
 847		xen_e820_table.nr_entries++;
 848#endif
 849	}
 850
 851	/* Make sure the Xen-supplied memory map is well-ordered. */
 852	e820__update_table(&xen_e820_table);
 853
 854	/*
 855	 * Check whether the kernel itself conflicts with the target E820 map.
 856	 * Failing now is better than running into weird problems later due
 857	 * to relocating (and even reusing) pages with kernel text or data.
 858	 */
 859	xen_chk_is_e820_usable(__pa_symbol(_text),
 860			       __pa_symbol(_end) - __pa_symbol(_text),
 861			       "kernel");
 862
 863	/*
 864	 * Check for a conflict of the xen_start_info memory with the target
 865	 * E820 map.
 866	 */
 867	xen_chk_is_e820_usable(__pa(xen_start_info), sizeof(*xen_start_info),
 868			       "xen_start_info");
 869
 870	/*
 871	 * Check for a conflict of the hypervisor supplied page tables with
 872	 * the target E820 map.
 873	 */
 874	xen_pt_check_e820();
 875
 876	max_pages = xen_get_max_pages();
 877
 878	/* How many extra pages do we need due to remapping? */
 879	max_pages += xen_foreach_remap_area(xen_count_remap_pages);
 880
 881	if (max_pages > ini_nr_pages)
 882		extra_pages += max_pages - ini_nr_pages;
 883
 884	/*
 885	 * Clamp the amount of extra memory to a EXTRA_MEM_RATIO
 886	 * factor the base size.
 887	 *
 888	 * Make sure we have no memory above max_pages, as this area
 889	 * isn't handled by the p2m management.
 890	 */
 891	maxmem_pages = EXTRA_MEM_RATIO * min(ini_nr_pages, PFN_DOWN(MAXMEM));
 892	extra_pages = min3(maxmem_pages, extra_pages, max_pages - ini_nr_pages);
 893	i = 0;
 894	addr = xen_e820_table.entries[0].addr;
 895	size = xen_e820_table.entries[0].size;
 896	while (i < xen_e820_table.nr_entries) {
 897		bool discard = false;
 898
 899		chunk_size = size;
 900		type = xen_e820_table.entries[i].type;
 901
 902		if (type == E820_TYPE_RESERVED)
 903			xen_pv_pci_possible = true;
 904
 905		if (type == E820_TYPE_RAM) {
 906			if (addr < mem_end) {
 907				chunk_size = min(size, mem_end - addr);
 908			} else if (extra_pages) {
 909				chunk_size = min(size, PFN_PHYS(extra_pages));
 910				pfn_s = PFN_UP(addr);
 911				n_pfns = PFN_DOWN(addr + chunk_size) - pfn_s;
 912				extra_pages -= n_pfns;
 913				xen_add_extra_mem(pfn_s, n_pfns);
 914				xen_max_p2m_pfn = pfn_s + n_pfns;
 915			} else
 916				discard = true;
 917		}
 918
 919		if (!discard)
 920			xen_align_and_add_e820_region(addr, chunk_size, type);
 921
 922		addr += chunk_size;
 923		size -= chunk_size;
 924		if (size == 0) {
 925			i++;
 926			if (i < xen_e820_table.nr_entries) {
 927				addr = xen_e820_table.entries[i].addr;
 928				size = xen_e820_table.entries[i].size;
 929			}
 930		}
 931	}
 932
 933	/*
 934	 * Set the rest as identity mapped, in case PCI BARs are
 935	 * located here.
 936	 */
 937	set_phys_range_identity(addr / PAGE_SIZE, ~0ul);
 938
 939	/*
 940	 * In domU, the ISA region is normal, usable memory, but we
 941	 * reserve ISA memory anyway because too many things poke
 942	 * about in there.
 943	 */
 944	e820__range_add(ISA_START_ADDRESS, ISA_END_ADDRESS - ISA_START_ADDRESS, E820_TYPE_RESERVED);
 945
 946	e820__update_table(e820_table);
 947
 948	xen_reserve_xen_mfnlist();
 949
 950	/* Check for a conflict of the initrd with the target E820 map. */
 951	if (xen_is_e820_reserved(boot_params.hdr.ramdisk_image,
 952				 boot_params.hdr.ramdisk_size)) {
 953		phys_addr_t new_area, start, size;
 954
 955		new_area = xen_find_free_area(boot_params.hdr.ramdisk_size);
 956		if (!new_area) {
 957			xen_raw_console_write("Can't find new memory area for initrd needed due to E820 map conflict\n");
 958			BUG();
 959		}
 960
 961		start = boot_params.hdr.ramdisk_image;
 962		size = boot_params.hdr.ramdisk_size;
 963		xen_phys_memcpy(new_area, start, size);
 964		pr_info("initrd moved from [mem %#010llx-%#010llx] to [mem %#010llx-%#010llx]\n",
 965			start, start + size, new_area, new_area + size);
 966		memblock_phys_free(start, size);
 967		boot_params.hdr.ramdisk_image = new_area;
 968		boot_params.ext_ramdisk_image = new_area >> 32;
 969	}
 970
 971	/*
 972	 * Set identity map on non-RAM pages and prepare remapping the
 973	 * underlying RAM.
 974	 */
 975	xen_foreach_remap_area(xen_set_identity_and_remap_chunk);
 976
 977	pr_info("Released %ld page(s)\n", xen_released_pages);
 978
 979	return "Xen";
 980}
 981
 982static int register_callback(unsigned type, const void *func)
 983{
 984	struct callback_register callback = {
 985		.type = type,
 986		.address = XEN_CALLBACK(__KERNEL_CS, func),
 987		.flags = CALLBACKF_mask_events,
 988	};
 989
 990	return HYPERVISOR_callback_op(CALLBACKOP_register, &callback);
 991}
 992
 993void xen_enable_sysenter(void)
 994{
 995	if (cpu_feature_enabled(X86_FEATURE_SYSENTER32) &&
 996	    register_callback(CALLBACKTYPE_sysenter, xen_entry_SYSENTER_compat))
 997		setup_clear_cpu_cap(X86_FEATURE_SYSENTER32);
 998}
 999
1000void xen_enable_syscall(void)
1001{
1002	int ret;
1003
1004	ret = register_callback(CALLBACKTYPE_syscall, xen_entry_SYSCALL_64);
1005	if (ret != 0) {
1006		printk(KERN_ERR "Failed to set syscall callback: %d\n", ret);
1007		/* Pretty fatal; 64-bit userspace has no other
1008		   mechanism for syscalls. */
1009	}
1010
1011	if (cpu_feature_enabled(X86_FEATURE_SYSCALL32) &&
1012	    register_callback(CALLBACKTYPE_syscall32, xen_entry_SYSCALL_compat))
1013		setup_clear_cpu_cap(X86_FEATURE_SYSCALL32);
1014}
1015
1016static void __init xen_pvmmu_arch_setup(void)
1017{
1018	HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_writable_pagetables);
1019
1020	if (register_callback(CALLBACKTYPE_event,
1021			      xen_asm_exc_xen_hypervisor_callback) ||
1022	    register_callback(CALLBACKTYPE_failsafe, xen_failsafe_callback))
1023		BUG();
1024
1025	xen_enable_sysenter();
1026	xen_enable_syscall();
1027}
1028
1029/* This function is not called for HVM domains */
1030void __init xen_arch_setup(void)
1031{
1032	xen_panic_handler_init();
1033	xen_pvmmu_arch_setup();
1034
1035#ifdef CONFIG_ACPI
1036	if (!(xen_start_info->flags & SIF_INITDOMAIN)) {
1037		printk(KERN_INFO "ACPI in unprivileged domain disabled\n");
1038		disable_acpi();
1039	}
1040#endif
1041
1042	memcpy(boot_command_line, xen_start_info->cmd_line,
1043	       MAX_GUEST_CMDLINE > COMMAND_LINE_SIZE ?
1044	       COMMAND_LINE_SIZE : MAX_GUEST_CMDLINE);
1045
1046	/* Set up idle, making sure it calls safe_halt() pvop */
1047	disable_cpuidle();
1048	disable_cpufreq();
1049	WARN_ON(xen_set_default_idle());
1050#ifdef CONFIG_NUMA
1051	numa_off = 1;
1052#endif
1053}