numa.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Author:  Xiang Gao <gaoxiang@loongson.cn>
  4 *          Huacai Chen <chenhuacai@loongson.cn>
  5 *
  6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
  7 */
  8#include <linux/init.h>
  9#include <linux/kernel.h>
 10#include <linux/mm.h>
 11#include <linux/mmzone.h>
 12#include <linux/export.h>
 13#include <linux/nodemask.h>
 14#include <linux/numa_memblks.h>
 15#include <linux/swap.h>
 16#include <linux/memblock.h>
 17#include <linux/pfn.h>
 18#include <linux/acpi.h>
 19#include <linux/efi.h>
 20#include <linux/irq.h>
 21#include <linux/pci.h>
 22#include <asm/bootinfo.h>
 23#include <asm/loongson.h>
 24#include <asm/numa.h>
 25#include <asm/page.h>
 26#include <asm/pgalloc.h>
 27#include <asm/sections.h>
 28#include <asm/time.h>
 29
 30int numa_off;
 31cpumask_t cpus_on_node[MAX_NUMNODES];
 32cpumask_t phys_cpus_on_node[MAX_NUMNODES];
 33EXPORT_SYMBOL(cpus_on_node);
 34
 35/*
 36 * apicid, cpu, node mappings
 37 */
 38s16 __cpuid_to_node[CONFIG_NR_CPUS] = {
 39	[0 ... CONFIG_NR_CPUS - 1] = NUMA_NO_NODE
 40};
 41EXPORT_SYMBOL(__cpuid_to_node);
 42
 43#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
 44unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
 45EXPORT_SYMBOL(__per_cpu_offset);
 46
 47static int __init pcpu_cpu_to_node(int cpu)
 48{
 49	return early_cpu_to_node(cpu);
 50}
 51
 52static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
 53{
 54	if (early_cpu_to_node(from) == early_cpu_to_node(to))
 55		return LOCAL_DISTANCE;
 56	else
 57		return REMOTE_DISTANCE;
 58}
 59
 60void __init pcpu_populate_pte(unsigned long addr)
 61{
 62	populate_kernel_pte(addr);
 63}
 64
 65void __init setup_per_cpu_areas(void)
 66{
 67	unsigned long delta;
 68	unsigned int cpu;
 69	int rc = -EINVAL;
 70
 71	if (pcpu_chosen_fc == PCPU_FC_AUTO) {
 72		if (nr_node_ids >= 8)
 73			pcpu_chosen_fc = PCPU_FC_PAGE;
 74		else
 75			pcpu_chosen_fc = PCPU_FC_EMBED;
 76	}
 77
 78	/*
 79	 * Always reserve area for module percpu variables.  That's
 80	 * what the legacy allocator did.
 81	 */
 82	if (pcpu_chosen_fc != PCPU_FC_PAGE) {
 83		rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
 84					    PERCPU_DYNAMIC_RESERVE, PMD_SIZE,
 85					    pcpu_cpu_distance, pcpu_cpu_to_node);
 86		if (rc < 0)
 87			pr_warn("%s allocator failed (%d), falling back to page size\n",
 88				pcpu_fc_names[pcpu_chosen_fc], rc);
 89	}
 90	if (rc < 0)
 91		rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, pcpu_cpu_to_node);
 92	if (rc < 0)
 93		panic("cannot initialize percpu area (err=%d)", rc);
 94
 95	delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
 96	for_each_possible_cpu(cpu)
 97		__per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
 98}
 99#endif
100
101/*
102 * Get nodeid by logical cpu number.
103 * __cpuid_to_node maps phyical cpu id to node, so we
104 * should use cpu_logical_map(cpu) to index it.
105 *
106 * This routine is only used in early phase during
107 * booting, after setup_per_cpu_areas calling and numa_node
108 * initialization, cpu_to_node will be used instead.
109 */
110int early_cpu_to_node(int cpu)
111{
112	int physid = cpu_logical_map(cpu);
113
114	if (physid < 0)
115		return NUMA_NO_NODE;
116
117	return __cpuid_to_node[physid];
118}
119
120void __init early_numa_add_cpu(int cpuid, s16 node)
121{
122	int cpu = __cpu_number_map[cpuid];
123
124	if (cpu < 0)
125		return;
126
127	cpumask_set_cpu(cpu, &cpus_on_node[node]);
128	cpumask_set_cpu(cpuid, &phys_cpus_on_node[node]);
129}
130
131void numa_add_cpu(unsigned int cpu)
132{
133	int nid = cpu_to_node(cpu);
134	cpumask_set_cpu(cpu, &cpus_on_node[nid]);
135}
136
137void numa_remove_cpu(unsigned int cpu)
138{
139	int nid = cpu_to_node(cpu);
140	cpumask_clear_cpu(cpu, &cpus_on_node[nid]);
141}
142
143static void __init node_mem_init(unsigned int node)
144{
145	unsigned long start_pfn, end_pfn;
146	unsigned long node_addrspace_offset;
147
148	node_addrspace_offset = nid_to_addrbase(node);
149	pr_info("Node%d's addrspace_offset is 0x%lx\n",
150			node, node_addrspace_offset);
151
152	get_pfn_range_for_nid(node, &start_pfn, &end_pfn);
153	pr_info("Node%d: start_pfn=0x%lx, end_pfn=0x%lx\n",
154		node, start_pfn, end_pfn);
155
156	alloc_node_data(node);
157}
158
159#ifdef CONFIG_ACPI_NUMA
160
161static unsigned long num_physpages;
162
163static void __init info_node_memblock(void)
164{
165	u32 mem_type;
166	u64 mem_end, mem_start, mem_size;
167	efi_memory_desc_t *md;
168
169	/* Parse memory information and activate */
170	for_each_efi_memory_desc(md) {
171		mem_type = md->type;
172		mem_start = md->phys_addr;
173		mem_size = md->num_pages << EFI_PAGE_SHIFT;
174		mem_end = mem_start + mem_size;
175
176		switch (mem_type) {
177		case EFI_LOADER_CODE:
178		case EFI_LOADER_DATA:
179		case EFI_BOOT_SERVICES_CODE:
180		case EFI_BOOT_SERVICES_DATA:
181		case EFI_PERSISTENT_MEMORY:
182		case EFI_CONVENTIONAL_MEMORY:
183			num_physpages += (mem_size >> PAGE_SHIFT);
184			pr_info("Node%d: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n",
185				(u32)pa_to_nid(mem_start), mem_type, mem_start, mem_size);
186			pr_info("       start_pfn:0x%llx, end_pfn:0x%llx, num_physpages:0x%lx\n",
187				mem_start >> PAGE_SHIFT, mem_end >> PAGE_SHIFT, num_physpages);
188			break;
189		case EFI_PAL_CODE:
190		case EFI_UNUSABLE_MEMORY:
191		case EFI_ACPI_RECLAIM_MEMORY:
192			num_physpages += (mem_size >> PAGE_SHIFT);
193			pr_info("Node%d: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n",
194				(u32)pa_to_nid(mem_start), mem_type, mem_start, mem_size);
195			pr_info("       start_pfn:0x%llx, end_pfn:0x%llx, num_physpages:0x%lx\n",
196				mem_start >> PAGE_SHIFT, mem_end >> PAGE_SHIFT, num_physpages);
197			fallthrough;
198		case EFI_RESERVED_TYPE:
199		case EFI_RUNTIME_SERVICES_CODE:
200		case EFI_RUNTIME_SERVICES_DATA:
201		case EFI_MEMORY_MAPPED_IO:
202		case EFI_MEMORY_MAPPED_IO_PORT_SPACE:
203			pr_info("Resvd: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n",
204					mem_type, mem_start, mem_size);
205			break;
206		}
207	}
208}
209
210/*
211 * fake_numa_init() - For Non-ACPI systems
212 * Return: 0 on success, -errno on failure.
213 */
214static int __init fake_numa_init(void)
215{
216	phys_addr_t start = memblock_start_of_DRAM();
217	phys_addr_t end = memblock_end_of_DRAM() - 1;
218
219	node_set(0, numa_nodes_parsed);
220	pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end);
221
222	return numa_add_memblk(0, start, end + 1);
223}
224
225int __init init_numa_memory(void)
226{
227	int i;
228	int ret;
229	int node;
230
231	for (i = 0; i < NR_CPUS; i++)
232		set_cpuid_to_node(i, NUMA_NO_NODE);
233
234	/* Parse SRAT and SLIT if provided by firmware. */
235	if (!acpi_disabled)
236		ret = numa_memblks_init(acpi_numa_init, false);
237	else
238		ret = numa_memblks_init(fake_numa_init, false);
239
240	if (ret < 0)
241		return ret;
242
243	info_node_memblock();
244	if (!memblock_validate_numa_coverage(SZ_1M))
245		return -EINVAL;
246
247	for_each_node_mask(node, node_possible_map) {
248		node_mem_init(node);
249		node_set_online(node);
250	}
251	max_pfn = PFN_DOWN(memblock_end_of_DRAM());
252	max_low_pfn = min(PFN_DOWN(HIGHMEM_START), max_pfn);
253
254	setup_nr_node_ids();
255	loongson_sysconf.nr_nodes = nr_node_ids;
256	loongson_sysconf.cores_per_node = cpumask_weight(&phys_cpus_on_node[0]);
257
258	return 0;
259}
260
261#endif
262
263int pcibus_to_node(struct pci_bus *bus)
264{
265	return dev_to_node(&bus->dev);
266}
267EXPORT_SYMBOL(pcibus_to_node);