mmu_context.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * This file contains the routines for handling the MMU on those
  4 * PowerPC implementations where the MMU is not using the hash
  5 * table, such as 8xx, 4xx, BookE's etc...
  6 *
  7 * Copyright 2008 Ben Herrenschmidt <benh@kernel.crashing.org>
  8 *                IBM Corp.
  9 *
 10 *  Derived from previous arch/powerpc/mm/mmu_context.c
 11 *  and arch/powerpc/include/asm/mmu_context.h
 12 *
 13 * TODO:
 14 *
 15 *   - The global context lock will not scale very well
 16 *   - The maps should be dynamically allocated to allow for processors
 17 *     that support more PID bits at runtime
 18 *   - Implement flush_tlb_mm() by making the context stale and picking
 19 *     a new one
 20 *   - More aggressively clear stale map bits and maybe find some way to
 21 *     also clear mm->cpu_vm_mask bits when processes are migrated
 22 */
 23
 24#include <linux/kernel.h>
 25#include <linux/mm.h>
 26#include <linux/init.h>
 27#include <linux/spinlock.h>
 28#include <linux/memblock.h>
 29#include <linux/notifier.h>
 30#include <linux/cpu.h>
 31#include <linux/slab.h>
 32
 33#include <asm/mmu_context.h>
 34#include <asm/tlbflush.h>
 35#include <asm/smp.h>
 36#include <asm/kup.h>
 37
 38#include <mm/mmu_decl.h>
 39
 40/*
 41 * Room for two PTE table pointers, usually the kernel and current user
 42 * pointer to their respective root page table (pgdir).
 43 */
 44void *abatron_pteptrs[2];
 45
 46/*
 47 * The MPC8xx has only 16 contexts. We rotate through them on each task switch.
 48 * A better way would be to keep track of tasks that own contexts, and implement
 49 * an LRU usage. That way very active tasks don't always have to pay the TLB
 50 * reload overhead. The kernel pages are mapped shared, so the kernel can run on
 51 * behalf of any task that makes a kernel entry. Shared does not mean they are
 52 * not protected, just that the ASID comparison is not performed. -- Dan
 53 *
 54 * The IBM4xx has 256 contexts, so we can just rotate through these as a way of
 55 * "switching" contexts. If the TID of the TLB is zero, the PID/TID comparison
 56 * is disabled, so we can use a TID of zero to represent all kernel pages as
 57 * shared among all contexts. -- Dan
 58 *
 59 * The IBM 47x core supports 16-bit PIDs, thus 65535 contexts. We should
 60 * normally never have to steal though the facility is present if needed.
 61 * -- BenH
 62 */
 63#define FIRST_CONTEXT 1
 64#if defined(CONFIG_PPC_8xx)
 65#define LAST_CONTEXT 16
 66#elif defined(CONFIG_PPC_47x)
 67#define LAST_CONTEXT 65535
 68#else
 69#define LAST_CONTEXT 255
 70#endif
 71
 72static unsigned int next_context, nr_free_contexts;
 73static unsigned long *context_map;
 74static unsigned long *stale_map[NR_CPUS];
 75static struct mm_struct **context_mm;
 76static DEFINE_RAW_SPINLOCK(context_lock);
 77
 78#define CTX_MAP_SIZE	\
 79	(sizeof(unsigned long) * (LAST_CONTEXT / BITS_PER_LONG + 1))
 80
 81
 82/* Steal a context from a task that has one at the moment.
 83 *
 84 * This is used when we are running out of available PID numbers
 85 * on the processors.
 86 *
 87 * This isn't an LRU system, it just frees up each context in
 88 * turn (sort-of pseudo-random replacement :).  This would be the
 89 * place to implement an LRU scheme if anyone was motivated to do it.
 90 *  -- paulus
 91 *
 92 * For context stealing, we use a slightly different approach for
 93 * SMP and UP. Basically, the UP one is simpler and doesn't use
 94 * the stale map as we can just flush the local CPU
 95 *  -- benh
 96 */
 97static unsigned int steal_context_smp(unsigned int id)
 98{
 99	struct mm_struct *mm;
100	unsigned int cpu, max, i;
101
102	max = LAST_CONTEXT - FIRST_CONTEXT;
103
104	/* Attempt to free next_context first and then loop until we manage */
105	while (max--) {
106		/* Pick up the victim mm */
107		mm = context_mm[id];
108
109		/* We have a candidate victim, check if it's active, on SMP
110		 * we cannot steal active contexts
111		 */
112		if (mm->context.active) {
113			id++;
114			if (id > LAST_CONTEXT)
115				id = FIRST_CONTEXT;
116			continue;
117		}
118
119		/* Mark this mm has having no context anymore */
120		mm->context.id = MMU_NO_CONTEXT;
121
122		/* Mark it stale on all CPUs that used this mm. For threaded
123		 * implementations, we set it on all threads on each core
124		 * represented in the mask. A future implementation will use
125		 * a core map instead but this will do for now.
126		 */
127		for_each_cpu(cpu, mm_cpumask(mm)) {
128			for (i = cpu_first_thread_sibling(cpu);
129			     i <= cpu_last_thread_sibling(cpu); i++) {
130				if (stale_map[i])
131					__set_bit(id, stale_map[i]);
132			}
133			cpu = i - 1;
134		}
135		return id;
136	}
137
138	/* This will happen if you have more CPUs than available contexts,
139	 * all we can do here is wait a bit and try again
140	 */
141	raw_spin_unlock(&context_lock);
142	cpu_relax();
143	raw_spin_lock(&context_lock);
144
145	/* This will cause the caller to try again */
146	return MMU_NO_CONTEXT;
147}
148
149static unsigned int steal_all_contexts(void)
150{
151	struct mm_struct *mm;
152	int cpu = smp_processor_id();
153	unsigned int id;
154
155	for (id = FIRST_CONTEXT; id <= LAST_CONTEXT; id++) {
156		/* Pick up the victim mm */
157		mm = context_mm[id];
158
159		/* Mark this mm as having no context anymore */
160		mm->context.id = MMU_NO_CONTEXT;
161		if (id != FIRST_CONTEXT) {
162			context_mm[id] = NULL;
163			__clear_bit(id, context_map);
164		}
165		if (IS_ENABLED(CONFIG_SMP))
166			__clear_bit(id, stale_map[cpu]);
167	}
168
169	/* Flush the TLB for all contexts (not to be used on SMP) */
170	_tlbil_all();
171
172	nr_free_contexts = LAST_CONTEXT - FIRST_CONTEXT;
173
174	return FIRST_CONTEXT;
175}
176
177/* Note that this will also be called on SMP if all other CPUs are
178 * offlined, which means that it may be called for cpu != 0. For
179 * this to work, we somewhat assume that CPUs that are onlined
180 * come up with a fully clean TLB (or are cleaned when offlined)
181 */
182static unsigned int steal_context_up(unsigned int id)
183{
184	struct mm_struct *mm;
185	int cpu = smp_processor_id();
186
187	/* Pick up the victim mm */
188	mm = context_mm[id];
189
190	/* Flush the TLB for that context */
191	local_flush_tlb_mm(mm);
192
193	/* Mark this mm has having no context anymore */
194	mm->context.id = MMU_NO_CONTEXT;
195
196	/* XXX This clear should ultimately be part of local_flush_tlb_mm */
197	if (IS_ENABLED(CONFIG_SMP))
198		__clear_bit(id, stale_map[cpu]);
199
200	return id;
201}
202
203static void set_context(unsigned long id, pgd_t *pgd)
204{
205	if (IS_ENABLED(CONFIG_PPC_8xx)) {
206		mtspr(SPRN_M_TWB, __pa(pgd));
207
208		/* Update context */
209		mtspr(SPRN_M_CASID, id - 1);
210
211		/* sync */
212		mb();
213	} else if (kuap_is_disabled()) {
214		mtspr(SPRN_PID, id);
215		isync();
216	}
217}
218
219void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
220			struct task_struct *tsk)
221{
222	unsigned int id;
223	unsigned int i, cpu = smp_processor_id();
224	unsigned long *map;
225
226	/* No lockless fast path .. yet */
227	raw_spin_lock(&context_lock);
228
229	if (IS_ENABLED(CONFIG_SMP)) {
230		/* Mark us active and the previous one not anymore */
231		next->context.active++;
232		if (prev) {
233			WARN_ON(prev->context.active < 1);
234			prev->context.active--;
235		}
236	}
237
238 again:
239
240	/* If we already have a valid assigned context, skip all that */
241	id = next->context.id;
242	if (likely(id != MMU_NO_CONTEXT))
243		goto ctxt_ok;
244
245	/* We really don't have a context, let's try to acquire one */
246	id = next_context;
247	if (id > LAST_CONTEXT)
248		id = FIRST_CONTEXT;
249	map = context_map;
250
251	/* No more free contexts, let's try to steal one */
252	if (nr_free_contexts == 0) {
253		if (num_online_cpus() > 1) {
254			id = steal_context_smp(id);
255			if (id == MMU_NO_CONTEXT)
256				goto again;
257			goto stolen;
258		}
259		if (IS_ENABLED(CONFIG_PPC_8xx))
260			id = steal_all_contexts();
261		else
262			id = steal_context_up(id);
263		goto stolen;
264	}
265	nr_free_contexts--;
266
267	/* We know there's at least one free context, try to find it */
268	while (__test_and_set_bit(id, map)) {
269		id = find_next_zero_bit(map, LAST_CONTEXT+1, id);
270		if (id > LAST_CONTEXT)
271			id = FIRST_CONTEXT;
272	}
273 stolen:
274	next_context = id + 1;
275	context_mm[id] = next;
276	next->context.id = id;
277
278 ctxt_ok:
279
280	/* If that context got marked stale on this CPU, then flush the
281	 * local TLB for it and unmark it before we use it
282	 */
283	if (IS_ENABLED(CONFIG_SMP) && test_bit(id, stale_map[cpu])) {
284		local_flush_tlb_mm(next);
285
286		/* XXX This clear should ultimately be part of local_flush_tlb_mm */
287		for (i = cpu_first_thread_sibling(cpu);
288		     i <= cpu_last_thread_sibling(cpu); i++) {
289			if (stale_map[i])
290				__clear_bit(id, stale_map[i]);
291		}
292	}
293
294	/* Flick the MMU and release lock */
295	if (IS_ENABLED(CONFIG_BDI_SWITCH))
296		abatron_pteptrs[1] = next->pgd;
297	set_context(id, next->pgd);
298#if defined(CONFIG_BOOKE) && defined(CONFIG_PPC_KUAP)
299	tsk->thread.pid = id;
300#endif
301	raw_spin_unlock(&context_lock);
302}
303
304/*
305 * Set up the context for a new address space.
306 */
307int init_new_context(struct task_struct *t, struct mm_struct *mm)
308{
309	mm->context.id = MMU_NO_CONTEXT;
310	mm->context.active = 0;
311	pte_frag_set(&mm->context, NULL);
312	return 0;
313}
314
315/*
316 * We're finished using the context for an address space.
317 */
318void destroy_context(struct mm_struct *mm)
319{
320	unsigned long flags;
321	unsigned int id;
322
323	if (mm->context.id == MMU_NO_CONTEXT)
324		return;
325
326	WARN_ON(mm->context.active != 0);
327
328	raw_spin_lock_irqsave(&context_lock, flags);
329	id = mm->context.id;
330	if (id != MMU_NO_CONTEXT) {
331		__clear_bit(id, context_map);
332		mm->context.id = MMU_NO_CONTEXT;
333		context_mm[id] = NULL;
334		nr_free_contexts++;
335	}
336	raw_spin_unlock_irqrestore(&context_lock, flags);
337}
338
339static int mmu_ctx_cpu_prepare(unsigned int cpu)
340{
341	/* We don't touch CPU 0 map, it's allocated at aboot and kept
342	 * around forever
343	 */
344	if (cpu == boot_cpuid)
345		return 0;
346
347	stale_map[cpu] = kzalloc(CTX_MAP_SIZE, GFP_KERNEL);
348	return 0;
349}
350
351static int mmu_ctx_cpu_dead(unsigned int cpu)
352{
353#ifdef CONFIG_HOTPLUG_CPU
354	if (cpu == boot_cpuid)
355		return 0;
356
357	kfree(stale_map[cpu]);
358	stale_map[cpu] = NULL;
359
360	/* We also clear the cpu_vm_mask bits of CPUs going away */
361	clear_tasks_mm_cpumask(cpu);
362#endif
363	return 0;
364}
365
366/*
367 * Initialize the context management stuff.
368 */
369void __init mmu_context_init(void)
370{
371	/* Mark init_mm as being active on all possible CPUs since
372	 * we'll get called with prev == init_mm the first time
373	 * we schedule on a given CPU
374	 */
375	init_mm.context.active = NR_CPUS;
376
377	/*
378	 * Allocate the maps used by context management
379	 */
380	context_map = memblock_alloc_or_panic(CTX_MAP_SIZE, SMP_CACHE_BYTES);
381	context_mm = memblock_alloc_or_panic(sizeof(void *) * (LAST_CONTEXT + 1),
382				    SMP_CACHE_BYTES);
383	if (IS_ENABLED(CONFIG_SMP)) {
384		stale_map[boot_cpuid] = memblock_alloc_or_panic(CTX_MAP_SIZE, SMP_CACHE_BYTES);
385		cpuhp_setup_state_nocalls(CPUHP_POWERPC_MMU_CTX_PREPARE,
386					  "powerpc/mmu/ctx:prepare",
387					  mmu_ctx_cpu_prepare, mmu_ctx_cpu_dead);
388	}
389
390	printk(KERN_INFO
391	       "MMU: Allocated %zu bytes of context maps for %d contexts\n",
392	       2 * CTX_MAP_SIZE + (sizeof(void *) * (LAST_CONTEXT + 1)),
393	       LAST_CONTEXT - FIRST_CONTEXT + 1);
394
395	/*
396	 * Some processors have too few contexts to reserve one for
397	 * init_mm, and require using context 0 for a normal task.
398	 * Other processors reserve the use of context zero for the kernel.
399	 * This code assumes FIRST_CONTEXT < 32.
400	 */
401	context_map[0] = (1 << FIRST_CONTEXT) - 1;
402	next_context = FIRST_CONTEXT;
403	nr_free_contexts = LAST_CONTEXT - FIRST_CONTEXT + 1;
404}