ttm_pool.c

   1// SPDX-License-Identifier: GPL-2.0 OR MIT
   2/*
   3 * Copyright 2020 Advanced Micro Devices, Inc.
   4 *
   5 * Permission is hereby granted, free of charge, to any person obtaining a
   6 * copy of this software and associated documentation files (the "Software"),
   7 * to deal in the Software without restriction, including without limitation
   8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9 * and/or sell copies of the Software, and to permit persons to whom the
  10 * Software is furnished to do so, subject to the following conditions:
  11 *
  12 * The above copyright notice and this permission notice shall be included in
  13 * all copies or substantial portions of the Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21 * OTHER DEALINGS IN THE SOFTWARE.
  22 *
  23 * Authors: Christian König
  24 */
  25
  26/* Pooling of allocated pages is necessary because changing the caching
  27 * attributes on x86 of the linear mapping requires a costly cross CPU TLB
  28 * invalidate for those addresses.
  29 *
  30 * Additional to that allocations from the DMA coherent API are pooled as well
  31 * cause they are rather slow compared to alloc_pages+map.
  32 */
  33
  34#include <linux/export.h>
  35#include <linux/module.h>
  36#include <linux/dma-mapping.h>
  37#include <linux/debugfs.h>
  38#include <linux/highmem.h>
  39#include <linux/sched/mm.h>
  40
  41#ifdef CONFIG_X86
  42#include <asm/set_memory.h>
  43#endif
  44
  45#include <drm/ttm/ttm_backup.h>
  46#include <drm/ttm/ttm_pool.h>
  47#include <drm/ttm/ttm_tt.h>
  48#include <drm/ttm/ttm_bo.h>
  49
  50#include "ttm_module.h"
  51#include "ttm_pool_internal.h"
  52
  53#ifdef CONFIG_FAULT_INJECTION
  54#include <linux/fault-inject.h>
  55static DECLARE_FAULT_ATTR(backup_fault_inject);
  56#else
  57#define should_fail(...) false
  58#endif
  59
  60/**
  61 * struct ttm_pool_dma - Helper object for coherent DMA mappings
  62 *
  63 * @addr: original DMA address returned for the mapping
  64 * @vaddr: original vaddr return for the mapping and order in the lower bits
  65 */
  66struct ttm_pool_dma {
  67	dma_addr_t addr;
  68	unsigned long vaddr;
  69};
  70
  71/**
  72 * struct ttm_pool_alloc_state - Current state of the tt page allocation process
  73 * @pages: Pointer to the next tt page pointer to populate.
  74 * @caching_divide: Pointer to the first page pointer whose page has a staged but
  75 * not committed caching transition from write-back to @tt_caching.
  76 * @dma_addr: Pointer to the next tt dma_address entry to populate if any.
  77 * @remaining_pages: Remaining pages to populate.
  78 * @tt_caching: The requested cpu-caching for the pages allocated.
  79 */
  80struct ttm_pool_alloc_state {
  81	struct page **pages;
  82	struct page **caching_divide;
  83	dma_addr_t *dma_addr;
  84	pgoff_t remaining_pages;
  85	enum ttm_caching tt_caching;
  86};
  87
  88/**
  89 * struct ttm_pool_tt_restore - State representing restore from backup
  90 * @pool: The pool used for page allocation while restoring.
  91 * @snapshot_alloc: A snapshot of the most recent struct ttm_pool_alloc_state.
  92 * @alloced_page: Pointer to the page most recently allocated from a pool or system.
  93 * @first_dma: The dma address corresponding to @alloced_page if dma_mapping
  94 * is requested.
  95 * @alloced_pages: The number of allocated pages present in the struct ttm_tt
  96 * page vector from this restore session.
  97 * @restored_pages: The number of 4K pages restored for @alloced_page (which
  98 * is typically a multi-order page).
  99 * @page_caching: The struct ttm_tt requested caching
 100 * @order: The order of @alloced_page.
 101 *
 102 * Recovery from backup might fail when we've recovered less than the
 103 * full ttm_tt. In order not to loose any data (yet), keep information
 104 * around that allows us to restart a failed ttm backup recovery.
 105 */
 106struct ttm_pool_tt_restore {
 107	struct ttm_pool *pool;
 108	struct ttm_pool_alloc_state snapshot_alloc;
 109	struct page *alloced_page;
 110	dma_addr_t first_dma;
 111	pgoff_t alloced_pages;
 112	pgoff_t restored_pages;
 113	enum ttm_caching page_caching;
 114	unsigned int order;
 115};
 116
 117static unsigned long page_pool_size;
 118
 119MODULE_PARM_DESC(page_pool_size, "Number of pages in the WC/UC/DMA pool");
 120module_param(page_pool_size, ulong, 0644);
 121
 122static atomic_long_t allocated_pages;
 123
 124static struct ttm_pool_type global_write_combined[NR_PAGE_ORDERS];
 125static struct ttm_pool_type global_uncached[NR_PAGE_ORDERS];
 126
 127static struct ttm_pool_type global_dma32_write_combined[NR_PAGE_ORDERS];
 128static struct ttm_pool_type global_dma32_uncached[NR_PAGE_ORDERS];
 129
 130static spinlock_t shrinker_lock;
 131static struct list_head shrinker_list;
 132static struct shrinker *mm_shrinker;
 133static DECLARE_RWSEM(pool_shrink_rwsem);
 134
 135/* Allocate pages of size 1 << order with the given gfp_flags */
 136static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags,
 137					unsigned int order)
 138{
 139	const unsigned int beneficial_order = ttm_pool_beneficial_order(pool);
 140	unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS;
 141	struct ttm_pool_dma *dma;
 142	struct page *p;
 143	void *vaddr;
 144
 145	/* Don't set the __GFP_COMP flag for higher order allocations.
 146	 * Mapping pages directly into an userspace process and calling
 147	 * put_page() on a TTM allocated page is illegal.
 148	 */
 149	if (order)
 150		gfp_flags |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN |
 151			__GFP_THISNODE;
 152
 153	/*
 154	 * Do not add latency to the allocation path for allocations orders
 155	 * device tolds us do not bring them additional performance gains.
 156	 */
 157	if (beneficial_order && order > beneficial_order)
 158		gfp_flags &= ~__GFP_DIRECT_RECLAIM;
 159
 160	if (!ttm_pool_uses_dma_alloc(pool)) {
 161		p = alloc_pages_node(pool->nid, gfp_flags, order);
 162		if (p)
 163			p->private = order;
 164		return p;
 165	}
 166
 167	dma = kmalloc(sizeof(*dma), GFP_KERNEL);
 168	if (!dma)
 169		return NULL;
 170
 171	if (order)
 172		attr |= DMA_ATTR_NO_WARN;
 173
 174	vaddr = dma_alloc_attrs(pool->dev, (1ULL << order) * PAGE_SIZE,
 175				&dma->addr, gfp_flags, attr);
 176	if (!vaddr)
 177		goto error_free;
 178
 179	/* TODO: This is an illegal abuse of the DMA API, but we need to rework
 180	 * TTM page fault handling and extend the DMA API to clean this up.
 181	 */
 182	if (is_vmalloc_addr(vaddr))
 183		p = vmalloc_to_page(vaddr);
 184	else
 185		p = virt_to_page(vaddr);
 186
 187	dma->vaddr = (unsigned long)vaddr | order;
 188	p->private = (unsigned long)dma;
 189	return p;
 190
 191error_free:
 192	kfree(dma);
 193	return NULL;
 194}
 195
 196/* Reset the caching and pages of size 1 << order */
 197static void ttm_pool_free_page(struct ttm_pool *pool, enum ttm_caching caching,
 198			       unsigned int order, struct page *p)
 199{
 200	unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS;
 201	struct ttm_pool_dma *dma;
 202	void *vaddr;
 203
 204#ifdef CONFIG_X86
 205	/* We don't care that set_pages_wb is inefficient here. This is only
 206	 * used when we have to shrink and CPU overhead is irrelevant then.
 207	 */
 208	if (caching != ttm_cached && !PageHighMem(p))
 209		set_pages_wb(p, 1 << order);
 210#endif
 211
 212	if (!pool || !ttm_pool_uses_dma_alloc(pool)) {
 213		__free_pages(p, order);
 214		return;
 215	}
 216
 217	if (order)
 218		attr |= DMA_ATTR_NO_WARN;
 219
 220	dma = (void *)p->private;
 221	vaddr = (void *)(dma->vaddr & PAGE_MASK);
 222	dma_free_attrs(pool->dev, (1UL << order) * PAGE_SIZE, vaddr, dma->addr,
 223		       attr);
 224	kfree(dma);
 225}
 226
 227/* Apply any cpu-caching deferred during page allocation */
 228static int ttm_pool_apply_caching(struct ttm_pool_alloc_state *alloc)
 229{
 230#ifdef CONFIG_X86
 231	unsigned int num_pages = alloc->pages - alloc->caching_divide;
 232
 233	if (!num_pages)
 234		return 0;
 235
 236	switch (alloc->tt_caching) {
 237	case ttm_cached:
 238		break;
 239	case ttm_write_combined:
 240		return set_pages_array_wc(alloc->caching_divide, num_pages);
 241	case ttm_uncached:
 242		return set_pages_array_uc(alloc->caching_divide, num_pages);
 243	}
 244#endif
 245	alloc->caching_divide = alloc->pages;
 246	return 0;
 247}
 248
 249/* DMA Map pages of 1 << order size and return the resulting dma_address. */
 250static int ttm_pool_map(struct ttm_pool *pool, unsigned int order,
 251			struct page *p, dma_addr_t *dma_addr)
 252{
 253	dma_addr_t addr;
 254
 255	if (ttm_pool_uses_dma_alloc(pool)) {
 256		struct ttm_pool_dma *dma = (void *)p->private;
 257
 258		addr = dma->addr;
 259	} else {
 260		size_t size = (1ULL << order) * PAGE_SIZE;
 261
 262		addr = dma_map_page(pool->dev, p, 0, size, DMA_BIDIRECTIONAL);
 263		if (dma_mapping_error(pool->dev, addr))
 264			return -EFAULT;
 265	}
 266
 267	*dma_addr = addr;
 268
 269	return 0;
 270}
 271
 272/* Unmap pages of 1 << order size */
 273static void ttm_pool_unmap(struct ttm_pool *pool, dma_addr_t dma_addr,
 274			   unsigned int num_pages)
 275{
 276	/* Unmapped while freeing the page */
 277	if (ttm_pool_uses_dma_alloc(pool))
 278		return;
 279
 280	dma_unmap_page(pool->dev, dma_addr, (long)num_pages << PAGE_SHIFT,
 281		       DMA_BIDIRECTIONAL);
 282}
 283
 284/* Give pages into a specific pool_type */
 285static void ttm_pool_type_give(struct ttm_pool_type *pt, struct page *p)
 286{
 287	unsigned int i, num_pages = 1 << pt->order;
 288
 289	for (i = 0; i < num_pages; ++i) {
 290		if (PageHighMem(p))
 291			clear_highpage(p + i);
 292		else
 293			clear_page(page_address(p + i));
 294	}
 295
 296	spin_lock(&pt->lock);
 297	list_add(&p->lru, &pt->pages);
 298	spin_unlock(&pt->lock);
 299	atomic_long_add(1 << pt->order, &allocated_pages);
 300}
 301
 302/* Take pages from a specific pool_type, return NULL when nothing available */
 303static struct page *ttm_pool_type_take(struct ttm_pool_type *pt)
 304{
 305	struct page *p;
 306
 307	spin_lock(&pt->lock);
 308	p = list_first_entry_or_null(&pt->pages, typeof(*p), lru);
 309	if (p) {
 310		atomic_long_sub(1 << pt->order, &allocated_pages);
 311		list_del(&p->lru);
 312	}
 313	spin_unlock(&pt->lock);
 314
 315	return p;
 316}
 317
 318/* Initialize and add a pool type to the global shrinker list */
 319static void ttm_pool_type_init(struct ttm_pool_type *pt, struct ttm_pool *pool,
 320			       enum ttm_caching caching, unsigned int order)
 321{
 322	pt->pool = pool;
 323	pt->caching = caching;
 324	pt->order = order;
 325	spin_lock_init(&pt->lock);
 326	INIT_LIST_HEAD(&pt->pages);
 327
 328	spin_lock(&shrinker_lock);
 329	list_add_tail(&pt->shrinker_list, &shrinker_list);
 330	spin_unlock(&shrinker_lock);
 331}
 332
 333/* Remove a pool_type from the global shrinker list and free all pages */
 334static void ttm_pool_type_fini(struct ttm_pool_type *pt)
 335{
 336	struct page *p;
 337
 338	spin_lock(&shrinker_lock);
 339	list_del(&pt->shrinker_list);
 340	spin_unlock(&shrinker_lock);
 341
 342	while ((p = ttm_pool_type_take(pt)))
 343		ttm_pool_free_page(pt->pool, pt->caching, pt->order, p);
 344}
 345
 346/* Return the pool_type to use for the given caching and order */
 347static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool,
 348						  enum ttm_caching caching,
 349						  unsigned int order)
 350{
 351	if (ttm_pool_uses_dma_alloc(pool))
 352		return &pool->caching[caching].orders[order];
 353
 354#ifdef CONFIG_X86
 355	switch (caching) {
 356	case ttm_write_combined:
 357		if (pool->nid != NUMA_NO_NODE)
 358			return &pool->caching[caching].orders[order];
 359
 360		if (ttm_pool_uses_dma32(pool))
 361			return &global_dma32_write_combined[order];
 362
 363		return &global_write_combined[order];
 364	case ttm_uncached:
 365		if (pool->nid != NUMA_NO_NODE)
 366			return &pool->caching[caching].orders[order];
 367
 368		if (ttm_pool_uses_dma32(pool))
 369			return &global_dma32_uncached[order];
 370
 371		return &global_uncached[order];
 372	default:
 373		break;
 374	}
 375#endif
 376
 377	return NULL;
 378}
 379
 380/* Free pages using the global shrinker list */
 381static unsigned int ttm_pool_shrink(void)
 382{
 383	struct ttm_pool_type *pt;
 384	unsigned int num_pages;
 385	struct page *p;
 386
 387	down_read(&pool_shrink_rwsem);
 388	spin_lock(&shrinker_lock);
 389	pt = list_first_entry(&shrinker_list, typeof(*pt), shrinker_list);
 390	list_move_tail(&pt->shrinker_list, &shrinker_list);
 391	spin_unlock(&shrinker_lock);
 392
 393	p = ttm_pool_type_take(pt);
 394	if (p) {
 395		ttm_pool_free_page(pt->pool, pt->caching, pt->order, p);
 396		num_pages = 1 << pt->order;
 397	} else {
 398		num_pages = 0;
 399	}
 400	up_read(&pool_shrink_rwsem);
 401
 402	return num_pages;
 403}
 404
 405/* Return the allocation order based for a page */
 406static unsigned int ttm_pool_page_order(struct ttm_pool *pool, struct page *p)
 407{
 408	if (ttm_pool_uses_dma_alloc(pool)) {
 409		struct ttm_pool_dma *dma = (void *)p->private;
 410
 411		return dma->vaddr & ~PAGE_MASK;
 412	}
 413
 414	return p->private;
 415}
 416
 417/*
 418 * Split larger pages so that we can free each PAGE_SIZE page as soon
 419 * as it has been backed up, in order to avoid memory pressure during
 420 * reclaim.
 421 */
 422static void ttm_pool_split_for_swap(struct ttm_pool *pool, struct page *p)
 423{
 424	unsigned int order = ttm_pool_page_order(pool, p);
 425	pgoff_t nr;
 426
 427	if (!order)
 428		return;
 429
 430	split_page(p, order);
 431	nr = 1UL << order;
 432	while (nr--)
 433		(p++)->private = 0;
 434}
 435
 436/**
 437 * DOC: Partial backup and restoration of a struct ttm_tt.
 438 *
 439 * Swapout using ttm_backup_backup_page() and swapin using
 440 * ttm_backup_copy_page() may fail.
 441 * The former most likely due to lack of swap-space or memory, the latter due
 442 * to lack of memory or because of signal interruption during waits.
 443 *
 444 * Backup failure is easily handled by using a ttm_tt pages vector that holds
 445 * both backup handles and page pointers. This has to be taken into account when
 446 * restoring such a ttm_tt from backup, and when freeing it while backed up.
 447 * When restoring, for simplicity, new pages are actually allocated from the
 448 * pool and the contents of any old pages are copied in and then the old pages
 449 * are released.
 450 *
 451 * For restoration failures, the struct ttm_pool_tt_restore holds sufficient state
 452 * to be able to resume an interrupted restore, and that structure is freed once
 453 * the restoration is complete. If the struct ttm_tt is destroyed while there
 454 * is a valid struct ttm_pool_tt_restore attached, that is also properly taken
 455 * care of.
 456 */
 457
 458/* Is restore ongoing for the currently allocated page? */
 459static bool ttm_pool_restore_valid(const struct ttm_pool_tt_restore *restore)
 460{
 461	return restore && restore->restored_pages < (1 << restore->order);
 462}
 463
 464/* DMA unmap and free a multi-order page, either to the relevant pool or to system. */
 465static pgoff_t ttm_pool_unmap_and_free(struct ttm_pool *pool, struct page *page,
 466				       const dma_addr_t *dma_addr, enum ttm_caching caching)
 467{
 468	struct ttm_pool_type *pt = NULL;
 469	unsigned int order;
 470	pgoff_t nr;
 471
 472	if (pool) {
 473		order = ttm_pool_page_order(pool, page);
 474		nr = (1UL << order);
 475		if (dma_addr)
 476			ttm_pool_unmap(pool, *dma_addr, nr);
 477
 478		pt = ttm_pool_select_type(pool, caching, order);
 479	} else {
 480		order = page->private;
 481		nr = (1UL << order);
 482	}
 483
 484	if (pt)
 485		ttm_pool_type_give(pt, page);
 486	else
 487		ttm_pool_free_page(pool, caching, order, page);
 488
 489	return nr;
 490}
 491
 492/* Populate the page-array using the most recent allocated multi-order page. */
 493static void ttm_pool_allocated_page_commit(struct page *allocated,
 494					   dma_addr_t first_dma,
 495					   struct ttm_pool_alloc_state *alloc,
 496					   pgoff_t nr)
 497{
 498	pgoff_t i;
 499
 500	for (i = 0; i < nr; ++i)
 501		*alloc->pages++ = allocated++;
 502
 503	alloc->remaining_pages -= nr;
 504
 505	if (!alloc->dma_addr)
 506		return;
 507
 508	for (i = 0; i < nr; ++i) {
 509		*alloc->dma_addr++ = first_dma;
 510		first_dma += PAGE_SIZE;
 511	}
 512}
 513
 514/*
 515 * When restoring, restore backed-up content to the newly allocated page and
 516 * if successful, populate the page-table and dma-address arrays.
 517 */
 518static int ttm_pool_restore_commit(struct ttm_pool_tt_restore *restore,
 519				   struct file *backup,
 520				   const struct ttm_operation_ctx *ctx,
 521				   struct ttm_pool_alloc_state *alloc)
 522
 523{
 524	pgoff_t i, nr = 1UL << restore->order;
 525	struct page **first_page = alloc->pages;
 526	struct page *p;
 527	int ret = 0;
 528
 529	for (i = restore->restored_pages; i < nr; ++i) {
 530		p = first_page[i];
 531		if (ttm_backup_page_ptr_is_handle(p)) {
 532			unsigned long handle = ttm_backup_page_ptr_to_handle(p);
 533
 534			if (IS_ENABLED(CONFIG_FAULT_INJECTION) && ctx->interruptible &&
 535			    should_fail(&backup_fault_inject, 1)) {
 536				ret = -EINTR;
 537				break;
 538			}
 539
 540			if (handle == 0) {
 541				restore->restored_pages++;
 542				continue;
 543			}
 544
 545			ret = ttm_backup_copy_page(backup, restore->alloced_page + i,
 546						   handle, ctx->interruptible);
 547			if (ret)
 548				break;
 549
 550			ttm_backup_drop(backup, handle);
 551		} else if (p) {
 552			/*
 553			 * We could probably avoid splitting the old page
 554			 * using clever logic, but ATM we don't care, as
 555			 * we prioritize releasing memory ASAP. Note that
 556			 * here, the old retained page is always write-back
 557			 * cached.
 558			 */
 559			ttm_pool_split_for_swap(restore->pool, p);
 560			copy_highpage(restore->alloced_page + i, p);
 561			__free_pages(p, 0);
 562		}
 563
 564		restore->restored_pages++;
 565		first_page[i] = ttm_backup_handle_to_page_ptr(0);
 566	}
 567
 568	if (ret) {
 569		if (!restore->restored_pages) {
 570			dma_addr_t *dma_addr = alloc->dma_addr ? &restore->first_dma : NULL;
 571
 572			ttm_pool_unmap_and_free(restore->pool, restore->alloced_page,
 573						dma_addr, restore->page_caching);
 574			restore->restored_pages = nr;
 575		}
 576		return ret;
 577	}
 578
 579	ttm_pool_allocated_page_commit(restore->alloced_page, restore->first_dma,
 580				       alloc, nr);
 581	if (restore->page_caching == alloc->tt_caching || PageHighMem(restore->alloced_page))
 582		alloc->caching_divide = alloc->pages;
 583	restore->snapshot_alloc = *alloc;
 584	restore->alloced_pages += nr;
 585
 586	return 0;
 587}
 588
 589/* If restoring, save information needed for ttm_pool_restore_commit(). */
 590static void
 591ttm_pool_page_allocated_restore(struct ttm_pool *pool, unsigned int order,
 592				struct page *p,
 593				enum ttm_caching page_caching,
 594				dma_addr_t first_dma,
 595				struct ttm_pool_tt_restore *restore,
 596				const struct ttm_pool_alloc_state *alloc)
 597{
 598	restore->pool = pool;
 599	restore->order = order;
 600	restore->restored_pages = 0;
 601	restore->page_caching = page_caching;
 602	restore->first_dma = first_dma;
 603	restore->alloced_page = p;
 604	restore->snapshot_alloc = *alloc;
 605}
 606
 607/*
 608 * Called when we got a page, either from a pool or newly allocated.
 609 * if needed, dma map the page and populate the dma address array.
 610 * Populate the page address array.
 611 * If the caching is consistent, update any deferred caching. Otherwise
 612 * stage this page for an upcoming deferred caching update.
 613 */
 614static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order,
 615				   struct page *p, enum ttm_caching page_caching,
 616				   struct ttm_pool_alloc_state *alloc,
 617				   struct ttm_pool_tt_restore *restore)
 618{
 619	bool caching_consistent;
 620	dma_addr_t first_dma;
 621	int r = 0;
 622
 623	caching_consistent = (page_caching == alloc->tt_caching) || PageHighMem(p);
 624
 625	if (caching_consistent) {
 626		r = ttm_pool_apply_caching(alloc);
 627		if (r)
 628			return r;
 629	}
 630
 631	if (alloc->dma_addr) {
 632		r = ttm_pool_map(pool, order, p, &first_dma);
 633		if (r)
 634			return r;
 635	}
 636
 637	if (restore) {
 638		ttm_pool_page_allocated_restore(pool, order, p, page_caching,
 639						first_dma, restore, alloc);
 640	} else {
 641		ttm_pool_allocated_page_commit(p, first_dma, alloc, 1UL << order);
 642
 643		if (caching_consistent)
 644			alloc->caching_divide = alloc->pages;
 645	}
 646
 647	return 0;
 648}
 649
 650/**
 651 * ttm_pool_free_range() - Free a range of TTM pages
 652 * @pool: The pool used for allocating.
 653 * @tt: The struct ttm_tt holding the page pointers.
 654 * @caching: The page caching mode used by the range.
 655 * @start_page: index for first page to free.
 656 * @end_page: index for last page to free + 1.
 657 *
 658 * During allocation the ttm_tt page-vector may be populated with ranges of
 659 * pages with different attributes if allocation hit an error without being
 660 * able to completely fulfill the allocation. This function can be used
 661 * to free these individual ranges.
 662 */
 663static void ttm_pool_free_range(struct ttm_pool *pool, struct ttm_tt *tt,
 664				enum ttm_caching caching,
 665				pgoff_t start_page, pgoff_t end_page)
 666{
 667	struct page **pages = &tt->pages[start_page];
 668	struct file *backup = tt->backup;
 669	pgoff_t i, nr;
 670
 671	for (i = start_page; i < end_page; i += nr, pages += nr) {
 672		struct page *p = *pages;
 673
 674		nr = 1;
 675		if (ttm_backup_page_ptr_is_handle(p)) {
 676			unsigned long handle = ttm_backup_page_ptr_to_handle(p);
 677
 678			if (handle != 0)
 679				ttm_backup_drop(backup, handle);
 680		} else if (p) {
 681			dma_addr_t *dma_addr = tt->dma_address ?
 682				tt->dma_address + i : NULL;
 683
 684			nr = ttm_pool_unmap_and_free(pool, p, dma_addr, caching);
 685		}
 686	}
 687}
 688
 689static void ttm_pool_alloc_state_init(const struct ttm_tt *tt,
 690				      struct ttm_pool_alloc_state *alloc)
 691{
 692	alloc->pages = tt->pages;
 693	alloc->caching_divide = tt->pages;
 694	alloc->dma_addr = tt->dma_address;
 695	alloc->remaining_pages = tt->num_pages;
 696	alloc->tt_caching = tt->caching;
 697}
 698
 699/*
 700 * Find a suitable allocation order based on highest desired order
 701 * and number of remaining pages
 702 */
 703static unsigned int ttm_pool_alloc_find_order(unsigned int highest,
 704					      const struct ttm_pool_alloc_state *alloc)
 705{
 706	return min_t(unsigned int, highest, __fls(alloc->remaining_pages));
 707}
 708
 709static int __ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
 710			    const struct ttm_operation_ctx *ctx,
 711			    struct ttm_pool_alloc_state *alloc,
 712			    struct ttm_pool_tt_restore *restore)
 713{
 714	enum ttm_caching page_caching;
 715	gfp_t gfp_flags = GFP_USER;
 716	pgoff_t caching_divide;
 717	unsigned int order;
 718	bool allow_pools;
 719	struct page *p;
 720	int r;
 721
 722	WARN_ON(!alloc->remaining_pages || ttm_tt_is_populated(tt));
 723	WARN_ON(alloc->dma_addr && !pool->dev);
 724
 725	if (tt->page_flags & TTM_TT_FLAG_ZERO_ALLOC)
 726		gfp_flags |= __GFP_ZERO;
 727
 728	if (ctx->gfp_retry_mayfail)
 729		gfp_flags |= __GFP_RETRY_MAYFAIL;
 730
 731	if (ttm_pool_uses_dma32(pool))
 732		gfp_flags |= GFP_DMA32;
 733	else
 734		gfp_flags |= GFP_HIGHUSER;
 735
 736	page_caching = tt->caching;
 737	allow_pools = true;
 738	for (order = ttm_pool_alloc_find_order(MAX_PAGE_ORDER, alloc);
 739	     alloc->remaining_pages;
 740	     order = ttm_pool_alloc_find_order(order, alloc)) {
 741		struct ttm_pool_type *pt;
 742
 743		/* First, try to allocate a page from a pool if one exists. */
 744		p = NULL;
 745		pt = ttm_pool_select_type(pool, page_caching, order);
 746		if (pt && allow_pools)
 747			p = ttm_pool_type_take(pt);
 748		/*
 749		 * If that fails or previously failed, allocate from system.
 750		 * Note that this also disallows additional pool allocations using
 751		 * write-back cached pools of the same order. Consider removing
 752		 * that behaviour.
 753		 */
 754		if (!p) {
 755			page_caching = ttm_cached;
 756			allow_pools = false;
 757			p = ttm_pool_alloc_page(pool, gfp_flags, order);
 758		}
 759		/* If that fails, lower the order if possible and retry. */
 760		if (!p) {
 761			if (order) {
 762				--order;
 763				page_caching = tt->caching;
 764				allow_pools = true;
 765				continue;
 766			}
 767			r = -ENOMEM;
 768			goto error_free_all;
 769		}
 770		r = ttm_pool_page_allocated(pool, order, p, page_caching, alloc,
 771					    restore);
 772		if (r)
 773			goto error_free_page;
 774
 775		if (ttm_pool_restore_valid(restore)) {
 776			r = ttm_pool_restore_commit(restore, tt->backup, ctx, alloc);
 777			if (r)
 778				goto error_free_all;
 779		}
 780	}
 781
 782	r = ttm_pool_apply_caching(alloc);
 783	if (r)
 784		goto error_free_all;
 785
 786	kfree(tt->restore);
 787	tt->restore = NULL;
 788
 789	return 0;
 790
 791error_free_page:
 792	ttm_pool_free_page(pool, page_caching, order, p);
 793
 794error_free_all:
 795	if (tt->restore)
 796		return r;
 797
 798	caching_divide = alloc->caching_divide - tt->pages;
 799	ttm_pool_free_range(pool, tt, tt->caching, 0, caching_divide);
 800	ttm_pool_free_range(pool, tt, ttm_cached, caching_divide,
 801			    tt->num_pages - alloc->remaining_pages);
 802
 803	return r;
 804}
 805
 806/**
 807 * ttm_pool_alloc - Fill a ttm_tt object
 808 *
 809 * @pool: ttm_pool to use
 810 * @tt: ttm_tt object to fill
 811 * @ctx: operation context
 812 *
 813 * Fill the ttm_tt object with pages and also make sure to DMA map them when
 814 * necessary.
 815 *
 816 * Returns: 0 on successe, negative error code otherwise.
 817 */
 818int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
 819		   struct ttm_operation_ctx *ctx)
 820{
 821	struct ttm_pool_alloc_state alloc;
 822
 823	if (WARN_ON(ttm_tt_is_backed_up(tt)))
 824		return -EINVAL;
 825
 826	ttm_pool_alloc_state_init(tt, &alloc);
 827
 828	return __ttm_pool_alloc(pool, tt, ctx, &alloc, NULL);
 829}
 830EXPORT_SYMBOL(ttm_pool_alloc);
 831
 832/**
 833 * ttm_pool_restore_and_alloc - Fill a ttm_tt, restoring previously backed-up
 834 * content.
 835 *
 836 * @pool: ttm_pool to use
 837 * @tt: ttm_tt object to fill
 838 * @ctx: operation context
 839 *
 840 * Fill the ttm_tt object with pages and also make sure to DMA map them when
 841 * necessary. Read in backed-up content.
 842 *
 843 * Returns: 0 on successe, negative error code otherwise.
 844 */
 845int ttm_pool_restore_and_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
 846			       const struct ttm_operation_ctx *ctx)
 847{
 848	struct ttm_pool_alloc_state alloc;
 849
 850	if (WARN_ON(!ttm_tt_is_backed_up(tt)))
 851		return -EINVAL;
 852
 853	if (!tt->restore) {
 854		gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
 855
 856		ttm_pool_alloc_state_init(tt, &alloc);
 857		if (ctx->gfp_retry_mayfail)
 858			gfp |= __GFP_RETRY_MAYFAIL;
 859
 860		tt->restore = kzalloc(sizeof(*tt->restore), gfp);
 861		if (!tt->restore)
 862			return -ENOMEM;
 863
 864		tt->restore->snapshot_alloc = alloc;
 865		tt->restore->pool = pool;
 866		tt->restore->restored_pages = 1;
 867	} else {
 868		struct ttm_pool_tt_restore *restore = tt->restore;
 869		int ret;
 870
 871		alloc = restore->snapshot_alloc;
 872		if (ttm_pool_restore_valid(tt->restore)) {
 873			ret = ttm_pool_restore_commit(restore, tt->backup, ctx, &alloc);
 874			if (ret)
 875				return ret;
 876		}
 877		if (!alloc.remaining_pages)
 878			return 0;
 879	}
 880
 881	return __ttm_pool_alloc(pool, tt, ctx, &alloc, tt->restore);
 882}
 883
 884/**
 885 * ttm_pool_free - Free the backing pages from a ttm_tt object
 886 *
 887 * @pool: Pool to give pages back to.
 888 * @tt: ttm_tt object to unpopulate
 889 *
 890 * Give the packing pages back to a pool or free them
 891 */
 892void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt)
 893{
 894	ttm_pool_free_range(pool, tt, tt->caching, 0, tt->num_pages);
 895
 896	while (atomic_long_read(&allocated_pages) > page_pool_size)
 897		ttm_pool_shrink();
 898}
 899EXPORT_SYMBOL(ttm_pool_free);
 900
 901/**
 902 * ttm_pool_drop_backed_up() - Release content of a swapped-out struct ttm_tt
 903 * @tt: The struct ttm_tt.
 904 *
 905 * Release handles with associated content or any remaining pages of
 906 * a backed-up struct ttm_tt.
 907 */
 908void ttm_pool_drop_backed_up(struct ttm_tt *tt)
 909{
 910	struct ttm_pool_tt_restore *restore;
 911	pgoff_t start_page = 0;
 912
 913	WARN_ON(!ttm_tt_is_backed_up(tt));
 914
 915	restore = tt->restore;
 916
 917	/*
 918	 * Unmap and free any uncommitted restore page.
 919	 * any tt page-array backup entries already read back has
 920	 * been cleared already
 921	 */
 922	if (ttm_pool_restore_valid(restore)) {
 923		dma_addr_t *dma_addr = tt->dma_address ? &restore->first_dma : NULL;
 924
 925		ttm_pool_unmap_and_free(restore->pool, restore->alloced_page,
 926					dma_addr, restore->page_caching);
 927		restore->restored_pages = 1UL << restore->order;
 928	}
 929
 930	/*
 931	 * If a restore is ongoing, part of the tt pages may have a
 932	 * caching different than writeback.
 933	 */
 934	if (restore) {
 935		pgoff_t mid = restore->snapshot_alloc.caching_divide - tt->pages;
 936
 937		start_page = restore->alloced_pages;
 938		WARN_ON(mid > start_page);
 939		/* Pages that might be dma-mapped and non-cached */
 940		ttm_pool_free_range(restore->pool, tt, tt->caching,
 941				    0, mid);
 942		/* Pages that might be dma-mapped but cached */
 943		ttm_pool_free_range(restore->pool, tt, ttm_cached,
 944				    mid, restore->alloced_pages);
 945		kfree(restore);
 946		tt->restore = NULL;
 947	}
 948
 949	ttm_pool_free_range(NULL, tt, ttm_cached, start_page, tt->num_pages);
 950}
 951
 952/**
 953 * ttm_pool_backup() - Back up or purge a struct ttm_tt
 954 * @pool: The pool used when allocating the struct ttm_tt.
 955 * @tt: The struct ttm_tt.
 956 * @flags: Flags to govern the backup behaviour.
 957 *
 958 * Back up or purge a struct ttm_tt. If @purge is true, then
 959 * all pages will be freed directly to the system rather than to the pool
 960 * they were allocated from, making the function behave similarly to
 961 * ttm_pool_free(). If @purge is false the pages will be backed up instead,
 962 * exchanged for handles.
 963 * A subsequent call to ttm_pool_restore_and_alloc() will then read back the content and
 964 * a subsequent call to ttm_pool_drop_backed_up() will drop it.
 965 * If backup of a page fails for whatever reason, @ttm will still be
 966 * partially backed up, retaining those pages for which backup fails.
 967 * In that case, this function can be retried, possibly after freeing up
 968 * memory resources.
 969 *
 970 * Return: Number of pages actually backed up or freed, or negative
 971 * error code on error.
 972 */
 973long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt,
 974		     const struct ttm_backup_flags *flags)
 975{
 976	struct file *backup = tt->backup;
 977	struct page *page;
 978	unsigned long handle;
 979	gfp_t alloc_gfp;
 980	gfp_t gfp;
 981	int ret = 0;
 982	pgoff_t shrunken = 0;
 983	pgoff_t i, num_pages;
 984
 985	if (WARN_ON(ttm_tt_is_backed_up(tt)))
 986		return -EINVAL;
 987
 988	if ((!ttm_backup_bytes_avail() && !flags->purge) ||
 989	    ttm_pool_uses_dma_alloc(pool) || ttm_tt_is_backed_up(tt))
 990		return -EBUSY;
 991
 992#ifdef CONFIG_X86
 993	/* Anything returned to the system needs to be cached. */
 994	if (tt->caching != ttm_cached)
 995		set_pages_array_wb(tt->pages, tt->num_pages);
 996#endif
 997
 998	if (tt->dma_address || flags->purge) {
 999		for (i = 0; i < tt->num_pages; i += num_pages) {
1000			unsigned int order;
1001
1002			page = tt->pages[i];
1003			if (unlikely(!page)) {
1004				num_pages = 1;
1005				continue;
1006			}
1007
1008			order = ttm_pool_page_order(pool, page);
1009			num_pages = 1UL << order;
1010			if (tt->dma_address)
1011				ttm_pool_unmap(pool, tt->dma_address[i],
1012					       num_pages);
1013			if (flags->purge) {
1014				shrunken += num_pages;
1015				page->private = 0;
1016				__free_pages(page, order);
1017				memset(tt->pages + i, 0,
1018				       num_pages * sizeof(*tt->pages));
1019			}
1020		}
1021	}
1022
1023	if (flags->purge)
1024		return shrunken;
1025
1026	if (ttm_pool_uses_dma32(pool))
1027		gfp = GFP_DMA32;
1028	else
1029		gfp = GFP_HIGHUSER;
1030
1031	alloc_gfp = GFP_KERNEL | __GFP_HIGH | __GFP_NOWARN | __GFP_RETRY_MAYFAIL;
1032
1033	num_pages = tt->num_pages;
1034
1035	/* Pretend doing fault injection by shrinking only half of the pages. */
1036	if (IS_ENABLED(CONFIG_FAULT_INJECTION) && should_fail(&backup_fault_inject, 1))
1037		num_pages = DIV_ROUND_UP(num_pages, 2);
1038
1039	for (i = 0; i < num_pages; ++i) {
1040		s64 shandle;
1041
1042		page = tt->pages[i];
1043		if (unlikely(!page))
1044			continue;
1045
1046		ttm_pool_split_for_swap(pool, page);
1047
1048		shandle = ttm_backup_backup_page(backup, page, flags->writeback, i,
1049						 gfp, alloc_gfp);
1050		if (shandle < 0) {
1051			/* We allow partially shrunken tts */
1052			ret = shandle;
1053			break;
1054		}
1055		handle = shandle;
1056		tt->pages[i] = ttm_backup_handle_to_page_ptr(handle);
1057		put_page(page);
1058		shrunken++;
1059	}
1060
1061	return shrunken ? shrunken : ret;
1062}
1063
1064/**
1065 * ttm_pool_init - Initialize a pool
1066 *
1067 * @pool: the pool to initialize
1068 * @dev: device for DMA allocations and mappings
1069 * @nid: NUMA node to use for allocations
1070 * @alloc_flags: TTM_ALLOCATION_POOL_* flags
1071 *
1072 * Initialize the pool and its pool types.
1073 */
1074void ttm_pool_init(struct ttm_pool *pool, struct device *dev,
1075		   int nid, unsigned int alloc_flags)
1076{
1077	unsigned int i, j;
1078
1079	WARN_ON(!dev && ttm_pool_uses_dma_alloc(pool));
1080
1081	pool->dev = dev;
1082	pool->nid = nid;
1083	pool->alloc_flags = alloc_flags;
1084
1085	for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) {
1086		for (j = 0; j < NR_PAGE_ORDERS; ++j) {
1087			struct ttm_pool_type *pt;
1088
1089			/* Initialize only pool types which are actually used */
1090			pt = ttm_pool_select_type(pool, i, j);
1091			if (pt != &pool->caching[i].orders[j])
1092				continue;
1093
1094			ttm_pool_type_init(pt, pool, i, j);
1095		}
1096	}
1097}
1098EXPORT_SYMBOL(ttm_pool_init);
1099
1100/**
1101 * ttm_pool_synchronize_shrinkers - Wait for all running shrinkers to complete.
1102 *
1103 * This is useful to guarantee that all shrinker invocations have seen an
1104 * update, before freeing memory, similar to rcu.
1105 */
1106static void ttm_pool_synchronize_shrinkers(void)
1107{
1108	down_write(&pool_shrink_rwsem);
1109	up_write(&pool_shrink_rwsem);
1110}
1111
1112/**
1113 * ttm_pool_fini - Cleanup a pool
1114 *
1115 * @pool: the pool to clean up
1116 *
1117 * Free all pages in the pool and unregister the types from the global
1118 * shrinker.
1119 */
1120void ttm_pool_fini(struct ttm_pool *pool)
1121{
1122	unsigned int i, j;
1123
1124	for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) {
1125		for (j = 0; j < NR_PAGE_ORDERS; ++j) {
1126			struct ttm_pool_type *pt;
1127
1128			pt = ttm_pool_select_type(pool, i, j);
1129			if (pt != &pool->caching[i].orders[j])
1130				continue;
1131
1132			ttm_pool_type_fini(pt);
1133		}
1134	}
1135
1136	/* We removed the pool types from the LRU, but we need to also make sure
1137	 * that no shrinker is concurrently freeing pages from the pool.
1138	 */
1139	ttm_pool_synchronize_shrinkers();
1140}
1141EXPORT_SYMBOL(ttm_pool_fini);
1142
1143/* Free average pool number of pages.  */
1144#define TTM_SHRINKER_BATCH ((1 << (MAX_PAGE_ORDER / 2)) * NR_PAGE_ORDERS)
1145
1146static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink,
1147					    struct shrink_control *sc)
1148{
1149	unsigned long num_freed = 0;
1150
1151	do
1152		num_freed += ttm_pool_shrink();
1153	while (num_freed < sc->nr_to_scan &&
1154	       atomic_long_read(&allocated_pages));
1155
1156	sc->nr_scanned = num_freed;
1157
1158	return num_freed ?: SHRINK_STOP;
1159}
1160
1161/* Return the number of pages available or SHRINK_EMPTY if we have none */
1162static unsigned long ttm_pool_shrinker_count(struct shrinker *shrink,
1163					     struct shrink_control *sc)
1164{
1165	unsigned long num_pages = atomic_long_read(&allocated_pages);
1166
1167	return num_pages ? num_pages : SHRINK_EMPTY;
1168}
1169
1170#ifdef CONFIG_DEBUG_FS
1171/* Count the number of pages available in a pool_type */
1172static unsigned int ttm_pool_type_count(struct ttm_pool_type *pt)
1173{
1174	unsigned int count = 0;
1175	struct page *p;
1176
1177	spin_lock(&pt->lock);
1178	/* Only used for debugfs, the overhead doesn't matter */
1179	list_for_each_entry(p, &pt->pages, lru)
1180		++count;
1181	spin_unlock(&pt->lock);
1182
1183	return count;
1184}
1185
1186/* Print a nice header for the order */
1187static void ttm_pool_debugfs_header(struct seq_file *m)
1188{
1189	unsigned int i;
1190
1191	seq_puts(m, "\t ");
1192	for (i = 0; i < NR_PAGE_ORDERS; ++i)
1193		seq_printf(m, " ---%2u---", i);
1194	seq_puts(m, "\n");
1195}
1196
1197/* Dump information about the different pool types */
1198static void ttm_pool_debugfs_orders(struct ttm_pool_type *pt,
1199				    struct seq_file *m)
1200{
1201	unsigned int i;
1202
1203	for (i = 0; i < NR_PAGE_ORDERS; ++i)
1204		seq_printf(m, " %8u", ttm_pool_type_count(&pt[i]));
1205	seq_puts(m, "\n");
1206}
1207
1208/* Dump the total amount of allocated pages */
1209static void ttm_pool_debugfs_footer(struct seq_file *m)
1210{
1211	seq_printf(m, "\ntotal\t: %8lu of %8lu\n",
1212		   atomic_long_read(&allocated_pages), page_pool_size);
1213}
1214
1215/* Dump the information for the global pools */
1216static int ttm_pool_debugfs_globals_show(struct seq_file *m, void *data)
1217{
1218	ttm_pool_debugfs_header(m);
1219
1220	spin_lock(&shrinker_lock);
1221	seq_puts(m, "wc\t:");
1222	ttm_pool_debugfs_orders(global_write_combined, m);
1223	seq_puts(m, "uc\t:");
1224	ttm_pool_debugfs_orders(global_uncached, m);
1225	seq_puts(m, "wc 32\t:");
1226	ttm_pool_debugfs_orders(global_dma32_write_combined, m);
1227	seq_puts(m, "uc 32\t:");
1228	ttm_pool_debugfs_orders(global_dma32_uncached, m);
1229	spin_unlock(&shrinker_lock);
1230
1231	ttm_pool_debugfs_footer(m);
1232
1233	return 0;
1234}
1235DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_globals);
1236
1237/**
1238 * ttm_pool_debugfs - Debugfs dump function for a pool
1239 *
1240 * @pool: the pool to dump the information for
1241 * @m: seq_file to dump to
1242 *
1243 * Make a debugfs dump with the per pool and global information.
1244 */
1245int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m)
1246{
1247	unsigned int i;
1248
1249	if (!ttm_pool_uses_dma_alloc(pool) && pool->nid == NUMA_NO_NODE) {
1250		seq_puts(m, "unused\n");
1251		return 0;
1252	}
1253
1254	ttm_pool_debugfs_header(m);
1255
1256	spin_lock(&shrinker_lock);
1257	for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) {
1258		if (!ttm_pool_select_type(pool, i, 0))
1259			continue;
1260		if (ttm_pool_uses_dma_alloc(pool))
1261			seq_puts(m, "DMA ");
1262		else
1263			seq_printf(m, "N%d ", pool->nid);
1264		switch (i) {
1265		case ttm_cached:
1266			seq_puts(m, "\t:");
1267			break;
1268		case ttm_write_combined:
1269			seq_puts(m, "wc\t:");
1270			break;
1271		case ttm_uncached:
1272			seq_puts(m, "uc\t:");
1273			break;
1274		}
1275		ttm_pool_debugfs_orders(pool->caching[i].orders, m);
1276	}
1277	spin_unlock(&shrinker_lock);
1278
1279	ttm_pool_debugfs_footer(m);
1280	return 0;
1281}
1282EXPORT_SYMBOL(ttm_pool_debugfs);
1283
1284/* Test the shrinker functions and dump the result */
1285static int ttm_pool_debugfs_shrink_show(struct seq_file *m, void *data)
1286{
1287	struct shrink_control sc = {
1288		.gfp_mask = GFP_NOFS,
1289		.nr_to_scan = TTM_SHRINKER_BATCH,
1290	};
1291	unsigned long count;
1292
1293	fs_reclaim_acquire(GFP_KERNEL);
1294	count = ttm_pool_shrinker_count(mm_shrinker, &sc);
1295	seq_printf(m, "%lu/%lu\n", count,
1296		   ttm_pool_shrinker_scan(mm_shrinker, &sc));
1297	fs_reclaim_release(GFP_KERNEL);
1298
1299	return 0;
1300}
1301DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_shrink);
1302
1303#endif
1304
1305/**
1306 * ttm_pool_mgr_init - Initialize globals
1307 *
1308 * @num_pages: default number of pages
1309 *
1310 * Initialize the global locks and lists for the MM shrinker.
1311 */
1312int ttm_pool_mgr_init(unsigned long num_pages)
1313{
1314	unsigned int i;
1315
1316	if (!page_pool_size)
1317		page_pool_size = num_pages;
1318
1319	spin_lock_init(&shrinker_lock);
1320	INIT_LIST_HEAD(&shrinker_list);
1321
1322	for (i = 0; i < NR_PAGE_ORDERS; ++i) {
1323		ttm_pool_type_init(&global_write_combined[i], NULL,
1324				   ttm_write_combined, i);
1325		ttm_pool_type_init(&global_uncached[i], NULL, ttm_uncached, i);
1326
1327		ttm_pool_type_init(&global_dma32_write_combined[i], NULL,
1328				   ttm_write_combined, i);
1329		ttm_pool_type_init(&global_dma32_uncached[i], NULL,
1330				   ttm_uncached, i);
1331	}
1332
1333#ifdef CONFIG_DEBUG_FS
1334	debugfs_create_file("page_pool", 0444, ttm_debugfs_root, NULL,
1335			    &ttm_pool_debugfs_globals_fops);
1336	debugfs_create_file("page_pool_shrink", 0400, ttm_debugfs_root, NULL,
1337			    &ttm_pool_debugfs_shrink_fops);
1338#ifdef CONFIG_FAULT_INJECTION
1339	fault_create_debugfs_attr("backup_fault_inject", ttm_debugfs_root,
1340				  &backup_fault_inject);
1341#endif
1342#endif
1343
1344	mm_shrinker = shrinker_alloc(0, "drm-ttm_pool");
1345	if (!mm_shrinker)
1346		return -ENOMEM;
1347
1348	mm_shrinker->count_objects = ttm_pool_shrinker_count;
1349	mm_shrinker->scan_objects = ttm_pool_shrinker_scan;
1350	mm_shrinker->batch = TTM_SHRINKER_BATCH;
1351	mm_shrinker->seeks = 1;
1352
1353	shrinker_register(mm_shrinker);
1354
1355	return 0;
1356}
1357
1358/**
1359 * ttm_pool_mgr_fini - Finalize globals
1360 *
1361 * Cleanup the global pools and unregister the MM shrinker.
1362 */
1363void ttm_pool_mgr_fini(void)
1364{
1365	unsigned int i;
1366
1367	for (i = 0; i < NR_PAGE_ORDERS; ++i) {
1368		ttm_pool_type_fini(&global_write_combined[i]);
1369		ttm_pool_type_fini(&global_uncached[i]);
1370
1371		ttm_pool_type_fini(&global_dma32_write_combined[i]);
1372		ttm_pool_type_fini(&global_dma32_uncached[i]);
1373	}
1374
1375	shrinker_free(mm_shrinker);
1376	WARN_ON(!list_empty(&shrinker_list));
1377}