i915_gem_shmem.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2014-2016 Intel Corporation
  4 */
  5
  6#include <linux/pagevec.h>
  7#include <linux/shmem_fs.h>
  8#include <linux/swap.h>
  9#include <linux/uio.h>
 10
 11#include <drm/drm_cache.h>
 12
 13#include "gem/i915_gem_region.h"
 14#include "i915_drv.h"
 15#include "i915_gem_object.h"
 16#include "i915_gem_tiling.h"
 17#include "i915_gemfs.h"
 18#include "i915_scatterlist.h"
 19#include "i915_trace.h"
 20
 21/*
 22 * Move folios to appropriate lru and release the batch, decrementing the
 23 * ref count of those folios.
 24 */
 25static void check_release_folio_batch(struct folio_batch *fbatch)
 26{
 27	check_move_unevictable_folios(fbatch);
 28	__folio_batch_release(fbatch);
 29	cond_resched();
 30}
 31
 32void shmem_sg_free_table(struct sg_table *st, struct address_space *mapping,
 33			 bool dirty, bool backup)
 34{
 35	struct sgt_iter sgt_iter;
 36	struct folio_batch fbatch;
 37	struct folio *last = NULL;
 38	struct page *page;
 39
 40	mapping_clear_unevictable(mapping);
 41
 42	folio_batch_init(&fbatch);
 43	for_each_sgt_page(page, sgt_iter, st) {
 44		struct folio *folio = page_folio(page);
 45
 46		if (folio == last)
 47			continue;
 48		last = folio;
 49		if (dirty)
 50			folio_mark_dirty(folio);
 51		if (backup)
 52			folio_mark_accessed(folio);
 53
 54		if (!folio_batch_add(&fbatch, folio))
 55			check_release_folio_batch(&fbatch);
 56	}
 57	if (fbatch.nr)
 58		check_release_folio_batch(&fbatch);
 59
 60	sg_free_table(st);
 61}
 62
 63int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st,
 64			 size_t size, struct intel_memory_region *mr,
 65			 struct address_space *mapping,
 66			 unsigned int max_segment)
 67{
 68	unsigned int page_count; /* restricted by sg_alloc_table */
 69	unsigned long i;
 70	struct scatterlist *sg;
 71	unsigned long next_pfn = 0;	/* suppress gcc warning */
 72	gfp_t noreclaim;
 73	int ret;
 74
 75	if (overflows_type(size / PAGE_SIZE, page_count))
 76		return -E2BIG;
 77
 78	page_count = size / PAGE_SIZE;
 79	/*
 80	 * If there's no chance of allocating enough pages for the whole
 81	 * object, bail early.
 82	 */
 83	if (size > resource_size(&mr->region))
 84		return -ENOMEM;
 85
 86	if (sg_alloc_table(st, page_count, GFP_KERNEL | __GFP_NOWARN))
 87		return -ENOMEM;
 88
 89	/*
 90	 * Get the list of pages out of our struct file.  They'll be pinned
 91	 * at this point until we release them.
 92	 *
 93	 * Fail silently without starting the shrinker
 94	 */
 95	mapping_set_unevictable(mapping);
 96	noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);
 97	noreclaim |= __GFP_NORETRY | __GFP_NOWARN;
 98
 99	sg = st->sgl;
100	st->nents = 0;
101	for (i = 0; i < page_count; i++) {
102		struct folio *folio;
103		unsigned long nr_pages;
104		const unsigned int shrink[] = {
105			I915_SHRINK_BOUND | I915_SHRINK_UNBOUND,
106			0,
107		}, *s = shrink;
108		gfp_t gfp = noreclaim;
109
110		do {
111			cond_resched();
112			folio = shmem_read_folio_gfp(mapping, i, gfp);
113			if (!IS_ERR(folio))
114				break;
115
116			if (!*s) {
117				ret = PTR_ERR(folio);
118				goto err_sg;
119			}
120
121			i915_gem_shrink(NULL, i915, 2 * page_count, NULL, *s++);
122
123			/*
124			 * We've tried hard to allocate the memory by reaping
125			 * our own buffer, now let the real VM do its job and
126			 * go down in flames if truly OOM.
127			 *
128			 * However, since graphics tend to be disposable,
129			 * defer the oom here by reporting the ENOMEM back
130			 * to userspace.
131			 */
132			if (!*s) {
133				/* reclaim and warn, but no oom */
134				gfp = mapping_gfp_mask(mapping);
135
136				/*
137				 * Our bo are always dirty and so we require
138				 * kswapd to reclaim our pages (direct reclaim
139				 * does not effectively begin pageout of our
140				 * buffers on its own). However, direct reclaim
141				 * only waits for kswapd when under allocation
142				 * congestion. So as a result __GFP_RECLAIM is
143				 * unreliable and fails to actually reclaim our
144				 * dirty pages -- unless you try over and over
145				 * again with !__GFP_NORETRY. However, we still
146				 * want to fail this allocation rather than
147				 * trigger the out-of-memory killer and for
148				 * this we want __GFP_RETRY_MAYFAIL.
149				 */
150				gfp |= __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
151			}
152		} while (1);
153
154		nr_pages = min_array(((unsigned long[]) {
155					folio_nr_pages(folio),
156					page_count - i,
157					max_segment / PAGE_SIZE,
158				      }), 3);
159
160		if (!i ||
161		    sg->length >= max_segment ||
162		    folio_pfn(folio) != next_pfn) {
163			if (i)
164				sg = sg_next(sg);
165
166			st->nents++;
167			sg_set_folio(sg, folio, nr_pages * PAGE_SIZE, 0);
168		} else {
169			nr_pages = min_t(unsigned long, nr_pages,
170					 (max_segment - sg->length) / PAGE_SIZE);
171
172			sg->length += nr_pages * PAGE_SIZE;
173		}
174		next_pfn = folio_pfn(folio) + nr_pages;
175		i += nr_pages - 1;
176
177		/* Check that the i965g/gm workaround works. */
178		GEM_BUG_ON(gfp & __GFP_DMA32 && next_pfn >= 0x00100000UL);
179	}
180	if (sg) /* loop terminated early; short sg table */
181		sg_mark_end(sg);
182
183	/* Trim unused sg entries to avoid wasting memory. */
184	i915_sg_trim(st);
185
186	return 0;
187err_sg:
188	sg_mark_end(sg);
189	if (sg != st->sgl) {
190		shmem_sg_free_table(st, mapping, false, false);
191	} else {
192		mapping_clear_unevictable(mapping);
193		sg_free_table(st);
194	}
195
196	/*
197	 * shmemfs first checks if there is enough memory to allocate the page
198	 * and reports ENOSPC should there be insufficient, along with the usual
199	 * ENOMEM for a genuine allocation failure.
200	 *
201	 * We use ENOSPC in our driver to mean that we have run out of aperture
202	 * space and so want to translate the error from shmemfs back to our
203	 * usual understanding of ENOMEM.
204	 */
205	if (ret == -ENOSPC)
206		ret = -ENOMEM;
207
208	return ret;
209}
210
211static int shmem_get_pages(struct drm_i915_gem_object *obj)
212{
213	struct drm_i915_private *i915 = to_i915(obj->base.dev);
214	struct intel_memory_region *mem = obj->mm.region;
215	struct address_space *mapping = obj->base.filp->f_mapping;
216	unsigned int max_segment = i915_sg_segment_size(i915->drm.dev);
217	struct sg_table *st;
218	int ret;
219
220	/*
221	 * Assert that the object is not currently in any GPU domain. As it
222	 * wasn't in the GTT, there shouldn't be any way it could have been in
223	 * a GPU cache
224	 */
225	GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
226	GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
227
228rebuild_st:
229	st = kmalloc(sizeof(*st), GFP_KERNEL | __GFP_NOWARN);
230	if (!st)
231		return -ENOMEM;
232
233	ret = shmem_sg_alloc_table(i915, st, obj->base.size, mem, mapping,
234				   max_segment);
235	if (ret)
236		goto err_st;
237
238	ret = i915_gem_gtt_prepare_pages(obj, st);
239	if (ret) {
240		/*
241		 * DMA remapping failed? One possible cause is that
242		 * it could not reserve enough large entries, asking
243		 * for PAGE_SIZE chunks instead may be helpful.
244		 */
245		if (max_segment > PAGE_SIZE) {
246			shmem_sg_free_table(st, mapping, false, false);
247			kfree(st);
248
249			max_segment = PAGE_SIZE;
250			goto rebuild_st;
251		} else {
252			dev_warn(i915->drm.dev,
253				 "Failed to DMA remap %zu pages\n",
254				 obj->base.size >> PAGE_SHIFT);
255			goto err_pages;
256		}
257	}
258
259	if (i915_gem_object_needs_bit17_swizzle(obj))
260		i915_gem_object_do_bit_17_swizzle(obj, st);
261
262	if (i915_gem_object_can_bypass_llc(obj))
263		obj->cache_dirty = true;
264
265	__i915_gem_object_set_pages(obj, st);
266
267	return 0;
268
269err_pages:
270	shmem_sg_free_table(st, mapping, false, false);
271	/*
272	 * shmemfs first checks if there is enough memory to allocate the page
273	 * and reports ENOSPC should there be insufficient, along with the usual
274	 * ENOMEM for a genuine allocation failure.
275	 *
276	 * We use ENOSPC in our driver to mean that we have run out of aperture
277	 * space and so want to translate the error from shmemfs back to our
278	 * usual understanding of ENOMEM.
279	 */
280err_st:
281	if (ret == -ENOSPC)
282		ret = -ENOMEM;
283
284	kfree(st);
285
286	return ret;
287}
288
289static int
290shmem_truncate(struct drm_i915_gem_object *obj)
291{
292	/*
293	 * Our goal here is to return as much of the memory as
294	 * is possible back to the system as we are called from OOM.
295	 * To do this we must instruct the shmfs to drop all of its
296	 * backing pages, *now*.
297	 */
298	shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
299	obj->mm.madv = __I915_MADV_PURGED;
300	obj->mm.pages = ERR_PTR(-EFAULT);
301
302	return 0;
303}
304
305void __shmem_writeback(size_t size, struct address_space *mapping)
306{
307	struct writeback_control wbc = {
308		.sync_mode = WB_SYNC_NONE,
309		.nr_to_write = SWAP_CLUSTER_MAX,
310		.range_start = 0,
311		.range_end = LLONG_MAX,
312	};
313	struct folio *folio = NULL;
314	int error = 0;
315
316	/*
317	 * Leave mmapings intact (GTT will have been revoked on unbinding,
318	 * leaving only CPU mmapings around) and add those folios to the LRU
319	 * instead of invoking writeback so they are aged and paged out
320	 * as normal.
321	 */
322	while ((folio = writeback_iter(mapping, &wbc, folio, &error))) {
323		if (folio_mapped(folio))
324			folio_redirty_for_writepage(&wbc, folio);
325		else
326			error = shmem_writeout(folio, NULL, NULL);
327	}
328}
329
330static void
331shmem_writeback(struct drm_i915_gem_object *obj)
332{
333	__shmem_writeback(obj->base.size, obj->base.filp->f_mapping);
334}
335
336static int shmem_shrink(struct drm_i915_gem_object *obj, unsigned int flags)
337{
338	switch (obj->mm.madv) {
339	case I915_MADV_DONTNEED:
340		return i915_gem_object_truncate(obj);
341	case __I915_MADV_PURGED:
342		return 0;
343	}
344
345	if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK)
346		shmem_writeback(obj);
347
348	return 0;
349}
350
351void
352__i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
353				struct sg_table *pages,
354				bool needs_clflush)
355{
356	struct drm_i915_private *i915 = to_i915(obj->base.dev);
357
358	GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
359
360	if (obj->mm.madv == I915_MADV_DONTNEED)
361		obj->mm.dirty = false;
362
363	if (needs_clflush &&
364	    (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
365	    !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
366		drm_clflush_sg(pages);
367
368	__start_cpu_write(obj);
369	/*
370	 * On non-LLC igfx platforms, force the flush-on-acquire if this is ever
371	 * swapped-in. Our async flush path is not trust worthy enough yet(and
372	 * happens in the wrong order), and with some tricks it's conceivable
373	 * for userspace to change the cache-level to I915_CACHE_NONE after the
374	 * pages are swapped-in, and since execbuf binds the object before doing
375	 * the async flush, we have a race window.
376	 */
377	if (!HAS_LLC(i915) && !IS_DGFX(i915))
378		obj->cache_dirty = true;
379}
380
381void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages)
382{
383	__i915_gem_object_release_shmem(obj, pages, true);
384
385	i915_gem_gtt_finish_pages(obj, pages);
386
387	if (i915_gem_object_needs_bit17_swizzle(obj))
388		i915_gem_object_save_bit_17_swizzle(obj, pages);
389
390	shmem_sg_free_table(pages, file_inode(obj->base.filp)->i_mapping,
391			    obj->mm.dirty, obj->mm.madv == I915_MADV_WILLNEED);
392	kfree(pages);
393	obj->mm.dirty = false;
394}
395
396static void
397shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages)
398{
399	if (likely(i915_gem_object_has_struct_page(obj)))
400		i915_gem_object_put_pages_shmem(obj, pages);
401	else
402		i915_gem_object_put_pages_phys(obj, pages);
403}
404
405static int
406shmem_pwrite(struct drm_i915_gem_object *obj,
407	     const struct drm_i915_gem_pwrite *arg)
408{
409	char __user *user_data = u64_to_user_ptr(arg->data_ptr);
410	struct file *file = obj->base.filp;
411	struct kiocb kiocb;
412	struct iov_iter iter;
413	ssize_t written;
414	u64 size = arg->size;
415
416	/* Caller already validated user args */
417	GEM_BUG_ON(!access_ok(user_data, arg->size));
418
419	if (!i915_gem_object_has_struct_page(obj))
420		return i915_gem_object_pwrite_phys(obj, arg);
421
422	/*
423	 * Before we instantiate/pin the backing store for our use, we
424	 * can prepopulate the shmemfs filp efficiently using a write into
425	 * the pagecache. We avoid the penalty of instantiating all the
426	 * pages, important if the user is just writing to a few and never
427	 * uses the object on the GPU, and using a direct write into shmemfs
428	 * allows it to avoid the cost of retrieving a page (either swapin
429	 * or clearing-before-use) before it is overwritten.
430	 */
431	if (i915_gem_object_has_pages(obj))
432		return -ENODEV;
433
434	if (obj->mm.madv != I915_MADV_WILLNEED)
435		return -EFAULT;
436
437	if (size > MAX_RW_COUNT)
438		return -EFBIG;
439
440	if (!file->f_op->write_iter)
441		return -EINVAL;
442
443	init_sync_kiocb(&kiocb, file);
444	kiocb.ki_pos = arg->offset;
445	iov_iter_ubuf(&iter, ITER_SOURCE, (void __user *)user_data, size);
446
447	written = file->f_op->write_iter(&kiocb, &iter);
448	BUG_ON(written == -EIOCBQUEUED);
449
450	/*
451	 * First, check if write_iter returned a negative error.
452	 * If the write failed, return the real error code immediately.
453	 * This prevents it from being overwritten by the short write check below.
454	 */
455	if (written < 0)
456		return written;
457	/*
458	 * Check for a short write (written bytes != requested size).
459	 * Even if some data was written, return -EIO to indicate that the
460	 * write was not fully completed.
461	 */
462	if (written != size)
463		return -EIO;
464
465	return 0;
466}
467
468static int
469shmem_pread(struct drm_i915_gem_object *obj,
470	    const struct drm_i915_gem_pread *arg)
471{
472	if (!i915_gem_object_has_struct_page(obj))
473		return i915_gem_object_pread_phys(obj, arg);
474
475	return -ENODEV;
476}
477
478static void shmem_release(struct drm_i915_gem_object *obj)
479{
480	if (i915_gem_object_has_struct_page(obj))
481		i915_gem_object_release_memory_region(obj);
482
483	fput(obj->base.filp);
484}
485
486const struct drm_i915_gem_object_ops i915_gem_shmem_ops = {
487	.name = "i915_gem_object_shmem",
488	.flags = I915_GEM_OBJECT_IS_SHRINKABLE,
489
490	.get_pages = shmem_get_pages,
491	.put_pages = shmem_put_pages,
492	.truncate = shmem_truncate,
493	.shrink = shmem_shrink,
494
495	.pwrite = shmem_pwrite,
496	.pread = shmem_pread,
497
498	.release = shmem_release,
499};
500
501static int __create_shmem(struct drm_i915_private *i915,
502			  struct drm_gem_object *obj,
503			  resource_size_t size)
504{
505	unsigned long flags = VM_NORESERVE;
506	struct file *filp;
507
508	drm_gem_private_object_init(&i915->drm, obj, size);
509
510	/* XXX: The __shmem_file_setup() function returns -EINVAL if size is
511	 * greater than MAX_LFS_FILESIZE.
512	 * To handle the same error as other code that returns -E2BIG when
513	 * the size is too large, we add a code that returns -E2BIG when the
514	 * size is larger than the size that can be handled.
515	 * If BITS_PER_LONG is 32, size > MAX_LFS_FILESIZE is always false,
516	 * so we only needs to check when BITS_PER_LONG is 64.
517	 * If BITS_PER_LONG is 32, E2BIG checks are processed when
518	 * i915_gem_object_size_2big() is called before init_object() callback
519	 * is called.
520	 */
521	if (BITS_PER_LONG == 64 && size > MAX_LFS_FILESIZE)
522		return -E2BIG;
523
524	if (i915->mm.gemfs)
525		filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size,
526						 flags);
527	else
528		filp = shmem_file_setup("i915", size, flags);
529	if (IS_ERR(filp))
530		return PTR_ERR(filp);
531
532	/*
533	 * Prevent -EFBIG by allowing large writes beyond MAX_NON_LFS on shmem
534	 * objects by setting O_LARGEFILE.
535	 */
536	if (force_o_largefile())
537		filp->f_flags |= O_LARGEFILE;
538
539	obj->filp = filp;
540	return 0;
541}
542
543static int shmem_object_init(struct intel_memory_region *mem,
544			     struct drm_i915_gem_object *obj,
545			     resource_size_t offset,
546			     resource_size_t size,
547			     resource_size_t page_size,
548			     unsigned int flags)
549{
550	static struct lock_class_key lock_class;
551	struct drm_i915_private *i915 = mem->i915;
552	struct address_space *mapping;
553	unsigned int cache_level;
554	gfp_t mask;
555	int ret;
556
557	ret = __create_shmem(i915, &obj->base, size);
558	if (ret)
559		return ret;
560
561	mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
562	if (IS_I965GM(i915) || IS_I965G(i915)) {
563		/* 965gm cannot relocate objects above 4GiB. */
564		mask &= ~__GFP_HIGHMEM;
565		mask |= __GFP_DMA32;
566	}
567
568	mapping = obj->base.filp->f_mapping;
569	mapping_set_gfp_mask(mapping, mask);
570	GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM));
571
572	i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class, flags);
573	obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE;
574	obj->write_domain = I915_GEM_DOMAIN_CPU;
575	obj->read_domains = I915_GEM_DOMAIN_CPU;
576
577	/*
578	 * MTL doesn't snoop CPU cache by default for GPU access (namely
579	 * 1-way coherency). However some UMD's are currently depending on
580	 * that. Make 1-way coherent the default setting for MTL. A follow
581	 * up patch will extend the GEM_CREATE uAPI to allow UMD's specify
582	 * caching mode at BO creation time
583	 */
584	if (HAS_LLC(i915) || (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)))
585		/* On some devices, we can have the GPU use the LLC (the CPU
586		 * cache) for about a 10% performance improvement
587		 * compared to uncached.  Graphics requests other than
588		 * display scanout are coherent with the CPU in
589		 * accessing this cache.  This means in this mode we
590		 * don't need to clflush on the CPU side, and on the
591		 * GPU side we only need to flush internal caches to
592		 * get data visible to the CPU.
593		 *
594		 * However, we maintain the display planes as UC, and so
595		 * need to rebind when first used as such.
596		 */
597		cache_level = I915_CACHE_LLC;
598	else
599		cache_level = I915_CACHE_NONE;
600
601	i915_gem_object_set_cache_coherency(obj, cache_level);
602
603	i915_gem_object_init_memory_region(obj, mem);
604
605	return 0;
606}
607
608struct drm_i915_gem_object *
609i915_gem_object_create_shmem(struct drm_i915_private *i915,
610			     resource_size_t size)
611{
612	return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM],
613					     size, 0, 0);
614}
615
616/* Allocate a new GEM object and fill it with the supplied data */
617struct drm_i915_gem_object *
618i915_gem_object_create_shmem_from_data(struct drm_i915_private *i915,
619				       const void *data, resource_size_t size)
620{
621	struct drm_i915_gem_object *obj;
622	struct file *file;
623	loff_t pos = 0;
624	ssize_t err;
625
626	GEM_WARN_ON(IS_DGFX(i915));
627	obj = i915_gem_object_create_shmem(i915, round_up(size, PAGE_SIZE));
628	if (IS_ERR(obj))
629		return obj;
630
631	GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);
632
633	file = obj->base.filp;
634	err = kernel_write(file, data, size, &pos);
635
636	if (err < 0)
637		goto fail;
638
639	if (err != size) {
640		err = -EIO;
641		goto fail;
642	}
643
644	return obj;
645
646fail:
647	i915_gem_object_put(obj);
648	return ERR_PTR(err);
649}
650
651static int init_shmem(struct intel_memory_region *mem)
652{
653	i915_gemfs_init(mem->i915);
654	intel_memory_region_set_name(mem, "system");
655
656	return 0; /* We have fallback to the kernel mnt if gemfs init failed. */
657}
658
659static int release_shmem(struct intel_memory_region *mem)
660{
661	i915_gemfs_fini(mem->i915);
662	return 0;
663}
664
665static const struct intel_memory_region_ops shmem_region_ops = {
666	.init = init_shmem,
667	.release = release_shmem,
668	.init_object = shmem_object_init,
669};
670
671struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_private *i915,
672						 u16 type, u16 instance)
673{
674	return intel_memory_region_create(i915, 0,
675					  totalram_pages() << PAGE_SHIFT,
676					  PAGE_SIZE, 0, 0,
677					  type, instance,
678					  &shmem_region_ops);
679}
680
681bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj)
682{
683	return obj->ops == &i915_gem_shmem_ops;
684}